diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index e1fd84a66..0bc77b2c0 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -8,6 +8,9 @@ import { Attribute } from 'vue-media-annotator/use/AttributeTypes'; import { CustomStyle } from 'vue-media-annotator/StyleManager'; import { AttributeTrackFilter } from 'vue-media-annotator/AttributeTrackFilterControls'; import { ImageEnhancements } from 'vue-media-annotator/use/useImageEnhancements'; +import { + CameraHomographies, CameraCorrespondences, CameraTransformTypes, CalibrationSource, +} from 'vue-media-annotator/CameraCalibrationStore'; import type { PercentileStretch } from 'vue-media-annotator/use/useImageEnhancements'; type DatasetType = 'image-sequence' | 'video' | 'multi' | 'large-image'; @@ -136,6 +139,12 @@ export interface MultiCamImportFolderArgs { sourceList: Record; // path/track file per camera @@ -186,9 +195,14 @@ interface DatasetMetaMutable { attributes?: Readonly>; attributeTrackFilters?: Readonly>; datasetInfo?: Record; + cameraHomographies?: CameraHomographies; + cameraCorrespondences?: CameraCorrespondences; + cameraTransformTypes?: CameraTransformTypes; + /** Producer provenance of the camera calibration (see CalibrationSource). */ + cameraCalibrationSource?: CalibrationSource | null; error?: string; } -const DatasetMetaMutableKeys = ['attributes', 'confidenceFilters', 'timeFilters', 'imageEnhancements', 'customTypeStyling', 'customGroupStyling', 'attributeTrackFilters', 'datasetInfo']; +const DatasetMetaMutableKeys = ['attributes', 'confidenceFilters', 'timeFilters', 'imageEnhancements', 'customTypeStyling', 'customGroupStyling', 'attributeTrackFilters', 'datasetInfo', 'cameraHomographies', 'cameraCorrespondences', 'cameraTransformTypes', 'cameraCalibrationSource']; interface DatasetMeta extends DatasetMetaMutable { id: Readonly; @@ -278,7 +292,7 @@ interface Api { saveAttributeTrackFilters(datasetId: string, args: SaveAttributeTrackFilterArgs): Promise; // Non-Endpoint shared functions - openFromDisk(datasetType: DatasetType | 'bulk' | 'calibration' | 'annotation' | 'text' | 'zip', directory?: boolean): + openFromDisk(datasetType: DatasetType | 'bulk' | 'calibration' | 'annotation' | 'text' | 'zip' | 'transform', directory?: boolean): Promise<{canceled?: boolean; filePaths: string[]; fileList?: File[]; root?: string}>; /** Desktop: immediate child directory names under a parent folder (multicam subfolder import). */ listImmediateSubfolders?(parentPath: string): Promise; @@ -294,6 +308,8 @@ interface Api { ): Promise; /** Desktop: stereoscopic calibration file in a parent folder root. */ findParentFolderCalibrationFile?(parentPath: string): Promise; + /** Desktop: DIVE camera-calibration .json (alignment transforms) in a parent folder root. */ + findParentFolderTransformFile?(parentPath: string): Promise; /** True when the dataset folder has an attached stereoscopic calibration file. */ hasCalibrationFile?(datasetId: string): Promise; /** Web: stash a calibration File for multicam upload lookup. */ diff --git a/client/dive-common/components/CameraCalibration/CalibrationTools.vue b/client/dive-common/components/CameraCalibration/CalibrationTools.vue new file mode 100644 index 000000000..39f79c67e --- /dev/null +++ b/client/dive-common/components/CameraCalibration/CalibrationTools.vue @@ -0,0 +1,677 @@ + + + diff --git a/client/dive-common/components/ImportMultiCamDialog/ImportMultiCamChooseTransform.vue b/client/dive-common/components/ImportMultiCamDialog/ImportMultiCamChooseTransform.vue new file mode 100644 index 000000000..9e1c3bd1a --- /dev/null +++ b/client/dive-common/components/ImportMultiCamDialog/ImportMultiCamChooseTransform.vue @@ -0,0 +1,46 @@ + + + + diff --git a/client/dive-common/components/ImportMultiCamDialog/ImportMultiCamDialog.vue b/client/dive-common/components/ImportMultiCamDialog/ImportMultiCamDialog.vue index b6471e36d..9ada86978 100644 --- a/client/dive-common/components/ImportMultiCamDialog/ImportMultiCamDialog.vue +++ b/client/dive-common/components/ImportMultiCamDialog/ImportMultiCamDialog.vue @@ -42,6 +42,11 @@ export default defineComponent({ type: Boolean, default: false, }, + /** Offer per-camera calibration .json transform pickers (desktop multicam only). */ + enableTransformImport: { + type: Boolean, + default: false, + }, registerSubfolderCameras: { type: Function as PropType<(assignments: { cameraName: string; diff --git a/client/dive-common/components/ImportMultiCamDialog/ImportMultiCamMultiFolder.vue b/client/dive-common/components/ImportMultiCamDialog/ImportMultiCamMultiFolder.vue index b661f653b..d0182cb41 100644 --- a/client/dive-common/components/ImportMultiCamDialog/ImportMultiCamMultiFolder.vue +++ b/client/dive-common/components/ImportMultiCamDialog/ImportMultiCamMultiFolder.vue @@ -8,6 +8,7 @@ import { DatasetType } from 'dive-common/apispec'; import ImportMultiCamCameraGroup from './ImportMultiCamCameraGroup.vue'; import ImportMultiCamChooseSource from './ImportMultiCamChooseSource.vue'; import ImportMultiCamChooseAnnotation from './ImportMultiCamChooseAnnotation.vue'; +import ImportMultiCamChooseTransform from './ImportMultiCamChooseTransform.vue'; import ImportMultiCamAddType from './ImportMultiCamAddType.vue'; import ImportMultiCamCameraOrderControls from './ImportMultiCamCameraOrderControls.vue'; import { importMultiCamContextProp } from './importMultiCamContext'; @@ -18,6 +19,7 @@ export default defineComponent({ ImportMultiCamCameraGroup, ImportMultiCamChooseSource, ImportMultiCamChooseAnnotation, + ImportMultiCamChooseTransform, ImportMultiCamAddType, ImportMultiCamCameraOrderControls, }, @@ -43,6 +45,9 @@ export default defineComponent({ addNewSet: props.ctx.addNewSet, open: props.ctx.open, openAnnotationFile: props.ctx.openAnnotationFile, + showTransformFileField: props.ctx.showTransformFileField, + openTransformFile: props.ctx.openTransformFile, + clearTransformFile: props.ctx.clearTransformFile, }; }, }); @@ -81,6 +86,14 @@ export default defineComponent({ @clear="folderList[key].trackFile = ''" @open="openAnnotationFile(key)" /> + {{ pendingImportPayloads[key].jsonMeta.originalImageFiles.length }} files + Promise; + /** + * Offer a per-camera calibration .json transform file picker for cameras + * after the first (desktop only; the file is parsed by the desktop backend + * at import time). Ignored for stereo imports, which use calibration files. + */ + enableTransformImport?: boolean; enableSubfolderImport?: boolean; registerSubfolderCameras?: (assignments: { cameraName: string; @@ -59,11 +65,13 @@ export function useImportMultiCamDialog( listParentFolderCameras, resolveMulticamCameraSourcePath, findParentFolderCalibrationFile, + findParentFolderTransformFile, } = useApi(); const importType: Ref = ref(''); const folderList: Ref> = ref({}); const parentFolderName = ref(''); @@ -107,6 +115,12 @@ export function useImportMultiCamDialog( && !!lastCalibrationPath.value, ); + // Per-camera transform pickers: multicam only (stereo uses calibration), + // folder-based modes only, and never for the first (reference) camera. + const transformImportEnabled = computed( + () => !!props.enableTransformImport && !props.stereo, + ); + const orderedCameraKeys = computed(() => { const keys = Object.keys(folderList.value); const ordered = cameraOrder.value.filter((key) => keys.includes(key)); @@ -138,8 +152,8 @@ export function useImportMultiCamDialog( defaultDisplay.value = props.stereo ? 'left' : 'center'; if (props.stereo && importType.value === 'multi') { folderList.value = { - left: { sourcePath: '', trackFile: '' }, - right: { sourcePath: '', trackFile: '' }, + left: { sourcePath: '', trackFile: '', transformFile: '' }, + right: { sourcePath: '', trackFile: '', transformFile: '' }, }; } else { folderList.value = {}; @@ -401,6 +415,7 @@ export function useImportMultiCamDialog( Vue.set(folderList.value, cameraName, { sourcePath, trackFile: '', + transformFile: '', type: props.registerSubfolderCameras && props.dataType !== VideoType ? inferSubfolderImportType(files, { largeImageForTiff: true }) : props.dataType, @@ -414,6 +429,7 @@ export function useImportMultiCamDialog( { preferLeftForStereo: props.stereo }, ); syncDefaultDisplay(); + await discoverParentFolderTransform(parentPath); }); } @@ -440,6 +456,7 @@ export function useImportMultiCamDialog( Vue.set(folderList.value, newKey, { sourcePath: (importType.value === 'subfolders' && !listParentFolderCameras) ? newKey : sourcePath, trackFile: entry.trackFile, + transformFile: entry.transformFile, }); Vue.delete(folderList.value, oldKey); @@ -479,6 +496,7 @@ export function useImportMultiCamDialog( Vue.set(subfolderOriginalNames.value, cameraKey, displayName); folderList.value[cameraKey].sourcePath = resolvedPath; folderList.value[cameraKey].trackFile = ''; + folderList.value[cameraKey].transformFile = ''; if (props.registerSubfolderCameras && files.length) { props.registerSubfolderCameras([{ cameraName: cameraKey, @@ -503,6 +521,23 @@ export function useImportMultiCamDialog( } } + function showTransformFileField(folder: string) { + return transformImportEnabled.value + && (importType.value === 'multi' || importType.value === 'subfolders') + && orderedCameraKeys.value.indexOf(folder) > 0; + } + + async function openTransformFile(folder: string) { + const ret = await openFromDisk('transform'); + if (!ret.canceled && ret.filePaths?.length) { + [folderList.value[folder].transformFile] = ret.filePaths; + } + } + + function clearTransformFile(folder: string) { + folderList.value[folder].transformFile = ''; + } + async function open( dstype: DatasetType | 'calibration' | 'text', folder: string | 'calibration', @@ -525,6 +560,7 @@ export function useImportMultiCamDialog( folderList.value[folder].sourcePath = path; } folderList.value[folder].trackFile = ''; + folderList.value[folder].transformFile = ''; const { sourcePath } = folderList.value[folder]; if (props.registerSubfolderCameras && ret.fileList?.length) { props.registerSubfolderCameras([{ @@ -583,7 +619,7 @@ export function useImportMultiCamDialog( const addNewSet = (name: string) => { if (importType.value === 'multi') { - Vue.set(folderList.value, name, { sourcePath: '', trackFile: '' }); + Vue.set(folderList.value, name, { sourcePath: '', trackFile: '', transformFile: '' }); Vue.set(pendingImportPayloads.value, name, null); cameraOrder.value = [...cameraOrder.value, name]; } else if (importType.value === 'keyword') { @@ -601,12 +637,22 @@ export function useImportMultiCamDialog( const sourceList: MultiCamImportFolderArgs['sourceList'] = {}; orderedCameraKeys.value.forEach((key) => { if (folderList.value[key]) { - const { sourcePath, trackFile, type } = folderList.value[key]; + const { + sourcePath, trackFile, transformFile, type, + } = folderList.value[key]; + if (type === 'multi') { + // Sub Cameras shouldn't be multi types + return; + } sourceList[key] = { sourcePath, trackFile, ...(type ? { type } : {}), }; + // Transforms only apply to cameras after the first (reference) one. + if (transformFile && showTransformFileField(key)) { + sourceList[key].transformFile = transformFile; + } } }); const args: MultiCamImportFolderArgs = { @@ -670,6 +716,29 @@ export function useImportMultiCamDialog( return folderName; } + /** + * Auto-attach a DIVE camera-calibration .json found in the parent folder + * root as the import's transform file (desktop multicam subfolder imports + * only). It is attached to the first camera after the reference -- the + * file's pairs name their own cameras, so which slot carries it doesn't + * matter -- and shows in that camera's (clearable) transform field. + */ + async function discoverParentFolderTransform(parentPath: string) { + if (!transformImportEnabled.value || !findParentFolderTransformFile) { + return; + } + const discovered = await findParentFolderTransformFile(parentPath); + if (!discovered) { + return; + } + const target = orderedCameraKeys.value.find( + (key, index) => index > 0 && folderList.value[key] && !folderList.value[key].transformFile, + ); + if (target) { + folderList.value[target].transformFile = discovered; + } + } + async function discoverParentFolderCalibration( parentPath: string, fileList?: File[], @@ -737,6 +806,10 @@ export function useImportMultiCamDialog( deleteSet, onRenameCamera, openAnnotationFile, + transformImportEnabled, + showTransformFileField, + openTransformFile, + clearTransformFile, clearCalibration, }; } diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index 24f459080..428fd5cbe 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -18,8 +18,11 @@ import { import { Track, Group, CameraStore, + CameraCalibrationStore, + AlignedViewStore, StyleManager, TrackFilterControls, GroupFilterControls, } from 'vue-media-annotator/index'; +import { resolveToReferenceTransforms, unresolvedCameras } from 'vue-media-annotator/alignedView'; import { provideAnnotator, LassoModeSymbol } from 'vue-media-annotator/provides'; import { @@ -28,6 +31,8 @@ import { LargeImageAnnotator, LayerManager, useMediaController, + useCalibrationNavigation, + useAlignedNavigation, TrackList, FilterList, } from 'vue-media-annotator/components'; @@ -83,6 +88,7 @@ import context from 'dive-common/store/context'; import { MarkChangesPendingFilter } from 'vue-media-annotator/BaseFilterControls'; import GroupSidebarVue from './GroupSidebar.vue'; import MultiCamToolsVue from './MultiCamTools.vue'; +import CalibrationToolsVue from './CameraCalibration/CalibrationTools.vue'; import MultiCamToolbar from './MultiCamToolbar.vue'; import PrimaryAttributeTrackFilter from './PrimaryAttributeTrackFilter.vue'; import UserSettingsDialog from './UserSettingsDialog.vue'; @@ -316,6 +322,29 @@ export default defineComponent({ }); const showUserSettingsDialog = ref(false); + // When the Manual Alignment panel opens, minimize the workspace chrome to + // give the picking view more room: collapse the left type-filter sidebar and + // the bottom detections graph. This is a soft default -- the normal sidebar + // and timeline toggles still work while calibrating, so the user can bring + // either back -- and whatever layout they had before is restored on close. + const calibrationActive = computed(() => context.state.active === CalibrationToolsVue.name); + let preCalibrationSidebarMode: 'left' | 'bottom' | 'collapsed' | null = null; + let preCalibrationControlsCollapsed = false; + watch(calibrationActive, (active) => { + if (active) { + preCalibrationSidebarMode = sidebarMode.value; + preCalibrationControlsCollapsed = controlsCollapsed.value; + if (sidebarMode.value === 'left') { + sidebarMode.value = 'collapsed'; + } + controlsCollapsed.value = true; + } else if (preCalibrationSidebarMode !== null) { + sidebarMode.value = preCalibrationSidebarMode; + controlsCollapsed.value = preCalibrationControlsCollapsed; + preCalibrationSidebarMode = null; + } + }); + watch(sidebarMode, (mode) => { if (mode === 'left' || mode === 'bottom') { clientSettings.layoutSettings.sidebarPosition = mode; @@ -395,6 +424,94 @@ export default defineComponent({ const groupStyleManager = new StyleManager({ markChangesPending, vuetify }); const cameraStore = new CameraStore({ markChangesPending }); + const cameraCalibration = new CameraCalibrationStore(); + + /** + * Aligned view (SEAL-TK features 2 + 3): when every non-reference camera + * has a usable transform into the reference camera's space, the user may + * warp displays and link pan/zoom across all cameras during normal + * review. Reference camera = first camera in display order + * (multiCamList[0]). Transforms come from the calibration store's pair + * homographies (picked in-app or loaded from a calibration file via the + * panel), composed through the pair graph -- the single calibration the + * panel edits and saves is exactly what the Align button applies. + */ + const alignedView = new AlignedViewStore(); + // The calibration pair link maps through the homography for UNWARPED + // panes, so it needs the aligned view state to stand down while displays + // are warped into reference space. + useCalibrationNavigation(aggregateController, cameraCalibration, alignedView); + const alignedResolution = computed(() => { + if (multiCamList.value.length < 2) { + return null; + } + const reference = multiCamList.value[0]; + const toReference = resolveToReferenceTransforms( + multiCamList.value, + reference, + cameraCalibration.homographies.value, + ); + return toReference ? { reference, toReference } : null; + }); + watch(alignedResolution, (resolution) => { + alignedView.setTransforms( + resolution?.reference ?? null, + resolution?.toReference ?? null, + ); + }, { immediate: true }); + // Calibration point picking records raw native-space clicks and renders + // its own aligned preview; suspend the general warp while it is active + // so picks are never taken against a warped display (risk K2). + watch(cameraCalibration.pickingEnabled, (picking) => { + alignedView.setSuspended(picking); + }, { immediate: true }); + useAlignedNavigation(aggregateController, alignedView, multiCamList); + const alignedViewAvailable = computed(() => alignedView.available.value); + const alignedViewEnabled = computed(() => alignedView.enabled.value); + const toggleAlignedView = () => { + alignedView.setEnabled(!alignedView.enabled.value); + }; + const alignedViewTooltip = computed(() => { + if (alignedViewEnabled.value) { + return 'Align View on (draw/edit on any camera)'; + } + const cams = multiCamList.value; + const unresolved = cams.length >= 2 + ? unresolvedCameras(cams, cams[0], cameraCalibration.homographies.value) + : []; + if (unresolved.length) { + return `Align View — ${cams.length - unresolved.length}/${cams.length} cameras calibrated`; + } + return 'Align View'; + }); + // The aligned view is suspended while picking, so the button reads as + // unavailable rather than accepting a toggle that has no visible effect. + const calibrationPickingEnabled = computed(() => cameraCalibration.pickingEnabled.value); + /** + * Camera panes currently displayed. While the Manual Alignment panel is + * open with an active pair on a 3+ camera dataset, only the pair's two + * panes show, so the left/right alignment flow reads without unrelated + * panes in between (regardless of whether Pick points is toggled on). + * Panes are hidden (v-show), not unmounted, so their viewers keep state. + */ + const displayedCameras = computed(() => { + const pair = cameraCalibration.activePair.value; + if (calibrationActive.value && pair) { + const pairCameras = multiCamList.value.filter( + (camera) => camera === pair.camA || camera === pair.camB, + ); + if (pairCameras.length === 2) { + return pairCameras; + } + } + return multiCamList.value; + }); + watch(displayedCameras, async () => { + // Hidden/shown siblings change the remaining panes' sizes; resize the + // geojs maps once the DOM has settled. + await nextTick(); + handleResize(); + }); // This context for removal const removeGroups = (id: AnnotationId) => { cameraStore.removeGroups(id); @@ -462,6 +579,7 @@ export default defineComponent({ cameraStore, aggregateController, readonlyState, + alignedView, isStereoscopicDataset: computed(() => subType.value === 'stereo'), onStereoAnnotationComplete: (params: StereoAnnotationCompleteParams) => { emit('stereo-annotation-complete', params); @@ -1377,6 +1495,17 @@ export default defineComponent({ if (meta.attributeTrackFilters) { trackFilters.loadTrackAttributesFilter(Object.values(meta.attributeTrackFilters)); } + // Rehydrate any saved camera-to-camera calibration homographies, points, + // transform types, and producer provenance + cameraCalibration.hydrate( + meta.cameraHomographies, + meta.cameraCorrespondences, + meta.cameraTransformTypes, + meta.cameraCalibrationSource, + ); + // Reset the aligned-view toggle for the newly loaded dataset (no + // persistence this phase). + alignedView.setEnabled(false); setImageEnhancements( imageEnhancementsByCamera.value[selectedCamera.value] ?? { ...defaultImageEnhancements }, ); @@ -1393,11 +1522,19 @@ export default defineComponent({ component: MultiCamToolsVue, description: 'Multi Camera Tools', }); + context.register({ + component: CalibrationToolsVue, + description: 'Manual Alignment', + }); } else { context.unregister({ component: MultiCamToolsVue, description: 'Multi Camera Tools', }); + context.unregister({ + component: CalibrationToolsVue, + description: 'Manual Alignment', + }); context.register({ description: 'Group Manager', component: GroupSidebarVue, @@ -1457,7 +1594,13 @@ export default defineComponent({ if (previous) observer.unobserve(previous.$el); if (controlsRef.value) observer.observe(controlsRef.value.$el); }); - watch([controlsCollapsed, sidebarMode], async () => { + // Opening/closing the context sidebar shrinks or widens the camera panes, + // but nothing else notices: the only ResizeObserver watches the controls + // bar, which is position:absolute in side layout and so keeps its content + // width when the panes resize. Trigger a resize explicitly so the panes' + // GeoJS size() stays in sync (an unnoticed shrink leaves content anchored + // in a corner). + watch([controlsCollapsed, sidebarMode, () => context.state.active], async () => { await nextTick(); handleResize(); }); @@ -1502,6 +1645,8 @@ export default defineComponent({ annotatorPreferences: toRef(clientSettings, 'annotatorPreferences'), attributes, cameraStore, + cameraCalibration, + alignedView, datasetId, editingMode, groupFilters, @@ -1800,6 +1945,12 @@ export default defineComponent({ deleteAttributeHandler, saveTooltipText, showMultiCamToolbar, + alignedViewAvailable, + alignedViewEnabled, + alignedViewTooltip, + calibrationPickingEnabled, + displayedCameras, + toggleAlignedView, seekToFrame, resetAggregateZoom, /* large image methods */ @@ -1983,6 +2134,37 @@ export default defineComponent({ {{ item }} {{ item === defaultCamera ? '(Default)' : '' }} + + + + {{ alignedViewTooltip }} + @@ -2112,10 +2294,16 @@ export default defineComponent({ class="d-flex flex-column grow" >
+
= { description: 'Multi Camera Tools', component: MultiCamTools, }, + [CalibrationTools.name]: { + description: 'Manual Alignment', + component: CalibrationTools, + }, [AttributesSideBar.name]: { description: 'Attribute Details', component: AttributesSideBar, diff --git a/client/dive-common/use/useModeManager.spec.ts b/client/dive-common/use/useModeManager.spec.ts new file mode 100644 index 000000000..40eae73af --- /dev/null +++ b/client/dive-common/use/useModeManager.spec.ts @@ -0,0 +1,161 @@ +/// +/** + * Functional tests for the Align View cross-camera mirror: drawing/editing a + * track on one camera while the aligned view is active re-projects the + * geometry onto every other calibrated camera under the same track id. + */ +import { ref } from 'vue'; +import CameraStore from 'vue-media-annotator/CameraStore'; +import AlignedViewStore from 'vue-media-annotator/AlignedViewStore'; +import TrackFilterControls from 'vue-media-annotator/TrackFilterControls'; +import GroupFilterControls from 'vue-media-annotator/GroupFilterControls'; +import { IDENTITY3 } from 'vue-media-annotator/alignedView'; +import type { Matrix3 } from 'vue-media-annotator/homography'; +import type { AggregateMediaController } from 'vue-media-annotator/components/annotators/mediaControllerType'; +import type { AnnotationId } from 'vue-media-annotator/BaseAnnotation'; +import useModeManager from './useModeManager'; + +function translation(tx: number, ty: number): Matrix3 { + return [[1, 0, tx], [0, 1, ty], [0, 0, 1]]; +} + +function makeHarness() { + const cameraStore = new CameraStore({ markChangesPending: () => undefined }); + cameraStore.removeCamera('singleCam'); + cameraStore.addCamera('left'); + cameraStore.addCamera('right'); + + // right -> reference(left) shifts x by -100, so left -> right adds +100. + const alignedView = new AlignedViewStore(); + alignedView.setTransforms('left', { + left: IDENTITY3, + right: translation(-100, 0), + }); + alignedView.setEnabled(true); + + const perCamera: Record; hasFrame: ReturnType }> = { + left: { frame: ref(0), hasFrame: ref(true) }, + right: { frame: ref(0), hasFrame: ref(true) }, + }; + const aggregateController = ref({ + frame: ref(0), + nextFrame: () => undefined, + seekCameraFrame: () => undefined, + getController: (name: string) => perCamera[name], + } as unknown as AggregateMediaController); + + const groupFilterControls = new GroupFilterControls({ + sorted: cameraStore.sortedGroups, + remove: () => undefined, + markChangesPending: () => undefined, + setType: () => undefined, + removeTypes: () => [], + }); + const trackFilterControls = new TrackFilterControls({ + sorted: cameraStore.sortedTracks, + remove: () => undefined, + markChangesPending: () => undefined, + lookupGroups: cameraStore.lookupGroups.bind(cameraStore), + getTrack: (id: AnnotationId, camera = 'singleCam') => cameraStore.getTrack(id, camera), + groupFilterControls, + setType: () => undefined, + removeTypes: () => [], + }); + + const modeManager = useModeManager({ + cameraStore, + trackFilterControls, + groupFilterControls, + aggregateController, + readonlyState: ref(false), + recipes: [], + alignedView, + }); + modeManager.selectedCamera.value = 'left'; + return { + cameraStore, alignedView, modeManager, perCamera, + }; +} + +describe('useModeManager aligned-view track mirroring', () => { + it('mirrors a drawn rectangle onto the other camera, creating the same-id track', () => { + const { cameraStore, modeManager } = makeHarness(); + const trackId = modeManager.handler.trackAdd(); + modeManager.handler.updateRectBounds(0, 0, [10, 20, 30, 40]); + + const mirrored = cameraStore.getPossibleTrack(trackId, 'right'); + expect(mirrored).toBeDefined(); + expect(mirrored?.features[0]?.bounds).toEqual([110, 20, 130, 40]); + expect(mirrored?.confidencePairs[0][0]) + .toEqual(cameraStore.getTrack(trackId, 'left').confidencePairs[0][0]); + }); + + it('continuously re-mirrors subsequent edits (continuous mirror)', () => { + const { cameraStore, modeManager } = makeHarness(); + const trackId = modeManager.handler.trackAdd(); + modeManager.handler.updateRectBounds(0, 0, [10, 20, 30, 40]); + modeManager.handler.trackSelect(trackId, true); + modeManager.handler.updateRectBounds(0, 0, [50, 60, 70, 80]); + + const mirrored = cameraStore.getTrack(trackId, 'right'); + expect(mirrored.features[0]?.bounds).toEqual([150, 60, 170, 80]); + }); + + it('mirrors polygon geometry coordinates into the target camera space', () => { + const { cameraStore, modeManager } = makeHarness(); + const trackId = modeManager.handler.trackAdd(); + modeManager.handler.setTrackFeature(0, [0, 0, 4, 4], [{ + type: 'Feature', + geometry: { type: 'Polygon', coordinates: [[[0, 0], [4, 0], [4, 4], [0, 0]]] }, + properties: { key: '' }, + }]); + + const mirrored = cameraStore.getTrack(trackId, 'right'); + expect(mirrored.features[0]?.bounds).toEqual([100, 0, 104, 4]); + const polygon = mirrored.features[0]?.geometry?.features[0]; + expect(polygon?.geometry).toEqual({ + type: 'Polygon', + coordinates: [[[100, 0], [104, 0], [104, 4], [100, 0]]], + }); + }); + + it('maps through per-camera local frames when they diverge (aligned timeline)', () => { + const { cameraStore, modeManager, perCamera } = makeHarness(); + perCamera.left.frame.value = 5; + perCamera.right.frame.value = 8; + const trackId = modeManager.handler.trackAdd(); + modeManager.handler.updateRectBounds(5, 0, [10, 20, 30, 40]); + + const mirrored = cameraStore.getTrack(trackId, 'right'); + expect(mirrored.features[8]?.bounds).toEqual([110, 20, 130, 40]); + expect(mirrored.features[5]).toBeUndefined(); + }); + + it('skips cameras with no frame at the current aligned slot', () => { + const { cameraStore, modeManager, perCamera } = makeHarness(); + perCamera.right.hasFrame.value = false; + const trackId = modeManager.handler.trackAdd(); + modeManager.handler.updateRectBounds(0, 0, [10, 20, 30, 40]); + + expect(cameraStore.getPossibleTrack(trackId, 'right')).toBeUndefined(); + }); + + it('does not mirror when the aligned view is disabled', () => { + const { cameraStore, alignedView, modeManager } = makeHarness(); + alignedView.setEnabled(false); + const trackId = modeManager.handler.trackAdd(); + modeManager.handler.updateRectBounds(0, 0, [10, 20, 30, 40]); + + expect(cameraStore.getPossibleTrack(trackId, 'right')).toBeUndefined(); + expect(cameraStore.getTrack(trackId, 'left').features[0]?.bounds).toEqual([10, 20, 30, 40]); + }); + + it('does not mirror while the aligned view is suspended (calibration picking)', () => { + const { cameraStore, alignedView, modeManager } = makeHarness(); + alignedView.setSuspended(true); + const trackId = modeManager.handler.trackAdd(); + modeManager.handler.updateRectBounds(0, 0, [10, 20, 30, 40]); + + expect(cameraStore.getPossibleTrack(trackId, 'right')).toBeUndefined(); + }); +}); diff --git a/client/dive-common/use/useModeManager.ts b/client/dive-common/use/useModeManager.ts index 45e8813fa..78a8e17ab 100644 --- a/client/dive-common/use/useModeManager.ts +++ b/client/dive-common/use/useModeManager.ts @@ -7,8 +7,13 @@ import { RectBounds, updateBounds, validateRotation, + getRotationFromAttributes, ROTATION_ATTRIBUTE_NAME, } from 'vue-media-annotator/utils'; +import type AlignedViewStore from 'vue-media-annotator/AlignedViewStore'; +import { + mapBounds, mapRotatedBounds, mapGeoJSONFeatures, +} from 'vue-media-annotator/alignedView'; import { EditAnnotationTypes, VisibleAnnotationTypes } from 'vue-media-annotator/layers'; import { AggregateMediaController } from 'vue-media-annotator/components/annotators/mediaControllerType'; @@ -97,6 +102,7 @@ export default function useModeManager({ aggregateController, readonlyState, recipes, + alignedView, isStereoscopicDataset, onStereoAnnotationComplete, onStereoAnnotationReset, @@ -108,6 +114,12 @@ export default function useModeManager({ aggregateController: Ref; readonlyState: Readonly>; recipes: Recipe[]; + /** + * When provided, geometry drawn/edited while the Align View is active is + * mirrored onto every other calibrated camera (see + * {@link mirrorFeatureToAlignedCameras}). + */ + alignedView?: AlignedViewStore; /** When set, interactive stereo only runs on stereoscopic datasets. */ isStereoscopicDataset?: Ref; onStereoAnnotationComplete?: (params: StereoAnnotationCompleteParams) => void; @@ -553,6 +565,123 @@ export default function useModeManager({ creating = newCreatingValue; } + /** + * Continuous cross-camera mirror: while the Align View is active (the rig + * is fully calibrated and the user toggled the aligned display), re-project + * the source camera's keyframe at `sourceFrame` onto every other camera + * through the calibrated camera-to-camera homographies, creating the + * same-id track on cameras where it doesn't exist yet. Stored geometry + * stays native per camera (decision D3): each camera receives coordinates + * already mapped into its own image space. + * + * Called after every geometry write/removal below; a no-op when the + * aligned view is off, so single-camera and unaligned multicam behavior is + * unchanged. The camera that received the edit is always the source, so + * later edits on any camera re-sync the others (continuous mirror -- + * per-camera fine-tuning is intentionally overwritten by the next edit). + */ + function mirrorFeatureToAlignedCameras(trackId: AnnotationId, sourceFrame: number) { + if (!alignedView?.active.value) { + return; + } + const sourceCamera = selectedCamera.value; + const sourceTrack = cameraStore.getPossibleTrack(trackId, sourceCamera); + if (!sourceTrack) { + return; + } + const sourceFeature = sourceTrack.features[sourceFrame]; + const hasKeyframe = Boolean(sourceFeature && sourceFeature.keyframe); + // Under an aligned timeline, cameras sit on different local frames for + // the same instant; the per-camera controller frames give the mapping + // for the CURRENT slot. Edits at any other frame (e.g. multi-frame + // segmentation) fall back to the same local frame number. + const sourceIsCurrentFrame = sourceFrame === selectedCameraFrame(); + cameraStore.camMap.value.forEach(({ trackStore }, cameraName) => { + if (cameraName === sourceCamera) { + return; + } + const matrix = alignedView.cameraToCamera(sourceCamera, cameraName); + if (!matrix) { + return; + } + let targetFrame = sourceFrame; + if (sourceIsCurrentFrame) { + try { + const controller = aggregateController.value.getController(cameraName); + if (!controller.hasFrame.value) { + // This camera has no frame at the current aligned slot. + return; + } + targetFrame = controller.frame.value; + } catch { + // No controller mounted for the camera; assume matching local frames. + } + } + let targetTrack = trackStore.getPossible(trackId); + if (!hasKeyframe) { + // The source keyframe was removed: drop the mirrored keyframe too, + // and the mirrored track itself when that leaves it empty. + if (targetTrack && targetTrack.features[targetFrame]?.keyframe) { + targetTrack.deleteFeature(targetFrame); + if (targetTrack.begin === targetTrack.end + && !targetTrack.getFeature(targetTrack.begin).some((item) => item !== null)) { + trackStore.remove(trackId); + } + } + return; + } + const mappedGeometry = sourceFeature.geometry + ? mapGeoJSONFeatures(matrix, sourceFeature.geometry.features) + : []; + let mappedBounds: RectBounds | undefined; + let mappedRotation: number | undefined; + const sourceRotation = getRotationFromAttributes(sourceFeature.attributes); + if (sourceFeature.bounds) { + if (sourceRotation !== undefined) { + const mapped = mapRotatedBounds(matrix, sourceFeature.bounds, sourceRotation); + mappedBounds = mapped.bounds; + mappedRotation = validateRotation(mapped.rotation); + } else { + mappedBounds = mapBounds(matrix, sourceFeature.bounds); + } + } + if (!targetTrack) { + targetTrack = trackStore.add( + targetFrame, + sourceTrack.confidencePairs[0][0], + undefined, + trackId, + ); + } + // setFeature only upserts geometry by (key, type): drop mirrored + // geometry the source no longer has so deletions propagate too. + const targetFeature = targetTrack.features[targetFrame]; + if (targetFeature?.geometry) { + const mappedKeys = new Set(mappedGeometry.map( + (geo) => `${geo.properties?.key ?? ''}|${geo.geometry.type}`, + )); + targetFeature.geometry.features = targetFeature.geometry.features.filter( + (geo) => mappedKeys.has(`${geo.properties?.key ?? ''}|${geo.geometry.type}`), + ); + } + targetTrack.setFeature({ + frame: targetFrame, + flick: sourceFeature.flick, + bounds: mappedBounds, + keyframe: true, + interpolate: sourceFeature.interpolate, + }, mappedGeometry); + if (mappedRotation !== undefined) { + targetTrack.setFeatureAttribute(targetFrame, ROTATION_ATTRIBUTE_NAME, mappedRotation); + } else { + const written = targetTrack.features[targetFrame]; + if (written?.attributes && ROTATION_ATTRIBUTE_NAME in written.attributes) { + targetTrack.setFeatureAttribute(targetFrame, ROTATION_ATTRIBUTE_NAME, undefined); + } + } + }); + } + function handleUpdateRectBounds(frameNum: number, flickNum: number, bounds: RectBounds, rotation?: number) { if (selectedTrackId.value !== null) { const track = cameraStore.getPossibleTrack(selectedTrackId.value, selectedCamera.value); @@ -588,6 +717,9 @@ export default function useModeManager({ if (isEditingExisting && track.attributes?.userCreated !== true) { track.setFeatureAttribute(frameNum, 'userModified', true); } + + mirrorFeatureToAlignedCameras(track.id, frameNum); + // Capture track ID before newTrackSettingsAfterLogic, which may // create a new track in continuous detection mode and change // selectedTrackId @@ -633,6 +765,8 @@ export default function useModeManager({ interpolate: _shouldInterpolate(interpolate), }, geometry); + mirrorFeatureToAlignedCameras(track.id, frameNum); + if (runAfterLogic) { newTrackSettingsAfterLogic(track); } @@ -759,6 +893,8 @@ export default function useModeManager({ track.setFeatureAttribute(frameNum, 'userModified', true); } + mirrorFeatureToAlignedCameras(track.id, frameNum); + // Only perform "initialization" after the first shape. // Treat this as a completed annotation if eventType is editing // Or none of the recieps reported that they were unfinished. @@ -833,6 +969,7 @@ export default function useModeManager({ ); } }); + mirrorFeatureToAlignedCameras(track.id, selectedCameraFrame()); } } handleSelectFeatureHandle(-1); @@ -869,6 +1006,8 @@ export default function useModeManager({ } } + mirrorFeatureToAlignedCameras(track.id, frameNum); + _nudgeEditingCanary(); } } @@ -1323,6 +1462,8 @@ export default function useModeManager({ interpolate, }, polygonGeometry); + mirrorFeatureToAlignedCameras(track.id, targetFrame); + _nudgeEditingCanary(); // Interactive stereo: as soon as the left polygon is predicted, generate @@ -1424,6 +1565,8 @@ export default function useModeManager({ interpolate, }, polygonGeometry); + mirrorFeatureToAlignedCameras(track.id, frameNum); + // Note: the other-camera (stereo) annotation is generated earlier, on // each fresh prediction (handleSegmentationPredictionReady), so there is // no need to regenerate it on confirm. @@ -1501,6 +1644,8 @@ export default function useModeManager({ : []); } + mirrorFeatureToAlignedCameras(track.id, data.frameNum); + if (onStereoAnnotationReset && stereoInteractiveActive()) { onStereoAnnotationReset({ trackId: selectedTrackId.value as number, diff --git a/client/platform/desktop/backend/ipcService.ts b/client/platform/desktop/backend/ipcService.ts index 070d4bf44..94f3bbf24 100644 --- a/client/platform/desktop/backend/ipcService.ts +++ b/client/platform/desktop/backend/ipcService.ts @@ -203,6 +203,11 @@ export default function register() { { path }: { path: string }, ) => common.findParentFolderCalibrationFile(path)); + ipcMain.handle('find-parent-folder-transform-file', async ( + event, + { path }: { path: string }, + ) => common.findParentFolderTransformFile(path)); + ipcMain.handle('dataset-has-calibration-file', async ( event, { datasetId }: { datasetId: string }, diff --git a/client/platform/desktop/backend/native/common.spec.ts b/client/platform/desktop/backend/native/common.spec.ts index 828acab62..da3c46d10 100644 --- a/client/platform/desktop/backend/native/common.spec.ts +++ b/client/platform/desktop/backend/native/common.spec.ts @@ -174,6 +174,21 @@ beforeEach(() => { }, '/home/user/testPairs': { ...fileSystemData }, '/home/user/output': {}, + '/home/user/transformDiscovery': { + exactName: { + 'aaa-stamped.json': JSON.stringify({ type: 'dive-camera-calibration', version: 1, pairs: [] }), + 'calibration.json': JSON.stringify({ type: 'dive-camera-calibration', version: 1, pairs: [] }), + }, + otherName: { + 'a-rig-calibration.json': JSON.stringify({ calibrations: {} }), + 'broken.json': '{not json', + 'z-transforms.json': JSON.stringify({ type: 'dive-camera-calibration', version: 1, pairs: [] }), + }, + none: { + 'rig.json': JSON.stringify({ some: 'thing' }), + 'notes.txt': 'not json', + }, + }, '/home/user/data': { annotationImport: { 'viame.csv': emptyCsvString, @@ -641,6 +656,167 @@ describe('native.common', () => { }); }); + it('saveMetadata writes calibration.json (pairs + points) and reloads it', async () => { + const payload = await common.beginMediaImport( + '/home/user/data/imageLists/success/image_list.txt', + ); + const res = await common.finalizeMediaImport(settings, payload); + const final = res.meta; + // Directional key: rgb is left, ir is right. + const cameraHomographies = { + 'rgb::ir': { + AtoB: [[1, 0, 5], [0, 1, -3], [0, 0, 1]], + BtoA: [[1, 0, -5], [0, 1, 3], [0, 0, 1]], + }, + }; + const cameraCorrespondences = { + 'rgb::ir': [ + { id: 1, a: [10, 20], b: [12, 22] }, + { id: 2, a: [30, 40], b: [33, 44] }, + ], + }; + + await common.saveMetadata(settings, final.id, { cameraHomographies, cameraCorrespondences }); + + // Persisted as a standalone calibration.json: pairs labeled left/right, with + // points laid out as leftX leftY rightX rightY. + const projectDir = npath.join(settings.dataPath, 'DIVE_Projects', final.id); + const calibrationPath = npath.join(projectDir, 'calibration.json'); + expect(await fs.pathExists(calibrationPath)).toBe(true); + const calibration = await fs.readJSON(calibrationPath); + expect(calibration.pairs).toStrictEqual([ + { + left: 'rgb', + right: 'ir', + points: [[10, 20, 12, 22], [30, 40, 33, 44]], + leftToRight: [[1, 0, 5], [0, 1, -3], [0, 0, 1]], + rightToLeft: [[1, 0, -5], [0, 1, 3], [0, 0, 1]], + // No explicit choice was saved, so persistence fills the default model. + transformType: 'similarity', + }, + ]); + + // Not embedded in meta.json. + const meta = await fs.readJSON(npath.join(projectDir, 'meta.json')); + expect(meta.cameraHomographies).toBeUndefined(); + expect(meta.cameraCorrespondences).toBeUndefined(); + expect(meta.cameraTransformTypes).toBeUndefined(); + + // Rehydrated on load back into the in-app shapes. + const reloaded = await common.loadMetadata(settings, final.id, urlMapper); + expect(reloaded.cameraHomographies).toStrictEqual(cameraHomographies); + expect(reloaded.cameraCorrespondences).toStrictEqual(cameraCorrespondences); + expect(reloaded.cameraTransformTypes).toStrictEqual({ 'rgb::ir': 'similarity' }); + }); + + it('saveMetadata persists a non-default transformType per pair and reloads it', async () => { + const payload = await common.beginMediaImport( + '/home/user/data/imageLists/success/image_list.txt', + ); + const res = await common.finalizeMediaImport(settings, payload); + const final = res.meta; + const cameraHomographies = { + 'rgb::ir': { + AtoB: [[1, 0, 5], [0, 1, -3], [0, 0, 1]], + BtoA: [[1, 0, -5], [0, 1, 3], [0, 0, 1]], + }, + }; + const cameraTransformTypes = { 'rgb::ir': 'rigid' as const }; + + await common.saveMetadata(settings, final.id, { cameraHomographies, cameraTransformTypes }); + + const projectDir = npath.join(settings.dataPath, 'DIVE_Projects', final.id); + const calibration = await fs.readJSON(npath.join(projectDir, 'calibration.json')); + expect(calibration.pairs[0].transformType).toBe('rigid'); + + const reloaded = await common.loadMetadata(settings, final.id, urlMapper); + expect(reloaded.cameraTransformTypes).toStrictEqual(cameraTransformTypes); + }); + + describe('findParentFolderTransformFile', () => { + it('prefers a file named calibration.json among marked candidates', async () => { + const found = await common.findParentFolderTransformFile('/home/user/transformDiscovery/exactName'); + expect(found).toBe(npath.join('/home/user/transformDiscovery/exactName', 'calibration.json')); + }); + + it('finds a marked file under any name, skipping unmarked and broken JSON', async () => { + const found = await common.findParentFolderTransformFile('/home/user/transformDiscovery/otherName'); + expect(found).toBe(npath.join('/home/user/transformDiscovery/otherName', 'z-transforms.json')); + }); + + it('returns null when no self-identified calibration json exists', async () => { + expect(await common.findParentFolderTransformFile('/home/user/transformDiscovery/none')).toBeNull(); + }); + + it('returns null for a missing directory', async () => { + expect(await common.findParentFolderTransformFile('/home/user/doesNotExist')).toBeNull(); + }); + }); + + it('fromCalibrationPairs derives a missing matrix direction by inversion', () => { + const { homographies } = common.fromCalibrationPairs([{ + left: 'eo', + right: 'ir', + points: [], + leftToRight: null, + rightToLeft: [[1, 0, -5], [0, 1, 3], [0, 0, 1]], + }]); + expect(homographies['eo::ir'].BtoA).toEqual([[1, 0, -5], [0, 1, 3], [0, 0, 1]]); + expect(homographies['eo::ir'].AtoB[0][2]).toBeCloseTo(5); + expect(homographies['eo::ir'].AtoB[1][2]).toBeCloseTo(-3); + }); + + it('fromCalibrationPairs keeps points but skips the matrix for singular input', () => { + const { homographies, correspondences } = common.fromCalibrationPairs([{ + left: 'eo', + right: 'ir', + points: [[1, 2, 3, 4]], + leftToRight: [[0, 0, 0], [0, 0, 0], [0, 0, 0]], + rightToLeft: null, + }]); + expect(homographies['eo::ir']).toBeUndefined(); + expect(correspondences['eo::ir']).toHaveLength(1); + }); + + it('saveMetadata persists the calibration source stamp and reloads it', async () => { + const payload = await common.beginMediaImport( + '/home/user/data/imageLists/success/image_list.txt', + ); + const res = await common.finalizeMediaImport(settings, payload); + const final = res.meta; + const cameraHomographies = { + 'rgb::ir': { + AtoB: [[1, 0, 5], [0, 1, -3], [0, 0, 1]], + BtoA: [[1, 0, -5], [0, 1, 3], [0, 0, 1]], + }, + }; + const source = { model: 'colmap-2026-07-01', swathe: 'fl07_C' }; + + await common.saveMetadata(settings, final.id, { + cameraHomographies, + cameraCalibrationSource: source, + }); + + const projectDir = npath.join(settings.dataPath, 'DIVE_Projects', final.id); + const calibrationPath = npath.join(projectDir, 'calibration.json'); + expect((await fs.readJSON(calibrationPath)).source).toStrictEqual(source); + const reloaded = await common.loadMetadata(settings, final.id, urlMapper); + expect(reloaded.cameraCalibrationSource).toStrictEqual(source); + + // A save that doesn't mention the stamp leaves it alone. + await common.saveMetadata(settings, final.id, { + cameraTransformTypes: { 'rgb::ir': 'rigid' }, + }); + expect((await fs.readJSON(calibrationPath)).source).toStrictEqual(source); + + // An explicit null clears it. + await common.saveMetadata(settings, final.id, { + cameraHomographies, + cameraCalibrationSource: null, + }); + expect('source' in (await fs.readJSON(calibrationPath))).toBe(false); + }); + it('import with CSV annotations without specifying track file', async () => { const payload = await common.beginMediaImport('/home/user/data/imageSuccessWithAnnotations'); payload.trackFileAbsPath = ''; //It returns null be default but users change it. diff --git a/client/platform/desktop/backend/native/common.ts b/client/platform/desktop/backend/native/common.ts index 0c890ba03..53229b3ca 100644 --- a/client/platform/desktop/backend/native/common.ts +++ b/client/platform/desktop/backend/native/common.ts @@ -19,6 +19,9 @@ import { import { DefaultConfidence } from 'vue-media-annotator/BaseFilterControls'; import { TrackData } from 'vue-media-annotator/track'; import { GroupData } from 'vue-media-annotator/Group'; +import { TransformType, DEFAULT_TRANSFORM_TYPE } from 'vue-media-annotator/transform'; +import { CALIBRATION_FILE_TYPE } from 'vue-media-annotator/CameraCalibrationStore'; +import { invert3, Matrix3 } from 'vue-media-annotator/homography'; import { DatasetType, Pipelines, SaveDetectionsArgs, FrameImage, DatasetMetaMutable, TrainingConfig, TrainingConfigs, SaveAttributeArgs, @@ -71,6 +74,10 @@ const AuxFolderName = 'auxiliary'; const JsonTrackFileName = /^result(_.*)?\.json$/i; const JsonFileName = /^.*\.json$/i; const JsonMetaFileName = 'meta.json'; +// Standalone camera-to-camera (modality-to-modality) alignment transforms, +// stored separately from meta.json in the dataset directory. +const CalibrationFileName = 'calibration.json'; +const CalibrationFileVersion = 1; const CsvFileName = /^.*\.csv$/i; const YAMLFileName = /^.*\.ya?ml$/i; /** @@ -368,6 +375,28 @@ async function loadMetadata( const projectDirData = await getValidatedProjectDir(settings, datasetId); const projectMetaData = await loadJsonMetadata(projectDirData.metaFileAbsPath); + // Load standalone camera calibration (transforms + correspondences), if present. + const calibrationFileAbsPath = npath.join(projectDirData.basePath, CalibrationFileName); + let { + cameraHomographies, cameraCorrespondences, cameraTransformTypes, cameraCalibrationSource, + } = projectMetaData; + if (await fs.pathExists(calibrationFileAbsPath)) { + try { + const calibration = await _loadAsJson(calibrationFileAbsPath); + if (calibration && Array.isArray(calibration.pairs)) { + ({ + homographies: cameraHomographies, + correspondences: cameraCorrespondences, + transformTypes: cameraTransformTypes, + } = fromCalibrationPairs(calibration.pairs)); + cameraCalibrationSource = readCalibrationSource(calibration.source); + } + } catch (err) { + // A malformed calibration.json should not block loading the dataset. + console.warn(`Unable to read ${calibrationFileAbsPath}: ${err}`); + } + } + let videoUrl = ''; let imageData = [] as FrameImage[]; let multiCamMedia: MultiCamMedia | null = null; @@ -435,6 +464,10 @@ async function loadMetadata( imageData, multiCamMedia, subType, + cameraHomographies, + cameraCorrespondences, + cameraTransformTypes, + cameraCalibrationSource, }; } @@ -690,6 +723,104 @@ async function _saveAsJson(absPath: string, data: unknown) { await fs.writeFile(absPath, serialized); } +type CameraHomographies = NonNullable; +type CameraCorrespondences = NonNullable; +type CameraTransformTypes = NonNullable; +type CalibrationSource = NonNullable; + +/** + * Best-effort read of the calibration file's producer provenance stamp: a + * plain object, or null for anything else. Preserved verbatim across + * load/refine/save round trips; never interpreted by DIVE. + */ +function readCalibrationSource(raw: unknown): CalibrationSource | null { + if (raw && typeof raw === 'object' && !Array.isArray(raw)) { + return raw as CalibrationSource; + } + return null; +} + +/** + * One camera pair in calibration.json. `left`/`right` are camera (folder) names; + * `points` are the picked correspondences as rows of `leftX leftY rightX rightY`; + * `leftToRight`/`rightToLeft` are the fitted + * 3x3 homographies, when a fit has been performed; `transformType` is the fit + * model used to compute them (defaults to {@link DEFAULT_TRANSFORM_TYPE} when + * absent, matching the in-app default so a pair fitted at the default resolves + * to the same model after a save/reload). + */ +interface CalibrationPair { + left: string; + right: string; + points: number[][]; + leftToRight: number[][] | null; + rightToLeft: number[][] | null; + transformType?: TransformType; +} + +/** + * Convert the in-app calibration state (keyed by directional "left::right") into + * the self-describing list of pairs persisted in calibration.json. + */ +function toCalibrationPairs( + homographies: CameraHomographies, + correspondences: CameraCorrespondences, + transformTypes: CameraTransformTypes, +): CalibrationPair[] { + const keys = new Set([ + ...Object.keys(homographies), ...Object.keys(correspondences), ...Object.keys(transformTypes), + ]); + return [...keys].map((key) => { + const [left, right] = key.split('::'); + const homography = homographies[key]; + return { + left, + right, + points: (correspondences[key] || []).map((c) => [c.a[0], c.a[1], c.b[0], c.b[1]]), + leftToRight: homography ? homography.AtoB : null, + rightToLeft: homography ? homography.BtoA : null, + transformType: transformTypes[key] || DEFAULT_TRANSFORM_TYPE, + }; + }); +} + +/** Rebuild the in-app homographies/correspondences/transform types from calibration.json pairs. */ +function fromCalibrationPairs( + pairs: CalibrationPair[], +): { + homographies: CameraHomographies; + correspondences: CameraCorrespondences; + transformTypes: CameraTransformTypes; + } { + const homographies: CameraHomographies = {}; + const correspondences: CameraCorrespondences = {}; + const transformTypes: CameraTransformTypes = {}; + pairs.forEach((pair) => { + const key = `${pair.left}::${pair.right}`; + // Mirror the panel loader (CameraCalibrationStore.loadCalibrationText): + // producer files may carry only one fitted direction, so derive the + // missing one by inversion. A singular matrix can't participate in the + // warp either way, so such pairs contribute points only. + if (pair.leftToRight || pair.rightToLeft) { + try { + homographies[key] = { + AtoB: pair.leftToRight ?? invert3(pair.rightToLeft as Matrix3), + BtoA: pair.rightToLeft ?? invert3(pair.leftToRight as Matrix3), + }; + } catch { + // Singular / non-invertible: skip the matrix, keep the points. + } + } + if (pair.points && pair.points.length) { + correspondences[key] = pair.points.map((p, i) => ({ + id: i + 1, a: [p[0], p[1]], b: [p[2], p[3]], + })); + } + transformTypes[key] = pair.transformType || DEFAULT_TRANSFORM_TYPE; + }); + return { homographies, correspondences, transformTypes }; +} + async function saveMetadata(settings: Settings, datasetId: string, args: DatasetMetaMutable) { const projectDirInfo = await getValidatedProjectDir(settings, datasetId); const release = await _acquireLock(projectDirInfo.basePath, projectDirInfo.metaFileAbsPath, 'meta'); @@ -719,6 +850,51 @@ async function saveMetadata(settings: Settings, datasetId: string, args: Dataset existing.datasetInfo = args.datasetInfo; } + // Camera calibration (transforms + the points behind them) is persisted as a + // standalone calibration.json in the dataset directory rather than embedded in + // meta.json, so it is easy to find, hand-edit, and consume as a self-contained + // artifact. + if (args.cameraHomographies || args.cameraCorrespondences || args.cameraTransformTypes + || args.cameraCalibrationSource) { + const calibrationFileAbsPath = npath.join(projectDirInfo.basePath, CalibrationFileName); + // Start from whatever is on disk so a partial update doesn't clobber the rest. + let homographies: CameraHomographies = {}; + let correspondences: CameraCorrespondences = {}; + let transformTypes: CameraTransformTypes = {}; + let source: CalibrationSource | null = null; + if (await fs.pathExists(calibrationFileAbsPath)) { + try { + const existingCalibration = await _loadAsJson(calibrationFileAbsPath); + if (existingCalibration && Array.isArray(existingCalibration.pairs)) { + ({ homographies, correspondences, transformTypes } = fromCalibrationPairs( + existingCalibration.pairs, + )); + source = readCalibrationSource(existingCalibration.source); + } + } catch (err) { + console.warn(`Unable to read existing ${calibrationFileAbsPath}: ${err}`); + } + } + if (args.cameraHomographies) { + homographies = args.cameraHomographies; + } + if (args.cameraCorrespondences) { + correspondences = args.cameraCorrespondences; + } + if (args.cameraTransformTypes) { + transformTypes = args.cameraTransformTypes; + } + // undefined leaves the on-disk stamp alone; null/object replaces it. + if (args.cameraCalibrationSource !== undefined) { + source = args.cameraCalibrationSource; + } + await _saveAsJson(calibrationFileAbsPath, { + version: CalibrationFileVersion, + ...(source ? { source } : {}), + pairs: toCalibrationPairs(homographies, correspondences, transformTypes), + }); + } + await _saveAsJson(projectDirInfo.metaFileAbsPath, existing); await release(); } @@ -1156,6 +1332,48 @@ async function findParentFolderCalibrationFile(parentPath: string): Promise { + if (!await fs.pathExists(parentPath)) { + return null; + } + const stat = await fs.stat(parentPath); + if (!stat.isDirectory()) { + return null; + } + const children = await fs.readdir(parentPath, { withFileTypes: true }); + const candidates = children + .filter((entry) => entry.isFile() && /\.json$/i.test(entry.name)) + .map((entry) => entry.name) + .sort((a, b) => { + const aExact = a.toLowerCase() === CalibrationFileName ? 0 : 1; + const bExact = b.toLowerCase() === CalibrationFileName ? 0 : 1; + return aExact - bExact || a.localeCompare(b); + }); + // eslint-disable-next-line no-restricted-syntax + for (const name of candidates) { + const absPath = npath.join(parentPath, name); + try { + // eslint-disable-next-line no-await-in-loop -- candidates checked in priority order + const data = await fs.readJson(absPath); + if (data && data.type === CALIBRATION_FILE_TYPE && Array.isArray(data.pairs)) { + return absPath; + } + } catch { + // Unreadable/non-JSON candidates are simply not matches. + } + } + return null; +} + /** * Resolve the import path for one camera subfolder (directory or first video file). */ @@ -2135,6 +2353,8 @@ export { listImmediateSubfolders, listParentFolderCameras, findParentFolderCalibrationFile, + findParentFolderTransformFile, + fromCalibrationPairs, resolveMulticamCameraSourcePath, findTrackandMetaFileinFolder, getLastCalibrationPath, diff --git a/client/platform/desktop/backend/native/multiCamImport.ts b/client/platform/desktop/backend/native/multiCamImport.ts index 8b3b1cf44..40af5c47b 100644 --- a/client/platform/desktop/backend/native/multiCamImport.ts +++ b/client/platform/desktop/backend/native/multiCamImport.ts @@ -3,6 +3,7 @@ import fs from 'fs-extra'; import mime from 'mime-types'; import { DatasetType, + DatasetMetaMutable, MultiCamImportFolderArgs, MultiCamImportKeywordArgs, MultiCamImportArgs, @@ -16,7 +17,13 @@ import { Camera, } from 'platform/desktop/constants'; import { checkMedia } from 'platform/desktop/backend/native/mediaJobs'; -import { findImagesInFolder } from './common'; +import { readTransformMatrix } from 'vue-media-annotator/alignedView'; +import { findImagesInFolder, fromCalibrationPairs } from './common'; + +type CameraHomographies = NonNullable; +type CameraCorrespondences = NonNullable; +type CameraTransformTypes = NonNullable; +type CalibrationSource = NonNullable; function isFolderArgs(s: MultiCamImportArgs): s is MultiCamImportFolderArgs { if ('sourceList' in s && 'defaultDisplay' in s) { @@ -89,6 +96,52 @@ async function beginMultiCamImport(args: MultiCamImportArgs): Promise { + if (!item.transformFile) { + return; + } + try { + // A DIVE calibration .json (the panel's save format / the project + // calibration.json shape): pairs name their own cameras, so merge + // them all in. + const data = await fs.readJson(item.transformFile); + if (!data || !Array.isArray(data.pairs)) { + throw new Error('not a DIVE calibration file (expected a "pairs" list)'); + } + const parsed = fromCalibrationPairs(data.pairs); + Object.entries(parsed.homographies).forEach(([key, homography]) => { + if (!readTransformMatrix(homography.AtoB) || !readTransformMatrix(homography.BtoA)) { + throw new Error(`pair "${key.split('::').join(' / ')}" has an invalid 3x3 transform matrix`); + } + seedHomographies[key] = homography; + }); + Object.assign(seedCorrespondences, parsed.correspondences); + Object.assign(seedTransformTypes, parsed.transformTypes); + // Producer provenance travels with the seed. With one transform file + // per dataset (the expected case) this is that file's stamp; with + // several, the last stamped file wins, matching the merge order of the + // pairs above. + if (data.source && typeof data.source === 'object' && !Array.isArray(data.source)) { + seedCalibrationSource = data.source as CalibrationSource; + } + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + throw new Error(`Camera "${cameraName}": invalid transform file: ${message}`); + } + }); + } + const jsonMeta: JsonMeta = { version: JsonMetaCurrentVersion, type: datasetType, @@ -111,6 +164,14 @@ async function beginMultiCamImport(args: MultiCamImportArgs): Promise ({ + checkMedia: vi.fn(() => Promise.resolve({ + websafe: true, + originalFpsString: '30/1', + originalFps: 30, + videoDimensions: { width: 1920, height: 1080 }, + })), +})); + +let tmpDir: string; +let calibrationJsonPath: string; +let badCalibrationJsonPath: string; + +function sourceListWith(transformFile?: string) { + return { + eo: { sourcePath: npath.join(tmpDir, 'eo'), trackFile: '' }, + ir: { + sourcePath: npath.join(tmpDir, 'ir'), + trackFile: '', + ...(transformFile ? { transformFile } : {}), + }, + }; +} + +beforeAll(() => { + tmpDir = fs.mkdtempSync(npath.join(os.tmpdir(), 'multicam-transform-import-')); + ['eo', 'ir'].forEach((camera) => { + const dir = npath.join(tmpDir, camera); + fs.mkdirSync(dir); + ['frame0.png', 'frame1.png'].forEach((name) => { + fs.writeFileSync(npath.join(dir, name), ''); + }); + }); + calibrationJsonPath = npath.join(tmpDir, 'calibration.json'); + fs.writeJsonSync(calibrationJsonPath, { + type: 'dive-camera-calibration', + version: 1, + source: { model: 'colmap-2026-07-01', swathe: 'fl07_C' }, + pairs: [{ + left: 'eo', + right: 'ir', + points: [[0, 0, 5, -3], [10, 0, 15, -3]], + leftToRight: [[1, 0, 5], [0, 1, -3], [0, 0, 1]], + rightToLeft: [[1, 0, -5], [0, 1, 3], [0, 0, 1]], + transformType: 'translation', + }], + }); + badCalibrationJsonPath = npath.join(tmpDir, 'not-a-calibration.json'); + fs.writeJsonSync(badCalibrationJsonPath, { some: 'other json' }); +}); + +afterAll(() => { + fs.removeSync(tmpDir); +}); + +describe('multiCamImport transform wire-through', () => { + it('seeds the saved calibration (points and all) from a DIVE calibration .json', async () => { + const output = await beginMultiCamImport({ + datasetName: 'calibration_json_test', + defaultDisplay: 'eo', + cameraOrder: ['eo', 'ir'], + sourceList: sourceListWith(calibrationJsonPath), + type: 'image-sequence', + }); + expect(output.jsonMeta.cameraHomographies?.['eo::ir'].AtoB).toEqual( + [[1, 0, 5], [0, 1, -3], [0, 0, 1]], + ); + expect(output.jsonMeta.cameraCorrespondences?.['eo::ir']).toHaveLength(2); + expect(output.jsonMeta.cameraTransformTypes?.['eo::ir']).toBe('translation'); + // The producer provenance stamp travels with the seed. + expect(output.jsonMeta.cameraCalibrationSource).toEqual( + { model: 'colmap-2026-07-01', swathe: 'fl07_C' }, + ); + }); + + it('leaves the calibration unset when no transform file is given', async () => { + const output = await beginMultiCamImport({ + datasetName: 'no_transform', + defaultDisplay: 'eo', + sourceList: sourceListWith(), + type: 'image-sequence', + }); + expect(output.jsonMeta.cameraHomographies).toBeUndefined(); + expect(output.jsonMeta.cameraCorrespondences).toBeUndefined(); + }); + + it('fails the import for a .json without a pairs list', async () => { + await expect(beginMultiCamImport({ + datasetName: 'bad_calibration_json', + defaultDisplay: 'eo', + sourceList: sourceListWith(badCalibrationJsonPath), + type: 'image-sequence', + })).rejects.toThrow(/Camera "ir": invalid transform file: not a DIVE calibration file/); + }); + + it('fails the import for a file that is not JSON at all', async () => { + const notJsonPath = npath.join(tmpDir, 'not-json.json'); + fs.writeFileSync(notJsonPath, 'not json content'); + await expect(beginMultiCamImport({ + datasetName: 'not_json_transform', + defaultDisplay: 'eo', + sourceList: sourceListWith(notJsonPath), + type: 'image-sequence', + })).rejects.toThrow(/Camera "ir": invalid transform file/); + }); + + it('fails the import when the transform file does not exist', async () => { + await expect(beginMultiCamImport({ + datasetName: 'missing_transform', + defaultDisplay: 'eo', + sourceList: sourceListWith(npath.join(tmpDir, 'does-not-exist.json')), + type: 'image-sequence', + })).rejects.toThrow(/Camera "ir": invalid transform file/); + }); +}); diff --git a/client/platform/desktop/frontend/api.ts b/client/platform/desktop/frontend/api.ts index e773257ee..807a512d8 100644 --- a/client/platform/desktop/frontend/api.ts +++ b/client/platform/desktop/frontend/api.ts @@ -13,7 +13,7 @@ import type { import { fileVideoTypes, calibrationFileTypes, inputAnnotationFileTypes, listFileTypes, - largeImageDesktopTypes, + largeImageDesktopTypes, transformFileTypes, } from 'dive-common/constants'; import { DesktopMetadata, NvidiaSmiReply, @@ -47,7 +47,7 @@ function joinPath(dir: string, filename: string) { * Native functions that run entirely in the renderer */ -async function openFromDisk(datasetType: DatasetType | 'bulk' | 'calibration' | 'annotation' | 'text', directory = false) { +async function openFromDisk(datasetType: DatasetType | 'bulk' | 'calibration' | 'annotation' | 'text' | 'transform', directory = false) { let filters: FileFilter[] = []; const allFiles = { name: 'All Files', extensions: ['*'] }; if (datasetType === 'video') { @@ -73,6 +73,12 @@ async function openFromDisk(datasetType: DatasetType | 'bulk' | 'calibration' | allFiles, ]; } + if (datasetType === 'transform') { + filters = [ + { name: 'Transform / calibration', extensions: transformFileTypes }, + allFiles, + ]; + } if (datasetType === 'text') { filters = [ { name: 'text', extensions: listFileTypes }, @@ -200,6 +206,10 @@ function findParentFolderCalibrationFile(parentPath: string): Promise { + return window.diveDesktop.invoke('find-parent-folder-transform-file', { path: parentPath }); +} + function hasCalibrationFile(datasetId: string): Promise { return window.diveDesktop.invoke('dataset-has-calibration-file', { datasetId }); } @@ -698,6 +708,7 @@ export { listParentFolderCameras, resolveMulticamCameraSourcePath, findParentFolderCalibrationFile, + findParentFolderTransformFile, hasCalibrationFile, bulkImportMedia, deleteDataset, diff --git a/client/platform/desktop/frontend/components/Recent.vue b/client/platform/desktop/frontend/components/Recent.vue index d0a076616..b701156cf 100644 --- a/client/platform/desktop/frontend/components/Recent.vue +++ b/client/platform/desktop/frontend/components/Recent.vue @@ -309,6 +309,7 @@ export default defineComponent({ :stereo="stereo" :data-type="multiCamOpenType" :enable-subfolder-import="true" + :enable-transform-import="true" :import-media="importMedia" @begin-multicam-import="multiCamImport($event)" @abort="importMultiCamDialog = false" diff --git a/client/platform/web-girder/utils.ts b/client/platform/web-girder/utils.ts index f30b21ebd..91870cbed 100644 --- a/client/platform/web-girder/utils.ts +++ b/client/platform/web-girder/utils.ts @@ -2,7 +2,8 @@ import { UploadManager, Location } from '@girder/components/src'; import { calibrationFileTypes, inputAnnotationFileTypes, inputAnnotationTypes, getLargeImageAllowedExtensions, getLargeImageFileAccept, - otherImageTypes, otherVideoTypes, websafeImageTypes, websafeVideoTypes, zipFileTypes, + otherImageTypes, otherVideoTypes, transformFileTypes, + websafeImageTypes, websafeVideoTypes, zipFileTypes, } from 'dive-common/constants'; import { DatasetType } from 'dive-common/apispec'; import type { LocationType, RootlessLocationType } from 'platform/web-girder/store/types'; @@ -38,7 +39,7 @@ function getRouteFromLocation(location: LocationType): string { } async function openFromDisk( - datasetType: DatasetType | 'calibration' | 'annotation' | 'text' | 'zip', + datasetType: DatasetType | 'calibration' | 'annotation' | 'text' | 'zip' | 'transform', directory = false, ): Promise<{ canceled: boolean; filePaths: string[]; fileList?: File[]; root?: string }> { const input: HTMLInputElement = document.createElement('input'); @@ -66,6 +67,9 @@ async function openFromDisk( .concat(inputAnnotationFileTypes.map((item) => `.${item}`)).join(','); } else if (datasetType === 'zip') { input.accept = zipFileTypes.map((item) => `.${item}`).join(','); + } else if (datasetType === 'transform') { + input.accept = transformFileTypes.map((item) => `.${item}`).join(','); + input.multiple = false; } else if (datasetType === 'text') { input.accept = '.txt,.text'; input.multiple = false; diff --git a/client/src/AlignedViewStore.ts b/client/src/AlignedViewStore.ts new file mode 100644 index 000000000..0a603aabc --- /dev/null +++ b/client/src/AlignedViewStore.ts @@ -0,0 +1,110 @@ +import { + computed, ref, ComputedRef, Ref, +} from 'vue'; +import type { Matrix3, Point } from './homography'; +import { + cameraPairTransform, isIdentityMatrix3, mapPoint, +} from './alignedView'; + +/** + * Shared reactive state for the multicam "aligned view" toggle: when every + * non-reference camera has a stored/fitted transform into the reference + * camera's space, the user may warp each camera's display into that shared + * space during normal annotation review and link pan/zoom across all cameras + * (SEAL-TK features 2 + 3). + * + * Stored annotation geometry ALWAYS remains in native image space (decision + * D3); the transforms exposed here are applied at draw time only. + */ +export default class AlignedViewStore { + /** User toggle: whether the aligned view is requested. */ + enabled: Ref; + + /** Reference camera name (first camera in display order), null when unresolved. */ + reference: Ref; + + /** + * Per-camera native->reference matrices covering EVERY loaded camera, or + * null when at least one camera lacks a usable transform (see + * {@link resolveToReferenceTransforms}). + */ + toReference: Ref | null>; + + /** + * Externally suspended (e.g. while calibration point picking is active, + * which records raw native-space clicks and manages its own aligned + * preview). Suspension un-warps the display without losing the toggle. + */ + suspended: Ref; + + /** A usable transform exists for every camera, so the toggle may be shown. */ + available: ComputedRef; + + /** The aligned view is currently applied to rendering and navigation. */ + active: ComputedRef; + + constructor() { + this.enabled = ref(false); + this.reference = ref(null); + this.toReference = ref(null); + this.suspended = ref(false); + this.available = computed(() => this.reference.value !== null + && this.toReference.value !== null + && Object.keys(this.toReference.value).length > 1); + this.active = computed(() => this.enabled.value + && !this.suspended.value + && this.available.value); + } + + /** Replace the resolved transform set (null when unavailable). */ + setTransforms(reference: string | null, toReference: Record | null) { + this.reference.value = reference; + this.toReference.value = toReference; + } + + setEnabled(enabled: boolean) { + this.enabled.value = enabled; + } + + setSuspended(suspended: boolean) { + this.suspended.value = suspended; + } + + /** + * Display transform for `camera` (native -> aligned/reference space), or + * null when the aligned view is inactive or the camera renders unwarped + * (the reference camera / identity). Callers treat null as "draw exactly + * as today", keeping the off state byte-identical to current behavior. + */ + cameraTransform(camera: string): Matrix3 | null { + if (!this.active.value || !this.toReference.value) { + return null; + } + const matrix = this.toReference.value[camera]; + if (!matrix || isIdentityMatrix3(matrix)) { + return null; + } + return matrix; + } + + /** + * Matrix mapping `from`-camera native pixels onto `to`-camera native + * pixels, composed through the reference space. Null when inactive or + * either camera is unresolved. + */ + cameraToCamera(from: string, to: string): Matrix3 | null { + if (!this.active.value || !this.toReference.value) { + return null; + } + return cameraPairTransform(this.toReference.value, from, to); + } + + /** Map a native-space point of `from` onto `to`'s native space (null when unavailable). */ + mapCameraPoint(from: string, to: string, point: Point): Point | null { + const matrix = this.cameraToCamera(from, to); + if (!matrix) { + return null; + } + return mapPoint(matrix, point); + } +} diff --git a/client/src/CameraCalibrationStore.spec.ts b/client/src/CameraCalibrationStore.spec.ts new file mode 100644 index 000000000..791ba249a --- /dev/null +++ b/client/src/CameraCalibrationStore.spec.ts @@ -0,0 +1,858 @@ +/// +import CameraCalibrationStore from './CameraCalibrationStore'; + +describe('CameraCalibrationStore', () => { + it('produces a directional pairKey that preserves left/right order', () => { + const store = new CameraCalibrationStore(); + expect(store.pairKey('rgb', 'ir')).toEqual('rgb::ir'); + expect(store.pairKey('rgb', 'ir')).not.toEqual(store.pairKey('ir', 'rgb')); + }); + + it('preserves the chosen left/right order on the active pair', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('rgb', 'ir'); + expect(store.activePair.value).toEqual({ camA: 'rgb', camB: 'ir' }); + }); + + it('clears the active pair for identical or empty cameras', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('cam', 'cam'); + expect(store.activePair.value).toBeNull(); + }); + + it('resets alignment to original when switching pairs', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + addFourTranslationPairs(store); + store.setAlignmentMode('AtoB'); + expect(store.alignment.value.mode).toBe('AtoB'); + store.setActivePair('left', 'other'); + expect(store.alignment.value).toMatchObject({ mode: 'original' }); + }); + + it('forms one correspondence from a blue->red two-click sequence', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.addPoint('left', [10, 20]); // pending (blue) + expect(store.pendingPoint.value).not.toBeNull(); + expect(store.correspondences.value[key]).toBeUndefined(); + store.addPoint('right', [30, 40]); // completes (red) + expect(store.pendingPoint.value).toBeNull(); + expect(store.correspondences.value[key]).toHaveLength(1); + expect(store.correspondences.value[key][0]).toMatchObject({ a: [10, 20], b: [30, 40] }); + }); + + it('maps points to a/b by camera regardless of click order', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.addPoint('right', [30, 40]); // click right first + store.addPoint('left', [10, 20]); + expect(store.correspondences.value[key][0]).toMatchObject({ a: [10, 20], b: [30, 40] }); + }); + + it('replaces the pending point when the same camera is clicked twice', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + store.addPoint('left', [1, 1]); + store.addPoint('left', [2, 2]); + expect(store.pendingPoint.value).toMatchObject({ camera: 'left', coord: [2, 2] }); + }); + + it('ignores points for cameras outside the active pair', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + store.addPoint('other', [1, 1]); + expect(store.pendingPoint.value).toBeNull(); + }); + + it('removes a correspondence by id', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.addPoint('left', [1, 1]); + store.addPoint('right', [2, 2]); + const { id } = store.correspondences.value[key][0]; + store.removeCorrespondence(id); + expect(store.correspondences.value[key]).toHaveLength(0); + }); + + it('moves one side of a correspondence via updateCorrespondencePoint', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.addPoint('left', [1, 1]); + store.addPoint('right', [2, 2]); + const { id } = store.correspondences.value[key][0]; + store.updateCorrespondencePoint(id, 'left', [5, 6]); + expect(store.correspondences.value[key][0].a).toEqual([5, 6]); + expect(store.correspondences.value[key][0].b).toEqual([2, 2]); + store.updateCorrespondencePoint(id, 'right', [7, 8]); + expect(store.correspondences.value[key][0].b).toEqual([7, 8]); + }); + + it('ignores updateCorrespondencePoint for unknown ids or cameras outside the pair', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.addPoint('left', [1, 1]); + store.addPoint('right', [2, 2]); + const { id } = store.correspondences.value[key][0]; + store.updateCorrespondencePoint(id + 99, 'left', [5, 6]); + store.updateCorrespondencePoint(id, 'other', [5, 6]); + expect(store.correspondences.value[key][0]).toMatchObject({ a: [1, 1], b: [2, 2] }); + }); + + it('refits the pair homography when a point is drag-refined while alignment is active', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + addFourTranslationPairs(store); + store.setAlignmentMode('AtoB'); + const before = store.homographies.value[key].AtoB[0][2]; + const { id } = store.correspondences.value[key][0]; + store.updateCorrespondencePoint(id, 'right', [40, 40]); + expect(store.homographies.value[key].AtoB[0][2]).not.toBeCloseTo(before, 5); + }); + + it('moves the pending point only for its own camera', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + store.addPoint('left', [1, 1]); + store.movePendingPoint('right', [9, 9]); + expect(store.pendingPoint.value).toMatchObject({ camera: 'left', coord: [1, 1] }); + store.movePendingPoint('left', [9, 9]); + expect(store.pendingPoint.value).toMatchObject({ camera: 'left', coord: [9, 9] }); + }); + + it('fits a homography from >= 4 pairs and stores both directions', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + // A pure translation by (5, -3). + const pts: [number, number][] = [[0, 0], [10, 0], [10, 10], [0, 10]]; + pts.forEach((p) => { + store.addPoint('left', p); + store.addPoint('right', [p[0] + 5, p[1] - 3]); + }); + const { AtoB, BtoA } = store.fitTransform(key); + expect(AtoB[0][2]).toBeCloseTo(5, 5); + expect(AtoB[1][2]).toBeCloseTo(-3, 5); + expect(BtoA[0][2]).toBeCloseTo(-5, 5); + expect(store.homographies.value[key]).toBeDefined(); + }); + + it('throws when fitting with fewer than 4 pairs (default homography type)', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.addPoint('left', [1, 1]); + store.addPoint('right', [2, 2]); + expect(() => store.fitTransform(key)).toThrow(); + }); + + it('surfaces a fitError instead of throwing when maybeFitPair hits a degenerate configuration', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.setTransformType(key, 'homography'); + // 4 collinear points satisfy the homography minimum count but are degenerate. + const pts: [number, number][] = [[0, 0], [1, 0], [2, 0], [3, 0]]; + pts.forEach((p) => { + store.addPoint('left', p); + store.addPoint('right', p); + }); + expect(() => store.maybeFitPair(key)).not.toThrow(); + expect(store.fitError.value).toMatch(/degenerate/i); + expect(store.homographies.value[key]).toBeUndefined(); + }); + + it('clears a stale fitError once the active pair fits successfully', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.setTransformType(key, 'homography'); + const collinear: [number, number][] = [[0, 0], [1, 0], [2, 0], [3, 0]]; + collinear.forEach((p) => { + store.addPoint('left', p); + store.addPoint('right', p); + }); + store.maybeFitPair(key); + expect(store.fitError.value).not.toBeNull(); + store.clearPair(); + addFourTranslationPairs(store); + store.maybeFitPair(key); + expect(store.fitError.value).toBeNull(); + expect(store.homographies.value[key]).toBeDefined(); + }); + + it('clears fitError when switching to a different active pair', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.setTransformType(key, 'homography'); + const collinear: [number, number][] = [[0, 0], [1, 0], [2, 0], [3, 0]]; + collinear.forEach((p) => { + store.addPoint('left', p); + store.addPoint('right', p); + }); + store.maybeFitPair(key); + expect(store.fitError.value).not.toBeNull(); + store.setActivePair('left', 'other'); + expect(store.fitError.value).toBeNull(); + }); + + function addFourTranslationPairs(store: CameraCalibrationStore) { + const pts: [number, number][] = [[0, 0], [10, 0], [10, 10], [0, 10]]; + pts.forEach((p) => { + store.addPoint('left', p); + store.addPoint('right', [p[0] + 5, p[1] - 3]); + }); + } + + it('fits when enabling alignment mode with >= 4 pairs', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + addFourTranslationPairs(store); + store.setAlignmentMode('AtoB'); + expect(store.alignment.value.mode).toBe('AtoB'); + expect(store.homographies.value[key]).toBeDefined(); + expect(store.homographies.value[key].AtoB[0][2]).toBeCloseTo(5, 5); + }); + + it('does not enable alignment mode with fewer than 4 pairs (default homography type)', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + store.addPoint('left', [1, 1]); + store.addPoint('right', [2, 2]); + store.setAlignmentMode('AtoB'); + expect(store.alignment.value.mode).toBe('original'); + expect(store.homographies.value).toEqual({}); + }); + + it('refits when correspondences change while alignment mode is active', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + addFourTranslationPairs(store); + store.setAlignmentMode('AtoB'); + const before = store.homographies.value[key].AtoB[0][2]; + store.addPoint('left', [20, 20]); + store.addPoint('right', [30, 14]); + expect(store.homographies.value[key].AtoB[0][2]).not.toBeCloseTo(before, 5); + }); + + it('reverts alignment to original when correspondences drop below the transform minimum', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + addFourTranslationPairs(store); + store.setAlignmentMode('AtoB'); + const { id } = store.correspondences.value[key][0]; + store.removeCorrespondence(id); + store.removeCorrespondence(store.correspondences.value[key][0].id); + store.removeCorrespondence(store.correspondences.value[key][0].id); + store.removeCorrespondence(store.correspondences.value[key][0].id); + expect(store.correspondences.value[key]).toHaveLength(0); + expect(store.alignment.value.mode).toBe('original'); + expect(store.homographies.value[key]).toBeUndefined(); + }); + + it('maybeFitActivePair fits without enabling alignment mode', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + addFourTranslationPairs(store); + store.maybeFitActivePair(); + expect(store.alignment.value.mode).toBe('original'); + expect(store.homographies.value[key]).toBeDefined(); + }); + + it('hydrates homographies and resets transient state', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + store.addPoint('left', [1, 1]); + const saved = { 'a::b': { AtoB: [[1, 0, 0], [0, 1, 0], [0, 0, 1]], BtoA: [[1, 0, 0], [0, 1, 0], [0, 0, 1]] } }; + store.hydrate(saved); + expect(store.homographies.value).toEqual(saved); + expect(store.activePair.value).toBeNull(); + expect(store.pendingPoint.value).toBeNull(); + expect(store.correspondences.value).toEqual({}); + expect(store.transformTypes.value).toEqual({}); + expect(store.alignment.value).toEqual({ mode: 'original', opacity: 0.5 }); + }); + + it('hydrates transform types alongside homographies', () => { + const store = new CameraCalibrationStore(); + store.hydrate({}, {}, { 'a::b': 'rigid' }); + expect(store.transformTypeForPair('a::b')).toBe('rigid'); + expect(store.transformTypeForPair('unset::pair')).toBe('similarity'); + }); + + it('hydrates correspondences and resumes id allocation', () => { + const store = new CameraCalibrationStore(); + const correspondences = { + 'rgb::ir': [ + { id: 1, a: [1, 2], b: [3, 4] }, + { id: 2, a: [5, 6], b: [7, 8] }, + ], + }; + store.hydrate({}, correspondences); + expect(store.correspondences.value).toEqual(correspondences); + // New points pick up after the highest restored id. + store.setActivePair('rgb', 'ir'); + store.addPoint('rgb', [9, 9]); + store.addPoint('ir', [10, 10]); + expect(store.correspondences.value['rgb::ir'][2].id).toBe(3); + }); + + describe('transform type selection', () => { + it('fits a rigid transform from 2 pairs where a homography would throw', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.addPoint('left', [0, 0]); + store.addPoint('right', [5, -3]); + store.addPoint('left', [10, 0]); + store.addPoint('right', [15, -3]); + store.setTransformType(key, 'homography'); + expect(() => store.fitTransform(key)).toThrow(); + store.setTransformType(key, 'rigid'); + expect(store.homographies.value[key]).toBeDefined(); + expect(store.homographies.value[key].AtoB[0][2]).toBeCloseTo(5, 4); + }); + + it('clears the fit and reverts alignment when switching to a type needing more points than are picked', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.addPoint('left', [0, 0]); + store.addPoint('right', [5, -3]); + store.addPoint('left', [10, 0]); + store.addPoint('right', [15, -3]); + store.setTransformType(key, 'rigid'); + store.setAlignmentMode('AtoB'); + expect(store.alignment.value.mode).toBe('AtoB'); + + store.setTransformType(key, 'homography'); // needs 4, only 2 picked + expect(store.homographies.value[key]).toBeUndefined(); + expect(store.alignment.value.mode).toBe('original'); + }); + }); + + describe('setAlignmentMode guards', () => { + it('setAlignmentMode leaves mode original when the pair lacks enough points', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + store.setAlignmentMode('BtoA'); + expect(store.alignment.value.mode).toBe('original'); + }); + }); + + describe('pickPoint', () => { + it('records a native pick like addPoint in the Picking (original) mode', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + store.pickPoint('right', [15, 7]); + expect(store.pendingPoint.value).toMatchObject({ camera: 'right', coord: [15, 7] }); + }); + + it('is a no-op while an overlay warp is active', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + addFourTranslationPairs(store); + store.setAlignmentMode('AtoB'); + store.pickPoint('right', [15, 7]); + // The warp mode blocks new picks; nothing is pending and the pairs are unchanged. + expect(store.pendingPoint.value).toBeNull(); + expect(store.correspondences.value[store.pairKey('left', 'right')]).toHaveLength(4); + }); + }); + + describe('correspondence selection', () => { + function storeWithOnePair() { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + store.addPoint('left', [1, 1]); + store.addPoint('right', [2, 2]); + const key = store.pairKey('left', 'right'); + const { id } = store.correspondences.value[key][0]; + return { store, key, id }; + } + + it('selects an active-pair correspondence and clears via null or unknown ids', () => { + const { store, id } = storeWithOnePair(); + store.selectCorrespondence(id); + expect(store.selectedCorrespondenceId.value).toBe(id); + store.selectCorrespondence(id + 99); + expect(store.selectedCorrespondenceId.value).toBeNull(); + store.selectCorrespondence(id); + store.selectCorrespondence(null); + expect(store.selectedCorrespondenceId.value).toBeNull(); + }); + + it('removeSelectedCorrespondence removes both cameras\' points and clears the selection', () => { + const { store, key, id } = storeWithOnePair(); + store.selectCorrespondence(id); + store.removeSelectedCorrespondence(); + expect(store.correspondences.value[key]).toHaveLength(0); + expect(store.selectedCorrespondenceId.value).toBeNull(); + // No selection: a further call is a no-op. + store.removeSelectedCorrespondence(); + expect(store.correspondences.value[key]).toHaveLength(0); + }); + + it('clears the selection when the selected pair is removed, undone, or the pair switches', () => { + const first = storeWithOnePair(); + first.store.selectCorrespondence(first.id); + first.store.removeCorrespondence(first.id); + expect(first.store.selectedCorrespondenceId.value).toBeNull(); + + const second = storeWithOnePair(); + second.store.selectCorrespondence(second.id); + second.store.clearLast(); + expect(second.store.selectedCorrespondenceId.value).toBeNull(); + + const third = storeWithOnePair(); + third.store.selectCorrespondence(third.id); + third.store.setActivePair('left', 'other'); + expect(third.store.selectedCorrespondenceId.value).toBeNull(); + }); + + it('clears the selection on clearPair, load, and hydrate', () => { + const { store, id } = storeWithOnePair(); + store.selectCorrespondence(id); + store.clearPair(); + expect(store.selectedCorrespondenceId.value).toBeNull(); + + const loaded = storeWithOnePair(); + loaded.store.selectCorrespondence(loaded.id); + loaded.store.loadCalibrationText(JSON.stringify({ version: 1, pairs: [] })); + expect(loaded.store.selectedCorrespondenceId.value).toBeNull(); + + const hydrated = storeWithOnePair(); + hydrated.store.selectCorrespondence(hydrated.id); + hydrated.store.hydrate(); + expect(hydrated.store.selectedCorrespondenceId.value).toBeNull(); + }); + }); + + describe('linked navigation', () => { + it('linkedPoint maps a point from camA to camB and back, via the fitted homography', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + addFourTranslationPairs(store); // left -> right is +5, -3 + store.maybeFitActivePair(); + const fromLeft = store.linkedPoint('left', [1, 1]); + expect(fromLeft).toMatchObject({ camera: 'right', coord: [6, -2] }); + const fromRight = store.linkedPoint('right', [6, -2]); + expect(fromRight).toMatchObject({ camera: 'left', coord: [1, 1] }); + }); + + it('linkedPoint returns null when the pair has no fitted homography yet', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + expect(store.linkedPoint('left', [1, 1])).toBeNull(); + }); + + it('linkedPoint returns null for a camera outside the active pair', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + addFourTranslationPairs(store); + expect(store.linkedPoint('other', [1, 1])).toBeNull(); + }); + }); + + describe('cursor coordinate readout', () => { + it('records and clears the cursor coordinate', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + store.setCursorCoord('left', [12, 34]); + expect(store.cursorCoord.value).toEqual({ camera: 'left', coord: [12, 34] }); + store.clearCursorCoord(); + expect(store.cursorCoord.value).toBeNull(); + }); + + it('clears the cursor coordinate when switching pairs', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + store.setCursorCoord('left', [12, 34]); + store.setActivePair('left', 'other'); + expect(store.cursorCoord.value).toBeNull(); + }); + }); + + describe('clearLast', () => { + it('drops the pending point without touching completed correspondences', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.addPoint('left', [1, 1]); + store.addPoint('right', [2, 2]); + store.addPoint('left', [3, 3]); // pending + store.clearLast(); + expect(store.pendingPoint.value).toBeNull(); + expect(store.correspondences.value[key]).toHaveLength(1); + }); + + it('removes the last completed correspondence when there is no pending point', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.addPoint('left', [1, 1]); + store.addPoint('right', [2, 2]); + store.addPoint('left', [3, 3]); + store.addPoint('right', [4, 4]); + store.clearLast(); + expect(store.correspondences.value[key]).toHaveLength(1); + expect(store.correspondences.value[key][0]).toMatchObject({ a: [1, 1], b: [2, 2] }); + }); + + it('is a no-op with nothing to undo', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + expect(() => store.clearLast()).not.toThrow(); + store.clearLast(); + expect(store.pendingPoint.value).toBeNull(); + }); + + it('refits when clearing last while alignment mode is active', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.setTransformType(key, 'homography'); + addFourTranslationPairs(store); + store.setAlignmentMode('AtoB'); + store.clearLast(); + expect(store.correspondences.value[key]).toHaveLength(3); + expect(store.alignment.value.mode).toBe('original'); + expect(store.homographies.value[key]).toBeUndefined(); + }); + }); + + describe('requestRecenter', () => { + it('records a recenter request for a camera in the active pair', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + store.requestRecenter('right', [7, 8]); + expect(store.recenterRequest.value).toMatchObject({ camera: 'right', coord: [7, 8] }); + }); + + it('ignores a recenter request for a camera outside the active pair', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + store.requestRecenter('other', [7, 8]); + expect(store.recenterRequest.value).toBeNull(); + }); + + it('assigns a new id to each request so repeated identical requests still change', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + store.requestRecenter('left', [1, 1]); + const firstId = store.recenterRequest.value?.id; + store.requestRecenter('left', [1, 1]); + expect(store.recenterRequest.value?.id).not.toBe(firstId); + }); + + it('clears the recenter request when switching pairs', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + store.requestRecenter('left', [1, 1]); + store.setActivePair('left', 'other'); + expect(store.recenterRequest.value).toBeNull(); + }); + }); + + describe('loaded (file-sourced) homographies', () => { + // Pure translation by (+5, -3): trivially invertible. + const translate = [[1, 0, 5], [0, 1, -3], [0, 0, 1]]; + + /** Load a matrix-only (point-less) pair from calibration JSON, marking it 'loaded'. */ + function loadMatrixOnlyPair(store: CameraCalibrationStore, left: string, right: string, rightToLeft: number[][]) { + store.loadCalibrationText(JSON.stringify({ + version: 1, + pairs: [{ + left, right, points: [], leftToRight: null, rightToLeft, + }], + })); + } + + it('loads a matrix-only pair as B->A with its inverse as A->B and no points', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + loadMatrixOnlyPair(store, 'left', 'right', translate); + expect(store.correspondences.value[key]).toHaveLength(0); + expect(store.isLoadedHomography(key)).toBe(true); + const homog = store.homographies.value[key]; + expect(homog.BtoA).toEqual(translate); + expect(homog.AtoB[0][2]).toBeCloseTo(-5); + expect(homog.AtoB[1][2]).toBeCloseTo(3); + }); + + it('keeps a loaded homography through refit checks with too few points', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + loadMatrixOnlyPair(store, 'left', 'right', translate); + store.maybeFitPair(key); + expect(store.homographies.value[key]).toBeDefined(); + // Alignment can activate directly off the loaded transform. + store.setAlignmentMode('AtoB'); + expect(store.alignment.value.mode).toBe('AtoB'); + }); + + it('replaces a loaded homography once enough points are picked and fitted', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + loadMatrixOnlyPair(store, 'left', 'right', [[1, 0, 100], [0, 1, 100], [0, 0, 1]]); + addFourTranslationPairs(store); + store.maybeFitPair(key); + expect(store.isLoadedHomography(key)).toBe(false); + // Fitted from points: right = left + (5, -3), so AtoB translates by (5, -3). + expect(store.homographies.value[key].AtoB[0][2]).toBeCloseTo(5); + expect(store.homographies.value[key].AtoB[1][2]).toBeCloseTo(-3); + }); + + it('clearPair removes a loaded homography', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + loadMatrixOnlyPair(store, 'left', 'right', translate); + store.clearPair(); + expect(store.homographies.value[key]).toBeUndefined(); + expect(store.isLoadedHomography(key)).toBe(false); + }); + + it('clearPair also removes a stale fitted homography immediately', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + addFourTranslationPairs(store); + store.fitTransform(key); + store.clearPair(); + expect(store.homographies.value[key]).toBeUndefined(); + }); + + it('rejects a singular loaded matrix', () => { + const store = new CameraCalibrationStore(); + expect(() => loadMatrixOnlyPair(store, 'left', 'right', [[0, 0, 0], [0, 0, 0], [0, 0, 0]])) + .toThrow(/singular/); + }); + + it('hydrate marks an under-pointed homography as loaded so it survives refit checks', () => { + const store = new CameraCalibrationStore(); + const key = store.pairKey('left', 'right'); + store.hydrate({ [key]: { AtoB: translate, BtoA: translate } }, {}, {}); + expect(store.isLoadedHomography(key)).toBe(true); + store.maybeFitPair(key); + expect(store.homographies.value[key]).toBeDefined(); + }); + }); + + describe('calibration JSON file round trip', () => { + it('serializes and reloads all pairs', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + addFourTranslationPairs(store); + store.setTransformType(key, 'translation'); + store.fitTransform(key); + const json = store.toCalibrationJson(); + + const restored = new CameraCalibrationStore(); + restored.setActivePair('left', 'right'); + const result = restored.loadCalibrationText(json); + expect(result.pairCount).toBe(1); + expect(result.cameras.sort()).toEqual(['left', 'right']); + expect(restored.correspondences.value[key]).toHaveLength(4); + expect(restored.transformTypeForPair(key)).toBe('translation'); + expect(restored.homographies.value[key].AtoB[0][2]).toBeCloseTo(5); + // Enough points back the homography, so it is treated as fitted. + expect(restored.isLoadedHomography(key)).toBe(false); + }); + + it('includes pairs that only have a loaded homography (no points)', () => { + const store = new CameraCalibrationStore(); + const key = store.pairKey('uv', 'ir'); + store.loadCalibrationText(JSON.stringify({ + version: 1, + pairs: [{ + left: 'uv', right: 'ir', points: [], leftToRight: [[1, 0, 5], [0, 1, -3], [0, 0, 1]], rightToLeft: null, + }], + })); + const restored = new CameraCalibrationStore(); + restored.loadCalibrationText(store.toCalibrationJson()); + expect(restored.homographies.value[key]).toBeDefined(); + expect(restored.isLoadedHomography(key)).toBe(true); + }); + + it('reverts alignment to original and keeps the active pair on load', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + addFourTranslationPairs(store); + store.setAlignmentMode('AtoB'); + const json = store.toCalibrationJson(); + store.loadCalibrationText(json); + expect(store.alignment.value.mode).toBe('original'); + expect(store.activePair.value).toEqual({ camA: 'left', camB: 'right' }); + }); + + it('loads a desktop-persisted calibration.json (no "type" field, one direction only)', () => { + const store = new CameraCalibrationStore(); + const result = store.loadCalibrationText(JSON.stringify({ + version: 1, + pairs: [{ + left: 'eo', + right: 'ir', + points: [[0, 0, 5, -3]], + leftToRight: [[1, 0, 5], [0, 1, -3], [0, 0, 1]], + rightToLeft: null, + transformType: 'translation', + }], + })); + expect(result.pairCount).toBe(1); + const key = store.pairKey('eo', 'ir'); + expect(store.correspondences.value[key]).toHaveLength(1); + // The missing direction is derived by inversion. + expect(store.homographies.value[key].BtoA[0][2]).toBeCloseTo(-5); + expect(store.transformTypeForPair(key)).toBe('translation'); + }); + + it('preserves the producer source stamp across load, refinement, and save', () => { + const store = new CameraCalibrationStore(); + const source = { model: 'colmap-2026-07-01', swathe: 'fl07_C' }; + store.setActivePair('left', 'right'); + store.loadCalibrationText(JSON.stringify({ + version: 1, + source, + pairs: [{ + left: 'left', right: 'right', points: [], leftToRight: null, rightToLeft: [[1, 0, 5], [0, 1, -3], [0, 0, 1]], + }], + })); + expect(store.source.value).toStrictEqual(source); + // In-app refinement replaces the transform but keeps the lineage stamp. + addFourTranslationPairs(store); + store.maybeFitPair(store.pairKey('left', 'right')); + expect(store.source.value).toStrictEqual(source); + const saved = JSON.parse(store.toCalibrationJson()); + expect(saved.source).toStrictEqual(source); + }); + + it('omits the source key when no stamp was loaded', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + addFourTranslationPairs(store); + expect('source' in JSON.parse(store.toCalibrationJson())).toBe(false); + }); + + it('clears a previous stamp when loading a file without one', () => { + const store = new CameraCalibrationStore(); + store.loadCalibrationText(JSON.stringify({ + version: 1, + source: { model: 'old' }, + pairs: [], + })); + store.loadCalibrationText(JSON.stringify({ version: 1, pairs: [] })); + expect(store.source.value).toBeNull(); + }); + + it('rejects a non-object source', () => { + const store = new CameraCalibrationStore(); + expect(() => store.loadCalibrationText(JSON.stringify({ + version: 1, source: 'colmap', pairs: [], + }))).toThrow(/"source" must be an object/); + }); + + it('hydrate restores the source stamp', () => { + const store = new CameraCalibrationStore(); + store.hydrate({}, {}, {}, { model: 'colmap-x' }); + expect(store.source.value).toStrictEqual({ model: 'colmap-x' }); + store.hydrate({}, {}, {}); + expect(store.source.value).toBeNull(); + }); + + it('flags a pair as refined once an in-app fit replaces a stamped matrix', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.loadCalibrationText(JSON.stringify({ + version: 1, + source: { model: 'colmap-x' }, + pairs: [{ + left: 'left', right: 'right', points: [], leftToRight: null, rightToLeft: [[1, 0, 5], [0, 1, -3], [0, 0, 1]], + }], + })); + // Fresh from the producer: loaded, not refined. + expect(store.isRefinedFromSource(key)).toBe(false); + addFourTranslationPairs(store); + store.maybeFitPair(key); + expect(store.isRefinedFromSource(key)).toBe(true); + }); + + it('does not flag fits as refined when no source stamp is loaded', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + addFourTranslationPairs(store); + store.fitTransform(key); + expect(store.isRefinedFromSource(key)).toBe(false); + }); + + it('keeps the refined flag across a save/load round trip', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.loadCalibrationText(JSON.stringify({ + version: 1, + source: { model: 'colmap-x' }, + pairs: [{ + left: 'left', right: 'right', points: [], leftToRight: null, rightToLeft: [[1, 0, 100], [0, 1, 100], [0, 0, 1]], + }], + })); + addFourTranslationPairs(store); + store.maybeFitPair(key); + + const restored = new CameraCalibrationStore(); + restored.loadCalibrationText(store.toCalibrationJson()); + // The refit pair saved with its backing points, so it re-marks as + // fitted (refined) rather than loaded. + expect(restored.isRefinedFromSource(key)).toBe(true); + }); + + it('rejects non-JSON, missing pairs, malformed pairs, and bad matrices without clobbering state', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + addFourTranslationPairs(store); + expect(() => store.loadCalibrationText('not json')).toThrow(/valid JSON/); + expect(() => store.loadCalibrationText('{"type": "other"}')).toThrow(/pairs/); + expect(() => store.loadCalibrationText(JSON.stringify({ + version: 1, pairs: [{ left: 'a', right: 'a' }], + }))).toThrow(/distinct/); + expect(() => store.loadCalibrationText(JSON.stringify({ + version: 1, + pairs: [{ + left: 'a', + right: 'b', + points: [], + leftToRight: [[1, 0], [0, 1]], + rightToLeft: null, + }], + }))).toThrow(/3x3/); + expect(() => store.loadCalibrationText(JSON.stringify({ + version: 1, + pairs: [{ left: 'a', right: 'b', points: [[1, 2, 3]] }], + }))).toThrow(/points row/); + expect(() => store.loadCalibrationText(JSON.stringify({ + version: 1, + pairs: [{ left: 'a', right: 'b', transformType: 'bogus' }], + }))).toThrow(/transformType/); + // Failed loads left the existing calibration alone. + expect(store.correspondences.value[key]).toHaveLength(4); + }); + }); +}); diff --git a/client/src/CameraCalibrationStore.ts b/client/src/CameraCalibrationStore.ts new file mode 100644 index 000000000..f0a6cf482 --- /dev/null +++ b/client/src/CameraCalibrationStore.ts @@ -0,0 +1,776 @@ +import { + ref, computed, Ref, ComputedRef, +} from 'vue'; +import { + invert3, applyHomography, Matrix3, Point, +} from './homography'; +import { + TransformType, TRANSFORM_TYPES, DEFAULT_TRANSFORM_TYPE, minPointsForTransform, estimateTransform, +} from './transform'; + +/** + * A single picked point pair. `a` is the point in the left camera (camA), `b` + * the point in the right camera (camB). Left/right is the order the user chose, + * which is preserved (not alphabetized) so it survives round trips through the + * calibration JSON's ordered `[leftX, leftY, rightX, rightY]` rows. + */ +export interface Correspondence { + id: number; + a: Point; + b: Point; +} + +/** Both directions of the fitted alignment transform for one camera pair. */ +export interface PairHomography { + /** Maps left (camA) image coordinates onto right (camB). */ + AtoB: Matrix3; + /** Maps right (camB) image coordinates onto left (camA). */ + BtoA: Matrix3; +} + +/** Fitted transforms keyed by {@link CameraCalibrationStore.pairKey}. */ +export type CameraHomographies = Record; + +/** + * Where a pair's homography came from: fitted in-app from picked points, or + * loaded from a calibration file (which may carry no points at all). Loaded + * homographies persist through refit checks that would otherwise clear an + * under-pointed pair, until enough points are picked to fit a replacement. + */ +type HomographySource = 'fit' | 'loaded'; + +/** + * Free-form provenance stamped into the calibration file by whatever produced + * the transforms (e.g. an external COLMAP/KAMERA model step: model version, + * swathe/flight id, generation time). DIVE never interprets it -- it is + * preserved verbatim through load/refine/save round trips so an external + * re-solver can tell which model version a returning file was refined against. + */ +export type CalibrationSource = Record; + +/** + * One camera pair in the portable calibration JSON file. This is the same + * self-describing shape the desktop platform persists as the project's + * standalone calibration.json (see desktop backend/native/common.ts), so a + * panel-saved file, the on-disk artifact, and an import-time seed are all + * interchangeable: correspondences flattened as [leftX, leftY, rightX, + * rightY] rows, plus both fitted directions (null when unfitted). + */ +export interface CalibrationFilePair { + left: string; + right: string; + points?: number[][]; + leftToRight?: Matrix3 | null; + rightToLeft?: Matrix3 | null; + transformType?: TransformType; +} + +/** Portable calibration file: everything needed to restore all pairs. */ +export interface CalibrationFile { + /** Written by panel saves for self-identification; optional on load. */ + type?: string; + version: number; + /** Producer provenance, preserved verbatim across round trips. */ + source?: CalibrationSource | null; + pairs: CalibrationFilePair[]; +} + +/** Identifying `type` value of the calibration JSON format. */ +export const CALIBRATION_FILE_TYPE = 'dive-camera-calibration'; + +/** Picked correspondences keyed by {@link CameraCalibrationStore.pairKey}. */ +export type CameraCorrespondences = Record; + +/** Chosen fit model per pair, keyed by {@link CameraCalibrationStore.pairKey}. Missing entries default to 'similarity'. */ +export type CameraTransformTypes = Record; + +/** Which image is warped onto which for the in-app aligned-picking preview. */ +export type AlignmentMode = 'original' | 'AtoB' | 'BtoA'; + +export interface AlignmentState { + mode: AlignmentMode; + opacity: number; +} + +/** Active pair. `camA` is the left camera, `camB` the right (user-chosen order). */ +export interface ActivePair { + camA: string; + camB: string; +} + +/** + * Shared, reactive state for the interactive camera-calibration tool. Lives in + * vue-media-annotator so both the geojs picking layer (client/src/layers) and the + * dive-common side panel can consume it via the provide/inject system. + * + * Implements the keypointgui blue->red pairing flow: the first click in one camera + * sets a pending point; the next click in the *other* camera completes a pair. + */ +export default class CameraCalibrationStore { + activePair: Ref; + + pickingEnabled: Ref; + + pendingPoint: Ref<{ camera: string; coord: Point } | null>; + + correspondences: Ref; + + homographies: Ref; + + transformTypes: Ref; + + alignment: Ref; + + /** + * Whether pan/zoom is linked between the active pair's two cameras through + * the fitted transform (see {@link useCalibrationNavigation}). Only has an + * effect once a transform is fitted; toggled from the panel's "Fit pan/zoom". + */ + linkedNav: Ref; + + /** + * Correspondence currently selected in the picking UI (grabbed marker / + * clicked table row), highlighted in BOTH cameras' panes and deletable via + * the panel or the Delete key. Authoring state only -- never persisted. + */ + selectedCorrespondenceId: Ref; + + /** Native-pixel coordinate under the cursor, for the calibration panel's live readout. */ + cursorCoord: Ref<{ camera: string; coord: Point } | null>; + + /** + * A one-shot "recenter here" request (e.g. from a right-click), keyed by an + * incrementing id so repeated requests at the same coordinate still trigger + * watchers. See {@link requestRecenter}. + */ + recenterRequest: Ref<{ camera: string; coord: Point; id: number } | null>; + + /** + * Message from the most recent failed fit attempt (e.g. collinear/degenerate + * points that satisfy the minimum count but can't be solved), or null if the + * active pair's last fit attempt (if any) succeeded. Surfaced by the + * calibration panel instead of letting the estimator's exception escape a + * geojs click handler. + */ + fitError: Ref; + + /** + * Provenance of the loaded calibration (see {@link CalibrationSource}). + * Deliberately NOT cleared by in-app edits or refits -- refinements are + * exactly what should travel back to the producer stamped with the model + * lineage they were made against. Replaced (or cleared) only when a + * calibration file is loaded or the store is re-hydrated. + */ + source: Ref; + + /** True when the calibration has unsaved changes since the last save or load. */ + dirty: ComputedRef; + + private nextId: number; + + private nextRecenterId: number; + + /** Provenance per homography key; missing entries behave like 'fit'. */ + private homographySources: Record; + + /** Serialized calibration at the last save/load, the baseline for {@link dirty}. */ + private savedSnapshot: Ref; + + constructor() { + this.activePair = ref(null); + this.pickingEnabled = ref(false); + this.pendingPoint = ref(null); + this.correspondences = ref({}); + this.homographies = ref({}); + this.transformTypes = ref({}); + this.alignment = ref({ mode: 'original', opacity: 0.5 }); + this.linkedNav = ref(true); + this.selectedCorrespondenceId = ref(null); + this.cursorCoord = ref(null); + this.recenterRequest = ref(null); + this.fitError = ref(null); + this.source = ref(null); + this.nextId = 1; + this.nextRecenterId = 1; + this.homographySources = {}; + this.savedSnapshot = ref(this.calibrationSnapshot()); + this.dirty = computed(() => this.calibrationSnapshot() !== this.savedSnapshot.value); + } + + /** Serialize the saved-to-dataset calibration state (points, transforms, provenance). */ + private calibrationSnapshot(): string { + return JSON.stringify({ + homographies: this.homographies.value, + correspondences: this.correspondences.value, + transformTypes: this.transformTypes.value, + source: this.source.value, + }); + } + + /** Capture the current calibration as the saved baseline, so {@link dirty} reads false. */ + markSaved() { + this.savedSnapshot.value = this.calibrationSnapshot(); + } + + /** + * Directional key for a camera pair: `left::right`. Order is significant and + * preserved so left/right (e.g. RGB vs IR) survives for ordered exports. + */ + // eslint-disable-next-line class-methods-use-this + pairKey(camA: string, camB: string): string { + return `${camA}::${camB}`; + } + + /** Key of the currently active pair, or null if none selected. */ + activePairKey(): string | null { + const pair = this.activePair.value; + return pair ? this.pairKey(pair.camA, pair.camB) : null; + } + + /** Select a camera pair. `left` becomes camA, `right` becomes camB. */ + setActivePair(left: string | null, right: string | null) { + if (!left || !right || left === right) { + this.activePair.value = null; + } else { + this.activePair.value = { camA: left, camB: right }; + } + this.pendingPoint.value = null; + // Switching pairs invalidates any active overlay warp: drop back to the + // unwarped Picking mode so the new pair starts from its own native views. + this.alignment.value = { mode: 'original', opacity: this.alignment.value.opacity }; + this.selectedCorrespondenceId.value = null; + this.cursorCoord.value = null; + this.recenterRequest.value = null; + this.fitError.value = null; + } + + /** + * Add a clicked image point for `camera`. The first click sets a pending point; + * a subsequent click in the *other* camera of the active pair completes a pair. + * Clicking the same camera again replaces the pending point. + */ + addPoint(camera: string, coord: Point) { + const pair = this.activePair.value; + if (!pair || (camera !== pair.camA && camera !== pair.camB)) { + return; + } + const pending = this.pendingPoint.value; + if (!pending || pending.camera === camera) { + this.pendingPoint.value = { camera, coord }; + return; + } + const key = this.pairKey(pair.camA, pair.camB); + const a = pending.camera === pair.camA ? pending.coord : coord; + const b = pending.camera === pair.camB ? pending.coord : coord; + const list = this.correspondences.value[key] + ? [...this.correspondences.value[key]] + : []; + // eslint-disable-next-line no-plusplus + list.push({ id: this.nextId++, a, b }); + this.correspondences.value = { ...this.correspondences.value, [key]: list }; + this.pendingPoint.value = null; + this.syncAlignmentHomography(); + } + + /** + * Record a click at `coord` (native pixel coords of `camera`'s own pane). + * New points are only picked in the unwarped 'original' (Picking) mode: while + * an overlay warp is active the panes show a warped preview rather than native + * coordinates, so clicks there are ignored. + */ + pickPoint(camera: string, coord: Point) { + if (this.alignment.value.mode !== 'original') { + return; + } + this.addPoint(camera, coord); + } + + /** + * Move one side of an existing correspondence (drag-to-refine). `camera` + * selects which side (a for camA, b for camB); the pair is refit live so + * the alignment ghost and linked navigation track the drag. + */ + updateCorrespondencePoint(id: number, camera: string, coord: Point) { + const pair = this.activePair.value; + if (!pair || (camera !== pair.camA && camera !== pair.camB)) { + return; + } + const key = this.pairKey(pair.camA, pair.camB); + const list = this.correspondences.value[key]; + if (!list || !list.some((c) => c.id === id)) { + return; + } + const side = camera === pair.camA ? 'a' : 'b'; + this.correspondences.value = { + ...this.correspondences.value, + [key]: list.map((c) => (c.id === id ? { ...c, [side]: coord } : c)), + }; + this.syncAlignmentHomography(); + } + + /** Move the pending (blue) point while it is being drag-refined. */ + movePendingPoint(camera: string, coord: Point) { + const pending = this.pendingPoint.value; + if (!pending || pending.camera !== camera) { + return; + } + this.pendingPoint.value = { camera, coord }; + } + + /** Remove a correspondence (by id) from the active pair -- both cameras' points at once. */ + removeCorrespondence(id: number) { + const key = this.activePairKey(); + if (!key) { + return; + } + const list = this.correspondences.value[key]; + if (!list) { + return; + } + this.correspondences.value = { + ...this.correspondences.value, + [key]: list.filter((c) => c.id !== id), + }; + if (this.selectedCorrespondenceId.value === id) { + this.selectedCorrespondenceId.value = null; + } + this.syncAlignmentHomography(); + } + + /** + * Select a correspondence marker for inspection/deletion (null clears). + * Only ids belonging to the active pair are selectable; anything else + * clears the selection. + */ + selectCorrespondence(id: number | null) { + if (id === null) { + this.selectedCorrespondenceId.value = null; + return; + } + const key = this.activePairKey(); + const list = key ? this.correspondences.value[key] : undefined; + this.selectedCorrespondenceId.value = (list && list.some((c) => c.id === id)) ? id : null; + } + + /** Remove the selected correspondence (both cameras' points). No-op without a selection. */ + removeSelectedCorrespondence() { + const id = this.selectedCorrespondenceId.value; + if (id !== null) { + this.removeCorrespondence(id); + } + } + + /** + * Drop all correspondences, the pending point, and any homography + * (fitted or file-loaded) for the active pair. + */ + clearPair() { + const key = this.activePairKey(); + this.pendingPoint.value = null; + this.selectedCorrespondenceId.value = null; + if (!key) { + return; + } + this.correspondences.value = { ...this.correspondences.value, [key]: [] }; + // Clearing is explicit: a file-loaded homography goes too. Dropping the + // 'loaded' mark lets maybeFitPair remove it through the normal path. + delete this.homographySources[key]; + this.maybeFitPair(key); + } + + /** + * Undo one step, mirroring keypointgui's Clear Last button: if there's a + * pending (blue) point, drop it; otherwise remove the most recently + * completed correspondence for the active pair. + */ + clearLast() { + if (this.pendingPoint.value) { + this.pendingPoint.value = null; + return; + } + const key = this.activePairKey(); + if (!key) { + return; + } + const list = this.correspondences.value[key]; + if (!list || list.length === 0) { + return; + } + if (this.selectedCorrespondenceId.value === list[list.length - 1].id) { + this.selectedCorrespondenceId.value = null; + } + this.correspondences.value = { ...this.correspondences.value, [key]: list.slice(0, -1) }; + this.syncAlignmentHomography(); + } + + /** + * True when `key`'s homography came from a calibration file rather than an + * in-app fit. Not independently reactive -- always read alongside + * {@link homographies} (provenance only changes when that map does). + */ + isLoadedHomography(key: string): boolean { + return this.homographySources[key] === 'loaded'; + } + + /** + * True when `key`'s transform was fitted from in-app picked points while a + * producer-stamped calibration is loaded -- i.e. the pair has diverged from + * what the stamped {@link source} shipped (producer files carry matrix-only + * pairs, so any point-backed fit is a human refinement). Derived rather + * than stored: it survives save/reload naturally, because point-backed + * pairs re-mark as fitted on hydrate. Read alongside {@link homographies}, + * same reactivity caveat as {@link isLoadedHomography}. + */ + isRefinedFromSource(key: string): boolean { + return this.source.value !== null + && this.homographies.value[key] !== undefined + && this.homographySources[key] === 'fit'; + } + + /** The chosen fit model for `key`, defaulting to {@link DEFAULT_TRANSFORM_TYPE} when unset. */ + transformTypeForPair(key: string): TransformType { + return this.transformTypes.value[key] || DEFAULT_TRANSFORM_TYPE; + } + + /** Choose the fit model for `key` and immediately (re)fit or clear as needed. */ + setTransformType(key: string, type: TransformType) { + this.transformTypes.value = { ...this.transformTypes.value, [key]: type }; + this.maybeFitPair(key); + } + + /** + * Fit `key` when it has enough points for its chosen transform type; otherwise + * clear its homography and, if it's the active (aligned) pair, revert + * alignment to 'original'. A fit can still fail past the minimum-count check + * (e.g. collinear/near-duplicate points make the system unsolvable); that's + * caught here and surfaced via {@link fitError} instead of throwing out of a + * geojs click handler, keeping any previously fitted homography in place. + */ + maybeFitPair(key: string) { + const list = this.correspondences.value[key]; + const required = minPointsForTransform(this.transformTypeForPair(key)); + if (!list || list.length < required) { + // A file-loaded homography has no backing points; it stays in place + // until enough points are picked to fit a replacement (or the pair is + // explicitly cleared, which drops its 'loaded' mark first). + if (this.homographySources[key] !== 'loaded') { + const rest = { ...this.homographies.value }; + delete rest[key]; + this.homographies.value = rest; + delete this.homographySources[key]; + if (this.activePairKey() === key && this.alignment.value.mode !== 'original') { + this.alignment.value = { ...this.alignment.value, mode: 'original' }; + } + } + if (this.activePairKey() === key) { + this.fitError.value = null; + } + return; + } + try { + this.fitTransform(key); + if (this.activePairKey() === key) { + this.fitError.value = null; + } + } catch (err) { + if (this.activePairKey() === key) { + this.fitError.value = err instanceof Error ? err.message : String(err); + } + } + } + + /** Fit the active pair when it has enough points; otherwise clear/revert as in {@link maybeFitPair}. */ + maybeFitActivePair() { + const key = this.activePairKey(); + if (!key) { + return; + } + this.maybeFitPair(key); + } + + /** Enable or change the alignment (ghost overlay) mode, fitting the pair first if needed. */ + setAlignmentMode(mode: AlignmentMode) { + if (mode !== 'original') { + this.maybeFitActivePair(); + const key = this.activePairKey(); + if (!key || !this.homographies.value[key]) { + // Not enough points for the active pair's transform type; stay original. + return; + } + } + this.alignment.value = { ...this.alignment.value, mode }; + } + + /** Ghost overlay opacity, independent of alignment mode. */ + setAlignmentOpacity(opacity: number) { + this.alignment.value = { ...this.alignment.value, opacity }; + } + + /** + * Map `coord` (native pixel space of `camera`) to the corresponding point in + * the *other* camera of the active pair, via the fitted homography. Returns + * `null` when `camera` isn't part of the active pair or the pair has no + * fitted homography yet (not enough correspondences) -- callers should treat + * that as "nothing to link to" rather than an error. + */ + linkedPoint(camera: string, coord: Point): { camera: string; coord: Point } | null { + const pair = this.activePair.value; + if (!pair || (camera !== pair.camA && camera !== pair.camB)) { + return null; + } + const homog = this.homographies.value[this.pairKey(pair.camA, pair.camB)]; + if (!homog) { + return null; + } + const other = camera === pair.camA ? pair.camB : pair.camA; + const matrix = camera === pair.camA ? homog.AtoB : homog.BtoA; + return { camera: other, coord: applyHomography(matrix, coord) }; + } + + /** Record the native-pixel coordinate under the cursor for `camera` (calibration panel readout). */ + setCursorCoord(camera: string, coord: Point) { + this.cursorCoord.value = { camera, coord }; + } + + /** Clear the cursor coordinate readout (e.g. on mouse leave). */ + clearCursorCoord() { + this.cursorCoord.value = null; + } + + /** + * Request that `camera` (native pixel coords `coord`) and, when the pair has + * a fitted homography, the other camera of the active pair (via + * {@link linkedPoint}) recenter their views on this location. Consumed by + * {@link useCalibrationNavigation}; a no-op if `camera` isn't part of the + * active pair. A one-shot "snap to this feature" action, distinct from the + * continuous pan/zoom link that is active while picking. + */ + requestRecenter(camera: string, coord: Point) { + const pair = this.activePair.value; + if (!pair || (camera !== pair.camA && camera !== pair.camB)) { + return; + } + // eslint-disable-next-line no-plusplus + this.recenterRequest.value = { camera, coord, id: this.nextRecenterId++ }; + } + + /** Re-fit the active pair while alignment is active (mode != 'original'). */ + private syncAlignmentHomography() { + if (this.alignment.value.mode !== 'original') { + this.maybeFitActivePair(); + } + } + + /** + * Fit `key`'s chosen transform type from its correspondences (see + * {@link minPointsForTransform} for the required count). Computes both + * directions and stores them. Returns the fitted pair. + */ + fitTransform(key: string): PairHomography { + const list = this.correspondences.value[key]; + const type = this.transformTypeForPair(key); + const required = minPointsForTransform(type); + if (!list || list.length < required) { + throw new Error(`At least ${required} point pair(s) are required to fit a ${type} transform`); + } + const AtoB = estimateTransform(type, list.map((c) => c.a), list.map((c) => c.b)); + const BtoA = invert3(AtoB); + this.homographies.value = { ...this.homographies.value, [key]: { AtoB, BtoA } }; + this.homographySources[key] = 'fit'; + return { AtoB, BtoA }; + } + + /** + * Serialize every pair with content (points and/or a homography) as the + * portable calibration JSON file (see {@link CalibrationFile}). Pairs whose + * only state is a transform-type choice are omitted. + */ + toCalibrationJson(): string { + const keys = new Set([ + ...Object.keys(this.correspondences.value).filter( + (key) => this.correspondences.value[key].length > 0, + ), + ...Object.keys(this.homographies.value), + ]); + const pairs: CalibrationFilePair[] = [...keys].sort().map((key) => { + const [left, right] = key.split('::'); + const homography = this.homographies.value[key] || null; + return { + left, + right, + points: (this.correspondences.value[key] || []).map((c) => [c.a[0], c.a[1], c.b[0], c.b[1]]), + leftToRight: homography ? homography.AtoB : null, + rightToLeft: homography ? homography.BtoA : null, + transformType: this.transformTypeForPair(key), + }; + }); + const file: CalibrationFile = { + type: CALIBRATION_FILE_TYPE, + version: 1, + ...(this.source.value ? { source: this.source.value } : {}), + pairs, + }; + return JSON.stringify(file, null, 2); + } + + /** + * Parse and load a calibration JSON file (the format written by + * {@link toCalibrationJson}), REPLACING all pairs' correspondences, + * homographies, and transform types. The active pair selection and picking + * toggle are left alone; the alignment ghost reverts to 'original' since + * the transform under it changed wholesale. Throws a descriptive Error on + * malformed input without touching current state. Returns the camera names + * referenced by the file so callers can warn about ones missing from the + * loaded dataset. + */ + loadCalibrationText(text: string): { cameras: string[]; pairCount: number } { + let data: unknown; + try { + data = JSON.parse(text); + } catch { + throw new Error('File is not valid JSON'); + } + const file = data as Partial; + if (!Array.isArray(file?.pairs)) { + throw new Error('Not a DIVE camera calibration file (expected a "pairs" list)'); + } + const source = CameraCalibrationStore.readSource(file.source); + const correspondences: CameraCorrespondences = {}; + const homographies: CameraHomographies = {}; + const transformTypes: CameraTransformTypes = {}; + const cameras = new Set(); + file.pairs.forEach((pair, i) => { + const context = `Pair ${i + 1}`; + if (typeof pair?.left !== 'string' || typeof pair?.right !== 'string' + || !pair.left || !pair.right || pair.left === pair.right) { + throw new Error(`${context}: "left" and "right" must be two distinct camera names`); + } + const key = this.pairKey(pair.left, pair.right); + cameras.add(pair.left); + cameras.add(pair.right); + if (pair.transformType !== undefined) { + if (!TRANSFORM_TYPES.some((t) => t.value === pair.transformType)) { + throw new Error( + `${context}: unknown transformType "${pair.transformType}" (expected one of ${TRANSFORM_TYPES.map((t) => t.value).join(', ')})`, + ); + } + transformTypes[key] = pair.transformType; + } + correspondences[key] = (pair.points || []).map((row, j) => { + const [ax, ay, bx, by] = CameraCalibrationStore.readPointsRow(row, `${context}, points row ${j + 1}`); + // eslint-disable-next-line no-plusplus + return { id: this.nextId++, a: [ax, ay] as Point, b: [bx, by] as Point }; + }); + const leftToRight = (pair.leftToRight === null || pair.leftToRight === undefined) + ? null + : CameraCalibrationStore.readMatrix(pair.leftToRight, `${context}, leftToRight`); + const rightToLeft = (pair.rightToLeft === null || pair.rightToLeft === undefined) + ? null + : CameraCalibrationStore.readMatrix(pair.rightToLeft, `${context}, rightToLeft`); + if (leftToRight || rightToLeft) { + // If only one direction is present, derive the other by inversion + // (readMatrix guarantees invertibility). + homographies[key] = { + AtoB: leftToRight ?? invert3(rightToLeft as Matrix3), + BtoA: rightToLeft ?? invert3(leftToRight as Matrix3), + }; + } + }); + this.correspondences.value = correspondences; + this.homographies.value = homographies; + this.transformTypes.value = transformTypes; + this.source.value = source; + this.markHomographySources(); + this.pendingPoint.value = null; + this.selectedCorrespondenceId.value = null; + this.fitError.value = null; + this.alignment.value = { ...this.alignment.value, mode: 'original' }; + return { cameras: [...cameras], pairCount: file.pairs.length }; + } + + /** Validate an untrusted `source` value: a plain object, or absent (-> null). */ + private static readSource(raw: unknown): CalibrationSource | null { + if (raw === undefined || raw === null) { + return null; + } + if (typeof raw !== 'object' || Array.isArray(raw)) { + throw new Error('"source" must be an object when present'); + } + return raw as CalibrationSource; + } + + /** Validate an untrusted value as a 4-element finite [leftX, leftY, rightX, rightY] row. */ + private static readPointsRow(raw: unknown, context: string): [number, number, number, number] { + if (!Array.isArray(raw) || raw.length !== 4) { + throw new Error(`${context}: expected [leftX, leftY, rightX, rightY]`); + } + const nums = raw.map(Number); + if (nums.some((n) => !Number.isFinite(n))) { + throw new Error(`${context}: expected [leftX, leftY, rightX, rightY] with finite numbers`); + } + return [nums[0], nums[1], nums[2], nums[3]]; + } + + /** Validate an untrusted value as an invertible row-major 3x3 matrix. */ + private static readMatrix(raw: unknown, context: string): Matrix3 { + if (!Array.isArray(raw) || raw.length !== 3 + || raw.some((row) => !Array.isArray(row) || row.length !== 3)) { + throw new Error(`${context}: expected a 3x3 matrix`); + } + const m = (raw as unknown[][]).map((row) => row.map(Number)); + if (m.some((row) => row.some((n) => !Number.isFinite(n)))) { + throw new Error(`${context}: matrix entries must be finite numbers`); + } + try { + invert3(m); + } catch { + throw new Error(`${context}: matrix is singular (not invertible)`); + } + return m; + } + + /** + * Reset homography provenance after bulk-loading state: a homography whose + * pair lacks enough points for its transform type can only have come from a + * file ('loaded', so refit checks preserve it); one with enough points is + * treated as fitted from them. + */ + private markHomographySources() { + this.homographySources = {}; + Object.keys(this.homographies.value).forEach((key) => { + const count = (this.correspondences.value[key] || []).length; + const required = minPointsForTransform(this.transformTypeForPair(key)); + this.homographySources[key] = count >= required ? 'fit' : 'loaded'; + }); + } + + /** Reset state and load saved homographies, correspondences, transform type choices, and provenance. */ + hydrate( + homographies?: CameraHomographies, + correspondences?: CameraCorrespondences, + transformTypes?: CameraTransformTypes, + source?: CalibrationSource | null, + ) { + this.homographies.value = homographies ? { ...homographies } : {}; + this.correspondences.value = correspondences ? { ...correspondences } : {}; + this.transformTypes.value = transformTypes ? { ...transformTypes } : {}; + this.source.value = source ?? null; + this.markHomographySources(); + this.activePair.value = null; + this.pendingPoint.value = null; + this.pickingEnabled.value = false; + this.alignment.value = { mode: 'original', opacity: 0.5 }; + this.selectedCorrespondenceId.value = null; + this.cursorCoord.value = null; + this.recenterRequest.value = null; + this.fitError.value = null; + // Resume id allocation past any restored correspondences. + let maxId = 0; + Object.values(this.correspondences.value).forEach((list) => { + list.forEach((c) => { maxId = Math.max(maxId, c.id); }); + }); + this.nextId = maxId + 1; + // The freshly loaded state is the saved baseline. + this.markSaved(); + } +} diff --git a/client/src/alignedView.spec.ts b/client/src/alignedView.spec.ts new file mode 100644 index 000000000..bb5c6acf3 --- /dev/null +++ b/client/src/alignedView.spec.ts @@ -0,0 +1,301 @@ +/// +import { + IDENTITY3, + isIdentityMatrix3, + readTransformMatrix, + composeThroughPairs, + resolveToReferenceTransforms, + unresolvedCameras, + cameraPairTransform, + mapPoint, + mapBounds, + mapRotatedBounds, + mapGeoJSONFeatures, +} from './alignedView'; +import { applyHomography, Matrix3, Point } from './homography'; +import type { CameraHomographies } from './CameraCalibrationStore'; +import AlignedViewStore from './AlignedViewStore'; + +/** Simple affine helpers for readable fixtures. */ +function translation(tx: number, ty: number): Matrix3 { + return [[1, 0, tx], [0, 1, ty], [0, 0, 1]]; +} +function scale(s: number): Matrix3 { + return [[s, 0, 0], [0, s, 0], [0, 0, 1]]; +} + +function expectPointClose(actual: Point, expected: Point) { + expect(actual[0]).toBeCloseTo(expected[0], 6); + expect(actual[1]).toBeCloseTo(expected[1], 6); +} + +describe('isIdentityMatrix3', () => { + it('accepts the identity and rejects non-identity', () => { + expect(isIdentityMatrix3(IDENTITY3)).toBe(true); + expect(isIdentityMatrix3(translation(0, 0))).toBe(true); + expect(isIdentityMatrix3(translation(1, 0))).toBe(false); + expect(isIdentityMatrix3(scale(2))).toBe(false); + }); +}); + +describe('readTransformMatrix', () => { + it('accepts a valid row-major 3x3', () => { + const m = readTransformMatrix([[1, 0, 5], [0, 1, -3], [0, 0, 1]]); + expect(m).toEqual([[1, 0, 5], [0, 1, -3], [0, 0, 1]]); + }); + it('rejects malformed shapes', () => { + expect(readTransformMatrix(undefined)).toBeNull(); + expect(readTransformMatrix(null)).toBeNull(); + expect(readTransformMatrix('matrix')).toBeNull(); + expect(readTransformMatrix([[1, 0], [0, 1]])).toBeNull(); + expect(readTransformMatrix([[1, 0, 0], [0, 1, 0]])).toBeNull(); + expect(readTransformMatrix([[1, 0, 0], [0, 1, 0], [0, 0]])).toBeNull(); + }); + it('rejects non-finite values', () => { + expect(readTransformMatrix([[1, 0, NaN], [0, 1, 0], [0, 0, 1]])).toBeNull(); + expect(readTransformMatrix([[1, 0, Infinity], [0, 1, 0], [0, 0, 1]])).toBeNull(); + expect(readTransformMatrix([[1, 0, '5'], [0, 1, 'abc'], [0, 0, 1]])).toBeNull(); + }); + it('rejects singular matrices', () => { + expect(readTransformMatrix([[1, 1, 0], [1, 1, 0], [0, 0, 1]])).toBeNull(); + expect(readTransformMatrix([[0, 0, 0], [0, 0, 0], [0, 0, 0]])).toBeNull(); + }); +}); + +describe('composeThroughPairs', () => { + const irToEo = translation(100, 50); + const uvToIr = scale(2); + const homographies: CameraHomographies = { + // eo::ir stored with AtoB = eo->ir, so BtoA = ir->eo. + 'eo::ir': { AtoB: [[1, 0, -100], [0, 1, -50], [0, 0, 1]], BtoA: irToEo }, + // uv::ir stored with AtoB = uv->ir. + 'uv::ir': { AtoB: uvToIr, BtoA: [[0.5, 0, 0], [0, 0.5, 0], [0, 0, 1]] }, + }; + it('returns identity for the reference itself', () => { + expect(composeThroughPairs('eo', 'eo', homographies)).toEqual(IDENTITY3); + }); + it('uses a direct pair edge in either direction', () => { + const m = composeThroughPairs('ir', 'eo', homographies); + expect(m).not.toBeNull(); + expectPointClose(applyHomography(m as Matrix3, [10, 20]), [110, 70]); + }); + it('composes multi-hop paths (uv -> ir -> eo)', () => { + const m = composeThroughPairs('uv', 'eo', homographies); + expect(m).not.toBeNull(); + // uv (x, y) -> ir (2x, 2y) -> eo (2x + 100, 2y + 50). + expectPointClose(applyHomography(m as Matrix3, [10, 20]), [120, 90]); + }); + it('returns null when no path exists', () => { + expect(composeThroughPairs('flir', 'eo', homographies)).toBeNull(); + expect(composeThroughPairs('ir', 'eo', {})).toBeNull(); + }); +}); + +describe('resolveToReferenceTransforms', () => { + const cameras = ['eo', 'ir', 'uv']; + const homographies: CameraHomographies = { + 'eo::ir': { AtoB: translation(-100, 0), BtoA: translation(100, 0) }, + 'uv::ir': { AtoB: scale(2), BtoA: scale(0.5) }, + }; + it('resolves every camera through calibration pairs', () => { + const result = resolveToReferenceTransforms(cameras, 'eo', homographies); + expect(result).not.toBeNull(); + const transforms = result as Record; + expect(transforms.eo).toEqual(IDENTITY3); + expectPointClose(applyHomography(transforms.ir, [5, 5]), [105, 5]); + expectPointClose(applyHomography(transforms.uv, [5, 5]), [110, 10]); + }); + it('is all-or-none: any unresolved camera fails the whole set', () => { + expect(resolveToReferenceTransforms(['eo', 'ir', 'flir'], 'eo', homographies)).toBeNull(); + expect(resolveToReferenceTransforms(cameras, 'eo', {})).toBeNull(); + }); + it('fails when the reference is not among the cameras or the list is empty', () => { + expect(resolveToReferenceTransforms([], 'eo', homographies)).toBeNull(); + expect(resolveToReferenceTransforms(['ir', 'uv'], 'eo', homographies)).toBeNull(); + }); +}); + +describe('unresolvedCameras', () => { + const homographies: CameraHomographies = { + 'eo::ir': { AtoB: translation(-100, 0), BtoA: translation(100, 0) }, + }; + it('names cameras with no path to the reference', () => { + expect(unresolvedCameras(['eo', 'ir', 'uv'], 'eo', homographies)).toEqual(['uv']); + expect(unresolvedCameras(['eo', 'ir', 'uv'], 'eo', {})).toEqual(['ir', 'uv']); + }); + it('is empty when every camera resolves', () => { + expect(unresolvedCameras(['eo', 'ir'], 'eo', homographies)).toEqual([]); + }); +}); + +describe('cameraPairTransform', () => { + it('composes from -> reference -> to', () => { + const toReference = { + eo: IDENTITY3, + ir: translation(100, 0), + uv: scale(2), + }; + // ir (x, y) -> ref (x + 100, y) -> uv ((x + 100) / 2, y / 2). + const m = cameraPairTransform(toReference, 'ir', 'uv'); + expect(m).not.toBeNull(); + expectPointClose(applyHomography(m as Matrix3, [10, 20]), [55, 10]); + }); + it('returns null for unresolved cameras', () => { + expect(cameraPairTransform({ eo: IDENTITY3 }, 'eo', 'ir')).toBeNull(); + expect(cameraPairTransform({ eo: IDENTITY3 }, 'ir', 'eo')).toBeNull(); + }); +}); + +describe('mapPoint', () => { + it('is identity for a null matrix', () => { + expect(mapPoint(null, [3, 4])).toEqual([3, 4]); + }); + it('applies the matrix otherwise', () => { + expectPointClose(mapPoint(translation(1, 2), [3, 4]), [4, 6]); + }); +}); + +describe('mapBounds', () => { + it('translates and scales an axis-aligned box', () => { + expect(mapBounds(translation(10, -5), [0, 0, 4, 6])).toEqual([10, -5, 14, 1]); + expect(mapBounds(scale(2), [1, 2, 3, 4])).toEqual([2, 4, 6, 8]); + }); + it('returns the AABB of the mapped corners under rotation', () => { + // 90-degree rotation about the origin: (x, y) -> (-y, x). + const rot90: Matrix3 = [[0, -1, 0], [1, 0, 0], [0, 0, 1]]; + const [x1, y1, x2, y2] = mapBounds(rot90, [0, 0, 4, 2]); + expect(x1).toBeCloseTo(-2, 6); + expect(y1).toBeCloseTo(0, 6); + expect(x2).toBeCloseTo(0, 6); + expect(y2).toBeCloseTo(4, 6); + }); +}); + +describe('mapRotatedBounds', () => { + const rot = (angle: number): Matrix3 => [ + [Math.cos(angle), -Math.sin(angle), 0], + [Math.sin(angle), Math.cos(angle), 0], + [0, 0, 1], + ]; + it('adds the transform rotation and keeps the rect size under similarity', () => { + const theta = Math.PI / 6; + const result = mapRotatedBounds(rot(theta), [10, 20, 50, 40], Math.PI / 8); + expect(result.rotation).toBeCloseTo(Math.PI / 8 + theta, 6); + // The rect's own dimensions (40 x 20) are preserved by a pure rotation. + expect(result.bounds[2] - result.bounds[0]).toBeCloseTo(40, 6); + expect(result.bounds[3] - result.bounds[1]).toBeCloseTo(20, 6); + }); + it('is a passthrough for the identity', () => { + const result = mapRotatedBounds(IDENTITY3, [10, 20, 50, 40], 0.3); + expect(result.rotation).toBeCloseTo(0.3, 6); + expect(result.bounds[0]).toBeCloseTo(10, 6); + expect(result.bounds[1]).toBeCloseTo(20, 6); + expect(result.bounds[2]).toBeCloseTo(50, 6); + expect(result.bounds[3]).toBeCloseTo(40, 6); + }); + it('scales the rect and centers under translation + scale', () => { + const matrix = [[2, 0, 100], [0, 2, -10], [0, 0, 1]]; + const result = mapRotatedBounds(matrix, [0, 0, 10, 20], 0.5); + expect(result.rotation).toBeCloseTo(0.5, 6); + expect(result.bounds[2] - result.bounds[0]).toBeCloseTo(20, 6); + expect(result.bounds[3] - result.bounds[1]).toBeCloseTo(40, 6); + // The center (5, 10) maps to (110, 10). + expect((result.bounds[0] + result.bounds[2]) / 2).toBeCloseTo(110, 6); + expect((result.bounds[1] + result.bounds[3]) / 2).toBeCloseTo(10, 6); + }); +}); + +describe('mapGeoJSONFeatures', () => { + it('maps points, lines, and polygon rings without mutating the source', () => { + const source: GeoJSON.Feature[] = [ + { + type: 'Feature', + geometry: { type: 'Point', coordinates: [1, 2] }, + properties: { key: 'head' }, + }, + { + type: 'Feature', + geometry: { type: 'LineString', coordinates: [[0, 0], [3, 4]] }, + properties: { key: 'HeadTails' }, + }, + { + type: 'Feature', + geometry: { + type: 'Polygon', + coordinates: [[[0, 0], [4, 0], [4, 4], [0, 0]], [[1, 1], [2, 1], [2, 2], [1, 1]]], + }, + properties: { key: '' }, + }, + ]; + const mapped = mapGeoJSONFeatures(translation(10, 20), source); + expect(mapped[0].geometry).toEqual({ type: 'Point', coordinates: [11, 22] }); + expect(mapped[1].geometry).toEqual({ type: 'LineString', coordinates: [[10, 20], [13, 24]] }); + expect(mapped[2].geometry.type).toBe('Polygon'); + expect((mapped[2].geometry as GeoJSON.Polygon).coordinates[1][2]).toEqual([12, 22]); + expect(mapped[0].properties).toEqual({ key: 'head' }); + // Deep copy: the source geometry/properties are untouched and unshared. + expect(source[0].geometry).toEqual({ type: 'Point', coordinates: [1, 2] }); + expect(mapped[0].properties).not.toBe(source[0].properties); + expect(mapped[2].geometry).not.toBe(source[2].geometry); + }); +}); + +describe('AlignedViewStore', () => { + function makeResolvedStore() { + const store = new AlignedViewStore(); + store.setTransforms('eo', { + eo: IDENTITY3, + ir: translation(100, 0), + }); + return store; + } + + it('is unavailable and inactive by default', () => { + const store = new AlignedViewStore(); + expect(store.available.value).toBe(false); + store.setEnabled(true); + expect(store.active.value).toBe(false); + expect(store.cameraTransform('ir')).toBeNull(); + expect(store.cameraToCamera('eo', 'ir')).toBeNull(); + }); + + it('activates only when enabled, available, and not suspended', () => { + const store = makeResolvedStore(); + expect(store.available.value).toBe(true); + expect(store.active.value).toBe(false); + store.setEnabled(true); + expect(store.active.value).toBe(true); + store.setSuspended(true); + expect(store.active.value).toBe(false); + expect(store.cameraTransform('ir')).toBeNull(); + store.setSuspended(false); + expect(store.active.value).toBe(true); + }); + + it('exposes a display transform for warped cameras and null for the reference', () => { + const store = makeResolvedStore(); + store.setEnabled(true); + expect(store.cameraTransform('eo')).toBeNull(); + const m = store.cameraTransform('ir'); + expect(m).not.toBeNull(); + expectPointClose(applyHomography(m as Matrix3, [1, 1]), [101, 1]); + // Unknown camera also renders unwarped. + expect(store.cameraTransform('uv')).toBeNull(); + }); + + it('maps cross-camera points through the reference', () => { + const store = makeResolvedStore(); + store.setEnabled(true); + expectPointClose(store.mapCameraPoint('ir', 'eo', [0, 0]) as Point, [100, 0]); + expectPointClose(store.mapCameraPoint('eo', 'ir', [100, 0]) as Point, [0, 0]); + expect(store.mapCameraPoint('eo', 'flir', [0, 0])).toBeNull(); + }); + + it('becomes unavailable when a camera set collapses to one entry', () => { + const store = new AlignedViewStore(); + store.setTransforms('eo', { eo: IDENTITY3 }); + store.setEnabled(true); + expect(store.available.value).toBe(false); + expect(store.active.value).toBe(false); + }); +}); diff --git a/client/src/alignedView.ts b/client/src/alignedView.ts new file mode 100644 index 000000000..eed028c0e --- /dev/null +++ b/client/src/alignedView.ts @@ -0,0 +1,305 @@ +/** + * Pure helpers for the multicam "aligned view" (SEAL-TK features 2 + 3): + * resolving a per-camera native->reference-space transform for every loaded + * camera from the available sources, and composing camera-to-camera mappings + * through the reference camera. + * + * Conventions (documented assumptions, see migration plan Q3): + * - The REFERENCE camera is the first camera in display order + * (`meta.multiCamMedia.cameraOrder[0]`, i.e. `multiCamList[0]`). Its + * transform is the identity. + * - Calibration-tool homographies are stored per directional pair key + * `camA::camB` with `AtoB` mapping camA pixels onto camB (and `BtoA` the + * inverse); a camera's path to the reference is found by composing pair + * edges (breadth-first, so up-to-3-camera rigs may chain e.g. UV->IR->EO). + * - The calibration store is the SINGLE source the viewer resolves from: + * whatever the calibration panel shows/saves is what the Align button + * applies. External calibration .json files enter that same store, either + * through the panel's Load calibration button or by seeding the saved + * calibration at multicam import time. + */ +import { + Matrix3, matMul3, invert3, applyHomography, Point, +} from './homography'; +import type { CameraHomographies } from './CameraCalibrationStore'; +import type { RectBounds } from './utils'; + +export const IDENTITY3: Matrix3 = [[1, 0, 0], [0, 1, 0], [0, 0, 1]]; + +/** True when `m` is (numerically) the identity transform. */ +export function isIdentityMatrix3(m: Matrix3, eps = 1e-9): boolean { + for (let i = 0; i < 3; i += 1) { + for (let j = 0; j < 3; j += 1) { + if (Math.abs(m[i][j] - IDENTITY3[i][j]) > eps) { + return false; + } + } + } + return true; +} + +/** + * Defensively validate an untrusted value (e.g. read from dataset meta JSON) + * as a row-major 3x3 matrix usable as a display transform. Returns null for + * anything malformed, non-finite, or singular (non-invertible) rather than + * throwing, so callers can treat "bad matrix" as "no matrix". + */ +export function readTransformMatrix(raw: unknown): Matrix3 | null { + if (!Array.isArray(raw) || raw.length !== 3) { + return null; + } + const m: number[][] = []; + for (let i = 0; i < 3; i += 1) { + const row = raw[i]; + if (!Array.isArray(row) || row.length !== 3) { + return null; + } + const nums = row.map(Number); + if (nums.some((v) => !Number.isFinite(v))) { + return null; + } + m.push(nums); + } + try { + invert3(m); + } catch { + return null; + } + return m; +} + +/** One directed edge of the calibration-pair graph: `matrix` maps `from` pixels onto `to`. */ +interface PairEdge { + to: string; + matrix: Matrix3; +} + +/** Build the bidirectional camera adjacency from calibration pair homographies. */ +function buildPairGraph(homographies: CameraHomographies): Record { + const graph: Record = {}; + const addEdge = (from: string, to: string, matrix: Matrix3) => { + if (!graph[from]) { + graph[from] = []; + } + graph[from].push({ to, matrix }); + }; + Object.entries(homographies).forEach(([key, pair]) => { + const [camA, camB] = key.split('::'); + if (!camA || !camB || camA === camB || !pair) { + return; + } + addEdge(camA, camB, pair.AtoB); + addEdge(camB, camA, pair.BtoA); + }); + return graph; +} + +/** + * Compose a `camera` -> `reference` matrix by walking the calibration pair + * graph breadth-first (shortest hop count). Returns null when no path exists. + */ +export function composeThroughPairs( + camera: string, + reference: string, + homographies: CameraHomographies, +): Matrix3 | null { + if (camera === reference) { + return IDENTITY3; + } + const graph = buildPairGraph(homographies); + // BFS queue of (camera, accumulated camera->node matrix). + const queue: { node: string; matrix: Matrix3 }[] = [{ node: camera, matrix: IDENTITY3 }]; + const visited = new Set([camera]); + while (queue.length) { + const { node, matrix } = queue.shift() as { node: string; matrix: Matrix3 }; + const edges = graph[node] || []; + for (let i = 0; i < edges.length; i += 1) { + const edge = edges[i]; + if (!visited.has(edge.to)) { + // p_to = edge.matrix * p_node and p_node = matrix * p_camera. + const composed = matMul3(edge.matrix, matrix); + if (edge.to === reference) { + return composed; + } + visited.add(edge.to); + queue.push({ node: edge.to, matrix: composed }); + } + } + } + return null; +} + +/** + * Resolve a native->reference matrix for EVERY camera in `cameras` from the + * calibration store's pair homographies, or null when any camera lacks a + * usable transform (the aligned view is all-or-none: a partially-aligned + * display would be misleading). The reference camera always maps by the + * identity. + */ +export function resolveToReferenceTransforms( + cameras: string[], + reference: string, + homographies: CameraHomographies, +): Record | null { + if (!cameras.length || !cameras.includes(reference)) { + return null; + } + const result: Record = {}; + for (let i = 0; i < cameras.length; i += 1) { + const camera = cameras[i]; + if (camera === reference) { + result[camera] = IDENTITY3; + } else { + const matrix = composeThroughPairs(camera, reference, homographies); + if (!matrix) { + return null; + } + result[camera] = matrix; + } + } + return result; +} + +/** + * The cameras in `cameras` with no composed transform path to `reference` -- + * the reason {@link resolveToReferenceTransforms} returned null, named so the + * UI can say exactly which pair(s) still need calibrating instead of silently + * hiding the aligned view. + */ +export function unresolvedCameras( + cameras: string[], + reference: string, + homographies: CameraHomographies, +): string[] { + return cameras.filter( + (camera) => camera !== reference + && composeThroughPairs(camera, reference, homographies) === null, + ); +} + +/** + * Matrix mapping `from`-camera native pixels onto `to`-camera native pixels, + * composed through the shared reference space: + * p_to = inv(T_to->ref) * T_from->ref * p_from. + * Returns null when either camera has no resolved transform. + */ +export function cameraPairTransform( + toReference: Record, + from: string, + to: string, +): Matrix3 | null { + const fromRef = toReference[from]; + const toRef = toReference[to]; + if (!fromRef || !toRef) { + return null; + } + try { + return matMul3(invert3(toRef), fromRef); + } catch { + return null; + } +} + +/** Map a point through a nullable transform (identity when null). */ +export function mapPoint(matrix: Matrix3 | null, point: Point): Point { + if (!matrix) { + return point; + } + return applyHomography(matrix, point); +} + +/** Axis-aligned bounding box of a set of points. */ +function pointsToBounds(points: Point[]): RectBounds { + const xs = points.map((p) => p[0]); + const ys = points.map((p) => p[1]); + return [Math.min(...xs), Math.min(...ys), Math.max(...xs), Math.max(...ys)]; +} + +/** The four corners of a RectBounds, clockwise from (x1, y1). */ +function boundsCorners(bounds: RectBounds): Point[] { + return [ + [bounds[0], bounds[1]], + [bounds[2], bounds[1]], + [bounds[2], bounds[3]], + [bounds[0], bounds[3]], + ]; +} + +/** + * Map an axis-aligned RectBounds through a homography. A non-affine transform + * turns a rectangle into a general quad, so the result is the axis-aligned + * bounding box of the four mapped corners. + */ +export function mapBounds(matrix: Matrix3, bounds: RectBounds): RectBounds { + return pointsToBounds(boundsCorners(bounds).map((c) => applyHomography(matrix, c))); +} + +/** + * Map a rotated rectangle (RectBounds + rotation in radians about the bounds + * center, the convention of utils.rotateGeoJSONCoordinates) through a + * homography, returning a best-fit rotated rectangle in the target space: + * the mapped direction of the rect's top edge gives the new rotation, and the + * mapped corners un-rotated about their centroid give the new bounds. Exact + * for similarity transforms; a least-surprise approximation for the rest. + */ +export function mapRotatedBounds( + matrix: Matrix3, + bounds: RectBounds, + rotation: number, +): { bounds: RectBounds; rotation: number } { + const cx = (bounds[0] + bounds[2]) / 2; + const cy = (bounds[1] + bounds[3]) / 2; + const rotate = (p: Point, angle: number, ox: number, oy: number): Point => { + const cos = Math.cos(angle); + const sin = Math.sin(angle); + const dx = p[0] - ox; + const dy = p[1] - oy; + return [ox + dx * cos - dy * sin, oy + dx * sin + dy * cos]; + }; + const mapped = boundsCorners(bounds) + .map((c) => applyHomography(matrix, rotate(c, rotation, cx, cy))); + // The top edge (corner 0 -> corner 1) points along +x at zero rotation, so + // its mapped direction is the combined rotation in the target space. + const newRotation = Math.atan2( + mapped[1][1] - mapped[0][1], + mapped[1][0] - mapped[0][0], + ); + const mcx = mapped.reduce((sum, p) => sum + p[0], 0) / mapped.length; + const mcy = mapped.reduce((sum, p) => sum + p[1], 0) / mapped.length; + const unrotated = mapped.map((p) => rotate(p, -newRotation, mcx, mcy)); + return { bounds: pointsToBounds(unrotated), rotation: newRotation }; +} + +type MappableGeometry = GeoJSON.Point | GeoJSON.LineString | GeoJSON.Polygon; + +/** + * Deep-copy GeoJSON features (the shapes tracks store per-frame: Point, + * LineString, Polygon) with every coordinate mapped through a homography. + * Unsupported geometry types are copied through unmapped. + */ +export function mapGeoJSONFeatures( + matrix: Matrix3, + features: GeoJSON.Feature[], +): GeoJSON.Feature[] { + const mapPosition = (p: GeoJSON.Position): GeoJSON.Position => ( + applyHomography(matrix, [p[0], p[1]]) + ); + return features.map((feature) => { + let geometry: MappableGeometry; + if (feature.geometry.type === 'Point') { + geometry = { type: 'Point', coordinates: mapPosition(feature.geometry.coordinates) }; + } else if (feature.geometry.type === 'LineString') { + geometry = { type: 'LineString', coordinates: feature.geometry.coordinates.map(mapPosition) }; + } else { + geometry = { + type: 'Polygon', + coordinates: feature.geometry.coordinates.map((ring) => ring.map(mapPosition)), + }; + } + return { + ...feature, + geometry: geometry as T, + properties: feature.properties ? { ...feature.properties } : feature.properties, + }; + }); +} diff --git a/client/src/components/LayerManager.vue b/client/src/components/LayerManager.vue index 376ef45cc..5b3951193 100644 --- a/client/src/components/LayerManager.vue +++ b/client/src/components/LayerManager.vue @@ -12,16 +12,22 @@ import PointLayer from '../layers/AnnotationLayers/PointLayer'; import LineLayer from '../layers/AnnotationLayers/LineLayer'; import TailLayer from '../layers/AnnotationLayers/TailLayer'; import OverlapLayer from '../layers/AnnotationLayers/OverlapLayer'; +import CalibrationKeypointLayer from '../layers/AnnotationLayers/CalibrationKeypointLayer'; import EditAnnotationLayer, { EditAnnotationTypes } from '../layers/EditAnnotationLayer'; import LassoSelectionLayer from '../layers/LassoSelectionLayer'; +import AlignedImageLayer from '../layers/AlignedImageLayer'; import { FrameDataTrack } from '../layers/LayerTypes'; +import { applyHomography, invert3, Matrix3 } from '../homography'; +import { mapBounds, mapRotatedBounds, mapGeoJSONFeatures } from '../alignedView'; +import type { Feature } from '../track'; import TextLayer, { FormatTextRow } from '../layers/AnnotationLayers/TextLayer'; import AttributeLayer from '../layers/AnnotationLayers/AttributeLayer'; import AttributeBoxLayer from '../layers/AnnotationLayers/AttributeBoxLayer'; import type { AnnotationId } from '../BaseAnnotation'; import { geojsonToBound, isRotationValue, ROTATION_ATTRIBUTE_NAME, featureHasSegmentationPolygon, + getRotationFromAttributes, } from '../utils'; import { VisibleAnnotationTypes } from '../layers'; import UILayer from '../layers/UILayers/UILayer'; @@ -39,6 +45,8 @@ import { useAnnotatorPreferences, useGroupStyleManager, useCameraStore, + useCameraCalibration, + useAlignedView, useSelectedCamera, useAttributes, useComparisonSets, @@ -77,6 +85,18 @@ export default defineComponent({ // Viewer may not provide lasso context in tests or minimal embeds. } const cameraStore = useCameraStore(); + let cameraCalibration: ReturnType | undefined; + try { + cameraCalibration = useCameraCalibration(); + } catch { + // calibration store may not be provided in tests or minimal embeds. + } + let alignedView: ReturnType | undefined; + try { + alignedView = useAlignedView(); + } catch { + // aligned view store may not be provided in tests or minimal embeds. + } const selectedCamera = useSelectedCamera(); const comparison = useComparisonSets(); const trackStore = cameraStore.camMap.value.get(props.camera)?.trackStore; @@ -107,6 +127,144 @@ export default defineComponent({ const flickNumberRef = annotator.flick; const hasFrameRef = annotator.hasFrame; + /** + * Resolve another camera's currently displayed frame image (for the ghost + * overlay and the aligned-view warp). Matches the `quad.image` data used + * by ImageAnnotator and the `quad.video` data used by VideoAnnotator -- + * geojs' canvas quad renderer supports both as texture sources. + * LargeImageAnnotator (tiled/geospatial imagery) has no single resolvable + * image element, so it returns null and the ghost overlay / display warp + * is simply unavailable for those datasets; picking itself + * (native-coordinate inverse mapping) is unaffected either way. + */ + const getCameraImage = (camera: string) => { + let viewer; + try { + // getController throws for an unknown/cleared camera; the ghost and + // aligned-warp rAF loops call this after a dataset reload has cleared + // the controllers, so swallow it here rather than let it escape into + // the animation-frame callback uncaught. + viewer = aggregateController.value.getController(camera)?.geoViewerRef?.value; + } catch { + return null; + } + if (!viewer || typeof viewer.layers !== 'function') { + return null; + } + const layerList = viewer.layers(); + for (let i = 0; i < layerList.length; i += 1) { + const layer = layerList[i]; + if (typeof layer.features === 'function') { + const features = layer.features(); + for (let j = 0; j < features.length; j += 1) { + const data = typeof features[j].data === 'function' ? features[j].data() : undefined; + const datum = Array.isArray(data) ? data[0] : undefined; + if (datum && datum.image) { + const image = datum.image as HTMLImageElement; + return { + source: image, kind: 'image' as const, width: image.naturalWidth, height: image.naturalHeight, + }; + } + if (datum && datum.video) { + const video = datum.video as HTMLVideoElement; + return { + source: video, kind: 'video' as const, width: video.videoWidth, height: video.videoHeight, + }; + } + } + } + } + return null; + }; + + /** + * Aligned view (SEAL-TK features 2 + 3): while active, this camera's + * display transform (native -> reference space, null when unwarped). + * Stored geometry stays native (decision D3); the transform is applied + * at draw time only. + */ + const alignedDisplayTransform = computed( + () => (alignedView ? alignedView.cameraTransform(props.camera) : null), + ); + /** + * Inverse of the display transform (reference/display space -> this + * camera's native space). The edit layer operates in geojs map + * coordinates -- display space -- so draws and edits made while the + * aligned view warps this camera must be mapped back through this before + * being committed to (native) track storage. + */ + const alignedDisplayInverse = computed(() => { + const matrix = alignedDisplayTransform.value; + if (!matrix) { + return null; + } + try { + return invert3(matrix); + } catch { + return null; + } + }); + /** Map a native-space location into display space for view centering. */ + const mapDisplayPoint = (x: number, y: number) => { + const matrix = alignedDisplayTransform.value; + if (!matrix) { + return { x, y }; + } + const [mx, my] = applyHomography(matrix, [x, y]); + return { x: mx, y: my }; + }; + /** + * Copy a native-space track feature into display space for the edit + * layer (identity passthrough when this camera renders unwarped), so + * edit handles land on the warped imagery. The stored feature is never + * mutated (decision D3: storage stays native). + */ + function featureToDisplay(feature: Feature | null): Feature | null { + const matrix = alignedDisplayTransform.value; + if (!matrix || !feature) { + return feature; + } + const mapped: Feature = { ...feature }; + const rotation = getRotationFromAttributes(feature.attributes); + if (feature.bounds) { + if (rotation !== undefined) { + const rotated = mapRotatedBounds(matrix, feature.bounds, rotation); + mapped.bounds = rotated.bounds; + mapped.attributes = { + ...feature.attributes, + [ROTATION_ATTRIBUTE_NAME]: rotated.rotation, + }; + } else { + mapped.bounds = mapBounds(matrix, feature.bounds); + } + } + if (feature.geometry) { + mapped.geometry = { + ...feature.geometry, + features: mapGeoJSONFeatures(matrix, feature.geometry.features), + }; + } + return mapped; + } + + // Created before the annotation layers below so its geojs layer z-orders + // beneath boxes/polygons/text (geojs stacks layers by creation order). + const alignedImageLayer = new AlignedImageLayer({ + annotator, + getImage: () => { + try { + return getCameraImage(props.camera); + } catch { + // Controllers may be cleared mid-poll during a dataset reload. + return null; + } + }, + getTransform: () => alignedDisplayTransform.value, + // Right-click means "remove last point" while creating/editing + // geometry; recenter everywhere else. + getRecenterEnabled: () => !editingModeRef.value, + }); + const rectAnnotationLayer = new RectangleLayer({ annotator, stateStyling: trackStyleManager.stateStyles, @@ -187,12 +345,68 @@ export default defineComponent({ watch([segmentationPointsRef, frameNumberRef, selectedCamera], ([newPoints, currentFrame, currentCamera]) => { if (newPoints.points.length > 0 && newPoints.frameNum === currentFrame && props.camera === currentCamera) { - segmentationPointsLayer.updatePoints(newPoints.points, newPoints.labels); + // Prompt points are stored in native image space; render them where + // the warped imagery actually is (identity when unwarped). + const displayPoints = newPoints.points.map((p): [number, number] => { + const { x, y } = mapDisplayPoint(p[0], p[1]); + return [x, y]; + }); + segmentationPointsLayer.updatePoints(displayPoints, newPoints.labels); } else { segmentationPointsLayer.clear(); } }, { deep: true }); + const calibrationLayer = cameraCalibration + ? new CalibrationKeypointLayer({ + annotator, + stateStyling: trackStyleManager.stateStyles, + typeStyling: typeStylingRef, + calibration: cameraCalibration, + getCameraImage, + }) + : undefined; + + if (cameraCalibration && calibrationLayer) { + const calibration = cameraCalibration; + /** + * Frame number of the camera whose image is being ghosted into another + * pane, or null when no ghost is active. Watched so the ghost re-renders + * when the *source* pane scrubs, not just this pane -- this pane's own + * frameNumberRef can update before (or without) the source's, and the + * source image element itself only swaps after its frame finishes + * loading (see CalibrationKeypointLayer.scheduleGhostRefresh). + */ + const ghostSourceFrame = computed(() => { + const { mode } = calibration.alignment.value; + const pair = calibration.activePair.value; + if (mode === 'original' || !pair) { + return null; + } + const srcCam = mode === 'BtoA' ? pair.camB : pair.camA; + try { + return aggregateController.value.getController(srcCam).frame.value; + } catch { + return null; + } + }); + watch( + [ + cameraCalibration.activePair, + cameraCalibration.pickingEnabled, + cameraCalibration.correspondences, + cameraCalibration.pendingPoint, + cameraCalibration.selectedCorrespondenceId, + cameraCalibration.homographies, + cameraCalibration.alignment, + frameNumberRef, + ghostSourceFrame, + ], + () => calibrationLayer.update(), + { deep: true }, + ); + } + const updateAttributes = () => { const newList = attributes.value.filter((item) => item.render).sort((a, b) => { if (a.render && b.render) { @@ -262,6 +476,11 @@ export default defineComponent({ selectedKey: string, colorBy: string, ) { + // Drawing and editing work on every camera while the aligned view is + // on: the edit layer operates in display (warped) space -- it is fed + // display-space copies of the geometry (featureToDisplay below) and its + // draws/edits are mapped back to native through alignedDisplayInverse + // in the update:geojson handler before committing to track storage. const currentFrameIds: AnnotationId[] | undefined = trackStore?.intervalTree .search([frame, frame]) .map((str) => parseInt(str, 10)); @@ -320,9 +539,13 @@ export default defineComponent({ } if (clientSettings.annotatorPreferences.lockedCamera.enabled) { if (trackFrame.features?.bounds) { + // Under the aligned view the display is warped, so center + // on the displayed (warped) location, not the native one. const coords = { - x: (trackFrame.features.bounds[0] + trackFrame.features.bounds[2]) / 2.0, - y: (trackFrame.features.bounds[1] + trackFrame.features.bounds[3]) / 2.0, + ...mapDisplayPoint( + (trackFrame.features.bounds[0] + trackFrame.features.bounds[2]) / 2.0, + (trackFrame.features.bounds[1] + trackFrame.features.bounds[3]) / 2.0, + ), z: 0, }; const [x0, y0, x1, y1] = trackFrame.features.bounds; @@ -341,10 +564,14 @@ export default defineComponent({ const halfWidth = (width * multiplyBoundsVal) / 2.0; const halfHeight = (height * multiplyBoundsVal) / 2.0; - const left = centerX - halfWidth; - const right = centerX + halfWidth; - const top = centerY - halfHeight; - const bottom = centerY + halfHeight; + // Map the zoom-target corners into display space too + // (identity when the aligned view is off). + const ulMapped = mapDisplayPoint(centerX - halfWidth, centerY - halfHeight); + const lrMapped = mapDisplayPoint(centerX + halfWidth, centerY + halfHeight); + const left = Math.min(ulMapped.x, lrMapped.x); + const right = Math.max(ulMapped.x, lrMapped.x); + const top = Math.min(ulMapped.y, lrMapped.y); + const bottom = Math.max(ulMapped.y, lrMapped.y); const zoomAndCenter = annotator.geoViewerRef.value.zoomAndCenterFromBounds({ left, top, right, bottom, @@ -384,7 +611,11 @@ export default defineComponent({ } else { lineLayer.disable(); } - if (visibleModes.includes('TrackTail')) { + // Track tails read multi-frame geometry straight from the trackStore + // (not FrameDataTrack) and are not routed through the display + // transform, so they are hidden for warped cameras while the aligned + // view is on rather than rendered in the wrong (native) space. + if (visibleModes.includes('TrackTail') && !alignedDisplayTransform.value) { tailLayer.updateSettings( frame, annotatorPrefs.value.trackTails.before, @@ -432,7 +663,13 @@ export default defineComponent({ if (editingTrack) { editAnnotationLayer.setType(editingTrack); editAnnotationLayer.setKey(selectedKey); - editAnnotationLayer.changeData(editingTracks); + // The edit layer works in display space: hand it display-space + // copies of the feature so its handles land on warped imagery + // (identity when this camera renders unwarped). + editAnnotationLayer.changeData(editingTracks.map((trackFrame) => ({ + ...trackFrame, + features: featureToDisplay(trackFrame.features), + }))); } } else if (editingTrack && props.camera !== selectedCamera.value && (isCreatingNewDetection(frame, selectedTrackId) @@ -506,6 +743,49 @@ export default defineComponent({ refreshLayers(); }); + /** Layers whose stored-geometry rendering follows the aligned-view warp. */ + const displayTransformedLayers = [ + rectAnnotationLayer, + overlapLayer, + polyAnnotationLayer, + lineLayer, + pointLayer, + textLayer, + attributeBoxLayer, + attributeLayer, + ]; + + /** + * Apply (or clear) the aligned-view display transform: warp the imagery + * quad and point every geometry layer's draw-time mapping at the same + * matrix, then re-render. Immediate so a LayerManager created while the + * aligned view is already on (e.g. a view-mode switch) starts warped. + */ + watch(alignedDisplayTransform, (matrix) => { + displayTransformedLayers.forEach((layer) => layer.setDisplayTransform(matrix)); + alignedImageLayer.update(); + updateLayers( + frameNumberRef.value, + editingModeRef.value, + selectedTrackIdRef.value, + multiSeletListRef.value, + enabledTracksRef.value, + visibleModesRef.value, + selectedKeyRef.value, + props.colorBy, + ); + }, { immediate: true }); + + // The warped imagery must follow frame changes: the annotator swaps its + // element asynchronously after each seek, and AlignedImageLayer + // polls briefly after every trigger to catch that swap. Guarded so this + // is a strict no-op whenever the camera renders unwarped. + watch(frameNumberRef, () => { + if (alignedDisplayTransform.value) { + alignedImageLayer.update(); + } + }); + /** Shallow watch */ watch( [ @@ -745,16 +1025,31 @@ export default defineComponent({ return; } } + // Under the aligned view this camera renders warped, so the draw/edit + // just made lives in display (reference) space: map it back to this + // camera's native space before committing to track storage (decision + // D3 -- storage stays native). Identity when the camera is unwarped. + const inverse = alignedDisplayInverse.value; if (type === 'rectangle') { - const bounds = geojsonToBound(data as GeoJSON.Feature); + let bounds = geojsonToBound(data as GeoJSON.Feature); // Extract rotation from properties if it exists - const rotation = data.properties && isRotationValue(data.properties?.[ROTATION_ATTRIBUTE_NAME]) + let rotation = data.properties && isRotationValue(data.properties?.[ROTATION_ATTRIBUTE_NAME]) ? data.properties[ROTATION_ATTRIBUTE_NAME] as number : undefined; + if (inverse) { + if (rotation !== undefined) { + const mapped = mapRotatedBounds(inverse, bounds, rotation); + bounds = mapped.bounds; + rotation = mapped.rotation; + } else { + bounds = mapBounds(inverse, bounds); + } + } cb(); handler.updateRectBounds(frameNumberRef.value, flickNumberRef.value, bounds, rotation); } else { - handler.updateGeoJSON(mode, frameNumberRef.value, flickNumberRef.value, data, key, cb); + const nativeData = inverse ? mapGeoJSONFeatures(inverse, [data])[0] : data; + handler.updateGeoJSON(mode, frameNumberRef.value, flickNumberRef.value, nativeData, key, cb); } // Jump into edit mode if we completed a new shape if (geometryCompleteEvent) { @@ -779,8 +1074,13 @@ export default defineComponent({ ); // Handle clicks outside the edit polygon to allow selecting other polygons editAnnotationLayer.bus.$on('click-outside-edit', (geo: { x: number; y: number }) => { - // Check which polygon was clicked by iterating through formatted data - const point: [number, number] = [geo.x, geo.y]; + // Check which polygon was clicked by iterating through formatted data. + // The click arrives in display space while formattedData is native, so + // map it back through the aligned-view inverse (identity when unwarped). + const inverse = alignedDisplayInverse.value; + const point: [number, number] = inverse + ? applyHomography(inverse, [geo.x, geo.y]) + : [geo.x, geo.y]; const polygonData = polyAnnotationLayer.formattedData; // Find the polygon that contains the click point diff --git a/client/src/components/annotators/mediaControllerType.ts b/client/src/components/annotators/mediaControllerType.ts index ece6d1d12..ff771830d 100644 --- a/client/src/components/annotators/mediaControllerType.ts +++ b/client/src/components/annotators/mediaControllerType.ts @@ -45,6 +45,16 @@ export interface AggregateMediaController { cameraSync: Readonly>; /** Incremented when the viewer is resized, used to trigger layer redraws */ resizeTrigger: Readonly>; + /** + * True only while onResize is applying its programmatic size()/resetZoom() + * to the panes. resetZoom emits GeoJS pan/zoom events synchronously; the + * linked-viewer navigation (useAlignedNavigation / useCalibrationNavigation) + * must ignore those so one pane's native-space reset isn't broadcast to the + * others as if it were a shared-space move (which parks warped panes on an + * empty corner). The resizeTrigger bump that follows re-snaps every pane from + * the reference once the reset has settled. + */ + resizing: Readonly>; /** * Global aligned-timeline slot indices with at least one camera missing a * frame (see AlignedFrameResolver's gapSlots); empty whenever alignment diff --git a/client/src/components/annotators/useAlignedNavigation.spec.ts b/client/src/components/annotators/useAlignedNavigation.spec.ts new file mode 100644 index 000000000..a28108171 --- /dev/null +++ b/client/src/components/annotators/useAlignedNavigation.spec.ts @@ -0,0 +1,195 @@ +/// +import { + ref, shallowRef, nextTick, Ref, +} from 'vue'; +import useAlignedNavigation from './useAlignedNavigation'; +import AlignedViewStore from '../../AlignedViewStore'; +import type { AggregateMediaController } from './mediaControllerType'; +import type { Matrix3 } from '../../homography'; + +vi.mock('geojs', () => ({ default: { event: { pan: 'geo_pan', zoom: 'geo_zoom' } } })); + +const IDENTITY: Matrix3 = [[1, 0, 0], [0, 1, 0], [0, 0, 1]]; + +/** Minimal stand-in for a geojs viewer: center/zoom state + geoOn events. */ +function fakeViewer(baseUnitsPerPixel: number) { + const state = { center: { x: 0, y: 0 }, zoom: 0 }; + const handlers: Record void>> = {}; + return { + geoOn(evt: string, handler: () => void) { + handlers[evt] = handlers[evt] || []; + handlers[evt].push(handler); + }, + geoOff(evt: string, handler: () => void) { + handlers[evt] = (handlers[evt] || []).filter((h) => h !== handler); + }, + center(c?: { x: number; y: number }) { + if (c) { + state.center = { ...c }; + } + return state.center; + }, + zoom(z?: number) { + if (z !== undefined) { + state.zoom = z; + } + return state.zoom; + }, + unitsPerPixel(z: number) { + return baseUnitsPerPixel / 2 ** z; + }, + trigger(evt: string) { + (handlers[evt] || []).forEach((h) => h()); + }, + }; +} + +function makeHarness() { + // EO: high-res pane (fine zoom-0 baseline); IR: low-res pane. + const eo = fakeViewer(1); + const ir = fakeViewer(8); + const resetZoom = vi.fn(); + const controllers: Record; resetZoom: () => void }> = { + eo: { geoViewerRef: ref(eo), resetZoom }, + ir: { geoViewerRef: ref(ir), resetZoom }, + }; + const cameraSync = ref(false); + const resizing = ref(false); + const resizeTrigger = ref(0); + // shallowRef: a plain ref would deep-unwrap the nested cameraSync / + // resizeTrigger refs, unlike the real aggregate controller object. + const aggregate = shallowRef({ + cameraSync, + resizeTrigger, + resizing, + getController: (name: string) => controllers[name], + }) as unknown as Ref; + const cameras = ref(['eo', 'ir']); + const alignedView = new AlignedViewStore(); + useAlignedNavigation(aggregate, alignedView, cameras); + return { + eo, ir, cameraSync, resizing, resizeTrigger, alignedView, resetZoom, + }; +} + +describe('useAlignedNavigation', () => { + it('snaps panes to the reference view immediately on activation', async () => { + const { eo, ir, alignedView } = makeHarness(); + eo.center({ x: 250, y: 150 }); + eo.zoom(1); // units-per-pixel = 0.5 + alignedView.setTransforms('eo', { + eo: IDENTITY, + ir: [[1, 0, 100], [0, 1, 0], [0, 0, 1]], + }); + alignedView.setEnabled(true); + await nextTick(); + + // No pan/zoom event fired: activation itself aligned the panes. + expect(ir.center()).toEqual({ x: 250, y: 150 }); + expect(ir.zoom()).toBeCloseTo(Math.log2(8 / 0.5), 6); + }); + + it('links panes by IDENTITY center in the shared reference space', async () => { + const { eo, ir, alignedView } = makeHarness(); + // ir's imagery renders warped through ir->eo (translation +100), so its + // pane coordinates ARE reference coordinates: the link must NOT map the + // center through the camera-to-camera transform again. + alignedView.setTransforms('eo', { + eo: IDENTITY, + ir: [[1, 0, 100], [0, 1, 0], [0, 0, 1]], + }); + alignedView.setEnabled(true); + await nextTick(); + + eo.center({ x: 500, y: 300 }); + eo.zoom(2); // source units-per-pixel = 1 / 2^2 = 0.25 reference units + eo.trigger('geo_pan'); + + expect(ir.center()).toEqual({ x: 500, y: 300 }); + // Same reference extent through ir's own zoom-0 baseline: + // log2(8 / 0.25) = 5. + expect(ir.zoom()).toBeCloseTo(5, 6); + }); + + it('propagates from the warped pane back to the reference pane identically', async () => { + const { eo, ir, alignedView } = makeHarness(); + alignedView.setTransforms('eo', { + eo: IDENTITY, + ir: [[2, 0, 0], [0, 2, 0], [0, 0, 1]], + }); + alignedView.setEnabled(true); + await nextTick(); + + ir.center({ x: 40, y: 60 }); + ir.trigger('geo_pan'); + + expect(eo.center()).toEqual({ x: 40, y: 60 }); + }); + + it('resets every pane to its native view when the aligned view deactivates', async () => { + const { alignedView, resetZoom } = makeHarness(); + alignedView.setTransforms('eo', { eo: IDENTITY, ir: IDENTITY }); + alignedView.setEnabled(true); + await nextTick(); + expect(resetZoom).not.toHaveBeenCalled(); + + alignedView.setEnabled(false); + await nextTick(); + // Once per pane (eo + ir). + expect(resetZoom).toHaveBeenCalledTimes(2); + }); + + it('ignores the native-space pan/zoom events onResize emits while resizing', async () => { + const { + eo, ir, resizing, resizeTrigger, alignedView, + } = makeHarness(); + alignedView.setTransforms('eo', { + eo: IDENTITY, + ir: [[1, 0, 100], [0, 1, 0], [0, 0, 1]], + }); + alignedView.setEnabled(true); + await nextTick(); + + // A good aligned view: both panes centered in the shared reference space. + eo.center({ x: 500, y: 300 }); + eo.zoom(2); + eo.trigger('geo_pan'); + expect(ir.center()).toEqual({ x: 500, y: 300 }); + + // onResize resets each pane to its OWN native bounds and fires pan events. + // ir's reset drops it to its native center; while resizing this must NOT be + // broadcast into the reference space (that is what parked panes in a black + // corner). eo -- the reference -- must stay put. + resizing.value = true; + ir.center({ x: 40, y: 60 }); + ir.trigger('geo_pan'); + expect(eo.center()).toEqual({ x: 500, y: 300 }); + resizing.value = false; + + // The resizeTrigger bump that follows re-snaps every pane from the + // reference, so ir lands back on the reference-space center. + resizeTrigger.value += 1; + await nextTick(); + expect(ir.center()).toEqual({ x: 500, y: 300 }); + }); + + it('stands down while inactive, suspended, or raw camera sync is on', async () => { + const { + eo, ir, cameraSync, alignedView, + } = makeHarness(); + alignedView.setTransforms('eo', { eo: IDENTITY, ir: IDENTITY }); + alignedView.setEnabled(true); + await nextTick(); + + cameraSync.value = true; + eo.center({ x: 9, y: 9 }); + eo.trigger('geo_pan'); + expect(ir.center()).toEqual({ x: 0, y: 0 }); + cameraSync.value = false; + + alignedView.setSuspended(true); + await nextTick(); + eo.trigger('geo_pan'); + expect(ir.center()).toEqual({ x: 0, y: 0 }); + }); +}); diff --git a/client/src/components/annotators/useAlignedNavigation.ts b/client/src/components/annotators/useAlignedNavigation.ts new file mode 100644 index 000000000..133e35749 --- /dev/null +++ b/client/src/components/annotators/useAlignedNavigation.ts @@ -0,0 +1,105 @@ +import { Ref, watch } from 'vue'; +import type { AggregateMediaController } from './mediaControllerType'; +import type AlignedViewStore from '../../AlignedViewStore'; +import useLinkedViewers from './useLinkedViewers'; + +/** + * Links pan/zoom recentering across ALL loaded cameras while the aligned view + * is active (SEAL-TK feature 3). + * + * While active, every pane RENDERS in the shared reference space, so the link + * between panes is the IDENTITY on coordinates: same center, same + * reference-units-per-screen-pixel. (Mapping centers through the + * camera-to-camera transforms here would re-apply a transform the rendering has + * already applied.) Distinct from the raw "sync cameras" toggle (Controls.vue), + * which forwards raw screen deltas for UNWARPED panes and is hidden whenever the + * aligned view is available, and from the calibration pair link + * ({@link useCalibrationNavigation}), which maps through the homography and + * stands down while the aligned view is active. + */ +export default function useAlignedNavigation( + aggregateController: Ref, + alignedView: AlignedViewStore, + cameras: Ref, +) { + const { + viewer, teardown, attach, guarded, applyView, + } = useLinkedViewers(aggregateController); + + function link(camera: string) { + return () => guarded(() => { + if (!alignedView.active.value) { + return; + } + // onResize resets each pane to its own native bounds and emits pan/zoom + // events; ignore them so a non-reference pane's native center doesn't get + // copied into the shared reference space. setup() re-snaps from the + // reference once the resize settles (via the resizeTrigger watch). + if (aggregateController.value.resizing.value) { + return; + } + // Never fight the raw screen-delta sync (unreachable while the aligned + // view is available, but be defensive about two handlers on one event). + if (aggregateController.value.cameraSync.value) { + return; + } + const source = viewer(camera); + if (!source) { + return; + } + // Shared reference space: copy the center verbatim and match the extent. + const center = source.center(); + const view = { + center: { x: center.x, y: center.y }, + unitsPerPixel: source.unitsPerPixel(source.zoom()), + }; + cameras.value.forEach((other) => { + if (other !== camera) { + const target = viewer(other); + if (target) { + applyView(target, view); + } + } + }); + }); + } + + function setup() { + teardown(); + if (!alignedView.active.value) { + return; + } + cameras.value.forEach((camera) => attach(camera, link(camera))); + // Snap immediately from the reference pane so hitting Align lines every pane + // up right away instead of waiting for the first pan/zoom event. + const reference = alignedView.reference.value; + if (reference && cameras.value.includes(reference)) { + link(reference)(); + } + } + + watch( + [ + alignedView.active, + alignedView.toReference, + cameras, + aggregateController.value.resizeTrigger, + ], + setup, + ); + + // Leaving the aligned view strands every pane at reference-space centers/zooms + // while content reverts to native coordinates -- reset each to its full native + // view so the imagery is back on-screen. + watch(alignedView.active, (active, wasActive) => { + if (!active && wasActive) { + cameras.value.forEach((camera) => { + try { + aggregateController.value.getController(camera).resetZoom(); + } catch { + // A pane may already be torn down during dataset unload. + } + }); + } + }); +} diff --git a/client/src/components/annotators/useCalibrationNavigation.spec.ts b/client/src/components/annotators/useCalibrationNavigation.spec.ts new file mode 100644 index 000000000..39f8ec20d --- /dev/null +++ b/client/src/components/annotators/useCalibrationNavigation.spec.ts @@ -0,0 +1,140 @@ +/// +import { + ref, shallowRef, nextTick, Ref, +} from 'vue'; +import useCalibrationNavigation from './useCalibrationNavigation'; +import type CameraCalibrationStore from '../../CameraCalibrationStore'; +import type { AggregateMediaController } from './mediaControllerType'; +import type { Point } from '../../homography'; + +vi.mock('geojs', () => ({ default: { event: { pan: 'geo_pan', zoom: 'geo_zoom' } } })); + +/** Minimal stand-in for a geojs viewer: center/zoom state + geoOn events. */ +function fakeViewer(baseUnitsPerPixel: number) { + const state = { center: { x: 0, y: 0 }, zoom: 0 }; + const handlers: Record void>> = {}; + return { + geoOn(evt: string, handler: () => void) { + handlers[evt] = handlers[evt] || []; + handlers[evt].push(handler); + }, + geoOff(evt: string, handler: () => void) { + handlers[evt] = (handlers[evt] || []).filter((h) => h !== handler); + }, + center(c?: { x: number; y: number }) { + if (c) { + state.center = { ...c }; + } + return state.center; + }, + zoom(z?: number) { + if (z !== undefined) { + state.zoom = z; + } + return state.zoom; + }, + unitsPerPixel(z: number) { + return baseUnitsPerPixel / 2 ** z; + }, + trigger(evt: string) { + (handlers[evt] || []).forEach((h) => h()); + }, + }; +} + +function makeHarness() { + const eo = fakeViewer(1); + const ir = fakeViewer(8); + const controllers: Record }> = { + eo: { geoViewerRef: ref(eo) }, + ir: { geoViewerRef: ref(ir) }, + }; + const resizing = ref(false); + const resizeTrigger = ref(0); + // shallowRef: a plain ref would deep-unwrap the nested resizing / + // resizeTrigger refs, unlike the real aggregate controller object. + const aggregate = shallowRef({ + resizing, + resizeTrigger, + getController: (name: string) => controllers[name], + }) as unknown as Ref; + + const pickingEnabled = ref(false); + const linkedNav = ref(false); + const homographies = ref>({}); + const fitted = ref(true); + // eo -> ir is a pure +100 x-translation (and ir -> eo its inverse), so the + // linked scale is 1 and centers map by simple offset. + const calibration = { + pickingEnabled, + linkedNav, + activePair: ref({ camA: 'eo', camB: 'ir' }), + homographies, + recenterRequest: ref(null), + linkedPoint(camera: string, coord: Point) { + if (!fitted.value) { + return null; + } + if (camera === 'eo') { + return { camera: 'ir', coord: [coord[0] + 100, coord[1]] as Point }; + } + return { camera: 'eo', coord: [coord[0] - 100, coord[1]] as Point }; + }, + } as unknown as CameraCalibrationStore; + + useCalibrationNavigation(aggregate, calibration); + return { + eo, ir, pickingEnabled, linkedNav, homographies, fitted, + }; +} + +describe('useCalibrationNavigation', () => { + it('snaps the pair immediately when "Fit pan/zoom" turns on', async () => { + const { + eo, ir, pickingEnabled, linkedNav, + } = makeHarness(); + pickingEnabled.value = true; + await nextTick(); + eo.center({ x: 250, y: 150 }); + eo.zoom(1); // units-per-pixel = 0.5 + + // Off: nothing linked yet. + expect(ir.center()).toEqual({ x: 0, y: 0 }); + + linkedNav.value = true; + await nextTick(); + + // No pan/zoom event fired: the toggle itself lined the pair up. + expect(ir.center()).toEqual({ x: 350, y: 150 }); + // Matching extent (scale 1) through ir's own zoom-0 baseline: log2(8 / 0.5). + expect(ir.zoom()).toBeCloseTo(4, 6); + }); + + it('re-snaps when the fitted homography changes under the link', async () => { + const { + eo, ir, pickingEnabled, linkedNav, homographies, + } = makeHarness(); + pickingEnabled.value = true; + linkedNav.value = true; + await nextTick(); + + eo.center({ x: 10, y: 20 }); + homographies.value = { 'eo|ir': 'refit' }; + await nextTick(); + + expect(ir.center()).toEqual({ x: 110, y: 20 }); + }); + + it('does not snap while no fit exists yet', async () => { + const { + ir, pickingEnabled, linkedNav, fitted, + } = makeHarness(); + fitted.value = false; + pickingEnabled.value = true; + linkedNav.value = true; + await nextTick(); + + expect(ir.center()).toEqual({ x: 0, y: 0 }); + expect(ir.zoom()).toBe(0); + }); +}); diff --git a/client/src/components/annotators/useCalibrationNavigation.ts b/client/src/components/annotators/useCalibrationNavigation.ts new file mode 100644 index 000000000..1f96ce778 --- /dev/null +++ b/client/src/components/annotators/useCalibrationNavigation.ts @@ -0,0 +1,121 @@ +import { Ref, watch } from 'vue'; +import type { AggregateMediaController } from './mediaControllerType'; +import type CameraCalibrationStore from '../../CameraCalibrationStore'; +import type AlignedViewStore from '../../AlignedViewStore'; +import { localLinkedScale } from '../../homography'; +import type { Point } from '../../homography'; +import useLinkedViewers from './useLinkedViewers'; + +/** + * Links pan/zoom recentering between the two cameras of the active calibration + * pair: panning or zooming one recenters the other on the same point, mapped + * through the pair's fitted homography ({@link CameraCalibrationStore.linkedPoint}). + * The mapping assumes UNWARPED panes showing native coordinates, so this is + * active only while point picking is (the aligned-view link, + * {@link useAlignedNavigation}, owns navigation otherwise) and stands down if + * the aligned view is somehow active. Distinct from the general "sync cameras" + * toggle (Controls.vue), which assumes identical pixel scale between panes. + */ +export default function useCalibrationNavigation( + aggregateController: Ref, + calibration: CameraCalibrationStore, + alignedView?: AlignedViewStore, +) { + const { + viewer, teardown, attach, guarded, applyView, + } = useLinkedViewers(aggregateController); + + function link(camera: string, otherCamera: string) { + return () => guarded(() => { + // The homography mapping assumes unwarped panes; the aligned-view link + // owns navigation while it is active. + if (alignedView?.active.value) { + return; + } + // Ignore the pan/zoom events onResize emits while resetting each pane to + // its own native bounds, so one pane's reset isn't mapped onto its pair. + if (aggregateController.value.resizing.value) { + return; + } + const source = viewer(camera); + const target = viewer(otherCamera); + if (!source || !target) { + return; + } + const center = source.center(); + const linked = calibration.linkedPoint(camera, [center.x, center.y]); + if (!linked || linked.camera !== otherCamera) { + return; + } + // Match the visible extent: one source pixel spans `scale` target pixels + // here (position-dependent for non-similarity fits, so sampled at center; + // null when unavailable -- leave the target's zoom alone). + const scale = localLinkedScale( + (p) => calibration.linkedPoint(camera, p)?.coord ?? null, + [center.x, center.y], + ); + applyView(target, { + center: { x: linked.coord[0], y: linked.coord[1] }, + unitsPerPixel: scale === null ? null : source.unitsPerPixel(source.zoom()) * scale, + }); + }); + } + + function setup() { + teardown(); + const pair = calibration.activePair.value; + // Authoring UI: active while picking and "Fit pan/zoom" is on (once a fit + // exists, linkedPoint returns matches). attach() no-ops for a not-yet-ready + // pane; the resizeTrigger watch re-runs setup once both viewers exist. + if (calibration.pickingEnabled.value && calibration.linkedNav.value && pair) { + attach(pair.camA, link(pair.camA, pair.camB)); + attach(pair.camB, link(pair.camB, pair.camA)); + // Snap immediately from camA so toggling "Fit pan/zoom" on (or a refit + // under it) lines the pair up right away instead of waiting for the + // first pan/zoom event. No-ops harmlessly while no fit exists yet + // (linkedPoint returns null). + link(pair.camA, pair.camB)(); + } + } + + watch( + [ + calibration.pickingEnabled, + calibration.linkedNav, + calibration.activePair, + calibration.homographies, + aggregateController.value.resizeTrigger, + ], + setup, + { deep: true }, + ); + + /** + * One-shot recenter (right-click while picking): center the clicked camera on + * the clicked point and, when the pair has a fitted homography, the other + * camera on the corresponding point. Guarded so it doesn't loop back through + * the continuous link above. + */ + function handleRecenterRequest( + request: { camera: string; coord: Point; id: number } | null, + ) { + if (!request || alignedView?.active.value) { + return; + } + const pair = calibration.activePair.value; + if (!pair || (request.camera !== pair.camA && request.camera !== pair.camB)) { + return; + } + const source = viewer(request.camera); + if (source) { + guarded(() => source.center({ x: request.coord[0], y: request.coord[1] })); + } + const linked = calibration.linkedPoint(request.camera, request.coord); + const target = linked && viewer(linked.camera); + if (linked && target) { + guarded(() => target.center({ x: linked.coord[0], y: linked.coord[1] })); + } + } + + watch(() => calibration.recenterRequest.value, handleRecenterRequest); +} diff --git a/client/src/components/annotators/useLinkedViewers.ts b/client/src/components/annotators/useLinkedViewers.ts new file mode 100644 index 000000000..fd757f434 --- /dev/null +++ b/client/src/components/annotators/useLinkedViewers.ts @@ -0,0 +1,94 @@ +import geo from 'geojs'; +import { onBeforeUnmount, Ref } from 'vue'; +import type { AggregateMediaController } from './mediaControllerType'; + +/** + * Where to move a linked pane: the world-space center to show, and how many + * world units one screen pixel should span there (null leaves the pane's + * current zoom untouched, for when a local scale can't be resolved). + */ +export interface LinkedView { + center: { x: number; y: number }; + unitsPerPixel: number | null; +} + +/** + * Shared plumbing for the two multicam pan/zoom links -- the calibration pair + * link ({@link useCalibrationNavigation}) and the aligned-view link + * ({@link useAlignedNavigation}). Both attach geojs pan/zoom listeners to a set + * of panes and, when one moves, drive the others to a matching view; they + * differ only in how a source pane's view maps onto a target pane. This owns + * the parts that don't: a re-entrancy guard so a driven update doesn't echo + * back, listener bookkeeping, and the zoom-baseline conversion (geojs zoom is + * log2-based, and panes sized to different-resolution images keep their own + * zoom-0 baselines, so matching an extent converts through the target's + * baseline rather than copying the zoom level). + */ +export default function useLinkedViewers( + aggregateController: Ref, +) { + // Setting a pane's center/zoom from a handler must not re-trigger its own listener. + let guard = false; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let attached: { viewer: any; handler: () => void }[] = []; + + /** This camera's geojs viewer, or null when it isn't initialized/known yet. */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + function viewer(camera: string): any { + try { + return aggregateController.value.getController(camera).geoViewerRef.value || null; + } catch { + return null; + } + } + + /** Detach every attached pan/zoom listener. */ + function teardown() { + attached.forEach(({ viewer: v, handler }) => { + v.geoOff(geo.event.pan, handler); + v.geoOff(geo.event.zoom, handler); + }); + attached = []; + } + + /** Attach a pan+zoom handler to `camera`'s pane; no-op when it has no viewer yet. */ + function attach(camera: string, handler: () => void) { + const v = viewer(camera); + if (!v) { + return; + } + v.geoOn(geo.event.pan, handler); + v.geoOn(geo.event.zoom, handler); + attached.push({ viewer: v, handler }); + } + + /** Run `fn` with the re-entrancy guard raised (a no-op while already guarded). */ + function guarded(fn: () => void) { + if (guard) { + return; + } + guard = true; + try { + fn(); + } finally { + guard = false; + } + } + + /** Drive `target` to `view`: center, plus a matching extent via the target's own zoom-0 baseline. */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + function applyView(target: any, view: LinkedView) { + if (view.unitsPerPixel !== null) { + const targetZoom = Math.log2(target.unitsPerPixel(0) / view.unitsPerPixel); + if (Number.isFinite(targetZoom)) { + target.zoom(targetZoom); + } + } + target.center(view.center); + } + + onBeforeUnmount(teardown); + return { + viewer, teardown, attach, guarded, applyView, + }; +} diff --git a/client/src/components/annotators/useMediaController.ts b/client/src/components/annotators/useMediaController.ts index 5b3317277..e81f884c1 100644 --- a/client/src/components/annotators/useMediaController.ts +++ b/client/src/components/annotators/useMediaController.ts @@ -93,6 +93,10 @@ export function useMediaController() { let cameraControllerSymbols: Record = {}; const synchronizeCameras: Ref = ref(false); const resizeTrigger: Ref = ref(0); + // Raised only while onResize applies its programmatic resetZoom, so the + // linked-viewer navigation ignores the resulting pan/zoom events (see + // AggregateMediaController.resizing). + const resizing: Ref = ref(false); // shallowRef: an AlignedFrameResolver carries nested Refs (slotCount, frameRate) // that must NOT be deep-reactive-converted/auto-unwrapped by a plain ref(). const alignedFrameResolver: Ref = shallowRef(null); @@ -127,6 +131,7 @@ export function useMediaController() { toggleSynchronizeCameras, cameraSync: synchronizeCameras, resizeTrigger, + resizing, alignedGapSlots, seekCameraFrame: aggregateSeekCameraFrame, }; @@ -226,7 +231,18 @@ export function useMediaController() { // Resize maps first, then redraw annotation layers once GeoJS has settled. if (resized) { window.requestAnimationFrame(() => { - pendingResizes.forEach((applyResize) => applyResize()); + // resetZoom recenters each pane on its OWN native bounds and emits + // pan/zoom events synchronously. Suppress the linked-viewer navigation + // for the duration so a non-reference pane's native center isn't + // broadcast into the shared/reference space (which would strand warped + // panes on an empty corner). The resizeTrigger bump below then re-snaps + // every pane from a clean reference view. + resizing.value = true; + try { + pendingResizes.forEach((applyResize) => applyResize()); + } finally { + resizing.value = false; + } window.requestAnimationFrame(() => { resizeTrigger.value += 1; }); @@ -686,6 +702,7 @@ export function useMediaController() { toggleSynchronizeCameras, cameraSync: synchronizeCameras, resizeTrigger, + resizing, alignedGapSlots, seekCameraFrame: aggregateSeekCameraFrame, }; diff --git a/client/src/components/controls/Controls.vue b/client/src/components/controls/Controls.vue index c580c5bfc..9223b4459 100644 --- a/client/src/components/controls/Controls.vue +++ b/client/src/components/controls/Controls.vue @@ -10,7 +10,9 @@ import { DatasetType, useApi } from 'dive-common/apispec'; import { computeGapGradient } from 'dive-common/alignedTimeline'; import { frameToTimestamp } from 'vue-media-annotator/utils'; import { injectAggregateController } from '../annotators/useMediaController'; -import { useTime, useTrackFilters, useDatasetId } from '../../provides'; +import { + useTime, useTrackFilters, useDatasetId, useAlignedView, +} from '../../provides'; export default defineComponent({ name: 'Controls', @@ -38,6 +40,35 @@ export default defineComponent({ dragging: false, }); const mediaController = injectAggregateController().value; + let alignedView: ReturnType | undefined; + try { + alignedView = useAlignedView(); + } catch { + // aligned view store may not be provided in tests or minimal embeds. + } + /** + * The raw screen-delta camera sync is only offered while the transform + * -aware aligned view (the Align button) is unavailable: once every + * camera has a calibration transform, Align is the single place to link + * pan/zoom, and this cruder link (which assumes identical pixel scale + * between panes) would just be a second, worse toggle for the same thing. + */ + const rawSyncAvailable = computed(() => mediaController.cameras.value.length > 1 + && !alignedView?.available.value); + const toggleRawSync = () => { + if (rawSyncAvailable.value) { + mediaController.toggleSynchronizeCameras(!mediaController.cameraSync.value); + } + }; + // If transforms become available while the raw sync is on, switch it off: + // its toggle is hidden from that point, and the aligned-view link stands + // down while raw sync is enabled, so a stuck-on raw sync would silently + // block the Align button's linking with no visible control to clear it. + watch(rawSyncAvailable, (available) => { + if (!available && mediaController.cameraSync.value) { + mediaController.toggleSynchronizeCameras(false); + } + }, { immediate: true }); const isVideo = computed(() => props.datasetType === 'video'); const { frameRate } = useTime(); const { visible } = usePrompt(); @@ -229,6 +260,8 @@ export default defineComponent({ activeTimeFilter, data, mediaController, + rawSyncAvailable, + toggleRawSync, dragHandler, input, alignedGapGradient, @@ -274,7 +307,7 @@ export default defineComponent({ { bind: 'd', handler: mediaController.prevFrame, disabled: visible() }, { bind: 'l', - handler: () => mediaController.toggleSynchronizeCameras(!mediaController.cameraSync.value), + handler: toggleRawSync, disabled: visible(), }, ]" @@ -626,12 +659,12 @@ export default defineComponent({ {{ mediaController.cameraSync.value ? 'mdi-link' : 'mdi-link-off' }} @@ -946,12 +979,12 @@ export default defineComponent({ {{ mediaController.cameraSync.value ? 'mdi-link' : 'mdi-link-off' }} @@ -1236,13 +1269,13 @@ export default defineComponent({ {{ mediaController.cameraSync.value ? 'mdi-link' : 'mdi-link-off' }} diff --git a/client/src/components/index.ts b/client/src/components/index.ts index ec4d30a9a..2aab7cfc2 100644 --- a/client/src/components/index.ts +++ b/client/src/components/index.ts @@ -28,6 +28,8 @@ import TypePicker from './TypePicker.vue'; export * from './annotators/useMediaController'; export { default as useAnnotatorImageCursor } from './annotators/useAnnotatorImageCursor'; +export { default as useCalibrationNavigation } from './annotators/useCalibrationNavigation'; +export { default as useAlignedNavigation } from './annotators/useAlignedNavigation'; export { /* Annotators */ AnnotatorImageCursor, diff --git a/client/src/homography.spec.ts b/client/src/homography.spec.ts new file mode 100644 index 000000000..9facf0512 --- /dev/null +++ b/client/src/homography.spec.ts @@ -0,0 +1,258 @@ +/// +import { + solveHomography, + applyHomography, + invert3, + matMul3, + subdivideWarpQuads, + warpGridSize, + localLinkedScale, + Point, + Matrix3, +} from './homography'; + +function expectMatrixClose(actual: Matrix3, expected: Matrix3, tol = 1e-6) { + for (let i = 0; i < 3; i += 1) { + for (let j = 0; j < 3; j += 1) { + expect(actual[i][j]).toBeCloseTo(expected[i][j], 5); + // tol referenced to keep signature explicit + expect(Math.abs(actual[i][j] - expected[i][j])).toBeLessThan(tol * 10); + } + } +} + +const unitSquare: Point[] = [[0, 0], [1, 0], [1, 1], [0, 1]]; + +describe('homography', () => { + it('recovers the identity from identical correspondences', () => { + const H = solveHomography(unitSquare, unitSquare); + expectMatrixClose(H, [[1, 0, 0], [0, 1, 0], [0, 0, 1]]); + }); + + it('recovers a pure translation', () => { + const dst = unitSquare.map(([x, y]): Point => [x + 5, y - 3]); + const H = solveHomography(unitSquare, dst); + expectMatrixClose(H, [[1, 0, 5], [0, 1, -3], [0, 0, 1]]); + }); + + it('recovers a scale + translation', () => { + const H = solveHomography(unitSquare, [[10, 10], [30, 10], [30, 30], [10, 30]]); + expectMatrixClose(H, [[20, 0, 10], [0, 20, 10], [0, 0, 1]]); + }); + + it('maps source points onto destination points (least-squares, >4 pts)', () => { + // A known projective transform applied to 5 points; solver should recover it. + const truth: Matrix3 = [[1.2, 0.1, 5], [0.05, 0.9, -4], [0.0008, -0.0005, 1]]; + const src: Point[] = [[0, 0], [100, 0], [100, 80], [0, 80], [50, 40]]; + const dst = src.map((p) => applyHomography(truth, p)); + const H = solveHomography(src, dst); + src.forEach((p) => { + const [u, v] = applyHomography(H, p); + const [eu, ev] = applyHomography(truth, p); + expect(u).toBeCloseTo(eu, 3); + expect(v).toBeCloseTo(ev, 3); + }); + }); + + it('round-trips through its inverse (H * H^-1 ~= I)', () => { + const H = solveHomography(unitSquare, [[10, 5], [40, 8], [38, 35], [9, 33]]); + const product = matMul3(H, invert3(H)); + const scale = 1 / product[2][2]; + const normalized = product.map((r) => r.map((c) => c * scale)) as Matrix3; + expectMatrixClose(normalized, [[1, 0, 0], [0, 1, 0], [0, 0, 1]]); + }); + + it('throws with fewer than 4 correspondences', () => { + expect(() => solveHomography(unitSquare.slice(0, 3), unitSquare.slice(0, 3))).toThrow(); + }); + + it('throws on a degenerate (collinear) point configuration despite having 4 points', () => { + const collinear: Point[] = [[0, 0], [1, 0], [2, 0], [3, 0]]; + expect(() => solveHomography(collinear, collinear)).toThrow(/degenerate/i); + }); +}); + +describe('warpGridSize', () => { + const affine: Matrix3 = [[1.5, 0.2, 30], [-0.1, 0.9, -12], [0, 0, 1]]; + const projective: Matrix3 = [[1.2, 0.1, 5], [0.05, 0.9, -4], [0.0008, -0.0005, 1]]; + + it('returns 1 for a pure affine transform', () => { + expect(warpGridSize(affine, 640, 480)).toBe(1); + }); + + it('returns maxN when perspective terms are non-negligible', () => { + expect(warpGridSize(projective, 640, 480)).toBe(8); + expect(warpGridSize(projective, 640, 480, 12)).toBe(12); + }); + + it('returns 1 for negligible perspective terms', () => { + // Perspective terms exist but vary w by only ~0.006% over the extent. + const nearAffine: Matrix3 = [[1.2, 0.1, 5], [0.05, 0.9, -4], [1e-7, 0, 1]]; + expect(warpGridSize(nearAffine, 640, 480)).toBe(1); + }); + + it('returns maxN when the horizon crosses the image (w changes sign)', () => { + const extreme: Matrix3 = [[1, 0, 0], [0, 1, 0], [-0.01, 0, 1]]; + // w at x=0 is 1, at x=640 is -5.4. + expect(warpGridSize(extreme, 640, 480)).toBe(8); + }); +}); + +describe('subdivideWarpQuads', () => { + const projective: Matrix3 = [[1.2, 0.1, 5], [0.05, 0.9, -4], [0.0008, -0.0005, 1]]; + + it('with n=1 produces a single quad matching the full image corners', () => { + const [quad, ...rest] = subdivideWarpQuads(projective, 640, 480, 1); + expect(rest).toHaveLength(0); + expect(quad.crop).toEqual({ + left: 0, top: 0, right: 640, bottom: 480, + }); + expect(quad.ul).toEqual(applyHomography(projective, [0, 0])); + expect(quad.ur).toEqual(applyHomography(projective, [640, 0])); + expect(quad.lr).toEqual(applyHomography(projective, [640, 480])); + expect(quad.ll).toEqual(applyHomography(projective, [0, 480])); + }); + + it('maps every sub-quad corner through the exact homography', () => { + const n = 8; + const quads = subdivideWarpQuads(projective, 640, 480, n); + expect(quads).toHaveLength(n * n); + quads.forEach((q) => { + expect(q.ul).toEqual(applyHomography(projective, [q.crop.left, q.crop.top])); + expect(q.ur).toEqual(applyHomography(projective, [q.crop.right, q.crop.top])); + expect(q.lr).toEqual(applyHomography(projective, [q.crop.right, q.crop.bottom])); + expect(q.ll).toEqual(applyHomography(projective, [q.crop.left, q.crop.bottom])); + }); + }); + + it('expands cells by the overlap (clamped to the image), corners still exact', () => { + const n = 4; + const overlap = 2; + const plain = subdivideWarpQuads(projective, 640, 480, n); + const padded = subdivideWarpQuads(projective, 640, 480, n, overlap); + expect(padded).toHaveLength(plain.length); + padded.forEach((q, i) => { + const base = plain[i].crop; + expect(q.crop.left).toBe(Math.max(0, base.left - overlap)); + expect(q.crop.right).toBe(Math.min(640, base.right + overlap)); + expect(q.crop.top).toBe(Math.max(0, base.top - overlap)); + expect(q.crop.bottom).toBe(Math.min(480, base.bottom + overlap)); + // Corners remain the exact projective mapping of the (expanded) crop, + // so overlapping regions of adjacent cells render identical content. + expect(q.ul).toEqual(applyHomography(projective, [q.crop.left, q.crop.top])); + expect(q.lr).toEqual(applyHomography(projective, [q.crop.right, q.crop.bottom])); + }); + // Interior edges overlap: cell 0's right crop passes cell 1's left crop. + expect(padded[0].crop.right).toBeGreaterThan(padded[1].crop.left); + }); + + it('tiles the source image exactly, with integer grid lines and no gaps', () => { + const n = 8; + const width = 641; // not divisible by n + const height = 479; + const quads = subdivideWarpQuads(projective, width, height, n); + const lefts = new Set(quads.map((q) => q.crop.left)); + const tops = new Set(quads.map((q) => q.crop.top)); + expect(lefts.size).toBe(n); + expect(tops.size).toBe(n); + quads.forEach((q) => { + [q.crop.left, q.crop.top, q.crop.right, q.crop.bottom].forEach((v) => { + expect(Number.isInteger(v)).toBe(true); + }); + expect(q.crop.right).toBeGreaterThan(q.crop.left); + expect(q.crop.bottom).toBeGreaterThan(q.crop.top); + // Each cell's right/bottom edge is another cell's left/top edge or the border. + expect(q.crop.right === width || lefts.has(q.crop.right)).toBe(true); + expect(q.crop.bottom === height || tops.has(q.crop.bottom)).toBe(true); + }); + // Crop areas sum to the full image area (exact tiling, no overlap/gap). + const area = quads.reduce( + (sum, q) => sum + (q.crop.right - q.crop.left) * (q.crop.bottom - q.crop.top), + 0, + ); + expect(area).toBe(width * height); + }); + + it('skips degenerate zero-area cells for images smaller than the grid', () => { + const quads = subdivideWarpQuads(projective, 3, 3, 8); + expect(quads.length).toBeGreaterThan(0); + quads.forEach((q) => { + expect(q.crop.right).toBeGreaterThan(q.crop.left); + expect(q.crop.bottom).toBeGreaterThan(q.crop.top); + }); + const area = quads.reduce( + (sum, q) => sum + (q.crop.right - q.crop.left) * (q.crop.bottom - q.crop.top), + 0, + ); + expect(area).toBe(9); + }); + + it('sub-quad centers stay close to the true projective warp (approximation quality)', () => { + // The deviation between the piecewise-affine render and the true warp is + // largest in cell interiors; measure it at the center of every cell as the + // distance between the true projective warp of the cell's source center + // and the average of the four warped corners (what an affine-ish renderer + // produces there). + const width = 640; + const height = 480; + const centerError = (quads: ReturnType) => Math.max( + ...quads.map((q) => { + const midSrc: Point = [ + (q.crop.left + q.crop.right) / 2, + (q.crop.top + q.crop.bottom) / 2, + ]; + const truth = applyHomography(projective, midSrc); + const approx: Point = [ + (q.ul[0] + q.ur[0] + q.lr[0] + q.ll[0]) / 4, + (q.ul[1] + q.ur[1] + q.lr[1] + q.ll[1]) / 4, + ]; + return Math.hypot(approx[0] - truth[0], approx[1] - truth[1]); + }), + ); + const singleQuadError = centerError(subdivideWarpQuads(projective, width, height, 1)); + const gridError = centerError(subdivideWarpQuads(projective, width, height, 8)); + // This matrix has strong perspective: a single (parallelogram-rendered) + // quad is tens of pixels off at the image center. + expect(singleQuadError).toBeGreaterThan(10); + // Error shrinks roughly with 1/n^2; at n=8 even this extreme perspective + // (w varies ~2x across the image) is down to a couple of pixels. + expect(gridError).toBeLessThan(3); + expect(gridError).toBeLessThan(singleQuadError / 20); + }); +}); + +describe('localLinkedScale', () => { + const mapper = (matrix: Matrix3) => (p: Point) => applyHomography(matrix, p); + + it('returns 1 for the identity mapping', () => { + const identity: Matrix3 = [[1, 0, 0], [0, 1, 0], [0, 0, 1]]; + expect(localLinkedScale(mapper(identity), [100, 200])).toBeCloseTo(1); + }); + + it('recovers a uniform similarity scale regardless of rotation', () => { + const s = 2.5; + const cos = Math.cos(Math.PI / 6) * s; + const sin = Math.sin(Math.PI / 6) * s; + const similarity: Matrix3 = [[cos, -sin, 10], [sin, cos, -4], [0, 0, 1]]; + expect(localLinkedScale(mapper(similarity), [50, 75])).toBeCloseTo(s); + }); + + it('samples the local scale of a projective transform at the given point', () => { + const homography: Matrix3 = [[1, 0, 0], [0, 1, 0], [0.001, 0, 1]]; + const nearOrigin = localLinkedScale(mapper(homography), [0, 0], 1); + const farRight = localLinkedScale(mapper(homography), [500, 0], 1); + expect(nearOrigin).toBeCloseTo(1, 1); + // At x=500 the perspective divide (w = 1.5) has shrunk the local scale + // well below 1; exact value differs per axis, so just assert the shrink. + expect(farRight).not.toBeNull(); + expect(farRight as number).toBeLessThan(0.7); + }); + + it('returns null when the mapping is unavailable', () => { + expect(localLinkedScale(() => null, [10, 10])).toBeNull(); + }); + + it('returns null for a degenerate (collapsing) mapping', () => { + expect(localLinkedScale(() => [3, 3], [10, 10])).toBeNull(); + }); +}); diff --git a/client/src/homography.ts b/client/src/homography.ts new file mode 100644 index 000000000..ff09188a7 --- /dev/null +++ b/client/src/homography.ts @@ -0,0 +1,309 @@ +/** + * Self-contained homography estimation via the normalized Direct Linear Transform. + * + * Given >= 4 point correspondences src[i] -> dst[i] (both [x, y] in image + * coordinates), {@link solveHomography} returns the 3x3 matrix H such that, in + * homogeneous coordinates, dst ~= H * src. Exact for 4 points, least-squares for + * more. This is the client-side analogue of OpenCV's cv2.findHomography used by + * the keypointgui reference app; the warp itself is done by geojs (quadFeature). + */ + +export type Point = [number, number]; +export type Matrix3 = number[][]; + +/** Multiply two 3x3 matrices. */ +export function matMul3(a: Matrix3, b: Matrix3): Matrix3 { + const out: Matrix3 = [[0, 0, 0], [0, 0, 0], [0, 0, 0]]; + for (let i = 0; i < 3; i += 1) { + for (let j = 0; j < 3; j += 1) { + let sum = 0; + for (let k = 0; k < 3; k += 1) { + sum += a[i][k] * b[k][j]; + } + out[i][j] = sum; + } + } + return out; +} + +/** Invert a 3x3 matrix. Throws if the matrix is singular. */ +export function invert3(m: Matrix3): Matrix3 { + const [a, b, c] = m[0]; + const [d, e, f] = m[1]; + const [g, h, i] = m[2]; + const A = e * i - f * h; + const B = -(d * i - f * g); + const C = d * h - e * g; + const det = a * A + b * B + c * C; + if (Math.abs(det) < 1e-12) { + throw new Error('Cannot invert singular matrix'); + } + const invDet = 1 / det; + return [ + [A * invDet, -(b * i - c * h) * invDet, (b * f - c * e) * invDet], + [B * invDet, (a * i - c * g) * invDet, -(a * f - c * d) * invDet], + [C * invDet, -(a * h - b * g) * invDet, (a * e - b * d) * invDet], + ]; +} + +/** Apply a 3x3 homography to a single point (perspective divide). */ +export function applyHomography(h: Matrix3, p: Point): Point { + const x = h[0][0] * p[0] + h[0][1] * p[1] + h[0][2]; + const y = h[1][0] * p[0] + h[1][1] * p[1] + h[1][2]; + const w = h[2][0] * p[0] + h[2][1] * p[1] + h[2][2]; + return [x / w, y / w]; +} + +/** + * Local scale factor of a point mapping around `center`: how many target-image + * pixels one source-image pixel spans there, estimated by probing `delta` + * pixels along each axis and averaging. For similarity/affine transforms this + * is constant; for homographies it varies with position, which is why it's + * sampled at a specific point (e.g. the current view center for linked + * pan/zoom). Returns null when the mapping is unavailable or degenerate at + * that point. + */ +export function localLinkedScale( + mapPoint: (p: Point) => Point | null, + center: Point, + delta = 10, +): number | null { + const mapped = mapPoint(center); + const mappedX = mapPoint([center[0] + delta, center[1]]); + const mappedY = mapPoint([center[0], center[1] + delta]); + if (!mapped || !mappedX || !mappedY) { + return null; + } + const scaleX = Math.hypot(mappedX[0] - mapped[0], mappedX[1] - mapped[1]) / delta; + const scaleY = Math.hypot(mappedY[0] - mapped[0], mappedY[1] - mapped[1]) / delta; + const scale = (scaleX + scaleY) / 2; + if (!Number.isFinite(scale) || scale <= 0) { + return null; + } + return scale; +} + +/** + * One cell of a subdivided image warp: the axis-aligned source-image rectangle + * `crop` (in source pixels) and the four destination corners it maps to under + * the exact projective transform. See {@link subdivideWarpQuads}. + */ +export interface WarpQuad { + ul: Point; + ur: Point; + lr: Point; + ll: Point; + crop: { left: number; top: number; right: number; bottom: number }; +} + +/** + * Choose a subdivision grid size for rendering the warp of a `width` x `height` + * image through `h` with an affine-only quad renderer (e.g. geojs' canvas + * renderer, which draws each quad from only three of its corners). A pure + * affine matrix (zero perspective row terms) warps exactly as a single + * parallelogram, so 1 is returned; when the perspective terms are + * non-negligible over the image extent, `maxN` is returned so each sub-quad is + * approximately affine. + */ +export function warpGridSize(h: Matrix3, width: number, height: number, maxN = 8): number { + // Homogeneous w at each image corner: constant w <=> affine transform. + const wAt = (x: number, y: number) => h[2][0] * x + h[2][1] * y + h[2][2]; + const ws = [wAt(0, 0), wAt(width, 0), wAt(width, height), wAt(0, height)]; + if (ws.some((w) => !Number.isFinite(w) || w === 0)) { + return maxN; + } + const absW = ws.map((w) => Math.abs(w)); + const maxAbs = Math.max(...absW); + const minAbs = Math.min(...absW); + // Sign change means the horizon line crosses the image: definitely projective. + if (ws.some((w) => w * ws[0] < 0)) { + return maxN; + } + // Relative variation of w across the quad; below ~0.1% the affine + // approximation is visually indistinguishable (sub-pixel for typical sizes). + return (maxAbs - minAbs) / maxAbs < 1e-3 ? 1 : maxN; +} + +/** + * Subdivide the warp of a `width` x `height` image through `h` into an n x n + * grid of {@link WarpQuad}s. Every sub-quad corner is mapped through the exact + * projective transform, so rendering each cell as an (approximately affine) + * textured quad converges to the true projective warp as n grows -- unlike + * rendering the whole image as a single quad, which an affine canvas renderer + * collapses to a parallelogram. Grid lines land on integer source pixels; + * degenerate (zero-area) cells from tiny images are skipped. + * + * `overlap` expands each cell by that many source pixels (clamped to the + * image). The canvas renderer antialiases every quad's border against the + * transparent background, so abutting cells meet as two half-transparent + * edges and show as dark seam lines along the grid; overlapping cells paint + * over each other's seams with pixel-identical content (corners still map + * through the same exact homography). Only for opaque drawing -- translucent + * quads would double-blend in the overlap, so semi-transparent consumers + * must apply opacity at the layer level and draw quads opaque. + */ +export function subdivideWarpQuads( + h: Matrix3, + width: number, + height: number, + n: number, + overlap = 0, +): WarpQuad[] { + const cells = Math.max(1, Math.floor(n)); + const xs: number[] = []; + const ys: number[] = []; + for (let i = 0; i <= cells; i += 1) { + xs.push(Math.round((i * width) / cells)); + ys.push(Math.round((i * height) / cells)); + } + const quads: WarpQuad[] = []; + for (let row = 0; row < cells; row += 1) { + for (let col = 0; col < cells; col += 1) { + const left = Math.max(0, xs[col] - overlap); + const right = Math.min(width, xs[col + 1] + overlap); + const top = Math.max(0, ys[row] - overlap); + const bottom = Math.min(height, ys[row + 1] + overlap); + if (right <= left || bottom <= top) { + // eslint-disable-next-line no-continue + continue; + } + quads.push({ + ul: applyHomography(h, [left, top]), + ur: applyHomography(h, [right, top]), + lr: applyHomography(h, [right, bottom]), + ll: applyHomography(h, [left, bottom]), + crop: { + left, top, right, bottom, + }, + }); + } + } + return quads; +} + +/** + * Hartley normalization: translate points to the centroid and scale so the + * mean distance from the origin is sqrt(2). Returns the normalized points and + * the 3x3 transform T such that normalized = T * original. + */ +function normalizePoints(pts: Point[]): { normalized: Point[]; transform: Matrix3 } { + const n = pts.length; + let cx = 0; + let cy = 0; + pts.forEach(([x, y]) => { cx += x; cy += y; }); + cx /= n; + cy /= n; + let meanDist = 0; + pts.forEach(([x, y]) => { meanDist += Math.hypot(x - cx, y - cy); }); + meanDist /= n; + const scale = meanDist > 1e-12 ? Math.SQRT2 / meanDist : 1; + const transform: Matrix3 = [ + [scale, 0, -scale * cx], + [0, scale, -scale * cy], + [0, 0, 1], + ]; + const normalized = pts.map(([x, y]): Point => [(x - cx) * scale, (y - cy) * scale]); + return { normalized, transform }; +} + +/** + * Solve the linear system A x = b for x using Gaussian elimination with partial + * pivoting. A is square (n x n), modified in place. + */ +/* eslint-disable no-param-reassign */ +export function solveLinearSystem(A: number[][], b: number[]): number[] { + const n = b.length; + for (let col = 0; col < n; col += 1) { + // Partial pivot: find the row with the largest magnitude in this column. + let pivot = col; + for (let row = col + 1; row < n; row += 1) { + if (Math.abs(A[row][col]) > Math.abs(A[pivot][col])) { + pivot = row; + } + } + if (Math.abs(A[pivot][col]) < 1e-12) { + throw new Error('Degenerate point configuration; cannot solve linear system'); + } + if (pivot !== col) { + [A[col], A[pivot]] = [A[pivot], A[col]]; + [b[col], b[pivot]] = [b[pivot], b[col]]; + } + // Eliminate below. + for (let row = col + 1; row < n; row += 1) { + const factor = A[row][col] / A[col][col]; + for (let k = col; k < n; k += 1) { + A[row][k] -= factor * A[col][k]; + } + b[row] -= factor * b[col]; + } + } + // Back-substitution. + const x = new Array(n).fill(0); + for (let row = n - 1; row >= 0; row -= 1) { + let sum = b[row]; + for (let k = row + 1; k < n; k += 1) { + sum -= A[row][k] * x[k]; + } + x[row] = sum / A[row][row]; + } + return x; +} +/* eslint-enable no-param-reassign */ + +/** + * Estimate the homography H (dst ~= H * src) from >= 4 correspondences. + * + * Uses the h33 = 1 formulation in Hartley-normalized coordinates, solved via the + * normal equations (least-squares for > 4 points, exact for 4). Normalization + * keeps the system well-conditioned for typical image-pixel magnitudes. + */ +export function solveHomography(src: Point[], dst: Point[]): Matrix3 { + if (src.length !== dst.length) { + throw new Error('src and dst must have the same number of points'); + } + if (src.length < 4) { + throw new Error('At least 4 point correspondences are required'); + } + + const { normalized: srcN, transform: T1 } = normalizePoints(src); + const { normalized: dstN, transform: T2 } = normalizePoints(dst); + + // Build the 2N x 8 design matrix for the unknowns [h11..h32] (h33 fixed to 1). + const rows: number[][] = []; + const rhs: number[] = []; + for (let i = 0; i < srcN.length; i += 1) { + const [x, y] = srcN[i]; + const [u, v] = dstN[i]; + rows.push([x, y, 1, 0, 0, 0, -x * u, -y * u]); + rhs.push(u); + rows.push([0, 0, 0, x, y, 1, -x * v, -y * v]); + rhs.push(v); + } + + // Normal equations: (A^T A) h = A^T b -> an 8x8 system. + const ata: number[][] = Array.from({ length: 8 }, () => new Array(8).fill(0)); + const atb: number[] = new Array(8).fill(0); + for (let r = 0; r < rows.length; r += 1) { + const row = rows[r]; + for (let i = 0; i < 8; i += 1) { + atb[i] += row[i] * rhs[r]; + for (let j = 0; j < 8; j += 1) { + ata[i][j] += row[i] * row[j]; + } + } + } + + const h = solveLinearSystem(ata, atb); + const hNorm: Matrix3 = [ + [h[0], h[1], h[2]], + [h[3], h[4], h[5]], + [h[6], h[7], 1], + ]; + + // Denormalize: H = inv(T2) * Hnorm * T1. + const denorm = matMul3(matMul3(invert3(T2), hNorm), T1); + + // Scale so H[2][2] == 1 for a canonical form. + const scale = Math.abs(denorm[2][2]) > 1e-12 ? 1 / denorm[2][2] : 1; + return denorm.map((row) => row.map((value) => value * scale)); +} diff --git a/client/src/index.ts b/client/src/index.ts index 33708c1c5..45f010cbd 100644 --- a/client/src/index.ts +++ b/client/src/index.ts @@ -2,9 +2,11 @@ import * as layers from './layers'; import * as components from './components'; +import AlignedViewStore from './AlignedViewStore'; import BaseAnnotation from './BaseAnnotation'; import BaseAnnotationStore from './BaseAnnotationStore'; import CameraStore from './CameraStore'; +import CameraCalibrationStore from './CameraCalibrationStore'; import Group from './Group'; import GroupFilterControls from './GroupFilterControls'; import GroupStore from './GroupStore'; @@ -25,9 +27,11 @@ export { layers, components, /* other */ + AlignedViewStore, BaseAnnotation, BaseAnnotationStore, CameraStore, + CameraCalibrationStore, Group, GroupFilterControls, GroupStore, diff --git a/client/src/layers/AlignedImageLayer.ts b/client/src/layers/AlignedImageLayer.ts new file mode 100644 index 000000000..22e680885 --- /dev/null +++ b/client/src/layers/AlignedImageLayer.ts @@ -0,0 +1,252 @@ +import geo from 'geojs'; +import type { MediaController } from '../components/annotators/mediaControllerType'; +import { Matrix3, subdivideWarpQuads, warpGridSize } from '../homography'; +import type { CameraImage } from './AnnotationLayers/CalibrationKeypointLayer'; + +/** + * How many animation frames to keep re-checking whether this camera's + * displayed image element has changed after an update trigger (image + * sequences swap their quad datum asynchronously once the new frame loads; + * see CalibrationKeypointLayer.scheduleGhostRefresh for the same pattern). + */ +const REFRESH_MAX_ATTEMPTS = 60; + +interface AlignedImageLayerParams { + annotator: MediaController; + /** Resolve this camera's currently displayed frame image element. */ + getImage: () => CameraImage | null; + /** Current native->reference display transform, or null when unwarped. */ + getTransform: () => Matrix3 | null; + /** + * Whether the right-click recenter is currently allowed. Right-click + * recenters in EVERY view mode (aligned or not, multicam or single); the + * caller disables it while annotation geometry is being created/edited, + * where right-click already means "remove last point". + */ + getRecenterEnabled: () => boolean; +} + +/** + * Renders this camera's own frame warped into the reference camera's space + * while the multicam aligned view is on (SEAL-TK feature 2, decision D1: the + * proven quad-corner warp). The warp is drawn as an n x n grid of geojs + * canvas quads whose corners are mapped through the exact projective + * transform ({@link subdivideWarpQuads}), and the annotator's own native + * image quad layer is hidden while the warp is shown -- so annotation layers + * (whose vertices are mapped through the same matrix at draw time) land + * exactly on the warped imagery. When no transform applies, everything is + * restored and the pane renders byte-identically to today. + * + * Construct BEFORE the annotation layers in LayerManager so this geojs layer + * z-orders below boxes/polygons/text (geojs stacks by creation order). + */ +export default class AlignedImageLayer { + private annotator: MediaController; + + private getImage: () => CameraImage | null; + + private getTransform: () => Matrix3 | null; + + private getRecenterEnabled: () => boolean; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + private quadLayer: any; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + private quadFeature: any; + + /** The source element currently rendered as the warp, if any. */ + private renderedSource: HTMLImageElement | HTMLVideoElement | null = null; + + /** Whether we hid the annotator's native image quad layer. */ + private nativeHidden = false; + + /** Pending requestAnimationFrame handle for the staleness re-check loop. */ + private retryHandle: number | null = null; + + private retryAttempts = 0; + + constructor(params: AlignedImageLayerParams) { + this.annotator = params.annotator; + this.getImage = params.getImage; + this.getTransform = params.getTransform; + this.getRecenterEnabled = params.getRecenterEnabled; + this.quadLayer = this.annotator.geoViewerRef.value.createLayer('feature', { + features: ['quad'], + autoshareRenderer: false, + renderer: 'canvas', + }); + this.quadFeature = this.quadLayer.createFeature('quad'); + // Right-click recenter: center this pane on the clicked location, in any + // view mode. While the aligned view is active, the aligned pan/zoom link + // then recenters every other pane on the same reference-space point; + // during calibration picking, CalibrationKeypointLayer additionally maps + // the recenter across the active pair. + this.annotator.geoViewerRef.value.geoOn( + geo.event.mouseclick, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (e: any) => this.handleClick(e), + ); + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + private handleClick(e: any) { + if (!this.getRecenterEnabled() || !e.geo) { + return; + } + const buttonsDown = e.buttonsDown || (e.mouse && e.mouse.buttonsDown); + if (!buttonsDown || !buttonsDown.right) { + return; + } + this.annotator.geoViewerRef.value.center({ x: e.geo.x, y: e.geo.y }); + } + + /** + * Find the annotator's own image/video quad layer (the one Image/Video + * Annotator draws each frame into). It is created before any LayerManager + * layer, so it is the first layer -- other than ours -- containing a quad + * datum with an `image`/`video` texture source. Returns null for + * annotators without one (e.g. tiled large-image datasets). + */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + private findNativeQuadLayer(): any | null { + const viewer = this.annotator.geoViewerRef.value; + if (!viewer || typeof viewer.layers !== 'function') { + return null; + } + const layerList = viewer.layers(); + for (let i = 0; i < layerList.length; i += 1) { + const layer = layerList[i]; + if (layer !== this.quadLayer && typeof layer.features === 'function') { + const features = layer.features(); + for (let j = 0; j < features.length; j += 1) { + const data = typeof features[j].data === 'function' ? features[j].data() : undefined; + const datum = Array.isArray(data) ? data[0] : undefined; + if (datum && (datum.image || datum.video)) { + return layer; + } + } + } + } + return null; + } + + private setNativeVisible(visible: boolean) { + if (visible === !this.nativeHidden) { + return; + } + const nativeLayer = this.findNativeQuadLayer(); + if (nativeLayer) { + nativeLayer.visible(visible); + if (visible) { + nativeLayer.draw(); + } + this.nativeHidden = !visible; + } else if (visible) { + this.nativeHidden = false; + } + } + + /** Clear the warp and restore the native image display. */ + clear() { + this.cancelRefresh(); + this.renderedSource = null; + this.quadFeature.data([]).draw(); + this.setNativeVisible(true); + } + + /** Recompute the warp from the current transform and frame image. */ + update() { + const transform = this.getTransform(); + if (!transform) { + this.clear(); + return; + } + const src = this.getImage(); + if (!src || !src.width || !src.height) { + // The frame may simply not have finished loading yet (or this + // annotator type exposes no image element, e.g. large-image tiles, in + // which case polling harmlessly expires and the native display stays). + this.cancelRefresh(); + this.renderedSource = null; + this.quadFeature.data([]).draw(); + this.setNativeVisible(true); + this.scheduleRefresh(); + return; + } + const { width: w, height: h } = src; + const grid = warpGridSize(transform, w, h); + // 2px cell overlap hides the canvas antialiasing seams between abutting + // sub-quads (dark grid lines); safe here because quads draw opaque. + const quads = subdivideWarpQuads(transform, w, h, grid, 2).map((q) => ({ + ul: { x: q.ul[0], y: q.ul[1] }, + ur: { x: q.ur[0], y: q.ur[1] }, + lr: { x: q.lr[0], y: q.lr[1] }, + ll: { x: q.ll[0], y: q.ll[1] }, + // geojs crop: left/top/right/bottom select the source-pixel region; + // x/y (the "size after crop") are set to the full source size so that + // region stretches across the whole sub-quad. + crop: { + ...q.crop, x: w, y: h, + }, + [src.kind]: src.source, + })); + this.renderedSource = src.source; + // Mirror the native layer's CSS filter (image enhancements) so toggling + // the warp doesn't change brightness/contrast rendering. + const nativeLayer = this.findNativeQuadLayer(); + if (nativeLayer) { + this.quadLayer.node().css('filter', nativeLayer.node().css('filter')); + } + this.quadFeature + .data(quads) + .style('opacity', 1) + .draw(); + this.setNativeVisible(false); + if (src.kind === 'image') { + // Image sequences swap the element asynchronously after the + // frame finishes loading, with no event reaching this layer; poll + // briefly so the warp catches up (video elements update in place). + this.scheduleRefresh(); + } else { + this.cancelRefresh(); + } + } + + private cancelRefresh() { + if (this.retryHandle !== null) { + cancelAnimationFrame(this.retryHandle); + this.retryHandle = null; + } + } + + /** + * Bounded requestAnimationFrame loop re-checking whether the displayed + * image element differs from the one the warp was rendered from, and + * re-rendering when it does (same pattern as the calibration ghost). + */ + private scheduleRefresh() { + this.cancelRefresh(); + this.retryAttempts = 0; + if (typeof requestAnimationFrame !== 'function') { + return; + } + const tick = () => { + this.retryHandle = null; + if (!this.getTransform()) { + return; + } + const src = this.getImage(); + if (src && src.source && src.width && src.height && src.source !== this.renderedSource) { + // Re-render with the new element; update() restarts this loop. + this.update(); + return; + } + this.retryAttempts += 1; + if (this.retryAttempts < REFRESH_MAX_ATTEMPTS) { + this.retryHandle = requestAnimationFrame(tick); + } + }; + this.retryHandle = requestAnimationFrame(tick); + } +} diff --git a/client/src/layers/AnnotationLayers/AttributeBoxLayer.ts b/client/src/layers/AnnotationLayers/AttributeBoxLayer.ts index eb14b5d6d..e3b8f6dc7 100644 --- a/client/src/layers/AnnotationLayers/AttributeBoxLayer.ts +++ b/client/src/layers/AnnotationLayers/AttributeBoxLayer.ts @@ -116,7 +116,7 @@ export default class AttributeBoxLayer extends BaseLayer { return { ...super.createStyle(), // Style conversion to get array objects to work in geoJS - position: (point) => ({ x: point[0], y: point[1] }), + position: (point) => this.transformPoint(point), strokeColor: (_point, _index, data) => data.lineColor, fill: (data) => { if (data.boxOpacity) { diff --git a/client/src/layers/AnnotationLayers/AttributeLayer.ts b/client/src/layers/AnnotationLayers/AttributeLayer.ts index 99fd1b9c3..1efb7b5d6 100644 --- a/client/src/layers/AnnotationLayers/AttributeLayer.ts +++ b/client/src/layers/AnnotationLayers/AttributeLayer.ts @@ -240,7 +240,7 @@ export default class AttributeLayer extends BaseLayer { this.featureLayer = layer .createFeature('text') .text((data: AttributeTextData) => data.text) - .position((data: AttributeTextData) => ({ x: data.x, y: data.y })); + .position((data: AttributeTextData) => this.transformXY(data)); super.initialize(); } diff --git a/client/src/layers/AnnotationLayers/CalibrationKeypointLayer.ts b/client/src/layers/AnnotationLayers/CalibrationKeypointLayer.ts new file mode 100644 index 000000000..994d39a98 --- /dev/null +++ b/client/src/layers/AnnotationLayers/CalibrationKeypointLayer.ts @@ -0,0 +1,519 @@ +import geo, { GeoEvent } from 'geojs'; +import BaseLayer, { BaseLayerParams, LayerStyle } from '../BaseLayer'; +import { FrameDataTrack } from '../LayerTypes'; +import CameraCalibrationStore from '../../CameraCalibrationStore'; +import { subdivideWarpQuads, warpGridSize } from '../../homography'; + +export interface CalibrationPointData { + x: number; + y: number; + label: string; + pending: boolean; + /** Marker belongs to the selected correspondence (highlighted in both panes). */ + selected: boolean; + /** Owning correspondence id; undefined for the pending (in-progress) point. */ + correspondenceId?: number; +} + +export interface CameraImage { + /** The texture source for the geojs quad feature: an `` for image sequences, a `