From ad58631378f023036dbf18ab33cbc52caccc7271 Mon Sep 17 00:00:00 2001 From: romleiaj Date: Wed, 1 Jul 2026 10:21:25 -0400 Subject: [PATCH 01/79] Add in-app multicam calibration tool modeled on keypointgui MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let users pick corresponding points between camera pairs inside DIVE, fit homographies for alignment preview, and persist results for VIAME workflows. This replaces the external keypointgui loop for the common EO/IR registration path. - Add CameraCalibrationStore with blue→red pairing, homography fit, and auto-fit when enabling the overlay or saving calibration - Add CalibrationTools sidebar: camera pair select, correspondence table, points.txt export, overlay preview with direction and opacity - Add CalibrationKeypointLayer for on-image markers and warped overlay - Add homography DLT solver and unit tests - Persist calibration to standalone calibration.json (pairs, points, leftToRight/rightToLeft matrices) and rehydrate on dataset load - Register Camera Calibration panel for multicam datasets in Viewer --- client/dive-common/apispec.ts | 5 +- .../CameraCalibration/CalibrationTools.vue | 270 ++++++++++++++++++ client/dive-common/components/Viewer.vue | 14 + client/dive-common/store/context.ts | 5 + .../desktop/backend/native/common.spec.ts | 49 ++++ .../platform/desktop/backend/native/common.ts | 114 ++++++++ client/src/CameraCalibrationStore.spec.ts | 203 +++++++++++++ client/src/CameraCalibrationStore.ts | 241 ++++++++++++++++ client/src/components/LayerManager.vue | 58 ++++ client/src/homography.spec.ts | 65 +++++ client/src/homography.ts | 182 ++++++++++++ client/src/index.ts | 2 + .../CalibrationKeypointLayer.ts | 225 +++++++++++++++ client/src/provides.ts | 9 + server/dive_utils/models.py | 6 + 15 files changed, 1447 insertions(+), 1 deletion(-) create mode 100644 client/dive-common/components/CameraCalibration/CalibrationTools.vue create mode 100644 client/src/CameraCalibrationStore.spec.ts create mode 100644 client/src/CameraCalibrationStore.ts create mode 100644 client/src/homography.spec.ts create mode 100644 client/src/homography.ts create mode 100644 client/src/layers/AnnotationLayers/CalibrationKeypointLayer.ts diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index 78c40b8ec..c9a104de6 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -8,6 +8,7 @@ import { Attribute } from 'vue-media-annotator/use/AttributeTypes'; import { CustomStyle } from 'vue-media-annotator/StyleManager'; import { AttributeTrackFilter } from 'vue-media-annotator/AttributeTrackFilterControls'; import { ImageEnhancements } from 'vue-media-annotator/use/useImageEnhancements'; +import { CameraHomographies, CameraCorrespondences } from 'vue-media-annotator/CameraCalibrationStore'; type DatasetType = 'image-sequence' | 'video' | 'multi' | 'large-image'; type MultiTrackRecord = Record; @@ -181,9 +182,11 @@ interface DatasetMetaMutable { attributes?: Readonly>; attributeTrackFilters?: Readonly>; datasetInfo?: Record; + cameraHomographies?: CameraHomographies; + cameraCorrespondences?: CameraCorrespondences; error?: string; } -const DatasetMetaMutableKeys = ['attributes', 'confidenceFilters', 'timeFilters', 'imageEnhancements', 'customTypeStyling', 'customGroupStyling', 'attributeTrackFilters', 'datasetInfo']; +const DatasetMetaMutableKeys = ['attributes', 'confidenceFilters', 'timeFilters', 'imageEnhancements', 'customTypeStyling', 'customGroupStyling', 'attributeTrackFilters', 'datasetInfo', 'cameraHomographies', 'cameraCorrespondences']; interface DatasetMeta extends DatasetMetaMutable { id: Readonly; diff --git a/client/dive-common/components/CameraCalibration/CalibrationTools.vue b/client/dive-common/components/CameraCalibration/CalibrationTools.vue new file mode 100644 index 000000000..2be3eed2a --- /dev/null +++ b/client/dive-common/components/CameraCalibration/CalibrationTools.vue @@ -0,0 +1,270 @@ + + + diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index 83a157ed2..ed9cec426 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -18,6 +18,7 @@ import { import { Track, Group, CameraStore, + CameraCalibrationStore, StyleManager, TrackFilterControls, GroupFilterControls, } from 'vue-media-annotator/index'; import { provideAnnotator, LassoModeSymbol } from 'vue-media-annotator/provides'; @@ -67,6 +68,7 @@ import context from 'dive-common/store/context'; import { MarkChangesPendingFilter } from 'vue-media-annotator/BaseFilterControls'; import GroupSidebarVue from './GroupSidebar.vue'; import MultiCamToolsVue from './MultiCamTools.vue'; +import CalibrationToolsVue from './CameraCalibration/CalibrationTools.vue'; import MultiCamToolbar from './MultiCamToolbar.vue'; import PrimaryAttributeTrackFilter from './PrimaryAttributeTrackFilter.vue'; import UserSettingsDialog from './UserSettingsDialog.vue'; @@ -312,6 +314,7 @@ export default defineComponent({ const groupStyleManager = new StyleManager({ markChangesPending, vuetify }); const cameraStore = new CameraStore({ markChangesPending }); + const cameraCalibration = new CameraCalibrationStore(); // This context for removal const removeGroups = (id: AnnotationId) => { cameraStore.removeGroups(id); @@ -1042,6 +1045,8 @@ export default defineComponent({ if (meta.attributeTrackFilters) { trackFilters.loadTrackAttributesFilter(Object.values(meta.attributeTrackFilters)); } + // Rehydrate any saved camera-to-camera calibration homographies and points + cameraCalibration.hydrate(meta.cameraHomographies, meta.cameraCorrespondences); progress.loaded = true; // If multiCam add Tools and remove group Tools if (cameraStore.camMap.value.size > 1) { @@ -1053,11 +1058,19 @@ export default defineComponent({ component: MultiCamToolsVue, description: 'Multi Camera Tools', }); + context.register({ + component: CalibrationToolsVue, + description: 'Camera Calibration', + }); } else { context.unregister({ component: MultiCamToolsVue, description: 'Multi Camera Tools', }); + context.unregister({ + component: CalibrationToolsVue, + description: 'Camera Calibration', + }); context.register({ description: 'Group Manager', component: GroupSidebarVue, @@ -1148,6 +1161,7 @@ export default defineComponent({ annotatorPreferences: toRef(clientSettings, 'annotatorPreferences'), attributes, cameraStore, + cameraCalibration, datasetId, editingMode, groupFilters, diff --git a/client/dive-common/store/context.ts b/client/dive-common/store/context.ts index c6ee24ff6..ce1c8246c 100644 --- a/client/dive-common/store/context.ts +++ b/client/dive-common/store/context.ts @@ -5,6 +5,7 @@ import ImageEnhancements from 'vue-media-annotator/components/ImageEnhancements. import GroupSidebar from 'dive-common/components/GroupSidebar.vue'; import AttributesSideBar from 'dive-common/components/Attributes/AttributesSideBar.vue'; import MultiCamTools from 'dive-common/components/MultiCamTools.vue'; +import CalibrationTools from 'dive-common/components/CameraCalibration/CalibrationTools.vue'; import AttributeTrackFilters from 'vue-media-annotator/components/AttributeTrackFilters.vue'; import DatasetInfo from 'dive-common/components/DatasetInfo.vue'; @@ -46,6 +47,10 @@ const componentMap: Record = { description: 'Multi Camera Tools', component: MultiCamTools, }, + [CalibrationTools.name]: { + description: 'Camera Calibration', + component: CalibrationTools, + }, [AttributesSideBar.name]: { description: 'Attribute Details', component: AttributesSideBar, diff --git a/client/platform/desktop/backend/native/common.spec.ts b/client/platform/desktop/backend/native/common.spec.ts index 4598d323c..57e6bf4f5 100644 --- a/client/platform/desktop/backend/native/common.spec.ts +++ b/client/platform/desktop/backend/native/common.spec.ts @@ -624,6 +624,55 @@ describe('native.common', () => { }); }); + it('saveMetadata writes calibration.json (pairs + points) and reloads it', async () => { + const payload = await common.beginMediaImport( + '/home/user/data/imageLists/success/image_list.txt', + ); + const res = await common.finalizeMediaImport(settings, payload); + const final = res.meta; + // Directional key: rgb is left, ir is right. + const cameraHomographies = { + 'rgb::ir': { + AtoB: [[1, 0, 5], [0, 1, -3], [0, 0, 1]], + BtoA: [[1, 0, -5], [0, 1, 3], [0, 0, 1]], + }, + }; + const cameraCorrespondences = { + 'rgb::ir': [ + { id: 1, a: [10, 20], b: [12, 22] }, + { id: 2, a: [30, 40], b: [33, 44] }, + ], + }; + + await common.saveMetadata(settings, final.id, { cameraHomographies, cameraCorrespondences }); + + // Persisted as a standalone calibration.json: pairs labeled left/right, with + // points laid out as leftX leftY rightX rightY. + const projectDir = npath.join(settings.dataPath, 'DIVE_Projects', final.id); + const calibrationPath = npath.join(projectDir, 'calibration.json'); + expect(await fs.pathExists(calibrationPath)).toBe(true); + const calibration = await fs.readJSON(calibrationPath); + expect(calibration.pairs).toStrictEqual([ + { + left: 'rgb', + right: 'ir', + points: [[10, 20, 12, 22], [30, 40, 33, 44]], + leftToRight: [[1, 0, 5], [0, 1, -3], [0, 0, 1]], + rightToLeft: [[1, 0, -5], [0, 1, 3], [0, 0, 1]], + }, + ]); + + // Not embedded in meta.json. + const meta = await fs.readJSON(npath.join(projectDir, 'meta.json')); + expect(meta.cameraHomographies).toBeUndefined(); + expect(meta.cameraCorrespondences).toBeUndefined(); + + // Rehydrated on load back into the in-app shapes. + const reloaded = await common.loadMetadata(settings, final.id, urlMapper); + expect(reloaded.cameraHomographies).toStrictEqual(cameraHomographies); + expect(reloaded.cameraCorrespondences).toStrictEqual(cameraCorrespondences); + }); + it('import with CSV annotations without specifying track file', async () => { const payload = await common.beginMediaImport('/home/user/data/imageSuccessWithAnnotations'); payload.trackFileAbsPath = ''; //It returns null be default but users change it. diff --git a/client/platform/desktop/backend/native/common.ts b/client/platform/desktop/backend/native/common.ts index 6939811e0..abd7e3a2a 100644 --- a/client/platform/desktop/backend/native/common.ts +++ b/client/platform/desktop/backend/native/common.ts @@ -70,6 +70,10 @@ const AuxFolderName = 'auxiliary'; const JsonTrackFileName = /^result(_.*)?\.json$/i; const JsonFileName = /^.*\.json$/i; const JsonMetaFileName = 'meta.json'; +// Standalone camera-to-camera (modality-to-modality) alignment transforms, +// stored separately from meta.json in the dataset directory. +const CalibrationFileName = 'calibration.json'; +const CalibrationFileVersion = 1; const CsvFileName = /^.*\.csv$/i; const YAMLFileName = /^.*\.ya?ml$/i; /** @@ -367,6 +371,24 @@ async function loadMetadata( const projectDirData = await getValidatedProjectDir(settings, datasetId); const projectMetaData = await loadJsonMetadata(projectDirData.metaFileAbsPath); + // Load standalone camera calibration (transforms + correspondences), if present. + const calibrationFileAbsPath = npath.join(projectDirData.basePath, CalibrationFileName); + let { cameraHomographies, cameraCorrespondences } = projectMetaData; + if (await fs.pathExists(calibrationFileAbsPath)) { + try { + const calibration = await _loadAsJson(calibrationFileAbsPath); + if (calibration && Array.isArray(calibration.pairs)) { + ({ + homographies: cameraHomographies, + correspondences: cameraCorrespondences, + } = fromCalibrationPairs(calibration.pairs)); + } + } catch (err) { + // A malformed calibration.json should not block loading the dataset. + console.warn(`Unable to read ${calibrationFileAbsPath}: ${err}`); + } + } + let videoUrl = ''; let imageData = [] as FrameImage[]; let multiCamMedia: MultiCamMedia | null = null; @@ -431,6 +453,8 @@ async function loadMetadata( imageData, multiCamMedia, subType, + cameraHomographies, + cameraCorrespondences, }; } @@ -686,6 +710,65 @@ async function _saveAsJson(absPath: string, data: unknown) { await fs.writeFile(absPath, serialized); } +type CameraHomographies = NonNullable; +type CameraCorrespondences = NonNullable; + +/** + * One camera pair in calibration.json. `left`/`right` are camera (folder) names; + * `points` are the picked correspondences as rows of `leftX leftY rightX rightY` + * (the keypointgui points.txt layout); `leftToRight`/`rightToLeft` are the fitted + * 3x3 homographies, when a fit has been performed. + */ +interface CalibrationPair { + left: string; + right: string; + points: number[][]; + leftToRight: number[][] | null; + rightToLeft: number[][] | null; +} + +/** + * Convert the in-app calibration state (keyed by directional "left::right") into + * the self-describing list of pairs persisted in calibration.json. + */ +function toCalibrationPairs( + homographies: CameraHomographies, + correspondences: CameraCorrespondences, +): CalibrationPair[] { + const keys = new Set([...Object.keys(homographies), ...Object.keys(correspondences)]); + return [...keys].map((key) => { + const [left, right] = key.split('::'); + const homography = homographies[key]; + return { + left, + right, + points: (correspondences[key] || []).map((c) => [c.a[0], c.a[1], c.b[0], c.b[1]]), + leftToRight: homography ? homography.AtoB : null, + rightToLeft: homography ? homography.BtoA : null, + }; + }); +} + +/** Rebuild the in-app homographies/correspondences from calibration.json pairs. */ +function fromCalibrationPairs( + pairs: CalibrationPair[], +): { homographies: CameraHomographies; correspondences: CameraCorrespondences } { + const homographies: CameraHomographies = {}; + const correspondences: CameraCorrespondences = {}; + pairs.forEach((pair) => { + const key = `${pair.left}::${pair.right}`; + if (pair.leftToRight && pair.rightToLeft) { + homographies[key] = { AtoB: pair.leftToRight, BtoA: pair.rightToLeft }; + } + if (pair.points && pair.points.length) { + correspondences[key] = pair.points.map((p, i) => ({ + id: i + 1, a: [p[0], p[1]], b: [p[2], p[3]], + })); + } + }); + return { homographies, correspondences }; +} + async function saveMetadata(settings: Settings, datasetId: string, args: DatasetMetaMutable) { const projectDirInfo = await getValidatedProjectDir(settings, datasetId); const release = await _acquireLock(projectDirInfo.basePath, projectDirInfo.metaFileAbsPath, 'meta'); @@ -715,6 +798,37 @@ async function saveMetadata(settings: Settings, datasetId: string, args: Dataset existing.datasetInfo = args.datasetInfo; } + // Camera calibration (transforms + the points behind them) is persisted as a + // standalone calibration.json in the dataset directory rather than embedded in + // meta.json, so it is easy to find, hand-edit, and consume as a self-contained + // artifact. + if (args.cameraHomographies || args.cameraCorrespondences) { + const calibrationFileAbsPath = npath.join(projectDirInfo.basePath, CalibrationFileName); + // Start from whatever is on disk so a partial update doesn't clobber the rest. + let homographies: CameraHomographies = {}; + let correspondences: CameraCorrespondences = {}; + if (await fs.pathExists(calibrationFileAbsPath)) { + try { + const existingCalibration = await _loadAsJson(calibrationFileAbsPath); + if (existingCalibration && Array.isArray(existingCalibration.pairs)) { + ({ homographies, correspondences } = fromCalibrationPairs(existingCalibration.pairs)); + } + } catch (err) { + console.warn(`Unable to read existing ${calibrationFileAbsPath}: ${err}`); + } + } + if (args.cameraHomographies) { + homographies = args.cameraHomographies; + } + if (args.cameraCorrespondences) { + correspondences = args.cameraCorrespondences; + } + await _saveAsJson(calibrationFileAbsPath, { + version: CalibrationFileVersion, + pairs: toCalibrationPairs(homographies, correspondences), + }); + } + await _saveAsJson(projectDirInfo.metaFileAbsPath, existing); await release(); } diff --git a/client/src/CameraCalibrationStore.spec.ts b/client/src/CameraCalibrationStore.spec.ts new file mode 100644 index 000000000..27ada9494 --- /dev/null +++ b/client/src/CameraCalibrationStore.spec.ts @@ -0,0 +1,203 @@ +/// +import CameraCalibrationStore from './CameraCalibrationStore'; + +describe('CameraCalibrationStore', () => { + it('produces a directional pairKey that preserves left/right order', () => { + const store = new CameraCalibrationStore(); + expect(store.pairKey('rgb', 'ir')).toEqual('rgb::ir'); + expect(store.pairKey('rgb', 'ir')).not.toEqual(store.pairKey('ir', 'rgb')); + }); + + it('preserves the chosen left/right order on the active pair', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('rgb', 'ir'); + expect(store.activePair.value).toEqual({ camA: 'rgb', camB: 'ir' }); + }); + + it('clears the active pair for identical or empty cameras', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('cam', 'cam'); + expect(store.activePair.value).toBeNull(); + }); + + it('forms one correspondence from a blue->red two-click sequence', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.addPoint('left', [10, 20]); // pending (blue) + expect(store.pendingPoint.value).not.toBeNull(); + expect(store.correspondences.value[key]).toBeUndefined(); + store.addPoint('right', [30, 40]); // completes (red) + expect(store.pendingPoint.value).toBeNull(); + expect(store.correspondences.value[key]).toHaveLength(1); + expect(store.correspondences.value[key][0]).toMatchObject({ a: [10, 20], b: [30, 40] }); + }); + + it('maps points to a/b by camera regardless of click order', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.addPoint('right', [30, 40]); // click right first + store.addPoint('left', [10, 20]); + expect(store.correspondences.value[key][0]).toMatchObject({ a: [10, 20], b: [30, 40] }); + }); + + it('replaces the pending point when the same camera is clicked twice', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + store.addPoint('left', [1, 1]); + store.addPoint('left', [2, 2]); + expect(store.pendingPoint.value).toMatchObject({ camera: 'left', coord: [2, 2] }); + }); + + it('ignores points for cameras outside the active pair', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + store.addPoint('other', [1, 1]); + expect(store.pendingPoint.value).toBeNull(); + }); + + it('removes a correspondence by id', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.addPoint('left', [1, 1]); + store.addPoint('right', [2, 2]); + const { id } = store.correspondences.value[key][0]; + store.removeCorrespondence(id); + expect(store.correspondences.value[key]).toHaveLength(0); + }); + + it('fits a homography from >= 4 pairs and stores both directions', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + // A pure translation by (5, -3). + const pts: [number, number][] = [[0, 0], [10, 0], [10, 10], [0, 10]]; + pts.forEach((p) => { + store.addPoint('left', p); + store.addPoint('right', [p[0] + 5, p[1] - 3]); + }); + const { AtoB, BtoA } = store.fitHomography(key); + expect(AtoB[0][2]).toBeCloseTo(5, 5); + expect(AtoB[1][2]).toBeCloseTo(-3, 5); + expect(BtoA[0][2]).toBeCloseTo(-5, 5); + expect(store.homographies.value[key]).toBeDefined(); + }); + + it('throws when fitting with fewer than 4 pairs', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.addPoint('left', [1, 1]); + store.addPoint('right', [2, 2]); + expect(() => store.fitHomography(key)).toThrow(); + }); + + function addFourTranslationPairs(store: CameraCalibrationStore) { + const pts: [number, number][] = [[0, 0], [10, 0], [10, 10], [0, 10]]; + pts.forEach((p) => { + store.addPoint('left', p); + store.addPoint('right', [p[0] + 5, p[1] - 3]); + }); + } + + it('fits when enabling the overlay with >= 4 pairs', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + addFourTranslationPairs(store); + store.setOverlayEnabled(true); + expect(store.overlay.value.enabled).toBe(true); + expect(store.homographies.value[key]).toBeDefined(); + expect(store.homographies.value[key].AtoB[0][2]).toBeCloseTo(5, 5); + }); + + it('does not enable the overlay with fewer than 4 pairs', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + store.addPoint('left', [1, 1]); + store.addPoint('right', [2, 2]); + store.setOverlayEnabled(true); + expect(store.overlay.value.enabled).toBe(false); + expect(store.homographies.value).toEqual({}); + }); + + it('refits when correspondences change while the overlay is enabled', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + addFourTranslationPairs(store); + store.setOverlayEnabled(true); + const before = store.homographies.value[key].AtoB[0][2]; + store.addPoint('left', [20, 20]); + store.addPoint('right', [30, 14]); + expect(store.homographies.value[key].AtoB[0][2]).not.toBeCloseTo(before, 5); + }); + + it('disables the overlay when correspondences drop below 4', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + addFourTranslationPairs(store); + store.setOverlayEnabled(true); + const { id } = store.correspondences.value[key][0]; + store.removeCorrespondence(id); + store.removeCorrespondence(store.correspondences.value[key][0].id); + store.removeCorrespondence(store.correspondences.value[key][0].id); + store.removeCorrespondence(store.correspondences.value[key][0].id); + expect(store.correspondences.value[key]).toHaveLength(0); + expect(store.overlay.value.enabled).toBe(false); + expect(store.homographies.value[key]).toBeUndefined(); + }); + + it('maybeFitActivePair fits without enabling the overlay', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + addFourTranslationPairs(store); + store.maybeFitActivePair(); + expect(store.overlay.value.enabled).toBe(false); + expect(store.homographies.value[key]).toBeDefined(); + }); + + it('hydrates homographies and resets transient state', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + store.addPoint('left', [1, 1]); + const saved = { 'a::b': { AtoB: [[1, 0, 0], [0, 1, 0], [0, 0, 1]], BtoA: [[1, 0, 0], [0, 1, 0], [0, 0, 1]] } }; + store.hydrate(saved); + expect(store.homographies.value).toEqual(saved); + expect(store.activePair.value).toBeNull(); + expect(store.pendingPoint.value).toBeNull(); + expect(store.correspondences.value).toEqual({}); + }); + + it('exports points as four columns "leftX leftY rightX rightY"', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('rgb', 'ir'); + const key = store.pairKey('rgb', 'ir'); + store.addPoint('rgb', [10, 20]); // left + store.addPoint('ir', [30, 40]); // right + store.addPoint('ir', [31, 41]); // left/right order independent of click order + store.addPoint('rgb', [11, 21]); + expect(store.toPointsText(key)).toBe('10 20 30 40\n11 21 31 41'); + }); + + it('hydrates correspondences and resumes id allocation', () => { + const store = new CameraCalibrationStore(); + const correspondences = { + 'rgb::ir': [ + { id: 1, a: [1, 2], b: [3, 4] }, + { id: 2, a: [5, 6], b: [7, 8] }, + ], + }; + store.hydrate({}, correspondences); + expect(store.correspondences.value).toEqual(correspondences); + // New points pick up after the highest restored id. + store.setActivePair('rgb', 'ir'); + store.addPoint('rgb', [9, 9]); + store.addPoint('ir', [10, 10]); + expect(store.correspondences.value['rgb::ir'][2].id).toBe(3); + }); +}); diff --git a/client/src/CameraCalibrationStore.ts b/client/src/CameraCalibrationStore.ts new file mode 100644 index 000000000..94911b43c --- /dev/null +++ b/client/src/CameraCalibrationStore.ts @@ -0,0 +1,241 @@ +import { ref, Ref } from 'vue'; +import { + solveHomography, invert3, Matrix3, Point, +} from './homography'; + +/** + * A single picked point pair. `a` is the point in the left camera (camA), `b` + * the point in the right camera (camB). Left/right is the order the user chose, + * which is preserved (not alphabetized) so it can drive ordered exports such as + * the keypointgui-style points.txt consumed by VIAME/SealTk. + */ +export interface Correspondence { + id: number; + a: Point; + b: Point; +} + +/** Both directions of the fitted homography for one camera pair. */ +export interface PairHomography { + /** Maps left (camA) image coordinates onto right (camB). */ + AtoB: Matrix3; + /** Maps right (camB) image coordinates onto left (camA). */ + BtoA: Matrix3; +} + +/** Fitted homographies keyed by {@link CameraCalibrationStore.pairKey}. */ +export type CameraHomographies = Record; + +/** Picked correspondences keyed by {@link CameraCalibrationStore.pairKey}. */ +export type CameraCorrespondences = Record; + +export interface OverlayState { + enabled: boolean; + opacity: number; + /** Which image to warp onto the other when overlaying. */ + direction: 'AtoB' | 'BtoA'; +} + +/** Active pair. `camA` is the left camera, `camB` the right (user-chosen order). */ +export interface ActivePair { + camA: string; + camB: string; +} + +/** + * Shared, reactive state for the interactive camera-calibration tool. Lives in + * vue-media-annotator so both the geojs picking layer (client/src/layers) and the + * dive-common side panel can consume it via the provide/inject system. + * + * Implements the keypointgui blue->red pairing flow: the first click in one camera + * sets a pending point; the next click in the *other* camera completes a pair. + */ +export default class CameraCalibrationStore { + activePair: Ref; + + pickingEnabled: Ref; + + pendingPoint: Ref<{ camera: string; coord: Point } | null>; + + correspondences: Ref; + + homographies: Ref; + + overlay: Ref; + + private nextId: number; + + constructor() { + this.activePair = ref(null); + this.pickingEnabled = ref(false); + this.pendingPoint = ref(null); + this.correspondences = ref({}); + this.homographies = ref({}); + this.overlay = ref({ enabled: false, opacity: 0.5, direction: 'AtoB' }); + this.nextId = 1; + } + + /** + * Directional key for a camera pair: `left::right`. Order is significant and + * preserved so left/right (e.g. RGB vs IR) survives for ordered exports. + */ + // eslint-disable-next-line class-methods-use-this + pairKey(camA: string, camB: string): string { + return `${camA}::${camB}`; + } + + /** Key of the currently active pair, or null if none selected. */ + activePairKey(): string | null { + const pair = this.activePair.value; + return pair ? this.pairKey(pair.camA, pair.camB) : null; + } + + /** Select a camera pair. `left` becomes camA, `right` becomes camB. */ + setActivePair(left: string | null, right: string | null) { + if (!left || !right || left === right) { + this.activePair.value = null; + } else { + this.activePair.value = { camA: left, camB: right }; + } + this.pendingPoint.value = null; + } + + /** + * Add a clicked image point for `camera`. The first click sets a pending point; + * a subsequent click in the *other* camera of the active pair completes a pair. + * Clicking the same camera again replaces the pending point. + */ + addPoint(camera: string, coord: Point) { + const pair = this.activePair.value; + if (!pair || (camera !== pair.camA && camera !== pair.camB)) { + return; + } + const pending = this.pendingPoint.value; + if (!pending || pending.camera === camera) { + this.pendingPoint.value = { camera, coord }; + return; + } + const key = this.pairKey(pair.camA, pair.camB); + const a = pending.camera === pair.camA ? pending.coord : coord; + const b = pending.camera === pair.camB ? pending.coord : coord; + const list = this.correspondences.value[key] + ? [...this.correspondences.value[key]] + : []; + // eslint-disable-next-line no-plusplus + list.push({ id: this.nextId++, a, b }); + this.correspondences.value = { ...this.correspondences.value, [key]: list }; + this.pendingPoint.value = null; + this.syncOverlayHomography(); + } + + /** Remove a correspondence (by id) from the active pair. */ + removeCorrespondence(id: number) { + const key = this.activePairKey(); + if (!key) { + return; + } + const list = this.correspondences.value[key]; + if (!list) { + return; + } + this.correspondences.value = { + ...this.correspondences.value, + [key]: list.filter((c) => c.id !== id), + }; + this.syncOverlayHomography(); + } + + /** Drop all correspondences and the pending point for the active pair. */ + clearPair() { + const key = this.activePairKey(); + this.pendingPoint.value = null; + if (!key) { + return; + } + this.correspondences.value = { ...this.correspondences.value, [key]: [] }; + this.syncOverlayHomography(); + } + + /** + * Fit the active pair when it has enough points; otherwise clear its homography + * and turn off the overlay preview. + */ + maybeFitActivePair() { + const key = this.activePairKey(); + if (!key) { + return; + } + const list = this.correspondences.value[key]; + if (!list || list.length < 4) { + const { [key]: _removed, ...rest } = this.homographies.value; + this.homographies.value = rest; + if (this.overlay.value.enabled) { + this.overlay.value = { ...this.overlay.value, enabled: false }; + } + return; + } + this.fitHomography(key); + } + + /** Enable or disable the overlay preview, fitting the homography when turning on. */ + setOverlayEnabled(enabled: boolean) { + if (enabled) { + this.maybeFitActivePair(); + const key = this.activePairKey(); + if (!key || !this.homographies.value[key]) { + return; + } + } + this.overlay.value = { ...this.overlay.value, enabled }; + } + + /** Re-fit the active pair while the overlay preview is enabled. */ + private syncOverlayHomography() { + if (this.overlay.value.enabled) { + this.maybeFitActivePair(); + } + } + + /** + * Fit a homography for `key` from its correspondences (>= 4 required). Computes + * both directions and stores them. Returns the fitted pair. + */ + fitHomography(key: string): PairHomography { + const list = this.correspondences.value[key]; + if (!list || list.length < 4) { + throw new Error('At least 4 point pairs are required to fit a homography'); + } + const AtoB = solveHomography(list.map((c) => c.a), list.map((c) => c.b)); + const BtoA = invert3(AtoB); + this.homographies.value = { ...this.homographies.value, [key]: { AtoB, BtoA } }; + return { AtoB, BtoA }; + } + + /** + * Serialize a pair's correspondences as keypointgui-style points text: one row + * per pair, four space-separated columns `leftX leftY rightX rightY`. This is + * the format consumed by VIAME's `itk_point_set_to_transform` to build the .h5. + */ + toPointsText(key: string): string { + const list = this.correspondences.value[key] || []; + return list + .map((c) => `${c.a[0]} ${c.a[1]} ${c.b[0]} ${c.b[1]}`) + .join('\n'); + } + + /** Reset state and load saved homographies and correspondences. */ + hydrate(homographies?: CameraHomographies, correspondences?: CameraCorrespondences) { + this.homographies.value = homographies ? { ...homographies } : {}; + this.correspondences.value = correspondences ? { ...correspondences } : {}; + this.activePair.value = null; + this.pendingPoint.value = null; + this.pickingEnabled.value = false; + this.overlay.value = { enabled: false, opacity: 0.5, direction: 'AtoB' }; + // Resume id allocation past any restored correspondences. + let maxId = 0; + Object.values(this.correspondences.value).forEach((list) => { + list.forEach((c) => { maxId = Math.max(maxId, c.id); }); + }); + this.nextId = maxId + 1; + } +} diff --git a/client/src/components/LayerManager.vue b/client/src/components/LayerManager.vue index ac0824c7d..bbf9d95bb 100644 --- a/client/src/components/LayerManager.vue +++ b/client/src/components/LayerManager.vue @@ -12,6 +12,7 @@ import PointLayer from '../layers/AnnotationLayers/PointLayer'; import LineLayer from '../layers/AnnotationLayers/LineLayer'; import TailLayer from '../layers/AnnotationLayers/TailLayer'; import OverlapLayer from '../layers/AnnotationLayers/OverlapLayer'; +import CalibrationKeypointLayer from '../layers/AnnotationLayers/CalibrationKeypointLayer'; import EditAnnotationLayer, { EditAnnotationTypes } from '../layers/EditAnnotationLayer'; import LassoSelectionLayer from '../layers/LassoSelectionLayer'; @@ -39,6 +40,7 @@ import { useAnnotatorPreferences, useGroupStyleManager, useCameraStore, + useCameraCalibration, useSelectedCamera, useAttributes, useComparisonSets, @@ -77,6 +79,12 @@ export default defineComponent({ // Viewer may not provide lasso context in tests or minimal embeds. } const cameraStore = useCameraStore(); + let cameraCalibration: ReturnType | undefined; + try { + cameraCalibration = useCameraCalibration(); + } catch { + // calibration store may not be provided in tests or minimal embeds. + } const selectedCamera = useSelectedCamera(); const comparison = useComparisonSets(); const trackStore = cameraStore.camMap.value.get(props.camera)?.trackStore; @@ -192,6 +200,56 @@ export default defineComponent({ } }, { deep: true }); + /** Resolve another camera's currently displayed frame image (for the overlay). */ + const getCameraImage = (camera: string) => { + const ctrl = aggregateController.value.getController(camera); + const viewer = ctrl?.geoViewerRef?.value; + if (!viewer || typeof viewer.layers !== 'function') { + return null; + } + const layerList = viewer.layers(); + for (let i = 0; i < layerList.length; i += 1) { + const layer = layerList[i]; + if (typeof layer.features === 'function') { + const features = layer.features(); + for (let j = 0; j < features.length; j += 1) { + const data = typeof features[j].data === 'function' ? features[j].data() : undefined; + if (Array.isArray(data) && data[0] && data[0].image) { + const image = data[0].image as HTMLImageElement; + return { image, width: image.naturalWidth, height: image.naturalHeight }; + } + } + } + } + return null; + }; + + const calibrationLayer = cameraCalibration + ? new CalibrationKeypointLayer({ + annotator, + stateStyling: trackStyleManager.stateStyles, + typeStyling: typeStylingRef, + calibration: cameraCalibration, + getCameraImage, + }) + : undefined; + + if (cameraCalibration && calibrationLayer) { + watch( + [ + cameraCalibration.activePair, + cameraCalibration.pickingEnabled, + cameraCalibration.correspondences, + cameraCalibration.pendingPoint, + cameraCalibration.homographies, + cameraCalibration.overlay, + frameNumberRef, + ], + () => calibrationLayer.update(), + { deep: true }, + ); + } + const updateAttributes = () => { const newList = attributes.value.filter((item) => item.render).sort((a, b) => { if (a.render && b.render) { diff --git a/client/src/homography.spec.ts b/client/src/homography.spec.ts new file mode 100644 index 000000000..25eb24278 --- /dev/null +++ b/client/src/homography.spec.ts @@ -0,0 +1,65 @@ +/// +import { + solveHomography, + applyHomography, + invert3, + matMul3, + Point, + Matrix3, +} from './homography'; + +function expectMatrixClose(actual: Matrix3, expected: Matrix3, tol = 1e-6) { + for (let i = 0; i < 3; i += 1) { + for (let j = 0; j < 3; j += 1) { + expect(actual[i][j]).toBeCloseTo(expected[i][j], 5); + // tol referenced to keep signature explicit + expect(Math.abs(actual[i][j] - expected[i][j])).toBeLessThan(tol * 10); + } + } +} + +const unitSquare: Point[] = [[0, 0], [1, 0], [1, 1], [0, 1]]; + +describe('homography', () => { + it('recovers the identity from identical correspondences', () => { + const H = solveHomography(unitSquare, unitSquare); + expectMatrixClose(H, [[1, 0, 0], [0, 1, 0], [0, 0, 1]]); + }); + + it('recovers a pure translation', () => { + const dst = unitSquare.map(([x, y]): Point => [x + 5, y - 3]); + const H = solveHomography(unitSquare, dst); + expectMatrixClose(H, [[1, 0, 5], [0, 1, -3], [0, 0, 1]]); + }); + + it('recovers a scale + translation', () => { + const H = solveHomography(unitSquare, [[10, 10], [30, 10], [30, 30], [10, 30]]); + expectMatrixClose(H, [[20, 0, 10], [0, 20, 10], [0, 0, 1]]); + }); + + it('maps source points onto destination points (least-squares, >4 pts)', () => { + // A known projective transform applied to 5 points; solver should recover it. + const truth: Matrix3 = [[1.2, 0.1, 5], [0.05, 0.9, -4], [0.0008, -0.0005, 1]]; + const src: Point[] = [[0, 0], [100, 0], [100, 80], [0, 80], [50, 40]]; + const dst = src.map((p) => applyHomography(truth, p)); + const H = solveHomography(src, dst); + src.forEach((p) => { + const [u, v] = applyHomography(H, p); + const [eu, ev] = applyHomography(truth, p); + expect(u).toBeCloseTo(eu, 3); + expect(v).toBeCloseTo(ev, 3); + }); + }); + + it('round-trips through its inverse (H * H^-1 ~= I)', () => { + const H = solveHomography(unitSquare, [[10, 5], [40, 8], [38, 35], [9, 33]]); + const product = matMul3(H, invert3(H)); + const scale = 1 / product[2][2]; + const normalized = product.map((r) => r.map((c) => c * scale)) as Matrix3; + expectMatrixClose(normalized, [[1, 0, 0], [0, 1, 0], [0, 0, 1]]); + }); + + it('throws with fewer than 4 correspondences', () => { + expect(() => solveHomography(unitSquare.slice(0, 3), unitSquare.slice(0, 3))).toThrow(); + }); +}); diff --git a/client/src/homography.ts b/client/src/homography.ts new file mode 100644 index 000000000..4af4f08f2 --- /dev/null +++ b/client/src/homography.ts @@ -0,0 +1,182 @@ +/** + * Self-contained homography estimation via the normalized Direct Linear Transform. + * + * Given >= 4 point correspondences src[i] -> dst[i] (both [x, y] in image + * coordinates), {@link solveHomography} returns the 3x3 matrix H such that, in + * homogeneous coordinates, dst ~= H * src. Exact for 4 points, least-squares for + * more. This is the client-side analogue of OpenCV's cv2.findHomography used by + * the keypointgui reference app; the warp itself is done by geojs (quadFeature). + */ + +export type Point = [number, number]; +export type Matrix3 = number[][]; + +/** Multiply two 3x3 matrices. */ +export function matMul3(a: Matrix3, b: Matrix3): Matrix3 { + const out: Matrix3 = [[0, 0, 0], [0, 0, 0], [0, 0, 0]]; + for (let i = 0; i < 3; i += 1) { + for (let j = 0; j < 3; j += 1) { + let sum = 0; + for (let k = 0; k < 3; k += 1) { + sum += a[i][k] * b[k][j]; + } + out[i][j] = sum; + } + } + return out; +} + +/** Invert a 3x3 matrix. Throws if the matrix is singular. */ +export function invert3(m: Matrix3): Matrix3 { + const [a, b, c] = m[0]; + const [d, e, f] = m[1]; + const [g, h, i] = m[2]; + const A = e * i - f * h; + const B = -(d * i - f * g); + const C = d * h - e * g; + const det = a * A + b * B + c * C; + if (Math.abs(det) < 1e-12) { + throw new Error('Cannot invert singular matrix'); + } + const invDet = 1 / det; + return [ + [A * invDet, -(b * i - c * h) * invDet, (b * f - c * e) * invDet], + [B * invDet, (a * i - c * g) * invDet, -(a * f - c * d) * invDet], + [C * invDet, -(a * h - b * g) * invDet, (a * e - b * d) * invDet], + ]; +} + +/** Apply a 3x3 homography to a single point (perspective divide). */ +export function applyHomography(h: Matrix3, p: Point): Point { + const x = h[0][0] * p[0] + h[0][1] * p[1] + h[0][2]; + const y = h[1][0] * p[0] + h[1][1] * p[1] + h[1][2]; + const w = h[2][0] * p[0] + h[2][1] * p[1] + h[2][2]; + return [x / w, y / w]; +} + +/** + * Hartley normalization: translate points to the centroid and scale so the + * mean distance from the origin is sqrt(2). Returns the normalized points and + * the 3x3 transform T such that normalized = T * original. + */ +function normalizePoints(pts: Point[]): { normalized: Point[]; transform: Matrix3 } { + const n = pts.length; + let cx = 0; + let cy = 0; + pts.forEach(([x, y]) => { cx += x; cy += y; }); + cx /= n; + cy /= n; + let meanDist = 0; + pts.forEach(([x, y]) => { meanDist += Math.hypot(x - cx, y - cy); }); + meanDist /= n; + const scale = meanDist > 1e-12 ? Math.SQRT2 / meanDist : 1; + const transform: Matrix3 = [ + [scale, 0, -scale * cx], + [0, scale, -scale * cy], + [0, 0, 1], + ]; + const normalized = pts.map(([x, y]): Point => [(x - cx) * scale, (y - cy) * scale]); + return { normalized, transform }; +} + +/** + * Solve the linear system A x = b for x using Gaussian elimination with partial + * pivoting. A is square (n x n), modified in place. + */ +/* eslint-disable no-param-reassign */ +function solveLinearSystem(A: number[][], b: number[]): number[] { + const n = b.length; + for (let col = 0; col < n; col += 1) { + // Partial pivot: find the row with the largest magnitude in this column. + let pivot = col; + for (let row = col + 1; row < n; row += 1) { + if (Math.abs(A[row][col]) > Math.abs(A[pivot][col])) { + pivot = row; + } + } + if (Math.abs(A[pivot][col]) < 1e-12) { + throw new Error('Degenerate point configuration; cannot solve homography'); + } + if (pivot !== col) { + [A[col], A[pivot]] = [A[pivot], A[col]]; + [b[col], b[pivot]] = [b[pivot], b[col]]; + } + // Eliminate below. + for (let row = col + 1; row < n; row += 1) { + const factor = A[row][col] / A[col][col]; + for (let k = col; k < n; k += 1) { + A[row][k] -= factor * A[col][k]; + } + b[row] -= factor * b[col]; + } + } + // Back-substitution. + const x = new Array(n).fill(0); + for (let row = n - 1; row >= 0; row -= 1) { + let sum = b[row]; + for (let k = row + 1; k < n; k += 1) { + sum -= A[row][k] * x[k]; + } + x[row] = sum / A[row][row]; + } + return x; +} +/* eslint-enable no-param-reassign */ + +/** + * Estimate the homography H (dst ~= H * src) from >= 4 correspondences. + * + * Uses the h33 = 1 formulation in Hartley-normalized coordinates, solved via the + * normal equations (least-squares for > 4 points, exact for 4). Normalization + * keeps the system well-conditioned for typical image-pixel magnitudes. + */ +export function solveHomography(src: Point[], dst: Point[]): Matrix3 { + if (src.length !== dst.length) { + throw new Error('src and dst must have the same number of points'); + } + if (src.length < 4) { + throw new Error('At least 4 point correspondences are required'); + } + + const { normalized: srcN, transform: T1 } = normalizePoints(src); + const { normalized: dstN, transform: T2 } = normalizePoints(dst); + + // Build the 2N x 8 design matrix for the unknowns [h11..h32] (h33 fixed to 1). + const rows: number[][] = []; + const rhs: number[] = []; + for (let i = 0; i < srcN.length; i += 1) { + const [x, y] = srcN[i]; + const [u, v] = dstN[i]; + rows.push([x, y, 1, 0, 0, 0, -x * u, -y * u]); + rhs.push(u); + rows.push([0, 0, 0, x, y, 1, -x * v, -y * v]); + rhs.push(v); + } + + // Normal equations: (A^T A) h = A^T b -> an 8x8 system. + const ata: number[][] = Array.from({ length: 8 }, () => new Array(8).fill(0)); + const atb: number[] = new Array(8).fill(0); + for (let r = 0; r < rows.length; r += 1) { + const row = rows[r]; + for (let i = 0; i < 8; i += 1) { + atb[i] += row[i] * rhs[r]; + for (let j = 0; j < 8; j += 1) { + ata[i][j] += row[i] * row[j]; + } + } + } + + const h = solveLinearSystem(ata, atb); + const hNorm: Matrix3 = [ + [h[0], h[1], h[2]], + [h[3], h[4], h[5]], + [h[6], h[7], 1], + ]; + + // Denormalize: H = inv(T2) * Hnorm * T1. + const denorm = matMul3(matMul3(invert3(T2), hNorm), T1); + + // Scale so H[2][2] == 1 for a canonical form. + const scale = Math.abs(denorm[2][2]) > 1e-12 ? 1 / denorm[2][2] : 1; + return denorm.map((row) => row.map((value) => value * scale)); +} diff --git a/client/src/index.ts b/client/src/index.ts index 33708c1c5..330cdc8be 100644 --- a/client/src/index.ts +++ b/client/src/index.ts @@ -5,6 +5,7 @@ import * as components from './components'; import BaseAnnotation from './BaseAnnotation'; import BaseAnnotationStore from './BaseAnnotationStore'; import CameraStore from './CameraStore'; +import CameraCalibrationStore from './CameraCalibrationStore'; import Group from './Group'; import GroupFilterControls from './GroupFilterControls'; import GroupStore from './GroupStore'; @@ -28,6 +29,7 @@ export { BaseAnnotation, BaseAnnotationStore, CameraStore, + CameraCalibrationStore, Group, GroupFilterControls, GroupStore, diff --git a/client/src/layers/AnnotationLayers/CalibrationKeypointLayer.ts b/client/src/layers/AnnotationLayers/CalibrationKeypointLayer.ts new file mode 100644 index 000000000..d79a7f39d --- /dev/null +++ b/client/src/layers/AnnotationLayers/CalibrationKeypointLayer.ts @@ -0,0 +1,225 @@ +import geo, { GeoEvent } from 'geojs'; +import BaseLayer, { BaseLayerParams, LayerStyle } from '../BaseLayer'; +import { FrameDataTrack } from '../LayerTypes'; +import CameraCalibrationStore from '../../CameraCalibrationStore'; +import { applyHomography } from '../../homography'; + +export interface CalibrationPointData { + x: number; + y: number; + label: string; + pending: boolean; +} + +export interface CameraImage { + image: HTMLImageElement; + width: number; + height: number; +} + +interface CalibrationLayerParams { + calibration: CameraCalibrationStore; + /** Resolve another camera's currently displayed frame image (for the overlay). */ + getCameraImage?: (camera: string) => CameraImage | null; +} + +/** + * Renders this camera's picked calibration points (numbered markers, the pending + * "blue" point highlighted) and, when enabled, an aligned overlay of the other + * camera's frame warped through the fitted homography (geojs quadFeature). One + * instance is created per camera in LayerManager. + */ +export default class CalibrationKeypointLayer extends BaseLayer { + calibration: CameraCalibrationStore; + + getCameraImage?: (camera: string) => CameraImage | null; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + textFeature: any; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + quadFeature: any; + + constructor(params: BaseLayerParams & CalibrationLayerParams) { + super(params); + this.calibration = params.calibration; + this.getCameraImage = params.getCameraImage; + // Listen on the map, which is what emits geo.event.mouseclick. The event + // exposes button state and image (gcs) coordinates at the top level. + this.annotator.geoViewerRef.value.geoOn( + geo.event.mouseclick, + (e: GeoEvent) => this.handleClick(e), + ); + this.update(); + } + + initialize() { + const geoViewer = this.annotator.geoViewerRef.value; + // quad, point, and text features require different geojs renderers, so each + // must live in its own feature layer. Create the quad first so the overlay + // image renders beneath the picked points and labels. + const quadLayer = geoViewer.createLayer('feature', { + features: ['quad'], + autoshareRenderer: false, + renderer: 'canvas', + }); + this.quadFeature = quadLayer.createFeature('quad'); + + const pointLayer = geoViewer.createLayer('feature', { features: ['point'] }); + this.featureLayer = pointLayer.createFeature('point'); + + const textLayer = geoViewer.createLayer('feature', { features: ['text'] }); + this.textFeature = textLayer + .createFeature('text') + .text((data: CalibrationPointData) => data.label) + .position((data: CalibrationPointData) => ({ x: data.x, y: data.y })) + .style({ + color: 'white', + fontSize: '14px', + textAlign: 'center', + textBaseline: 'bottom', + offset: { x: 0, y: -10 }, + }); + super.initialize(); + } + + /** Map-level click handler: records a point when picking is active for this camera. */ + handleClick(e: GeoEvent) { + if (!this.calibration || !this.calibration.pickingEnabled.value) { + return; + } + // Map-level mouseclick exposes buttonsDown at the top level; feature-level + // events nest it under `mouse`. Respond to left clicks only. + const buttonsDown = e.buttonsDown || (e.mouse && e.mouse.buttonsDown); + if (buttonsDown && !buttonsDown.left) { + return; + } + const pair = this.calibration.activePair.value; + const cam = this.annotator.cameraName.value; + if (!pair || (cam !== pair.camA && cam !== pair.camB)) { + return; + } + if (!e.geo) { + return; + } + // e.geo is already in image (gcs) coordinates. + this.calibration.addPoint(cam, [e.geo.x, e.geo.y]); + } + + // eslint-disable-next-line class-methods-use-this, @typescript-eslint/no-unused-vars + formatData(_frameData: FrameDataTrack[]): CalibrationPointData[] { + const result: CalibrationPointData[] = []; + if (!this.calibration) { + return result; + } + const pair = this.calibration.activePair.value; + const cam = this.annotator.cameraName.value; + if (!pair || (cam !== pair.camA && cam !== pair.camB)) { + return result; + } + const key = this.calibration.pairKey(pair.camA, pair.camB); + const list = this.calibration.correspondences.value[key] || []; + list.forEach((c, i) => { + const coord = cam === pair.camA ? c.a : c.b; + result.push({ + x: coord[0], y: coord[1], label: `${i + 1}`, pending: false, + }); + }); + const pending = this.calibration.pendingPoint.value; + if (pending && pending.camera === cam) { + result.push({ + x: pending.coord[0], + y: pending.coord[1], + label: `${list.length + 1}`, + pending: true, + }); + } + return result; + } + + /** + * Render (or clear) the aligned overlay quad. The overlay is drawn in the + * destination camera's pane: the source camera's image is warped through the + * fitted homography for the selected direction. + */ + updateOverlay() { + if (!this.quadFeature) { + return; + } + const clear = () => this.quadFeature.data([]).draw(); + const overlay = this.calibration?.overlay.value; + const pair = this.calibration?.activePair.value; + if (!overlay || !overlay.enabled || !pair || !this.getCameraImage) { + clear(); + return; + } + const key = this.calibration.pairKey(pair.camA, pair.camB); + const homog = this.calibration.homographies.value[key]; + if (!homog) { + clear(); + return; + } + const { direction } = overlay; + const srcCam = direction === 'BtoA' ? pair.camB : pair.camA; + const dstCam = direction === 'BtoA' ? pair.camA : pair.camB; + if (this.annotator.cameraName.value !== dstCam) { + clear(); + return; + } + const src = this.getCameraImage(srcCam); + if (!src || !src.width || !src.height) { + clear(); + return; + } + const h = homog[direction]; + const { width: w, height: hgt } = src; + const ul = applyHomography(h, [0, 0]); + const ur = applyHomography(h, [w, 0]); + const lr = applyHomography(h, [w, hgt]); + const ll = applyHomography(h, [0, hgt]); + this.quadFeature + .data([{ + ul: { x: ul[0], y: ul[1] }, + ur: { x: ur[0], y: ur[1] }, + lr: { x: lr[0], y: lr[1] }, + ll: { x: ll[0], y: ll[1] }, + image: src.image, + }]) + .style('opacity', overlay.opacity) + .draw(); + } + + /** Recompute points and overlay from the store and redraw. */ + update() { + this.formattedData = this.formatData([]); + this.redraw(); + this.updateOverlay(); + } + + redraw(): null { + this.featureLayer.data(this.formattedData).draw(); + this.textFeature.data(this.formattedData).draw(); + return null; + } + + disable() { + this.featureLayer.data([]).draw(); + this.textFeature.data([]).draw(); + if (this.quadFeature) { + this.quadFeature.data([]).draw(); + } + } + + // eslint-disable-next-line class-methods-use-this + createStyle(): LayerStyle { + return { + ...super.createStyle(), + fill: true, + fillColor: (data: CalibrationPointData) => (data.pending ? 'cyan' : 'yellow'), + fillOpacity: 1, + radius: 6, + strokeColor: (data: CalibrationPointData) => (data.pending ? 'blue' : 'red'), + strokeWidth: 2, + }; + } +} diff --git a/client/src/provides.ts b/client/src/provides.ts index c30aad99a..c561c881e 100644 --- a/client/src/provides.ts +++ b/client/src/provides.ts @@ -20,6 +20,7 @@ import type { ImageEnhancements } from './use/useImageEnhancements'; import TrackFilterControls from './TrackFilterControls'; import GroupFilterControls from './GroupFilterControls'; import CameraStore from './CameraStore'; +import CameraCalibrationStore from './CameraCalibrationStore'; /** * Type definitions are read only because injectors may mutate internal state, @@ -113,6 +114,7 @@ type ImageEnhancementsType = Readonly>; /** Class-based symbols */ const CameraStoreSymbol = Symbol('cameraStore'); +const CameraCalibrationSymbol = Symbol('cameraCalibration'); const TrackStyleManagerSymbol = Symbol('trackTypeStyling'); const GroupStyleManagerSymbol = Symbol('groupTypeStyling'); @@ -284,6 +286,7 @@ export interface State { annotatorPreferences: AnnotatorPreferences; attributes: AttributesType; cameraStore: CameraStore; + cameraCalibration: CameraCalibrationStore; datasetId: DatasetIdType; editingMode: EditingModeType; groupFilters: GroupFilterControls; @@ -352,6 +355,7 @@ function dummyState(): State { annotatorPreferences: ref({ trackTails: { before: 20, after: 10 }, lockedCamera: { enabled: false } }), attributes: ref([]), cameraStore, + cameraCalibration: new CameraCalibrationStore(), datasetId: ref(''), editingMode: ref(false), multiSelectList: ref([]), @@ -403,6 +407,7 @@ function provideAnnotator(state: State, handler: Handler, attributesFilters: Att provide(AnnotatorPreferencesSymbol, state.annotatorPreferences); provide(AttributesSymbol, state.attributes); provide(CameraStoreSymbol, state.cameraStore); + provide(CameraCalibrationSymbol, state.cameraCalibration); provide(DatasetIdSymbol, state.datasetId); provide(EditingModeSymbol, state.editingMode); provide(GroupFilterControlsSymbol, state.groupFilters); @@ -458,6 +463,9 @@ function useAttributesFilters() { function useCameraStore() { return use(CameraStoreSymbol); } +function useCameraCalibration() { + return use(CameraCalibrationSymbol); +} function useDatasetId() { return use(DatasetIdSymbol); } @@ -569,6 +577,7 @@ export { useAnnotatorPreferences, useAttributes, useCameraStore, + useCameraCalibration, useDatasetId, useEditingMode, useHandler, diff --git a/server/dive_utils/models.py b/server/dive_utils/models.py index 75b33383a..5ec1c1440 100644 --- a/server/dive_utils/models.py +++ b/server/dive_utils/models.py @@ -240,6 +240,12 @@ class MetadataMutable(BaseModel): attributes: Optional[Dict[str, Attribute]] attributeTrackFilters: Optional[Dict[str, AttributeTrackFilter]] datasetInfo: Optional[types.DatasetInfo] + # Per-camera-pair alignment homographies, keyed by directional "left::right". + # Each value holds the 3x3 AtoB / BtoA matrices. + cameraHomographies: Optional[Dict[str, Dict[str, List[List[float]]]]] + # The picked point correspondences behind those homographies, keyed the same + # way. Each entry is a list of {id, a: [x, y], b: [x, y]} pairs. + cameraCorrespondences: Optional[Dict[str, List[Dict[str, Any]]]] fps: Optional[float] @staticmethod From f20cea5e7693cb36eb1c858ea91e3593532b6c60 Mon Sep 17 00:00:00 2001 From: romleiaj Date: Wed, 1 Jul 2026 11:18:45 -0400 Subject: [PATCH 02/79] Add aligned picking and transform-type selection to camera calibration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Picking was always on native/unwarped viewers, so users couldn't refine correspondences while visually verifying alignment. DIVE also only ever fit a full homography, when near-rigid EO/IR rigs often need fewer points and a more stable low-DOF fit. - Add client/src/transform.ts: translation/rigid/similarity/affine estimators alongside the existing homography DLT solver, all sharing the same Matrix3 primitives so warping/inverse-mapping code doesn't need to special-case the transform type - Replace the passive overlay toggle with an alignment mode (Original / A→B / B→A) that warps the destination camera's pane; add a Picking --- client/dive-common/apispec.ts | 7 +- .../CameraCalibration/CalibrationTools.vue | 116 ++++++++---- client/dive-common/components/Viewer.vue | 4 +- .../desktop/backend/native/common.spec.ts | 27 +++ .../platform/desktop/backend/native/common.ts | 41 ++++- client/src/CameraCalibrationStore.spec.ts | 140 +++++++++++++-- client/src/CameraCalibrationStore.ts | 165 ++++++++++++----- client/src/components/LayerManager.vue | 2 +- client/src/homography.ts | 2 +- .../CalibrationKeypointLayer.ts | 35 ++-- client/src/transform.spec.ts | 163 +++++++++++++++++ client/src/transform.ts | 170 ++++++++++++++++++ 12 files changed, 755 insertions(+), 117 deletions(-) create mode 100644 client/src/transform.spec.ts create mode 100644 client/src/transform.ts diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index c9a104de6..765cab5e5 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -8,7 +8,9 @@ import { Attribute } from 'vue-media-annotator/use/AttributeTypes'; import { CustomStyle } from 'vue-media-annotator/StyleManager'; import { AttributeTrackFilter } from 'vue-media-annotator/AttributeTrackFilterControls'; import { ImageEnhancements } from 'vue-media-annotator/use/useImageEnhancements'; -import { CameraHomographies, CameraCorrespondences } from 'vue-media-annotator/CameraCalibrationStore'; +import { + CameraHomographies, CameraCorrespondences, CameraTransformTypes, +} from 'vue-media-annotator/CameraCalibrationStore'; type DatasetType = 'image-sequence' | 'video' | 'multi' | 'large-image'; type MultiTrackRecord = Record; @@ -184,9 +186,10 @@ interface DatasetMetaMutable { datasetInfo?: Record; cameraHomographies?: CameraHomographies; cameraCorrespondences?: CameraCorrespondences; + cameraTransformTypes?: CameraTransformTypes; error?: string; } -const DatasetMetaMutableKeys = ['attributes', 'confidenceFilters', 'timeFilters', 'imageEnhancements', 'customTypeStyling', 'customGroupStyling', 'attributeTrackFilters', 'datasetInfo', 'cameraHomographies', 'cameraCorrespondences']; +const DatasetMetaMutableKeys = ['attributes', 'confidenceFilters', 'timeFilters', 'imageEnhancements', 'customTypeStyling', 'customGroupStyling', 'attributeTrackFilters', 'datasetInfo', 'cameraHomographies', 'cameraCorrespondences', 'cameraTransformTypes']; interface DatasetMeta extends DatasetMetaMutable { id: Readonly; diff --git a/client/dive-common/components/CameraCalibration/CalibrationTools.vue b/client/dive-common/components/CameraCalibration/CalibrationTools.vue index 2be3eed2a..92c5b3f91 100644 --- a/client/dive-common/components/CameraCalibration/CalibrationTools.vue +++ b/client/dive-common/components/CameraCalibration/CalibrationTools.vue @@ -7,6 +7,7 @@ import { useCameraCalibration, useDatasetId, } from 'vue-media-annotator/provides'; +import { TransformType, TRANSFORM_TYPES, minPointsForTransform } from 'vue-media-annotator/transform'; import TooltipBtn from 'vue-media-annotator/components/TooltipButton.vue'; import { useApi } from 'dive-common/apispec'; @@ -39,11 +40,32 @@ export default defineComponent({ const key = activeKey.value; return key ? (calibration.correspondences.value[key] || []) : []; }); - const canFit = computed(() => correspondences.value.length >= 4); + const transformType = computed( + () => (activeKey.value ? calibration.transformTypeForPair(activeKey.value) : 'homography'), + ); + const minPoints = computed(() => minPointsForTransform(transformType.value)); + const canFit = computed(() => correspondences.value.length >= minPoints.value); const canExport = computed(() => correspondences.value.length >= 1); - function setOverlayEnabled(enabled: boolean) { - calibration.setOverlayEnabled(enabled); + const alignmentModeItems = computed(() => [ + { text: 'Original (no alignment)', value: 'original' }, + { text: `Warp ${camLeft.value ?? 'A'} onto ${camRight.value ?? 'B'}`, value: 'AtoB', disabled: !canFit.value }, + { text: `Warp ${camRight.value ?? 'B'} onto ${camLeft.value ?? 'A'}`, value: 'BtoA', disabled: !canFit.value }, + ]); + + function setTransformType(type: TransformType) { + const key = activeKey.value; + if (key) { + calibration.setTransformType(key, type); + } + } + + function setAlignmentMode(mode: 'original' | 'AtoB' | 'BtoA') { + calibration.setAlignmentMode(mode); + } + + function setPickTarget(target: 'native' | 'ghost') { + calibration.setPickTarget(target); } async function save() { @@ -53,6 +75,7 @@ export default defineComponent({ await saveMetadata(datasetId.value, { cameraHomographies: calibration.homographies.value, cameraCorrespondences: calibration.correspondences.value, + cameraTransformTypes: calibration.transformTypes.value, }); } finally { saving.value = false; @@ -87,12 +110,18 @@ export default defineComponent({ camRight, calibration, pickingEnabled: calibration.pickingEnabled, - overlay: calibration.overlay, + alignment: calibration.alignment, correspondences, + transformType, + transformTypeItems: TRANSFORM_TYPES, + minPoints, + alignmentModeItems, canFit, canExport, saving, - setOverlayEnabled, + setTransformType, + setAlignmentMode, + setPickTarget, save, exportPoints, }; @@ -103,7 +132,7 @@ export default defineComponent({ diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index ed9cec426..c2f92b790 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -1045,8 +1045,8 @@ export default defineComponent({ if (meta.attributeTrackFilters) { trackFilters.loadTrackAttributesFilter(Object.values(meta.attributeTrackFilters)); } - // Rehydrate any saved camera-to-camera calibration homographies and points - cameraCalibration.hydrate(meta.cameraHomographies, meta.cameraCorrespondences); + // Rehydrate any saved camera-to-camera calibration homographies, points, and transform types + cameraCalibration.hydrate(meta.cameraHomographies, meta.cameraCorrespondences, meta.cameraTransformTypes); progress.loaded = true; // If multiCam add Tools and remove group Tools if (cameraStore.camMap.value.size > 1) { diff --git a/client/platform/desktop/backend/native/common.spec.ts b/client/platform/desktop/backend/native/common.spec.ts index 57e6bf4f5..46dbe9c2f 100644 --- a/client/platform/desktop/backend/native/common.spec.ts +++ b/client/platform/desktop/backend/native/common.spec.ts @@ -659,6 +659,7 @@ describe('native.common', () => { points: [[10, 20, 12, 22], [30, 40, 33, 44]], leftToRight: [[1, 0, 5], [0, 1, -3], [0, 0, 1]], rightToLeft: [[1, 0, -5], [0, 1, 3], [0, 0, 1]], + transformType: 'homography', }, ]); @@ -666,11 +667,37 @@ describe('native.common', () => { const meta = await fs.readJSON(npath.join(projectDir, 'meta.json')); expect(meta.cameraHomographies).toBeUndefined(); expect(meta.cameraCorrespondences).toBeUndefined(); + expect(meta.cameraTransformTypes).toBeUndefined(); // Rehydrated on load back into the in-app shapes. const reloaded = await common.loadMetadata(settings, final.id, urlMapper); expect(reloaded.cameraHomographies).toStrictEqual(cameraHomographies); expect(reloaded.cameraCorrespondences).toStrictEqual(cameraCorrespondences); + expect(reloaded.cameraTransformTypes).toStrictEqual({ 'rgb::ir': 'homography' }); + }); + + it('saveMetadata persists a non-default transformType per pair and reloads it', async () => { + const payload = await common.beginMediaImport( + '/home/user/data/imageLists/success/image_list.txt', + ); + const res = await common.finalizeMediaImport(settings, payload); + const final = res.meta; + const cameraHomographies = { + 'rgb::ir': { + AtoB: [[1, 0, 5], [0, 1, -3], [0, 0, 1]], + BtoA: [[1, 0, -5], [0, 1, 3], [0, 0, 1]], + }, + }; + const cameraTransformTypes = { 'rgb::ir': 'rigid' as const }; + + await common.saveMetadata(settings, final.id, { cameraHomographies, cameraTransformTypes }); + + const projectDir = npath.join(settings.dataPath, 'DIVE_Projects', final.id); + const calibration = await fs.readJSON(npath.join(projectDir, 'calibration.json')); + expect(calibration.pairs[0].transformType).toBe('rigid'); + + const reloaded = await common.loadMetadata(settings, final.id, urlMapper); + expect(reloaded.cameraTransformTypes).toStrictEqual(cameraTransformTypes); }); it('import with CSV annotations without specifying track file', async () => { diff --git a/client/platform/desktop/backend/native/common.ts b/client/platform/desktop/backend/native/common.ts index abd7e3a2a..0ce0a9f68 100644 --- a/client/platform/desktop/backend/native/common.ts +++ b/client/platform/desktop/backend/native/common.ts @@ -19,6 +19,7 @@ import { import { DefaultConfidence } from 'vue-media-annotator/BaseFilterControls'; import { TrackData } from 'vue-media-annotator/track'; import { GroupData } from 'vue-media-annotator/Group'; +import { TransformType } from 'vue-media-annotator/transform'; import { DatasetType, Pipelines, SaveDetectionsArgs, FrameImage, DatasetMetaMutable, TrainingConfig, TrainingConfigs, SaveAttributeArgs, @@ -373,7 +374,7 @@ async function loadMetadata( // Load standalone camera calibration (transforms + correspondences), if present. const calibrationFileAbsPath = npath.join(projectDirData.basePath, CalibrationFileName); - let { cameraHomographies, cameraCorrespondences } = projectMetaData; + let { cameraHomographies, cameraCorrespondences, cameraTransformTypes } = projectMetaData; if (await fs.pathExists(calibrationFileAbsPath)) { try { const calibration = await _loadAsJson(calibrationFileAbsPath); @@ -381,6 +382,7 @@ async function loadMetadata( ({ homographies: cameraHomographies, correspondences: cameraCorrespondences, + transformTypes: cameraTransformTypes, } = fromCalibrationPairs(calibration.pairs)); } } catch (err) { @@ -455,6 +457,7 @@ async function loadMetadata( subType, cameraHomographies, cameraCorrespondences, + cameraTransformTypes, }; } @@ -712,12 +715,15 @@ async function _saveAsJson(absPath: string, data: unknown) { type CameraHomographies = NonNullable; type CameraCorrespondences = NonNullable; +type CameraTransformTypes = NonNullable; /** * One camera pair in calibration.json. `left`/`right` are camera (folder) names; * `points` are the picked correspondences as rows of `leftX leftY rightX rightY` * (the keypointgui points.txt layout); `leftToRight`/`rightToLeft` are the fitted - * 3x3 homographies, when a fit has been performed. + * 3x3 homographies, when a fit has been performed; `transformType` is the fit + * model used to compute them (defaults to 'homography' when absent, matching the + * in-app default so older calibration.json files still load correctly). */ interface CalibrationPair { left: string; @@ -725,6 +731,7 @@ interface CalibrationPair { points: number[][]; leftToRight: number[][] | null; rightToLeft: number[][] | null; + transformType?: TransformType; } /** @@ -734,8 +741,11 @@ interface CalibrationPair { function toCalibrationPairs( homographies: CameraHomographies, correspondences: CameraCorrespondences, + transformTypes: CameraTransformTypes, ): CalibrationPair[] { - const keys = new Set([...Object.keys(homographies), ...Object.keys(correspondences)]); + const keys = new Set([ + ...Object.keys(homographies), ...Object.keys(correspondences), ...Object.keys(transformTypes), + ]); return [...keys].map((key) => { const [left, right] = key.split('::'); const homography = homographies[key]; @@ -745,16 +755,22 @@ function toCalibrationPairs( points: (correspondences[key] || []).map((c) => [c.a[0], c.a[1], c.b[0], c.b[1]]), leftToRight: homography ? homography.AtoB : null, rightToLeft: homography ? homography.BtoA : null, + transformType: transformTypes[key] || 'homography', }; }); } -/** Rebuild the in-app homographies/correspondences from calibration.json pairs. */ +/** Rebuild the in-app homographies/correspondences/transform types from calibration.json pairs. */ function fromCalibrationPairs( pairs: CalibrationPair[], -): { homographies: CameraHomographies; correspondences: CameraCorrespondences } { +): { + homographies: CameraHomographies; + correspondences: CameraCorrespondences; + transformTypes: CameraTransformTypes; + } { const homographies: CameraHomographies = {}; const correspondences: CameraCorrespondences = {}; + const transformTypes: CameraTransformTypes = {}; pairs.forEach((pair) => { const key = `${pair.left}::${pair.right}`; if (pair.leftToRight && pair.rightToLeft) { @@ -765,8 +781,9 @@ function fromCalibrationPairs( id: i + 1, a: [p[0], p[1]], b: [p[2], p[3]], })); } + transformTypes[key] = pair.transformType || 'homography'; }); - return { homographies, correspondences }; + return { homographies, correspondences, transformTypes }; } async function saveMetadata(settings: Settings, datasetId: string, args: DatasetMetaMutable) { @@ -802,16 +819,19 @@ async function saveMetadata(settings: Settings, datasetId: string, args: Dataset // standalone calibration.json in the dataset directory rather than embedded in // meta.json, so it is easy to find, hand-edit, and consume as a self-contained // artifact. - if (args.cameraHomographies || args.cameraCorrespondences) { + if (args.cameraHomographies || args.cameraCorrespondences || args.cameraTransformTypes) { const calibrationFileAbsPath = npath.join(projectDirInfo.basePath, CalibrationFileName); // Start from whatever is on disk so a partial update doesn't clobber the rest. let homographies: CameraHomographies = {}; let correspondences: CameraCorrespondences = {}; + let transformTypes: CameraTransformTypes = {}; if (await fs.pathExists(calibrationFileAbsPath)) { try { const existingCalibration = await _loadAsJson(calibrationFileAbsPath); if (existingCalibration && Array.isArray(existingCalibration.pairs)) { - ({ homographies, correspondences } = fromCalibrationPairs(existingCalibration.pairs)); + ({ homographies, correspondences, transformTypes } = fromCalibrationPairs( + existingCalibration.pairs, + )); } } catch (err) { console.warn(`Unable to read existing ${calibrationFileAbsPath}: ${err}`); @@ -823,9 +843,12 @@ async function saveMetadata(settings: Settings, datasetId: string, args: Dataset if (args.cameraCorrespondences) { correspondences = args.cameraCorrespondences; } + if (args.cameraTransformTypes) { + transformTypes = args.cameraTransformTypes; + } await _saveAsJson(calibrationFileAbsPath, { version: CalibrationFileVersion, - pairs: toCalibrationPairs(homographies, correspondences), + pairs: toCalibrationPairs(homographies, correspondences, transformTypes), }); } diff --git a/client/src/CameraCalibrationStore.spec.ts b/client/src/CameraCalibrationStore.spec.ts index 27ada9494..6b130a73d 100644 --- a/client/src/CameraCalibrationStore.spec.ts +++ b/client/src/CameraCalibrationStore.spec.ts @@ -20,6 +20,17 @@ describe('CameraCalibrationStore', () => { expect(store.activePair.value).toBeNull(); }); + it('resets alignment to original when switching pairs', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + addFourTranslationPairs(store); + store.setAlignmentMode('AtoB'); + store.setPickTarget('ghost'); + expect(store.alignment.value.mode).toBe('AtoB'); + store.setActivePair('left', 'other'); + expect(store.alignment.value).toMatchObject({ mode: 'original', pickTarget: 'native' }); + }); + it('forms one correspondence from a blue->red two-click sequence', () => { const store = new CameraCalibrationStore(); store.setActivePair('left', 'right'); @@ -78,20 +89,20 @@ describe('CameraCalibrationStore', () => { store.addPoint('left', p); store.addPoint('right', [p[0] + 5, p[1] - 3]); }); - const { AtoB, BtoA } = store.fitHomography(key); + const { AtoB, BtoA } = store.fitTransform(key); expect(AtoB[0][2]).toBeCloseTo(5, 5); expect(AtoB[1][2]).toBeCloseTo(-3, 5); expect(BtoA[0][2]).toBeCloseTo(-5, 5); expect(store.homographies.value[key]).toBeDefined(); }); - it('throws when fitting with fewer than 4 pairs', () => { + it('throws when fitting with fewer than 4 pairs (default homography type)', () => { const store = new CameraCalibrationStore(); store.setActivePair('left', 'right'); const key = store.pairKey('left', 'right'); store.addPoint('left', [1, 1]); store.addPoint('right', [2, 2]); - expect(() => store.fitHomography(key)).toThrow(); + expect(() => store.fitTransform(key)).toThrow(); }); function addFourTranslationPairs(store: CameraCalibrationStore) { @@ -102,62 +113,62 @@ describe('CameraCalibrationStore', () => { }); } - it('fits when enabling the overlay with >= 4 pairs', () => { + it('fits when enabling alignment mode with >= 4 pairs', () => { const store = new CameraCalibrationStore(); store.setActivePair('left', 'right'); const key = store.pairKey('left', 'right'); addFourTranslationPairs(store); - store.setOverlayEnabled(true); - expect(store.overlay.value.enabled).toBe(true); + store.setAlignmentMode('AtoB'); + expect(store.alignment.value.mode).toBe('AtoB'); expect(store.homographies.value[key]).toBeDefined(); expect(store.homographies.value[key].AtoB[0][2]).toBeCloseTo(5, 5); }); - it('does not enable the overlay with fewer than 4 pairs', () => { + it('does not enable alignment mode with fewer than 4 pairs (default homography type)', () => { const store = new CameraCalibrationStore(); store.setActivePair('left', 'right'); store.addPoint('left', [1, 1]); store.addPoint('right', [2, 2]); - store.setOverlayEnabled(true); - expect(store.overlay.value.enabled).toBe(false); + store.setAlignmentMode('AtoB'); + expect(store.alignment.value.mode).toBe('original'); expect(store.homographies.value).toEqual({}); }); - it('refits when correspondences change while the overlay is enabled', () => { + it('refits when correspondences change while alignment mode is active', () => { const store = new CameraCalibrationStore(); store.setActivePair('left', 'right'); const key = store.pairKey('left', 'right'); addFourTranslationPairs(store); - store.setOverlayEnabled(true); + store.setAlignmentMode('AtoB'); const before = store.homographies.value[key].AtoB[0][2]; store.addPoint('left', [20, 20]); store.addPoint('right', [30, 14]); expect(store.homographies.value[key].AtoB[0][2]).not.toBeCloseTo(before, 5); }); - it('disables the overlay when correspondences drop below 4', () => { + it('reverts alignment to original when correspondences drop below the transform minimum', () => { const store = new CameraCalibrationStore(); store.setActivePair('left', 'right'); const key = store.pairKey('left', 'right'); addFourTranslationPairs(store); - store.setOverlayEnabled(true); + store.setAlignmentMode('AtoB'); const { id } = store.correspondences.value[key][0]; store.removeCorrespondence(id); store.removeCorrespondence(store.correspondences.value[key][0].id); store.removeCorrespondence(store.correspondences.value[key][0].id); store.removeCorrespondence(store.correspondences.value[key][0].id); expect(store.correspondences.value[key]).toHaveLength(0); - expect(store.overlay.value.enabled).toBe(false); + expect(store.alignment.value.mode).toBe('original'); expect(store.homographies.value[key]).toBeUndefined(); }); - it('maybeFitActivePair fits without enabling the overlay', () => { + it('maybeFitActivePair fits without enabling alignment mode', () => { const store = new CameraCalibrationStore(); store.setActivePair('left', 'right'); const key = store.pairKey('left', 'right'); addFourTranslationPairs(store); store.maybeFitActivePair(); - expect(store.overlay.value.enabled).toBe(false); + expect(store.alignment.value.mode).toBe('original'); expect(store.homographies.value[key]).toBeDefined(); }); @@ -171,6 +182,15 @@ describe('CameraCalibrationStore', () => { expect(store.activePair.value).toBeNull(); expect(store.pendingPoint.value).toBeNull(); expect(store.correspondences.value).toEqual({}); + expect(store.transformTypes.value).toEqual({}); + expect(store.alignment.value).toEqual({ mode: 'original', opacity: 0.5, pickTarget: 'native' }); + }); + + it('hydrates transform types alongside homographies', () => { + const store = new CameraCalibrationStore(); + store.hydrate({}, {}, { 'a::b': 'rigid' }); + expect(store.transformTypeForPair('a::b')).toBe('rigid'); + expect(store.transformTypeForPair('unset::pair')).toBe('homography'); }); it('exports points as four columns "leftX leftY rightX rightY"', () => { @@ -200,4 +220,92 @@ describe('CameraCalibrationStore', () => { store.addPoint('ir', [10, 10]); expect(store.correspondences.value['rgb::ir'][2].id).toBe(3); }); + + describe('transform type selection', () => { + it('fits a rigid transform from 2 pairs where the default homography type would throw', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.addPoint('left', [0, 0]); + store.addPoint('right', [5, -3]); + store.addPoint('left', [10, 0]); + store.addPoint('right', [15, -3]); + expect(() => store.fitTransform(key)).toThrow(); + store.setTransformType(key, 'rigid'); + expect(store.homographies.value[key]).toBeDefined(); + expect(store.homographies.value[key].AtoB[0][2]).toBeCloseTo(5, 4); + }); + + it('clears the fit and reverts alignment when switching to a type needing more points than are picked', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.addPoint('left', [0, 0]); + store.addPoint('right', [5, -3]); + store.addPoint('left', [10, 0]); + store.addPoint('right', [15, -3]); + store.setTransformType(key, 'rigid'); + store.setAlignmentMode('AtoB'); + expect(store.alignment.value.mode).toBe('AtoB'); + + store.setTransformType(key, 'homography'); // needs 4, only 2 picked + expect(store.homographies.value[key]).toBeUndefined(); + expect(store.alignment.value.mode).toBe('original'); + }); + }); + + describe('setAlignmentMode / setPickTarget guards', () => { + it('setPickTarget is a no-op while alignment mode is original', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + store.setPickTarget('ghost'); + expect(store.alignment.value.pickTarget).toBe('native'); + }); + + it('setAlignmentMode leaves mode original when the pair lacks enough points', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + store.setAlignmentMode('BtoA'); + expect(store.alignment.value.mode).toBe('original'); + }); + }); + + describe('pickPoint', () => { + it('attributes a ghost-pane click to the source camera via the inverse homography', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + addFourTranslationPairs(store); // left -> right is +5, -3 + store.setAlignmentMode('AtoB'); // ghost of left shown in right's pane + store.setPickTarget('ghost'); + store.pickPoint('right', [15, 7]); + expect(store.pendingPoint.value).toMatchObject({ camera: 'left', coord: [10, 10] }); + }); + + it('always records a native pick in the non-ghosted (source) pane', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + addFourTranslationPairs(store); + store.setAlignmentMode('AtoB'); // right is the ghosted/destination pane + store.setPickTarget('ghost'); + store.pickPoint('left', [3, 4]); // clicking the source pane, not ghosted + expect(store.pendingPoint.value).toMatchObject({ camera: 'left', coord: [3, 4] }); + }); + + it('records a native pick in the ghosted pane when pick target is native', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + addFourTranslationPairs(store); + store.setAlignmentMode('AtoB'); + // pickTarget defaults to 'native' + store.pickPoint('right', [15, 7]); + expect(store.pendingPoint.value).toMatchObject({ camera: 'right', coord: [15, 7] }); + }); + + it('behaves exactly like addPoint when alignment mode is original', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + store.pickPoint('right', [15, 7]); + expect(store.pendingPoint.value).toMatchObject({ camera: 'right', coord: [15, 7] }); + }); + }); }); diff --git a/client/src/CameraCalibrationStore.ts b/client/src/CameraCalibrationStore.ts index 94911b43c..1ce9a7d08 100644 --- a/client/src/CameraCalibrationStore.ts +++ b/client/src/CameraCalibrationStore.ts @@ -1,7 +1,8 @@ import { ref, Ref } from 'vue'; import { - solveHomography, invert3, Matrix3, Point, + invert3, applyHomography, Matrix3, Point, } from './homography'; +import { TransformType, minPointsForTransform, estimateTransform } from './transform'; /** * A single picked point pair. `a` is the point in the left camera (camA), `b` @@ -15,7 +16,7 @@ export interface Correspondence { b: Point; } -/** Both directions of the fitted homography for one camera pair. */ +/** Both directions of the fitted alignment transform for one camera pair. */ export interface PairHomography { /** Maps left (camA) image coordinates onto right (camB). */ AtoB: Matrix3; @@ -23,17 +24,25 @@ export interface PairHomography { BtoA: Matrix3; } -/** Fitted homographies keyed by {@link CameraCalibrationStore.pairKey}. */ +/** Fitted transforms keyed by {@link CameraCalibrationStore.pairKey}. */ export type CameraHomographies = Record; /** Picked correspondences keyed by {@link CameraCalibrationStore.pairKey}. */ export type CameraCorrespondences = Record; -export interface OverlayState { - enabled: boolean; +/** Chosen fit model per pair, keyed by {@link CameraCalibrationStore.pairKey}. Missing entries default to 'homography'. */ +export type CameraTransformTypes = Record; + +/** Which image is warped onto which for the in-app aligned-picking preview. */ +export type AlignmentMode = 'original' | 'AtoB' | 'BtoA'; + +/** Whether a click in an aligned (ghosted) pane is attributed to that pane's own camera, or the ghosted source camera. */ +export type PickTarget = 'native' | 'ghost'; + +export interface AlignmentState { + mode: AlignmentMode; opacity: number; - /** Which image to warp onto the other when overlaying. */ - direction: 'AtoB' | 'BtoA'; + pickTarget: PickTarget; } /** Active pair. `camA` is the left camera, `camB` the right (user-chosen order). */ @@ -61,7 +70,9 @@ export default class CameraCalibrationStore { homographies: Ref; - overlay: Ref; + transformTypes: Ref; + + alignment: Ref; private nextId: number; @@ -71,7 +82,8 @@ export default class CameraCalibrationStore { this.pendingPoint = ref(null); this.correspondences = ref({}); this.homographies = ref({}); - this.overlay = ref({ enabled: false, opacity: 0.5, direction: 'AtoB' }); + this.transformTypes = ref({}); + this.alignment = ref({ mode: 'original', opacity: 0.5, pickTarget: 'native' }); this.nextId = 1; } @@ -98,6 +110,10 @@ export default class CameraCalibrationStore { this.activePair.value = { camA: left, camB: right }; } this.pendingPoint.value = null; + // Switching pairs invalidates any active alignment/ghost picking state: a + // stale 'ghost' pick target could otherwise silently misattribute the next + // click to the wrong camera once the new pair has its own homography fitted. + this.alignment.value = { mode: 'original', opacity: this.alignment.value.opacity, pickTarget: 'native' }; } /** @@ -125,7 +141,34 @@ export default class CameraCalibrationStore { list.push({ id: this.nextId++, a, b }); this.correspondences.value = { ...this.correspondences.value, [key]: list }; this.pendingPoint.value = null; - this.syncOverlayHomography(); + this.syncAlignmentHomography(); + } + + /** + * Record a click at `coord` (native pixel coords of `camera`'s own pane). When + * alignment is active, the pick target is 'ghost', and `camera` is the pane + * currently showing the ghost overlay (the alignment "destination" pane), the + * coordinate is inverse-mapped through the fitted homography and attributed to + * the *source* camera being ghosted instead of `camera` -- letting the user + * complete a correspondence pair from a single pane. Clicking the source + * (non-ghosted) pane, or clicking with the pick target set to 'native', always + * records a native point for `camera` itself, same as {@link addPoint}. + */ + pickPoint(camera: string, coord: Point) { + const pair = this.activePair.value; + const { mode, pickTarget } = this.alignment.value; + if (pair && mode !== 'original' && pickTarget === 'ghost') { + const srcCam = mode === 'BtoA' ? pair.camB : pair.camA; + const dstCam = mode === 'BtoA' ? pair.camA : pair.camB; + if (camera === dstCam) { + const homog = this.homographies.value[this.pairKey(pair.camA, pair.camB)]; + if (homog) { + this.addPoint(srcCam, applyHomography(invert3(homog[mode]), coord)); + return; + } + } + } + this.addPoint(camera, coord); } /** Remove a correspondence (by id) from the active pair. */ @@ -142,7 +185,7 @@ export default class CameraCalibrationStore { ...this.correspondences.value, [key]: list.filter((c) => c.id !== id), }; - this.syncOverlayHomography(); + this.syncAlignmentHomography(); } /** Drop all correspondences and the pending point for the active pair. */ @@ -153,59 +196,96 @@ export default class CameraCalibrationStore { return; } this.correspondences.value = { ...this.correspondences.value, [key]: [] }; - this.syncOverlayHomography(); + this.syncAlignmentHomography(); + } + + /** The chosen fit model for `key`, defaulting to 'homography' when unset. */ + transformTypeForPair(key: string): TransformType { + return this.transformTypes.value[key] || 'homography'; + } + + /** Choose the fit model for `key` and immediately (re)fit or clear as needed. */ + setTransformType(key: string, type: TransformType) { + this.transformTypes.value = { ...this.transformTypes.value, [key]: type }; + this.maybeFitPair(key); } /** - * Fit the active pair when it has enough points; otherwise clear its homography - * and turn off the overlay preview. + * Fit `key` when it has enough points for its chosen transform type; otherwise + * clear its homography and, if it's the active (aligned) pair, revert + * alignment to 'original'. */ - maybeFitActivePair() { - const key = this.activePairKey(); - if (!key) { - return; - } + maybeFitPair(key: string) { const list = this.correspondences.value[key]; - if (!list || list.length < 4) { - const { [key]: _removed, ...rest } = this.homographies.value; + const required = minPointsForTransform(this.transformTypeForPair(key)); + if (!list || list.length < required) { + const rest = { ...this.homographies.value }; + delete rest[key]; this.homographies.value = rest; - if (this.overlay.value.enabled) { - this.overlay.value = { ...this.overlay.value, enabled: false }; + if (this.activePairKey() === key && this.alignment.value.mode !== 'original') { + this.alignment.value = { ...this.alignment.value, mode: 'original', pickTarget: 'native' }; } return; } - this.fitHomography(key); + this.fitTransform(key); + } + + /** Fit the active pair when it has enough points; otherwise clear/revert as in {@link maybeFitPair}. */ + maybeFitActivePair() { + const key = this.activePairKey(); + if (!key) { + return; + } + this.maybeFitPair(key); } - /** Enable or disable the overlay preview, fitting the homography when turning on. */ - setOverlayEnabled(enabled: boolean) { - if (enabled) { + /** Enable or change the alignment (ghost overlay) mode, fitting the pair first if needed. */ + setAlignmentMode(mode: AlignmentMode) { + if (mode !== 'original') { this.maybeFitActivePair(); const key = this.activePairKey(); if (!key || !this.homographies.value[key]) { + // Not enough points for the active pair's transform type; stay original. return; } } - this.overlay.value = { ...this.overlay.value, enabled }; + const pickTarget = mode === 'original' ? 'native' : this.alignment.value.pickTarget; + this.alignment.value = { ...this.alignment.value, mode, pickTarget }; + } + + /** Ghost overlay opacity, independent of alignment mode. */ + setAlignmentOpacity(opacity: number) { + this.alignment.value = { ...this.alignment.value, opacity }; + } + + /** Choose whether a click in an aligned pane is native or attributed to the ghosted camera. No-op while alignment is 'original'. */ + setPickTarget(pickTarget: PickTarget) { + if (this.alignment.value.mode === 'original') { + return; + } + this.alignment.value = { ...this.alignment.value, pickTarget }; } - /** Re-fit the active pair while the overlay preview is enabled. */ - private syncOverlayHomography() { - if (this.overlay.value.enabled) { + /** Re-fit the active pair while alignment is active (mode != 'original'). */ + private syncAlignmentHomography() { + if (this.alignment.value.mode !== 'original') { this.maybeFitActivePair(); } } /** - * Fit a homography for `key` from its correspondences (>= 4 required). Computes - * both directions and stores them. Returns the fitted pair. + * Fit `key`'s chosen transform type from its correspondences (see + * {@link minPointsForTransform} for the required count). Computes both + * directions and stores them. Returns the fitted pair. */ - fitHomography(key: string): PairHomography { + fitTransform(key: string): PairHomography { const list = this.correspondences.value[key]; - if (!list || list.length < 4) { - throw new Error('At least 4 point pairs are required to fit a homography'); + const type = this.transformTypeForPair(key); + const required = minPointsForTransform(type); + if (!list || list.length < required) { + throw new Error(`At least ${required} point pair(s) are required to fit a ${type} transform`); } - const AtoB = solveHomography(list.map((c) => c.a), list.map((c) => c.b)); + const AtoB = estimateTransform(type, list.map((c) => c.a), list.map((c) => c.b)); const BtoA = invert3(AtoB); this.homographies.value = { ...this.homographies.value, [key]: { AtoB, BtoA } }; return { AtoB, BtoA }; @@ -223,14 +303,19 @@ export default class CameraCalibrationStore { .join('\n'); } - /** Reset state and load saved homographies and correspondences. */ - hydrate(homographies?: CameraHomographies, correspondences?: CameraCorrespondences) { + /** Reset state and load saved homographies, correspondences, and transform type choices. */ + hydrate( + homographies?: CameraHomographies, + correspondences?: CameraCorrespondences, + transformTypes?: CameraTransformTypes, + ) { this.homographies.value = homographies ? { ...homographies } : {}; this.correspondences.value = correspondences ? { ...correspondences } : {}; + this.transformTypes.value = transformTypes ? { ...transformTypes } : {}; this.activePair.value = null; this.pendingPoint.value = null; this.pickingEnabled.value = false; - this.overlay.value = { enabled: false, opacity: 0.5, direction: 'AtoB' }; + this.alignment.value = { mode: 'original', opacity: 0.5, pickTarget: 'native' }; // Resume id allocation past any restored correspondences. let maxId = 0; Object.values(this.correspondences.value).forEach((list) => { diff --git a/client/src/components/LayerManager.vue b/client/src/components/LayerManager.vue index bbf9d95bb..b9cb97fc7 100644 --- a/client/src/components/LayerManager.vue +++ b/client/src/components/LayerManager.vue @@ -242,7 +242,7 @@ export default defineComponent({ cameraCalibration.correspondences, cameraCalibration.pendingPoint, cameraCalibration.homographies, - cameraCalibration.overlay, + cameraCalibration.alignment, frameNumberRef, ], () => calibrationLayer.update(), diff --git a/client/src/homography.ts b/client/src/homography.ts index 4af4f08f2..3fc633397 100644 --- a/client/src/homography.ts +++ b/client/src/homography.ts @@ -84,7 +84,7 @@ function normalizePoints(pts: Point[]): { normalized: Point[]; transform: Matrix * pivoting. A is square (n x n), modified in place. */ /* eslint-disable no-param-reassign */ -function solveLinearSystem(A: number[][], b: number[]): number[] { +export function solveLinearSystem(A: number[][], b: number[]): number[] { const n = b.length; for (let col = 0; col < n; col += 1) { // Partial pivot: find the row with the largest magnitude in this column. diff --git a/client/src/layers/AnnotationLayers/CalibrationKeypointLayer.ts b/client/src/layers/AnnotationLayers/CalibrationKeypointLayer.ts index d79a7f39d..77d3b9004 100644 --- a/client/src/layers/AnnotationLayers/CalibrationKeypointLayer.ts +++ b/client/src/layers/AnnotationLayers/CalibrationKeypointLayer.ts @@ -25,9 +25,9 @@ interface CalibrationLayerParams { /** * Renders this camera's picked calibration points (numbered markers, the pending - * "blue" point highlighted) and, when enabled, an aligned overlay of the other - * camera's frame warped through the fitted homography (geojs quadFeature). One - * instance is created per camera in LayerManager. + * "blue" point highlighted) and, when alignment mode is active, a ghost overlay + * of the other camera's frame warped through the fitted homography (geojs + * quadFeature). One instance is created per camera in LayerManager. */ export default class CalibrationKeypointLayer extends BaseLayer { calibration: CameraCalibrationStore; @@ -103,7 +103,7 @@ export default class CalibrationKeypointLayer extends BaseLayer this.quadFeature.data([]).draw(); - const overlay = this.calibration?.overlay.value; + const alignment = this.calibration?.alignment.value; const pair = this.calibration?.activePair.value; - if (!overlay || !overlay.enabled || !pair || !this.getCameraImage) { + if (!alignment || alignment.mode === 'original' || !pair || !this.getCameraImage) { clear(); return; } @@ -159,9 +162,9 @@ export default class CalibrationKeypointLayer extends BaseLayer +import { applyHomography, solveHomography, Matrix3 } from './homography'; +import { + TransformType, + minPointsForTransform, + estimateTranslation, + estimateRigid, + estimateSimilarity, + estimateAffine, + estimateTransform, + Point, +} from './transform'; + +function expectMatrixClose(actual: Matrix3, expected: Matrix3, precision = 4) { + for (let i = 0; i < 3; i += 1) { + for (let j = 0; j < 3; j += 1) { + expect(actual[i][j]).toBeCloseTo(expected[i][j], precision); + } + } +} + +function expectRoundTrip(h: Matrix3, src: Point[], dst: Point[], precision = 4) { + src.forEach((p, i) => { + const [x, y] = applyHomography(h, p); + expect(x).toBeCloseTo(dst[i][0], precision); + expect(y).toBeCloseTo(dst[i][1], precision); + }); +} + +const fixtureSrc: Point[] = [[0, 0], [10, 0], [10, 10], [0, 10], [5, 3]]; + +describe('minPointsForTransform', () => { + it('returns the keypointgui-matching minimum for each type', () => { + const expected: Record = { + translation: 1, rigid: 2, similarity: 2, affine: 3, homography: 4, + }; + (Object.keys(expected) as TransformType[]).forEach((type) => { + expect(minPointsForTransform(type)).toBe(expected[type]); + }); + }); +}); + +describe('estimateTranslation', () => { + it('recovers an exact translation from a single point', () => { + const H = estimateTranslation([[3, 4]], [[8, 1]]); + expectMatrixClose(H, [[1, 0, 5], [0, 1, -3], [0, 0, 1]]); + }); + + it('recovers the mean translation from multiple points', () => { + const dst = fixtureSrc.map(([x, y]): Point => [x + 5, y - 3]); + const H = estimateTranslation(fixtureSrc, dst); + expectMatrixClose(H, [[1, 0, 5], [0, 1, -3], [0, 0, 1]]); + }); +}); + +describe('estimateRigid', () => { + it('recovers a known rotation + translation from 2 points', () => { + const theta = Math.PI / 6; + const cos = Math.cos(theta); + const sin = Math.sin(theta); + const truth: Matrix3 = [[cos, -sin, 5], [sin, cos, -3], [0, 0, 1]]; + const src = fixtureSrc.slice(0, 2); + const dst = src.map((p) => applyHomography(truth, p)); + const H = estimateRigid(src, dst); + expectMatrixClose(H, truth); + // 2x2 block is a proper rotation (unit determinant, no scale/reflection). + const det = H[0][0] * H[1][1] - H[0][1] * H[1][0]; + expect(det).toBeCloseTo(1, 5); + }); + + it('least-squares fits a rigid transform from more than 2 noisy points', () => { + const theta = Math.PI / 4; + const cos = Math.cos(theta); + const sin = Math.sin(theta); + const truth: Matrix3 = [[cos, -sin, 2], [sin, cos, 7], [0, 0, 1]]; + const dst = fixtureSrc.map((p) => applyHomography(truth, p)); + const H = estimateRigid(fixtureSrc, dst); + expectRoundTrip(H, fixtureSrc, dst); + }); +}); + +describe('estimateSimilarity', () => { + it('recovers a known rotation + uniform scale + translation from 2 points', () => { + const theta = Math.PI / 3; + const scale = 2.5; + const cos = Math.cos(theta) * scale; + const sin = Math.sin(theta) * scale; + const truth: Matrix3 = [[cos, -sin, 4], [sin, cos, -6], [0, 0, 1]]; + const src = fixtureSrc.slice(0, 2); + const dst = src.map((p) => applyHomography(truth, p)); + const H = estimateSimilarity(src, dst); + expectMatrixClose(H, truth); + }); + + it('least-squares fits a similarity transform from more points', () => { + const theta = -Math.PI / 5; + const scale = 0.75; + const cos = Math.cos(theta) * scale; + const sin = Math.sin(theta) * scale; + const truth: Matrix3 = [[cos, -sin, -1], [sin, cos, 3], [0, 0, 1]]; + const dst = fixtureSrc.map((p) => applyHomography(truth, p)); + const H = estimateSimilarity(fixtureSrc, dst); + expectRoundTrip(H, fixtureSrc, dst); + }); +}); + +describe('estimateAffine', () => { + const truth: Matrix3 = [[1.2, 0.3, 5], [0.1, 0.9, -4], [0, 0, 1]]; + + it('recovers a known affine transform exactly from 3 non-collinear points', () => { + const src = fixtureSrc.slice(0, 3); + const dst = src.map((p) => applyHomography(truth, p)); + const H = estimateAffine(src, dst); + expectMatrixClose(H, truth); + }); + + it('recovers the same affine transform from more than 3 points', () => { + const dst = fixtureSrc.map((p) => applyHomography(truth, p)); + const H = estimateAffine(fixtureSrc, dst); + expectMatrixClose(H, truth); + }); +}); + +describe('estimateTransform', () => { + it('delegates to solveHomography for the homography type', () => { + const truth: Matrix3 = [[1.2, 0.1, 5], [0.05, 0.9, -4], [0.0008, -0.0005, 1]]; + const src: Point[] = [[0, 0], [100, 0], [100, 80], [0, 80], [50, 40]]; + const dst = src.map((p) => applyHomography(truth, p)); + expectMatrixClose(estimateTransform('homography', src, dst), solveHomography(src, dst)); + }); + + it('throws below the minimum point count for each type', () => { + const one: Point[] = [[0, 0]]; + const two: Point[] = [[0, 0], [1, 0]]; + expect(() => estimateTransform('translation', [], [])).toThrow(); + expect(() => estimateTransform('rigid', one, one)).toThrow(); + expect(() => estimateTransform('similarity', one, one)).toThrow(); + expect(() => estimateTransform('affine', two, two)).toThrow(); + expect(() => estimateTransform('homography', fixtureSrc.slice(0, 3), fixtureSrc.slice(0, 3))).toThrow(); + }); + + it('throws when src and dst lengths differ', () => { + expect(() => estimateTransform('translation', [[0, 0]], [])).toThrow(); + }); + + it('round-trips src -> dst for every transform type', () => { + const cases: { type: TransformType; truth: Matrix3 }[] = [ + { type: 'translation', truth: [[1, 0, 5], [0, 1, -3], [0, 0, 1]] }, + { type: 'rigid', truth: [[Math.cos(0.4), -Math.sin(0.4), 1], [Math.sin(0.4), Math.cos(0.4), 2], [0, 0, 1]] }, + { + type: 'similarity', + truth: [[1.5 * Math.cos(0.2), -1.5 * Math.sin(0.2), 3], [1.5 * Math.sin(0.2), 1.5 * Math.cos(0.2), 4], [0, 0, 1]], + }, + { type: 'affine', truth: [[1.1, 0.2, 2], [-0.1, 0.95, 6], [0, 0, 1]] }, + { type: 'homography', truth: [[1.2, 0.1, 5], [0.05, 0.9, -4], [0.0008, -0.0005, 1]] }, + ]; + cases.forEach(({ type, truth }) => { + const dst = fixtureSrc.map((p) => applyHomography(truth, p)); + const H = estimateTransform(type, fixtureSrc, dst); + expectRoundTrip(H, fixtureSrc, dst, 3); + }); + }); +}); diff --git a/client/src/transform.ts b/client/src/transform.ts new file mode 100644 index 000000000..db128cf3b --- /dev/null +++ b/client/src/transform.ts @@ -0,0 +1,170 @@ +/** + * Alignment transform models for camera calibration, layered on top of the matrix + * primitives in {@link "./homography"} (`Matrix3`, `applyHomography`, `invert3`, + * `solveLinearSystem`). Mirrors keypointgui's `fit_homography` transform types + * (translation / rigid / similarity / affine / homography) so near-rigid EO/IR + * rigs can be fit with fewer, more stable point pairs than a full homography + * requires. Every estimator returns a plain `Matrix3`, so callers (warping, + * inverse-mapping) never need to special-case the transform type. + */ + +import { + Point, Matrix3, solveLinearSystem, solveHomography, +} from './homography'; + +export type TransformType = 'translation' | 'rigid' | 'similarity' | 'affine' | 'homography'; + +/** UI-friendly ordered list of transform types, for dropdowns. */ +export const TRANSFORM_TYPES: { value: TransformType; text: string }[] = [ + { value: 'translation', text: 'Translation' }, + { value: 'rigid', text: 'Rigid (rotation + translation)' }, + { value: 'similarity', text: 'Similarity (rotation + uniform scale + translation)' }, + { value: 'affine', text: 'Affine' }, + { value: 'homography', text: 'Homography (full projective)' }, +]; + +const MIN_POINTS: Record = { + translation: 1, + rigid: 2, + similarity: 2, + affine: 3, + homography: 4, +}; + +/** Minimum correspondence count to fit `type`, matching keypointgui's fit_homography. */ +export function minPointsForTransform(type: TransformType): number { + return MIN_POINTS[type]; +} + +/** Pure translation: H = identity with translation = mean(dst - src). Exact for 1+ points. */ +export function estimateTranslation(src: Point[], dst: Point[]): Matrix3 { + const n = src.length; + let dx = 0; + let dy = 0; + for (let i = 0; i < n; i += 1) { + dx += dst[i][0] - src[i][0]; + dy += dst[i][1] - src[i][1]; + } + dx /= n; + dy /= n; + return [[1, 0, dx], [0, 1, dy], [0, 0, 1]]; +} + +/** + * Closed-form 2D Procrustes/Umeyama fit: rotation (plus optional uniform scale) + * and translation minimizing sum |R*src + t - dst|^2. SVD-free because a 2D + * rotation is a single angle: `theta = atan2(B, A)` is the angle that maximizes + * the projected alignment sum, and (for similarity) the optimal uniform scale is + * the ratio of that alignment's magnitude to the source points' spread. + */ +function estimateRotationScaleTranslation( + src: Point[], + dst: Point[], + allowScale: boolean, +): Matrix3 { + const n = src.length; + let csx = 0; + let csy = 0; + let cdx = 0; + let cdy = 0; + for (let i = 0; i < n; i += 1) { + csx += src[i][0]; + csy += src[i][1]; + cdx += dst[i][0]; + cdy += dst[i][1]; + } + csx /= n; + csy /= n; + cdx /= n; + cdy /= n; + + let a = 0; + let b = 0; + let srcVar = 0; + for (let i = 0; i < n; i += 1) { + const sx = src[i][0] - csx; + const sy = src[i][1] - csy; + const dx = dst[i][0] - cdx; + const dy = dst[i][1] - cdy; + a += sx * dx + sy * dy; + b += sx * dy - sy * dx; + srcVar += sx * sx + sy * sy; + } + + const theta = Math.atan2(b, a); + const scale = allowScale && srcVar > 1e-12 ? Math.hypot(a, b) / srcVar : 1; + const r00 = Math.cos(theta) * scale; + const r01 = -Math.sin(theta) * scale; + const r10 = Math.sin(theta) * scale; + const r11 = Math.cos(theta) * scale; + const tx = cdx - (r00 * csx + r01 * csy); + const ty = cdy - (r10 * csx + r11 * csy); + return [[r00, r01, tx], [r10, r11, ty], [0, 0, 1]]; +} + +/** Rotation + translation only (no scale, no shear). Matches keypointgui's rigid fit. */ +export function estimateRigid(src: Point[], dst: Point[]): Matrix3 { + return estimateRotationScaleTranslation(src, dst, false); +} + +/** Rotation + uniform scale + translation. Matches keypointgui's similarity fit. */ +export function estimateSimilarity(src: Point[], dst: Point[]): Matrix3 { + return estimateRotationScaleTranslation(src, dst, true); +} + +/** + * Full 6-DOF affine fit via two independent 3-unknown least-squares solves (one + * output row at a time): dst_x = a*src_x + b*src_y + tx, dst_y = c*src_x + d*src_y + * + ty. Exact for 3 non-collinear points, least-squares for more. + */ +export function estimateAffine(src: Point[], dst: Point[]): Matrix3 { + const n = src.length; + const ata: number[][] = [[0, 0, 0], [0, 0, 0], [0, 0, 0]]; + const atbX: number[] = [0, 0, 0]; + const atbY: number[] = [0, 0, 0]; + for (let i = 0; i < n; i += 1) { + const row = [src[i][0], src[i][1], 1]; + const dx = dst[i][0]; + const dy = dst[i][1]; + for (let r = 0; r < 3; r += 1) { + atbX[r] += row[r] * dx; + atbY[r] += row[r] * dy; + for (let c = 0; c < 3; c += 1) { + ata[r][c] += row[r] * row[c]; + } + } + } + // solveLinearSystem mutates its arguments in place, so each solve gets its own copy. + const rowX = solveLinearSystem(ata.map((row) => [...row]), [...atbX]); + const rowY = solveLinearSystem(ata.map((row) => [...row]), [...atbY]); + return [ + [rowX[0], rowX[1], rowX[2]], + [rowY[0], rowY[1], rowY[2]], + [0, 0, 1], + ]; +} + +/** Estimate a `Matrix3` mapping src -> dst using `type`, enforcing its minimum point count. */ +export function estimateTransform(type: TransformType, src: Point[], dst: Point[]): Matrix3 { + if (src.length !== dst.length) { + throw new Error('src and dst must have the same number of points'); + } + const required = minPointsForTransform(type); + if (src.length < required) { + throw new Error(`At least ${required} point pair(s) are required for a ${type} transform`); + } + switch (type) { + case 'translation': + return estimateTranslation(src, dst); + case 'rigid': + return estimateRigid(src, dst); + case 'similarity': + return estimateSimilarity(src, dst); + case 'affine': + return estimateAffine(src, dst); + case 'homography': + return solveHomography(src, dst); + default: + throw new Error(`Unknown transform type: ${type}`); + } +} From d8e3568162a8e7bb809aa9bc7de24d1a49ad4258 Mon Sep 17 00:00:00 2001 From: romleiaj Date: Wed, 1 Jul 2026 11:28:33 -0400 Subject: [PATCH 03/79] Support web as well --- server/dive_utils/models.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/dive_utils/models.py b/server/dive_utils/models.py index 5ec1c1440..363754aa7 100644 --- a/server/dive_utils/models.py +++ b/server/dive_utils/models.py @@ -246,6 +246,10 @@ class MetadataMutable(BaseModel): # The picked point correspondences behind those homographies, keyed the same # way. Each entry is a list of {id, a: [x, y], b: [x, y]} pairs. cameraCorrespondences: Optional[Dict[str, List[Dict[str, Any]]]] + # The fit model used to compute each pair's homography (translation / rigid / + # similarity / affine / homography), keyed the same way. Missing entries + # default to 'homography' client-side. + cameraTransformTypes: Optional[Dict[str, str]] fps: Optional[float] @staticmethod From de74f7177413b729cd0243f6fff959f9c090e462 Mon Sep 17 00:00:00 2001 From: romleiaj Date: Wed, 1 Jul 2026 11:51:18 -0400 Subject: [PATCH 04/79] Add linked pan/zoom and right-click recenter to camera calibration Picking correspondences at default zoom is imprecise for cross-modality (EO/IR) rigs since the two panes differ in resolution and scale. When a pair has a fitted transform, panning or zooming either camera now recenters the other on the corresponding point via the homography, and a right-click on either pane snaps both cameras to that location. Also adds a live cursor coordinate readout in the calibration panel. --- .../CameraCalibration/CalibrationTools.vue | 40 +++++ client/dive-common/components/Viewer.vue | 2 + client/src/CameraCalibrationStore.spec.ts | 94 ++++++++++ client/src/CameraCalibrationStore.ts | 77 ++++++++ .../annotators/useCalibrationNavigation.ts | 169 ++++++++++++++++++ client/src/components/index.ts | 1 + .../CalibrationKeypointLayer.ts | 36 +++- 7 files changed, 414 insertions(+), 5 deletions(-) create mode 100644 client/src/components/annotators/useCalibrationNavigation.ts diff --git a/client/dive-common/components/CameraCalibration/CalibrationTools.vue b/client/dive-common/components/CameraCalibration/CalibrationTools.vue index 92c5b3f91..69d004ece 100644 --- a/client/dive-common/components/CameraCalibration/CalibrationTools.vue +++ b/client/dive-common/components/CameraCalibration/CalibrationTools.vue @@ -68,6 +68,22 @@ export default defineComponent({ calibration.setPickTarget(target); } + /** Live cursor readout text: this camera's coord, and its linked point in the other camera. */ + const cursorReadout = computed(() => { + const cursor = calibration.cursorCoord.value; + if (!cursor) { + return null; + } + const [x, y] = cursor.coord; + const other = calibration.linkedPoint(cursor.camera, cursor.coord); + const here = `${cursor.camera}: (${x.toFixed(1)}, ${y.toFixed(1)})`; + if (!other) { + return here; + } + const [ox, oy] = other.coord; + return `${here} -> ${other.camera}: (${ox.toFixed(1)}, ${oy.toFixed(1)})`; + }); + async function save() { saving.value = true; try { @@ -111,6 +127,8 @@ export default defineComponent({ calibration, pickingEnabled: calibration.pickingEnabled, alignment: calibration.alignment, + linkedNav: calibration.linkedNav, + cursorReadout, correspondences, transformType, transformTypeItems: TRANSFORM_TYPES, @@ -164,8 +182,30 @@ export default defineComponent({ /> Click a point in one camera, then the matching point in the other. + Right-click to recenter both cameras on that point (requires a fitted + transform). + + + Panning or zooming either camera recenters the other on the same point + (requires a fitted transform). + + +
+ {{ cursorReadout || 'Move the cursor over a camera to see its coordinates.' }} +
+
diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index c2f92b790..56596dc26 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -29,6 +29,7 @@ import { LargeImageAnnotator, LayerManager, useMediaController, + useCalibrationNavigation, TrackList, FilterList, } from 'vue-media-annotator/components'; @@ -315,6 +316,7 @@ export default defineComponent({ const cameraStore = new CameraStore({ markChangesPending }); const cameraCalibration = new CameraCalibrationStore(); + useCalibrationNavigation(aggregateController, cameraCalibration); // This context for removal const removeGroups = (id: AnnotationId) => { cameraStore.removeGroups(id); diff --git a/client/src/CameraCalibrationStore.spec.ts b/client/src/CameraCalibrationStore.spec.ts index 6b130a73d..0e2ebeb52 100644 --- a/client/src/CameraCalibrationStore.spec.ts +++ b/client/src/CameraCalibrationStore.spec.ts @@ -308,4 +308,98 @@ describe('CameraCalibrationStore', () => { expect(store.pendingPoint.value).toMatchObject({ camera: 'right', coord: [15, 7] }); }); }); + + describe('linked navigation', () => { + it('setLinkedNav toggles the flag', () => { + const store = new CameraCalibrationStore(); + expect(store.linkedNav.value).toBe(false); + store.setLinkedNav(true); + expect(store.linkedNav.value).toBe(true); + store.setLinkedNav(false); + expect(store.linkedNav.value).toBe(false); + }); + + it('linkedPoint maps a point from camA to camB and back, via the fitted homography', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + addFourTranslationPairs(store); // left -> right is +5, -3 + store.maybeFitActivePair(); + const fromLeft = store.linkedPoint('left', [1, 1]); + expect(fromLeft).toMatchObject({ camera: 'right', coord: [6, -2] }); + const fromRight = store.linkedPoint('right', [6, -2]); + expect(fromRight).toMatchObject({ camera: 'left', coord: [1, 1] }); + }); + + it('linkedPoint returns null when the pair has no fitted homography yet', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + expect(store.linkedPoint('left', [1, 1])).toBeNull(); + }); + + it('linkedPoint returns null for a camera outside the active pair', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + addFourTranslationPairs(store); + expect(store.linkedPoint('other', [1, 1])).toBeNull(); + }); + + it('resets linkedNav on hydrate', () => { + const store = new CameraCalibrationStore(); + store.setLinkedNav(true); + store.hydrate(); + expect(store.linkedNav.value).toBe(false); + }); + }); + + describe('cursor coordinate readout', () => { + it('records and clears the cursor coordinate', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + store.setCursorCoord('left', [12, 34]); + expect(store.cursorCoord.value).toEqual({ camera: 'left', coord: [12, 34] }); + store.clearCursorCoord(); + expect(store.cursorCoord.value).toBeNull(); + }); + + it('clears the cursor coordinate when switching pairs', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + store.setCursorCoord('left', [12, 34]); + store.setActivePair('left', 'other'); + expect(store.cursorCoord.value).toBeNull(); + }); + }); + + describe('requestRecenter', () => { + it('records a recenter request for a camera in the active pair', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + store.requestRecenter('right', [7, 8]); + expect(store.recenterRequest.value).toMatchObject({ camera: 'right', coord: [7, 8] }); + }); + + it('ignores a recenter request for a camera outside the active pair', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + store.requestRecenter('other', [7, 8]); + expect(store.recenterRequest.value).toBeNull(); + }); + + it('assigns a new id to each request so repeated identical requests still change', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + store.requestRecenter('left', [1, 1]); + const firstId = store.recenterRequest.value?.id; + store.requestRecenter('left', [1, 1]); + expect(store.recenterRequest.value?.id).not.toBe(firstId); + }); + + it('clears the recenter request when switching pairs', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + store.requestRecenter('left', [1, 1]); + store.setActivePair('left', 'other'); + expect(store.recenterRequest.value).toBeNull(); + }); + }); }); diff --git a/client/src/CameraCalibrationStore.ts b/client/src/CameraCalibrationStore.ts index 1ce9a7d08..09a1fad51 100644 --- a/client/src/CameraCalibrationStore.ts +++ b/client/src/CameraCalibrationStore.ts @@ -74,8 +74,23 @@ export default class CameraCalibrationStore { alignment: Ref; + /** Whether pan/zoom recentering is linked between the active pair's two cameras. */ + linkedNav: Ref; + + /** Native-pixel coordinate under the cursor, for the calibration panel's live readout. */ + cursorCoord: Ref<{ camera: string; coord: Point } | null>; + + /** + * A one-shot "recenter here" request (e.g. from a right-click), keyed by an + * incrementing id so repeated requests at the same coordinate still trigger + * watchers. See {@link requestRecenter}. + */ + recenterRequest: Ref<{ camera: string; coord: Point; id: number } | null>; + private nextId: number; + private nextRecenterId: number; + constructor() { this.activePair = ref(null); this.pickingEnabled = ref(false); @@ -84,7 +99,11 @@ export default class CameraCalibrationStore { this.homographies = ref({}); this.transformTypes = ref({}); this.alignment = ref({ mode: 'original', opacity: 0.5, pickTarget: 'native' }); + this.linkedNav = ref(false); + this.cursorCoord = ref(null); + this.recenterRequest = ref(null); this.nextId = 1; + this.nextRecenterId = 1; } /** @@ -114,6 +133,8 @@ export default class CameraCalibrationStore { // stale 'ghost' pick target could otherwise silently misattribute the next // click to the wrong camera once the new pair has its own homography fitted. this.alignment.value = { mode: 'original', opacity: this.alignment.value.opacity, pickTarget: 'native' }; + this.cursorCoord.value = null; + this.recenterRequest.value = null; } /** @@ -266,6 +287,59 @@ export default class CameraCalibrationStore { this.alignment.value = { ...this.alignment.value, pickTarget }; } + /** Enable or disable linked pan/zoom recentering between the active pair's cameras. */ + setLinkedNav(enabled: boolean) { + this.linkedNav.value = enabled; + } + + /** + * Map `coord` (native pixel space of `camera`) to the corresponding point in + * the *other* camera of the active pair, via the fitted homography. Returns + * `null` when `camera` isn't part of the active pair or the pair has no + * fitted homography yet (not enough correspondences) -- callers should treat + * that as "nothing to link to" rather than an error. + */ + linkedPoint(camera: string, coord: Point): { camera: string; coord: Point } | null { + const pair = this.activePair.value; + if (!pair || (camera !== pair.camA && camera !== pair.camB)) { + return null; + } + const homog = this.homographies.value[this.pairKey(pair.camA, pair.camB)]; + if (!homog) { + return null; + } + const other = camera === pair.camA ? pair.camB : pair.camA; + const matrix = camera === pair.camA ? homog.AtoB : homog.BtoA; + return { camera: other, coord: applyHomography(matrix, coord) }; + } + + /** Record the native-pixel coordinate under the cursor for `camera` (calibration panel readout). */ + setCursorCoord(camera: string, coord: Point) { + this.cursorCoord.value = { camera, coord }; + } + + /** Clear the cursor coordinate readout (e.g. on mouse leave). */ + clearCursorCoord() { + this.cursorCoord.value = null; + } + + /** + * Request that `camera` (native pixel coords `coord`) and, when the pair has + * a fitted homography, the other camera of the active pair (via + * {@link linkedPoint}) recenter their views on this location. Consumed by + * {@link useCalibrationNavigation}; a no-op if `camera` isn't part of the + * active pair. Works independent of {@link linkedNav} -- this is a one-shot + * "snap to this feature" action, not the continuous pan/zoom link. + */ + requestRecenter(camera: string, coord: Point) { + const pair = this.activePair.value; + if (!pair || (camera !== pair.camA && camera !== pair.camB)) { + return; + } + // eslint-disable-next-line no-plusplus + this.recenterRequest.value = { camera, coord, id: this.nextRecenterId++ }; + } + /** Re-fit the active pair while alignment is active (mode != 'original'). */ private syncAlignmentHomography() { if (this.alignment.value.mode !== 'original') { @@ -316,6 +390,9 @@ export default class CameraCalibrationStore { this.pendingPoint.value = null; this.pickingEnabled.value = false; this.alignment.value = { mode: 'original', opacity: 0.5, pickTarget: 'native' }; + this.linkedNav.value = false; + this.cursorCoord.value = null; + this.recenterRequest.value = null; // Resume id allocation past any restored correspondences. let maxId = 0; Object.values(this.correspondences.value).forEach((list) => { diff --git a/client/src/components/annotators/useCalibrationNavigation.ts b/client/src/components/annotators/useCalibrationNavigation.ts new file mode 100644 index 000000000..8e68cb9ad --- /dev/null +++ b/client/src/components/annotators/useCalibrationNavigation.ts @@ -0,0 +1,169 @@ +import geo from 'geojs'; +import { + Ref, onBeforeUnmount, watch, +} from 'vue'; +import type { AggregateMediaController } from './mediaControllerType'; +import type CameraCalibrationStore from '../../CameraCalibrationStore'; +import type { Point } from '../../homography'; + +/** + * Links pan/zoom recentering between the two cameras of the active calibration + * pair: panning or zooming one camera recenters the other on the same + * reference-space point, mapped through the pair's fitted homography (see + * {@link CameraCalibrationStore.linkedPoint}). This is distinct from the + * general "sync cameras" toggle (Controls.vue), which forwards raw screen + * deltas/zoom levels and assumes identical pixel scale between panes -- not + * true for cross-modality (e.g. EO/IR) rigs with differing resolutions. + */ +export default function useCalibrationNavigation( + aggregateController: Ref, + calibration: CameraCalibrationStore, +) { + // Re-entrancy guard: setting a camera's center from a link handler must not + // itself trigger that camera's own pan/zoom listener. + let guard = false; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let attached: { viewer: any; handler: () => void }[] = []; + + function teardown() { + attached.forEach(({ viewer, handler }) => { + viewer.geoOff(geo.event.pan, handler); + viewer.geoOff(geo.event.zoom, handler); + }); + attached = []; + } + + function link(camera: string, otherCamera: string) { + return () => { + if (guard) { + return; + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let sourceViewer: any; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let targetViewer: any; + try { + sourceViewer = aggregateController.value.getController(camera).geoViewerRef.value; + targetViewer = aggregateController.value.getController(otherCamera).geoViewerRef.value; + } catch { + return; + } + if (!sourceViewer || !targetViewer) { + return; + } + const center = sourceViewer.center(); + const linked = calibration.linkedPoint(camera, [center.x, center.y]); + if (!linked || linked.camera !== otherCamera) { + return; + } + guard = true; + try { + targetViewer.center({ x: linked.coord[0], y: linked.coord[1] }); + } finally { + guard = false; + } + }; + } + + function setup() { + teardown(); + const pair = calibration.activePair.value; + if (!calibration.linkedNav.value || !pair) { + return; + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let viewerA: any; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let viewerB: any; + try { + viewerA = aggregateController.value.getController(pair.camA).geoViewerRef.value; + viewerB = aggregateController.value.getController(pair.camB).geoViewerRef.value; + } catch { + // One or both cameras haven't initialized a viewer yet; the resizeTrigger + // dependency below re-runs setup once they do. + return; + } + if (!viewerA || !viewerB) { + return; + } + const handlerA = link(pair.camA, pair.camB); + const handlerB = link(pair.camB, pair.camA); + viewerA.geoOn(geo.event.pan, handlerA); + viewerA.geoOn(geo.event.zoom, handlerA); + viewerB.geoOn(geo.event.pan, handlerB); + viewerB.geoOn(geo.event.zoom, handlerB); + attached = [ + { viewer: viewerA, handler: handlerA }, + { viewer: viewerB, handler: handlerB }, + ]; + } + + watch( + [ + calibration.linkedNav, + calibration.activePair, + calibration.homographies, + aggregateController.value.resizeTrigger, + ], + setup, + { deep: true }, + ); + + /** + * One-shot recenter (right-click): center the clicked camera on the clicked + * point, and -- when the pair has a fitted homography -- center the other + * camera on the corresponding point. Independent of {@link + * CameraCalibrationStore.linkedNav}; the `guard` flag still applies so this + * doesn't loop back through the continuous pan/zoom link above when it's on. + */ + function handleRecenterRequest( + request: { camera: string; coord: Point; id: number } | null, + ) { + if (!request) { + return; + } + const pair = calibration.activePair.value; + if (!pair || (request.camera !== pair.camA && request.camera !== pair.camB)) { + return; + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let sourceViewer: any; + try { + sourceViewer = aggregateController.value.getController(request.camera).geoViewerRef.value; + } catch { + return; + } + if (sourceViewer) { + guard = true; + try { + sourceViewer.center({ x: request.coord[0], y: request.coord[1] }); + } finally { + guard = false; + } + } + const linked = calibration.linkedPoint(request.camera, request.coord); + if (!linked) { + return; + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let targetViewer: any; + try { + targetViewer = aggregateController.value.getController(linked.camera).geoViewerRef.value; + } catch { + return; + } + if (!targetViewer) { + return; + } + guard = true; + try { + targetViewer.center({ x: linked.coord[0], y: linked.coord[1] }); + } finally { + guard = false; + } + } + + watch(() => calibration.recenterRequest.value, handleRecenterRequest); + + onBeforeUnmount(teardown); +} diff --git a/client/src/components/index.ts b/client/src/components/index.ts index ec4d30a9a..5e84e189a 100644 --- a/client/src/components/index.ts +++ b/client/src/components/index.ts @@ -28,6 +28,7 @@ import TypePicker from './TypePicker.vue'; export * from './annotators/useMediaController'; export { default as useAnnotatorImageCursor } from './annotators/useAnnotatorImageCursor'; +export { default as useCalibrationNavigation } from './annotators/useCalibrationNavigation'; export { /* Annotators */ AnnotatorImageCursor, diff --git a/client/src/layers/AnnotationLayers/CalibrationKeypointLayer.ts b/client/src/layers/AnnotationLayers/CalibrationKeypointLayer.ts index 77d3b9004..3ae598e3f 100644 --- a/client/src/layers/AnnotationLayers/CalibrationKeypointLayer.ts +++ b/client/src/layers/AnnotationLayers/CalibrationKeypointLayer.ts @@ -50,6 +50,10 @@ export default class CalibrationKeypointLayer extends BaseLayer this.handleClick(e), ); + this.annotator.geoViewerRef.value.geoOn( + geo.event.mousemove, + (e: GeoEvent) => this.handleMouseMove(e), + ); this.update(); } @@ -83,17 +87,19 @@ export default class CalibrationKeypointLayer extends BaseLayer Date: Wed, 1 Jul 2026 11:59:19 -0400 Subject: [PATCH 05/79] Add Clear Last, points.txt loading, and homography.txt export to calibration Closes the remaining keypointgui point-workflow gaps: Clear Last mirrors keypointgui's undo-pending/undo-last-pair semantics, Load points.txt parses the same four-column format with a replace/merge prompt, and the new homography.txt export matches keypointgui's Save Homography output. --- .../CameraCalibration/CalibrationTools.vue | 206 ++++++++++++++++-- client/src/CameraCalibrationStore.spec.ts | 136 ++++++++++++ client/src/CameraCalibrationStore.ts | 66 ++++++ 3 files changed, 392 insertions(+), 16 deletions(-) diff --git a/client/dive-common/components/CameraCalibration/CalibrationTools.vue b/client/dive-common/components/CameraCalibration/CalibrationTools.vue index 69d004ece..55ad5eb86 100644 --- a/client/dive-common/components/CameraCalibration/CalibrationTools.vue +++ b/client/dive-common/components/CameraCalibration/CalibrationTools.vue @@ -46,6 +46,9 @@ export default defineComponent({ const minPoints = computed(() => minPointsForTransform(transformType.value)); const canFit = computed(() => correspondences.value.length >= minPoints.value); const canExport = computed(() => correspondences.value.length >= 1); + const canClearLast = computed( + () => calibration.pendingPoint.value !== null || correspondences.value.length > 0, + ); const alignmentModeItems = computed(() => [ { text: 'Original (no alignment)', value: 'original' }, @@ -98,6 +101,19 @@ export default defineComponent({ } } + /** Trigger a browser download of `text` as `filename`. */ + function downloadText(text: string, filename: string) { + const blob = new Blob([text], { type: 'text/plain' }); + const url = URL.createObjectURL(blob); + const anchor = document.createElement('a'); + anchor.href = url; + anchor.download = filename; + document.body.appendChild(anchor); + anchor.click(); + document.body.removeChild(anchor); + URL.revokeObjectURL(url); + } + /** * Download the active pair's correspondences as a keypointgui-style * points.txt (leftX leftY rightX rightY), for VIAME's @@ -109,15 +125,73 @@ export default defineComponent({ if (!key) { return; } - const blob = new Blob([calibration.toPointsText(key)], { type: 'text/plain' }); - const url = URL.createObjectURL(blob); - const anchor = document.createElement('a'); - anchor.href = url; - anchor.download = `${camLeft.value}_to_${camRight.value}_points.txt`; - document.body.appendChild(anchor); - anchor.click(); - document.body.removeChild(anchor); - URL.revokeObjectURL(url); + downloadText(calibration.toPointsText(key), `${camLeft.value}_to_${camRight.value}_points.txt`); + } + + /** + * Download the fitted homography (fitting first if needed) as a + * whitespace-separated 3x3 matrix .txt, matching keypointgui's Save + * Left->Right / Right->Left Homography. + */ + function exportHomography(direction: 'AtoB' | 'BtoA') { + const key = activeKey.value; + if (!key) { + return; + } + calibration.maybeFitActivePair(); + const text = calibration.toHomographyText(key, direction); + if (!text) { + return; + } + const [from, to] = direction === 'AtoB' ? [camLeft.value, camRight.value] : [camRight.value, camLeft.value]; + downloadText(text, `${from}_to_${to}_homography.txt`); + } + + const pointsFileInput = ref(null); + const loadPointsDialog = ref(false); + const loadPointsError = ref(null); + const pendingLoadText = ref(null); + + function applyLoadedPoints(text: string, mode: 'replace' | 'merge') { + const key = activeKey.value; + if (!key) { + return; + } + loadPointsError.value = null; + try { + calibration.loadPointsFromText(key, text, mode); + } catch (e) { + loadPointsError.value = e instanceof Error ? e.message : String(e); + } + } + + /** File input change handler for Load points.txt; prompts for replace/merge if the pair already has points. */ + function onPointsFileSelected(event: Event) { + const input = event.target as HTMLInputElement; + const file = input.files?.[0]; + input.value = ''; + if (!file) { + return; + } + const reader = new FileReader(); + reader.onload = () => { + const text = String(reader.result ?? ''); + if (correspondences.value.length > 0) { + pendingLoadText.value = text; + loadPointsDialog.value = true; + } else { + applyLoadedPoints(text, 'replace'); + } + }; + reader.readAsText(file); + } + + function confirmLoadPoints(mode: 'replace' | 'merge') { + loadPointsDialog.value = false; + if (pendingLoadText.value !== null) { + applyLoadedPoints(pendingLoadText.value, mode); + pendingLoadText.value = null; + } } return { @@ -136,12 +210,19 @@ export default defineComponent({ alignmentModeItems, canFit, canExport, + canClearLast, saving, + pointsFileInput, + loadPointsDialog, + loadPointsError, setTransformType, setAlignmentMode, setPickTarget, save, exportPoints, + exportHomography, + onPointsFileSelected, + confirmLoadPoints, }; }, }); @@ -212,13 +293,21 @@ export default defineComponent({

Correspondences ({{ correspondences.length }})

- +
+ + +
+ + + Load points.txt + + + {{ loadPointsError }} + + + + + Load points.txt + + This pair already has {{ correspondences.length }} correspondence(s). + Replace them with the loaded points, or merge the loaded points in? + + + + + Cancel + + + Merge + + + Replace + + + + + + + + + Export A→B homography.txt + + + Export B→A homography.txt + + + Saves the fitted 3x3 matrix as whitespace-separated rows, matching + keypointgui's Save Homography. + + { }); }); + describe('clearLast', () => { + it('drops the pending point without touching completed correspondences', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.addPoint('left', [1, 1]); + store.addPoint('right', [2, 2]); + store.addPoint('left', [3, 3]); // pending + store.clearLast(); + expect(store.pendingPoint.value).toBeNull(); + expect(store.correspondences.value[key]).toHaveLength(1); + }); + + it('removes the last completed correspondence when there is no pending point', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + store.addPoint('left', [1, 1]); + store.addPoint('right', [2, 2]); + store.addPoint('left', [3, 3]); + store.addPoint('right', [4, 4]); + store.clearLast(); + expect(store.correspondences.value[key]).toHaveLength(1); + expect(store.correspondences.value[key][0]).toMatchObject({ a: [1, 1], b: [2, 2] }); + }); + + it('is a no-op with nothing to undo', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + expect(() => store.clearLast()).not.toThrow(); + store.clearLast(); + expect(store.pendingPoint.value).toBeNull(); + }); + + it('refits when clearing last while alignment mode is active', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + addFourTranslationPairs(store); + store.setAlignmentMode('AtoB'); + store.clearLast(); + expect(store.correspondences.value[key]).toHaveLength(3); + expect(store.alignment.value.mode).toBe('original'); + expect(store.homographies.value[key]).toBeUndefined(); + }); + }); + + describe('loadPointsFromText', () => { + it('replaces existing correspondences by default', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('rgb', 'ir'); + const key = store.pairKey('rgb', 'ir'); + store.addPoint('rgb', [0, 0]); + store.addPoint('ir', [0, 0]); + store.loadPointsFromText(key, '10 20 30 40\n11 21 31 41'); + expect(store.correspondences.value[key]).toHaveLength(2); + expect(store.correspondences.value[key][0]).toMatchObject({ a: [10, 20], b: [30, 40] }); + }); + + it('merges with existing correspondences when mode is merge', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('rgb', 'ir'); + const key = store.pairKey('rgb', 'ir'); + store.addPoint('rgb', [0, 0]); + store.addPoint('ir', [1, 1]); + store.loadPointsFromText(key, '10 20 30 40', 'merge'); + expect(store.correspondences.value[key]).toHaveLength(2); + expect(store.correspondences.value[key][0]).toMatchObject({ a: [0, 0], b: [1, 1] }); + expect(store.correspondences.value[key][1]).toMatchObject({ a: [10, 20], b: [30, 40] }); + }); + + it('ignores blank lines', () => { + const store = new CameraCalibrationStore(); + const key = store.pairKey('rgb', 'ir'); + store.loadPointsFromText(key, '\n10 20 30 40\n\n11 21 31 41\n'); + expect(store.correspondences.value[key]).toHaveLength(2); + }); + + it('throws on a malformed row and leaves correspondences untouched', () => { + const store = new CameraCalibrationStore(); + const key = store.pairKey('rgb', 'ir'); + expect(() => store.loadPointsFromText(key, '10 20 30 40\n1 2 3')).toThrow(); + expect(store.correspondences.value[key]).toBeUndefined(); + }); + + it('round-trips with toPointsText', () => { + const store = new CameraCalibrationStore(); + const key = store.pairKey('rgb', 'ir'); + const text = '10 20 30 40\n11 21 31 41'; + store.loadPointsFromText(key, text); + expect(store.toPointsText(key)).toBe(text); + }); + + it('resumes id allocation after loaded points', () => { + const store = new CameraCalibrationStore(); + const key = store.pairKey('rgb', 'ir'); + store.loadPointsFromText(key, '10 20 30 40\n11 21 31 41'); + store.setActivePair('rgb', 'ir'); + store.addPoint('rgb', [1, 1]); + store.addPoint('ir', [2, 2]); + expect(store.correspondences.value[key][2].id).toBe(3); + }); + }); + + describe('toHomographyText', () => { + it('returns null when the pair has no fitted homography', () => { + const store = new CameraCalibrationStore(); + const key = store.pairKey('left', 'right'); + expect(store.toHomographyText(key, 'AtoB')).toBeNull(); + }); + + it('serializes the fitted matrix as whitespace-separated rows', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + addFourTranslationPairs(store); // left -> right is +5, -3 + store.fitTransform(key); + const text = store.toHomographyText(key, 'AtoB'); + const rows = text?.split('\n').map((row) => row.split(' ').map(Number)); + expect(rows).toHaveLength(3); + expect(rows?.[0][2]).toBeCloseTo(5, 5); + expect(rows?.[1][2]).toBeCloseTo(-3, 5); + }); + + it('serializes the inverse direction', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + addFourTranslationPairs(store); + store.fitTransform(key); + const text = store.toHomographyText(key, 'BtoA'); + const rows = text?.split('\n').map((row) => row.split(' ').map(Number)); + expect(rows?.[0][2]).toBeCloseTo(-5, 5); + }); + }); + describe('requestRecenter', () => { it('records a recenter request for a camera in the active pair', () => { const store = new CameraCalibrationStore(); diff --git a/client/src/CameraCalibrationStore.ts b/client/src/CameraCalibrationStore.ts index 09a1fad51..9f2a7c529 100644 --- a/client/src/CameraCalibrationStore.ts +++ b/client/src/CameraCalibrationStore.ts @@ -220,6 +220,28 @@ export default class CameraCalibrationStore { this.syncAlignmentHomography(); } + /** + * Undo one step, mirroring keypointgui's Clear Last button: if there's a + * pending (blue) point, drop it; otherwise remove the most recently + * completed correspondence for the active pair. + */ + clearLast() { + if (this.pendingPoint.value) { + this.pendingPoint.value = null; + return; + } + const key = this.activePairKey(); + if (!key) { + return; + } + const list = this.correspondences.value[key]; + if (!list || list.length === 0) { + return; + } + this.correspondences.value = { ...this.correspondences.value, [key]: list.slice(0, -1) }; + this.syncAlignmentHomography(); + } + /** The chosen fit model for `key`, defaulting to 'homography' when unset. */ transformTypeForPair(key: string): TransformType { return this.transformTypes.value[key] || 'homography'; @@ -377,6 +399,50 @@ export default class CameraCalibrationStore { .join('\n'); } + /** + * Parse keypointgui-style points text (rows of "leftX leftY rightX rightY", + * the format written by {@link toPointsText} and by keypointgui's + * `np.savetxt`) into correspondences for `key`. Blank lines are ignored. + * Throws if any non-blank line doesn't parse to exactly 4 finite numbers. + * + * `mode: 'replace'` (default) discards `key`'s existing correspondences + * first, matching keypointgui's own Load behavior; `'merge'` appends to + * them instead. + */ + loadPointsFromText(key: string, text: string, mode: 'replace' | 'merge' = 'replace') { + const rows = text.split('\n').map((line) => line.trim()).filter((line) => line.length > 0); + const parsed = rows.map((line, i) => { + const parts = line.split(/\s+/).map(Number); + if (parts.length !== 4 || parts.some((n) => !Number.isFinite(n))) { + throw new Error(`Line ${i + 1} of points file is not "leftX leftY rightX rightY": "${line}"`); + } + return parts as [number, number, number, number]; + }); + const existing = mode === 'merge' ? (this.correspondences.value[key] || []) : []; + const added: Correspondence[] = parsed.map(([ax, ay, bx, by]) => ({ + // eslint-disable-next-line no-plusplus + id: this.nextId++, + a: [ax, ay] as Point, + b: [bx, by] as Point, + })); + this.correspondences.value = { ...this.correspondences.value, [key]: [...existing, ...added] }; + this.syncAlignmentHomography(); + } + + /** + * Serialize the fitted homography for `key` in `direction` as whitespace + * -separated rows (matching keypointgui's `np.savetxt` output for + * `on_save_left_to_right_homography` / `on_save_right_to_left_homography`). + * Returns null if `key` has no fitted homography yet. + */ + toHomographyText(key: string, direction: 'AtoB' | 'BtoA'): string | null { + const homog = this.homographies.value[key]; + if (!homog) { + return null; + } + return homog[direction].map((row) => row.join(' ')).join('\n'); + } + /** Reset state and load saved homographies, correspondences, and transform type choices. */ hydrate( homographies?: CameraHomographies, From a8f3ea4e815af4ef970d5575d635ad89858206c8 Mon Sep 17 00:00:00 2001 From: romleiaj Date: Wed, 1 Jul 2026 12:20:56 -0400 Subject: [PATCH 06/79] Fix calibration ghost overlay for video and surface degenerate-fit errors Ghost overlay resolution only matched image-sequence quads, silently disabling aligned picking's visual aid for video datasets; now resolves video quads too. Fitting a transform from enough points that are degenerate (e.g. collinear) threw uncaught out of the geojs click handler; failures are now caught and surfaced via a new fitError state instead of crashing the picking flow. --- .../CameraCalibration/CalibrationTools.vue | 7 +++ client/src/CameraCalibrationStore.spec.ts | 48 +++++++++++++++++++ client/src/CameraCalibrationStore.ts | 31 +++++++++++- client/src/components/LayerManager.vue | 25 ++++++++-- client/src/homography.spec.ts | 5 ++ .../CalibrationKeypointLayer.ts | 7 ++- 6 files changed, 115 insertions(+), 8 deletions(-) diff --git a/client/dive-common/components/CameraCalibration/CalibrationTools.vue b/client/dive-common/components/CameraCalibration/CalibrationTools.vue index 55ad5eb86..542136e0e 100644 --- a/client/dive-common/components/CameraCalibration/CalibrationTools.vue +++ b/client/dive-common/components/CameraCalibration/CalibrationTools.vue @@ -201,6 +201,7 @@ export default defineComponent({ calibration, pickingEnabled: calibration.pickingEnabled, alignment: calibration.alignment, + fitError: calibration.fitError, linkedNav: calibration.linkedNav, cursorReadout, correspondences, @@ -349,6 +350,12 @@ export default defineComponent({ > No correspondences yet. At least {{ minPoints }} required for the selected transform. + + Could not fit transform: {{ fitError }} + diff --git a/client/src/CameraCalibrationStore.spec.ts b/client/src/CameraCalibrationStore.spec.ts index b73ba7a78..88418b72b 100644 --- a/client/src/CameraCalibrationStore.spec.ts +++ b/client/src/CameraCalibrationStore.spec.ts @@ -105,6 +105,54 @@ describe('CameraCalibrationStore', () => { expect(() => store.fitTransform(key)).toThrow(); }); + it('surfaces a fitError instead of throwing when maybeFitPair hits a degenerate configuration', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + // 4 collinear points satisfy the homography minimum count but are degenerate. + const pts: [number, number][] = [[0, 0], [1, 0], [2, 0], [3, 0]]; + pts.forEach((p) => { + store.addPoint('left', p); + store.addPoint('right', p); + }); + expect(() => store.maybeFitPair(key)).not.toThrow(); + expect(store.fitError.value).toMatch(/degenerate/i); + expect(store.homographies.value[key]).toBeUndefined(); + }); + + it('clears a stale fitError once the active pair fits successfully', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + const collinear: [number, number][] = [[0, 0], [1, 0], [2, 0], [3, 0]]; + collinear.forEach((p) => { + store.addPoint('left', p); + store.addPoint('right', p); + }); + store.maybeFitPair(key); + expect(store.fitError.value).not.toBeNull(); + store.clearPair(); + addFourTranslationPairs(store); + store.maybeFitPair(key); + expect(store.fitError.value).toBeNull(); + expect(store.homographies.value[key]).toBeDefined(); + }); + + it('clears fitError when switching to a different active pair', () => { + const store = new CameraCalibrationStore(); + store.setActivePair('left', 'right'); + const key = store.pairKey('left', 'right'); + const collinear: [number, number][] = [[0, 0], [1, 0], [2, 0], [3, 0]]; + collinear.forEach((p) => { + store.addPoint('left', p); + store.addPoint('right', p); + }); + store.maybeFitPair(key); + expect(store.fitError.value).not.toBeNull(); + store.setActivePair('left', 'other'); + expect(store.fitError.value).toBeNull(); + }); + function addFourTranslationPairs(store: CameraCalibrationStore) { const pts: [number, number][] = [[0, 0], [10, 0], [10, 10], [0, 10]]; pts.forEach((p) => { diff --git a/client/src/CameraCalibrationStore.ts b/client/src/CameraCalibrationStore.ts index 9f2a7c529..65e0cdbd0 100644 --- a/client/src/CameraCalibrationStore.ts +++ b/client/src/CameraCalibrationStore.ts @@ -87,6 +87,15 @@ export default class CameraCalibrationStore { */ recenterRequest: Ref<{ camera: string; coord: Point; id: number } | null>; + /** + * Message from the most recent failed fit attempt (e.g. collinear/degenerate + * points that satisfy the minimum count but can't be solved), or null if the + * active pair's last fit attempt (if any) succeeded. Surfaced by the + * calibration panel instead of letting the estimator's exception escape a + * geojs click handler. + */ + fitError: Ref; + private nextId: number; private nextRecenterId: number; @@ -102,6 +111,7 @@ export default class CameraCalibrationStore { this.linkedNav = ref(false); this.cursorCoord = ref(null); this.recenterRequest = ref(null); + this.fitError = ref(null); this.nextId = 1; this.nextRecenterId = 1; } @@ -135,6 +145,7 @@ export default class CameraCalibrationStore { this.alignment.value = { mode: 'original', opacity: this.alignment.value.opacity, pickTarget: 'native' }; this.cursorCoord.value = null; this.recenterRequest.value = null; + this.fitError.value = null; } /** @@ -256,7 +267,10 @@ export default class CameraCalibrationStore { /** * Fit `key` when it has enough points for its chosen transform type; otherwise * clear its homography and, if it's the active (aligned) pair, revert - * alignment to 'original'. + * alignment to 'original'. A fit can still fail past the minimum-count check + * (e.g. collinear/near-duplicate points make the system unsolvable); that's + * caught here and surfaced via {@link fitError} instead of throwing out of a + * geojs click handler, keeping any previously fitted homography in place. */ maybeFitPair(key: string) { const list = this.correspondences.value[key]; @@ -268,9 +282,21 @@ export default class CameraCalibrationStore { if (this.activePairKey() === key && this.alignment.value.mode !== 'original') { this.alignment.value = { ...this.alignment.value, mode: 'original', pickTarget: 'native' }; } + if (this.activePairKey() === key) { + this.fitError.value = null; + } return; } - this.fitTransform(key); + try { + this.fitTransform(key); + if (this.activePairKey() === key) { + this.fitError.value = null; + } + } catch (err) { + if (this.activePairKey() === key) { + this.fitError.value = err instanceof Error ? err.message : String(err); + } + } } /** Fit the active pair when it has enough points; otherwise clear/revert as in {@link maybeFitPair}. */ @@ -459,6 +485,7 @@ export default class CameraCalibrationStore { this.linkedNav.value = false; this.cursorCoord.value = null; this.recenterRequest.value = null; + this.fitError.value = null; // Resume id allocation past any restored correspondences. let maxId = 0; Object.values(this.correspondences.value).forEach((list) => { diff --git a/client/src/components/LayerManager.vue b/client/src/components/LayerManager.vue index b9cb97fc7..b4d2fec8d 100644 --- a/client/src/components/LayerManager.vue +++ b/client/src/components/LayerManager.vue @@ -200,7 +200,15 @@ export default defineComponent({ } }, { deep: true }); - /** Resolve another camera's currently displayed frame image (for the overlay). */ + /** + * Resolve another camera's currently displayed frame image (for the ghost + * overlay). Matches the `quad.image` data used by ImageAnnotator and the + * `quad.video` data used by VideoAnnotator -- geojs' canvas quad renderer + * supports both as texture sources. LargeImageAnnotator (tiled/geospatial + * imagery) has no single resolvable image element, so it returns null and + * the ghost overlay is simply unavailable for those datasets; picking + * itself (native-coordinate inverse mapping) is unaffected either way. + */ const getCameraImage = (camera: string) => { const ctrl = aggregateController.value.getController(camera); const viewer = ctrl?.geoViewerRef?.value; @@ -214,9 +222,18 @@ export default defineComponent({ const features = layer.features(); for (let j = 0; j < features.length; j += 1) { const data = typeof features[j].data === 'function' ? features[j].data() : undefined; - if (Array.isArray(data) && data[0] && data[0].image) { - const image = data[0].image as HTMLImageElement; - return { image, width: image.naturalWidth, height: image.naturalHeight }; + const datum = Array.isArray(data) ? data[0] : undefined; + if (datum && datum.image) { + const image = datum.image as HTMLImageElement; + return { + source: image, kind: 'image' as const, width: image.naturalWidth, height: image.naturalHeight, + }; + } + if (datum && datum.video) { + const video = datum.video as HTMLVideoElement; + return { + source: video, kind: 'video' as const, width: video.videoWidth, height: video.videoHeight, + }; } } } diff --git a/client/src/homography.spec.ts b/client/src/homography.spec.ts index 25eb24278..ec6d85ae4 100644 --- a/client/src/homography.spec.ts +++ b/client/src/homography.spec.ts @@ -62,4 +62,9 @@ describe('homography', () => { it('throws with fewer than 4 correspondences', () => { expect(() => solveHomography(unitSquare.slice(0, 3), unitSquare.slice(0, 3))).toThrow(); }); + + it('throws on a degenerate (collinear) point configuration despite having 4 points', () => { + const collinear: Point[] = [[0, 0], [1, 0], [2, 0], [3, 0]]; + expect(() => solveHomography(collinear, collinear)).toThrow(/degenerate/i); + }); }); diff --git a/client/src/layers/AnnotationLayers/CalibrationKeypointLayer.ts b/client/src/layers/AnnotationLayers/CalibrationKeypointLayer.ts index 3ae598e3f..7332e24fe 100644 --- a/client/src/layers/AnnotationLayers/CalibrationKeypointLayer.ts +++ b/client/src/layers/AnnotationLayers/CalibrationKeypointLayer.ts @@ -12,7 +12,10 @@ export interface CalibrationPointData { } export interface CameraImage { - image: HTMLImageElement; + /** The texture source for the geojs quad feature: an `` for image sequences, a ` + + Saves with the dataset; restored automatically when it is reopened. + + + Export calibration (.json) + diff --git a/client/src/components/controls/Controls.vue b/client/src/components/controls/Controls.vue index 3bf2239a2..8d6982930 100644 --- a/client/src/components/controls/Controls.vue +++ b/client/src/components/controls/Controls.vue @@ -9,7 +9,9 @@ import { clientSettings } from 'dive-common/store/settings'; import { DatasetType, useApi } from 'dive-common/apispec'; import { frameToTimestamp } from 'vue-media-annotator/utils'; import { injectAggregateController } from '../annotators/useMediaController'; -import { useTime, useTrackFilters, useDatasetId } from '../../provides'; +import { + useTime, useTrackFilters, useDatasetId, useAlignedView, +} from '../../provides'; export default defineComponent({ name: 'Controls', @@ -37,6 +39,35 @@ export default defineComponent({ dragging: false, }); const mediaController = injectAggregateController().value; + let alignedView: ReturnType | undefined; + try { + alignedView = useAlignedView(); + } catch { + // aligned view store may not be provided in tests or minimal embeds. + } + /** + * The raw screen-delta camera sync is only offered while the transform + * -aware aligned view (the Align button) is unavailable: once every + * camera has a calibration transform, Align is the single place to link + * pan/zoom, and this cruder link (which assumes identical pixel scale + * between panes) would just be a second, worse toggle for the same thing. + */ + const rawSyncAvailable = computed(() => mediaController.cameras.value.length > 1 + && !alignedView?.available.value); + const toggleRawSync = () => { + if (rawSyncAvailable.value) { + mediaController.toggleSynchronizeCameras(!mediaController.cameraSync.value); + } + }; + // If transforms become available while the raw sync is on, switch it off: + // its toggle is hidden from that point, and the aligned-view link stands + // down while raw sync is enabled, so a stuck-on raw sync would silently + // block the Align button's linking with no visible control to clear it. + watch(rawSyncAvailable, (available) => { + if (!available && mediaController.cameraSync.value) { + mediaController.toggleSynchronizeCameras(false); + } + }, { immediate: true }); const isVideo = computed(() => props.datasetType === 'video'); const { frameRate } = useTime(); const { visible } = usePrompt(); @@ -213,6 +244,8 @@ export default defineComponent({ activeTimeFilter, data, mediaController, + rawSyncAvailable, + toggleRawSync, dragHandler, input, togglePlay, @@ -256,7 +289,7 @@ export default defineComponent({ { bind: 'd', handler: mediaController.prevFrame, disabled: visible() }, { bind: 'l', - handler: () => mediaController.toggleSynchronizeCameras(!mediaController.cameraSync.value), + handler: toggleRawSync, disabled: visible(), }, ]" @@ -602,12 +635,12 @@ export default defineComponent({ {{ mediaController.cameraSync.value ? 'mdi-link' : 'mdi-link-off' }} @@ -922,12 +955,12 @@ export default defineComponent({ {{ mediaController.cameraSync.value ? 'mdi-link' : 'mdi-link-off' }} @@ -1212,13 +1245,13 @@ export default defineComponent({ {{ mediaController.cameraSync.value ? 'mdi-link' : 'mdi-link-off' }} From 85fb51389c8726fa2731701f69ee09957b1699ad Mon Sep 17 00:00:00 2001 From: romleiaj Date: Mon, 6 Jul 2026 15:48:00 -0400 Subject: [PATCH 31/79] Keep the Align button visible and explain why it is unavailable The button previously vanished whenever any camera lacked a transform to the reference (Align is all-or-none), which reads as "align is broken" on a 3-camera rig with one calibrated pair. Show it disabled instead, with the tooltip naming exactly which cameras still need a pair, and call out the pick-points suspension when the view is on but inert. Co-Authored-By: Claude Fable 5 --- client/dive-common/components/Viewer.vue | 71 ++++++++++++++++-------- client/src/alignedView.spec.ts | 14 +++++ client/src/alignedView.ts | 17 ++++++ 3 files changed, 80 insertions(+), 22 deletions(-) diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index 671737184..dd448e7d2 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -22,7 +22,7 @@ import { AlignedViewStore, StyleManager, TrackFilterControls, GroupFilterControls, } from 'vue-media-annotator/index'; -import { resolveToReferenceTransforms } from 'vue-media-annotator/alignedView'; +import { resolveToReferenceTransforms, unresolvedCameras } from 'vue-media-annotator/alignedView'; import { provideAnnotator, LassoModeSymbol } from 'vue-media-annotator/provides'; import { @@ -387,6 +387,31 @@ export default defineComponent({ const toggleAlignedView = () => { alignedView.setEnabled(!alignedView.enabled.value); }; + /** + * Why the Align button is disabled/inert right now, or the normal usage + * text: the button stays visible for every multicam dataset so an + * incomplete calibration or the picking suspension is explained rather + * than silently hiding it. + */ + const alignedViewTooltip = computed(() => { + if (!alignedViewAvailable.value) { + const missing = unresolvedCameras( + multiCamList.value, + multiCamList.value[0], + cameraCalibration.homographies.value, + ); + const cameraText = missing.length ? missing.join(', ') : 'some cameras'; + return `Aligned view unavailable: no transform maps ${cameraText} onto ` + + `${multiCamList.value[0]}. Pick points (or load a calibration) for the missing pair(s), ` + + 'then Save calibration.'; + } + if (alignedViewEnabled.value && !alignedViewActive.value) { + return 'Aligned view is on but suspended while "Pick points" is active in the ' + + 'calibration panel. Turn off point picking to apply it.'; + } + return `Aligned view: warp cameras into ${alignedViewReference.value}'s space and ` + + 'link pan/zoom. Editing is disabled while on.'; + }); // This context for removal const removeGroups = (id: AnnotationId) => { cameraStore.removeGroups(id); @@ -1502,6 +1527,7 @@ export default defineComponent({ alignedViewEnabled, alignedViewActive, alignedViewReference, + alignedViewTooltip, toggleAlignedView, seekToFrame, resetAggregateZoom, @@ -1689,33 +1715,34 @@ export default defineComponent({ - - Aligned view: warp cameras into {{ alignedViewReference }}'s space and - link pan/zoom. Editing is disabled while on. - + {{ alignedViewTooltip }} { }); }); +describe('unresolvedCameras', () => { + const homographies: CameraHomographies = { + 'eo::ir': { AtoB: translation(-100, 0), BtoA: translation(100, 0) }, + }; + it('names cameras with no path to the reference', () => { + expect(unresolvedCameras(['eo', 'ir', 'uv'], 'eo', homographies)).toEqual(['uv']); + expect(unresolvedCameras(['eo', 'ir', 'uv'], 'eo', {})).toEqual(['ir', 'uv']); + }); + it('is empty when every camera resolves', () => { + expect(unresolvedCameras(['eo', 'ir'], 'eo', homographies)).toEqual([]); + }); +}); + describe('cameraPairTransform', () => { it('composes from -> reference -> to', () => { const toReference = { diff --git a/client/src/alignedView.ts b/client/src/alignedView.ts index 3e19921cf..468868b65 100644 --- a/client/src/alignedView.ts +++ b/client/src/alignedView.ts @@ -159,6 +159,23 @@ export function resolveToReferenceTransforms( return result; } +/** + * The cameras in `cameras` with no composed transform path to `reference` -- + * the reason {@link resolveToReferenceTransforms} returned null, named so the + * UI can say exactly which pair(s) still need calibrating instead of silently + * hiding the aligned view. + */ +export function unresolvedCameras( + cameras: string[], + reference: string, + homographies: CameraHomographies, +): string[] { + return cameras.filter( + (camera) => camera !== reference + && composeThroughPairs(camera, reference, homographies) === null, + ); +} + /** * Matrix mapping `from`-camera native pixels onto `to`-camera native pixels, * composed through the shared reference space: From 8c5ed9be625910a0290cf57d0d5b228c74e36359 Mon Sep 17 00:00:00 2001 From: romleiaj Date: Mon, 6 Jul 2026 15:57:51 -0400 Subject: [PATCH 32/79] Link aligned-view panes by identity, not camera-to-camera transforms While the aligned view is active every pane renders its imagery and geometry warped into the shared reference space, so linked navigation must copy centers verbatim and match reference-units-per-pixel. Mapping the center through the native camera-to-camera transform (as the link did) re-applied a transform the rendering had already applied, leaving every non-reference pane offset by exactly its pair transform -- the warp itself and the calibration panel's ghost/pair link (which operate on unwarped panes) looked correct while "Align" appeared wrong. The calibration pair link now stands down while the aligned view is active (its homography mapping assumes unwarped panes) instead of the aligned link deferring to it. Co-Authored-By: Claude Fable 5 --- client/dive-common/components/Viewer.vue | 7 +- .../annotators/useAlignedNavigation.spec.ts | 128 ++++++++++++++++++ .../annotators/useAlignedNavigation.ts | 66 ++++----- .../annotators/useCalibrationNavigation.ts | 18 +++ 4 files changed, 181 insertions(+), 38 deletions(-) create mode 100644 client/src/components/annotators/useAlignedNavigation.spec.ts diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index dd448e7d2..c60cf0cc7 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -342,7 +342,6 @@ export default defineComponent({ const cameraStore = new CameraStore({ markChangesPending }); const cameraCalibration = new CameraCalibrationStore(); - useCalibrationNavigation(aggregateController, cameraCalibration); /** * Aligned view (SEAL-TK features 2 + 3): when every non-reference camera @@ -355,6 +354,10 @@ export default defineComponent({ * panel edits and saves is exactly what the Align button applies. */ const alignedView = new AlignedViewStore(); + // The calibration pair link maps through the homography for UNWARPED + // panes, so it needs the aligned view state to stand down while displays + // are warped into reference space. + useCalibrationNavigation(aggregateController, cameraCalibration, alignedView); const alignedResolution = computed(() => { if (multiCamList.value.length < 2) { return null; @@ -379,7 +382,7 @@ export default defineComponent({ watch(cameraCalibration.pickingEnabled, (picking) => { alignedView.setSuspended(picking); }, { immediate: true }); - useAlignedNavigation(aggregateController, alignedView, multiCamList, cameraCalibration); + useAlignedNavigation(aggregateController, alignedView, multiCamList); const alignedViewAvailable = computed(() => alignedView.available.value); const alignedViewEnabled = computed(() => alignedView.enabled.value); const alignedViewActive = computed(() => alignedView.active.value); diff --git a/client/src/components/annotators/useAlignedNavigation.spec.ts b/client/src/components/annotators/useAlignedNavigation.spec.ts new file mode 100644 index 000000000..fd8adb3c3 --- /dev/null +++ b/client/src/components/annotators/useAlignedNavigation.spec.ts @@ -0,0 +1,128 @@ +/// +import { + ref, shallowRef, nextTick, Ref, +} from 'vue'; +import useAlignedNavigation from './useAlignedNavigation'; +import AlignedViewStore from '../../AlignedViewStore'; +import type { AggregateMediaController } from './mediaControllerType'; +import type { Matrix3 } from '../../homography'; + +vi.mock('geojs', () => ({ default: { event: { pan: 'geo_pan', zoom: 'geo_zoom' } } })); + +const IDENTITY: Matrix3 = [[1, 0, 0], [0, 1, 0], [0, 0, 1]]; + +/** Minimal stand-in for a geojs viewer: center/zoom state + geoOn events. */ +function fakeViewer(baseUnitsPerPixel: number) { + const state = { center: { x: 0, y: 0 }, zoom: 0 }; + const handlers: Record void>> = {}; + return { + geoOn(evt: string, handler: () => void) { + handlers[evt] = handlers[evt] || []; + handlers[evt].push(handler); + }, + geoOff(evt: string, handler: () => void) { + handlers[evt] = (handlers[evt] || []).filter((h) => h !== handler); + }, + center(c?: { x: number; y: number }) { + if (c) { + state.center = { ...c }; + } + return state.center; + }, + zoom(z?: number) { + if (z !== undefined) { + state.zoom = z; + } + return state.zoom; + }, + unitsPerPixel(z: number) { + return baseUnitsPerPixel / 2 ** z; + }, + trigger(evt: string) { + (handlers[evt] || []).forEach((h) => h()); + }, + }; +} + +function makeHarness() { + // EO: high-res pane (fine zoom-0 baseline); IR: low-res pane. + const eo = fakeViewer(1); + const ir = fakeViewer(8); + const controllers: Record }> = { + eo: { geoViewerRef: ref(eo) }, + ir: { geoViewerRef: ref(ir) }, + }; + const cameraSync = ref(false); + // shallowRef: a plain ref would deep-unwrap the nested cameraSync / + // resizeTrigger refs, unlike the real aggregate controller object. + const aggregate = shallowRef({ + cameraSync, + resizeTrigger: ref(0), + getController: (name: string) => controllers[name], + }) as unknown as Ref; + const cameras = ref(['eo', 'ir']); + const alignedView = new AlignedViewStore(); + useAlignedNavigation(aggregate, alignedView, cameras); + return { + eo, ir, cameraSync, alignedView, + }; +} + +describe('useAlignedNavigation', () => { + it('links panes by IDENTITY center in the shared reference space', async () => { + const { eo, ir, alignedView } = makeHarness(); + // ir's imagery renders warped through ir->eo (translation +100), so its + // pane coordinates ARE reference coordinates: the link must NOT map the + // center through the camera-to-camera transform again. + alignedView.setTransforms('eo', { + eo: IDENTITY, + ir: [[1, 0, 100], [0, 1, 0], [0, 0, 1]], + }); + alignedView.setEnabled(true); + await nextTick(); + + eo.center({ x: 500, y: 300 }); + eo.zoom(2); // source units-per-pixel = 1 / 2^2 = 0.25 reference units + eo.trigger('geo_pan'); + + expect(ir.center()).toEqual({ x: 500, y: 300 }); + // Same reference extent through ir's own zoom-0 baseline: + // log2(8 / 0.25) = 5. + expect(ir.zoom()).toBeCloseTo(5, 6); + }); + + it('propagates from the warped pane back to the reference pane identically', async () => { + const { eo, ir, alignedView } = makeHarness(); + alignedView.setTransforms('eo', { + eo: IDENTITY, + ir: [[2, 0, 0], [0, 2, 0], [0, 0, 1]], + }); + alignedView.setEnabled(true); + await nextTick(); + + ir.center({ x: 40, y: 60 }); + ir.trigger('geo_pan'); + + expect(eo.center()).toEqual({ x: 40, y: 60 }); + }); + + it('stands down while inactive, suspended, or raw camera sync is on', async () => { + const { + eo, ir, cameraSync, alignedView, + } = makeHarness(); + alignedView.setTransforms('eo', { eo: IDENTITY, ir: IDENTITY }); + alignedView.setEnabled(true); + await nextTick(); + + cameraSync.value = true; + eo.center({ x: 9, y: 9 }); + eo.trigger('geo_pan'); + expect(ir.center()).toEqual({ x: 0, y: 0 }); + cameraSync.value = false; + + alignedView.setSuspended(true); + await nextTick(); + eo.trigger('geo_pan'); + expect(ir.center()).toEqual({ x: 0, y: 0 }); + }); +}); diff --git a/client/src/components/annotators/useAlignedNavigation.ts b/client/src/components/annotators/useAlignedNavigation.ts index 60de75930..bbe2e61f5 100644 --- a/client/src/components/annotators/useAlignedNavigation.ts +++ b/client/src/components/annotators/useAlignedNavigation.ts @@ -4,29 +4,31 @@ import { } from 'vue'; import type { AggregateMediaController } from './mediaControllerType'; import type AlignedViewStore from '../../AlignedViewStore'; -import type CameraCalibrationStore from '../../CameraCalibrationStore'; -import { localLinkedScale } from '../../homography'; /** * Links pan/zoom recentering across ALL loaded cameras while the aligned - * view is active (SEAL-TK feature 3): panning or zooming any camera - * recenters every other camera on the same world location, mapped through - * the stored transforms composed via the reference camera - * ({@link AlignedViewStore.mapCameraPoint}). + * view is active (SEAL-TK feature 3). + * + * While the aligned view is active, every pane RENDERS in the shared + * reference space: AlignedImageLayer draws each camera's imagery -- and + * LayerManager maps its geometry -- through that camera's native->reference + * transform. The link between panes is therefore the IDENTITY on + * coordinates: same center, same reference-units-per-screen-pixel. Mapping + * centers through the camera-to-camera transforms here would re-apply a + * transform the rendering has already applied, offsetting every non-reference + * pane by exactly its pair transform. * * Distinct from (a) the general "sync cameras" toggle (Controls.vue), which - * forwards raw screen deltas/zoom levels and assumes identical pixel scale - * between panes, and (b) the calibration pair link - * (useCalibrationNavigation.ts), which is scoped to the active calibration - * pair while authoring correspondences. To avoid three handlers fighting - * over the same pan event, this link stands down while either of those is - * turned on. + * forwards raw screen deltas/zoom levels for UNWARPED panes (it is hidden + * and disabled whenever the aligned view is available), and (b) the + * calibration pair link (useCalibrationNavigation.ts), which maps through + * the homography because it operates on unwarped panes while authoring + * correspondences -- it stands down while the aligned view is active. */ export default function useAlignedNavigation( aggregateController: Ref, alignedView: AlignedViewStore, cameras: Ref, - calibration?: CameraCalibrationStore, ) { // Re-entrancy guard: setting a camera's center from a link handler must not // itself trigger that camera's own pan/zoom listener. @@ -50,8 +52,10 @@ export default function useAlignedNavigation( if (!alignedView.active.value) { return; } - // Stand down while another camera-linking mechanism is driving. - if (aggregateController.value.cameraSync.value || calibration?.linkedNav.value) { + // Stand down while the raw screen-delta sync is driving (it should be + // unreachable while the aligned view is available, but never let two + // handlers fight over the same pan event). + if (aggregateController.value.cameraSync.value) { return; } // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -64,17 +68,20 @@ export default function useAlignedNavigation( if (!sourceViewer) { return; } + // Shared reference space: copy the center verbatim and match the + // visible extent. Viewers keep their own zoom-0 baselines (sized to + // each camera's native image), so an equal units-per-pixel still needs + // converting through the target's baseline rather than copying the + // zoom level (geojs zoom is log2-based: + // unitsPerPixel(z) = unitsPerPixel(0) / 2^z). const center = sourceViewer.center(); + const desiredUnitsPerPixel = sourceViewer.unitsPerPixel(sourceViewer.zoom()); guard = true; try { cameras.value.forEach((otherCamera) => { if (otherCamera === camera) { return; } - const mapped = alignedView.mapCameraPoint(camera, otherCamera, [center.x, center.y]); - if (!mapped) { - return; - } // eslint-disable-next-line @typescript-eslint/no-explicit-any let targetViewer: any; try { @@ -83,24 +90,11 @@ export default function useAlignedNavigation( return; } if (targetViewer) { - // Match the visible extent, not just the center: one source pixel - // spans `scale` target pixels around this point, so the target - // needs `scale`x the source's units-per-pixel. geojs zoom is - // log2-based (unitsPerPixel(z) = unitsPerPixel(0) / 2^z); invert - // through the target's own zoom-0 baseline since the two images - // may differ in resolution (e.g. EO vs IR). - const scale = localLinkedScale( - (p) => alignedView.mapCameraPoint(camera, otherCamera, p), - [center.x, center.y], - ); - if (scale !== null) { - const desiredUnitsPerPixel = sourceViewer.unitsPerPixel(sourceViewer.zoom()) * scale; - const targetZoom = Math.log2(targetViewer.unitsPerPixel(0) / desiredUnitsPerPixel); - if (Number.isFinite(targetZoom)) { - targetViewer.zoom(targetZoom); - } + const targetZoom = Math.log2(targetViewer.unitsPerPixel(0) / desiredUnitsPerPixel); + if (Number.isFinite(targetZoom)) { + targetViewer.zoom(targetZoom); } - targetViewer.center({ x: mapped[0], y: mapped[1] }); + targetViewer.center({ x: center.x, y: center.y }); } }); } finally { diff --git a/client/src/components/annotators/useCalibrationNavigation.ts b/client/src/components/annotators/useCalibrationNavigation.ts index 0319cb582..b87a746de 100644 --- a/client/src/components/annotators/useCalibrationNavigation.ts +++ b/client/src/components/annotators/useCalibrationNavigation.ts @@ -4,6 +4,7 @@ import { } from 'vue'; import type { AggregateMediaController } from './mediaControllerType'; import type CameraCalibrationStore from '../../CameraCalibrationStore'; +import type AlignedViewStore from '../../AlignedViewStore'; import { localLinkedScale } from '../../homography'; import type { Point } from '../../homography'; @@ -15,10 +16,17 @@ import type { Point } from '../../homography'; * general "sync cameras" toggle (Controls.vue), which forwards raw screen * deltas/zoom levels and assumes identical pixel scale between panes -- not * true for cross-modality (e.g. EO/IR) rigs with differing resolutions. + * + * The homography mapping assumes UNWARPED panes displaying native + * coordinates, so this link stands down while the aligned view is actively + * warping displays into reference space (useAlignedNavigation links all + * panes by identity there); it takes over whenever picking suspends the + * aligned view during authoring. */ export default function useCalibrationNavigation( aggregateController: Ref, calibration: CameraCalibrationStore, + alignedView?: AlignedViewStore, ) { // Re-entrancy guard: setting a camera's center from a link handler must not // itself trigger that camera's own pan/zoom listener. @@ -39,6 +47,12 @@ export default function useCalibrationNavigation( if (guard) { return; } + // Panes render in reference space while the aligned view is active; + // this link's native-coordinate mapping would double-apply the + // transform there. useAlignedNavigation owns linking in that state. + if (alignedView?.active.value) { + return; + } // eslint-disable-next-line @typescript-eslint/no-explicit-any let sourceViewer: any; // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -140,6 +154,10 @@ export default function useCalibrationNavigation( if (!request) { return; } + // Same native-coordinate assumption as the continuous link above. + if (alignedView?.active.value) { + return; + } const pair = calibration.activePair.value; if (!pair || (request.camera !== pair.camA && request.camera !== pair.camB)) { return; From 55605cc0ed8d35282693cd29c4f3112b68785f9b Mon Sep 17 00:00:00 2001 From: romleiaj Date: Mon, 6 Jul 2026 16:10:58 -0400 Subject: [PATCH 33/79] Snap panes to the reference view the moment the aligned view activates The linked navigation only reacted to pan/zoom events, so after hitting Align nothing lined up until the first drag. Activation now runs one link pass from the reference pane immediately. Co-Authored-By: Claude Fable 5 --- .../annotators/useAlignedNavigation.spec.ts | 16 ++++++++++++++++ .../annotators/useAlignedNavigation.ts | 8 ++++++++ 2 files changed, 24 insertions(+) diff --git a/client/src/components/annotators/useAlignedNavigation.spec.ts b/client/src/components/annotators/useAlignedNavigation.spec.ts index fd8adb3c3..1968f49b7 100644 --- a/client/src/components/annotators/useAlignedNavigation.spec.ts +++ b/client/src/components/annotators/useAlignedNavigation.spec.ts @@ -69,6 +69,22 @@ function makeHarness() { } describe('useAlignedNavigation', () => { + it('snaps panes to the reference view immediately on activation', async () => { + const { eo, ir, alignedView } = makeHarness(); + eo.center({ x: 250, y: 150 }); + eo.zoom(1); // units-per-pixel = 0.5 + alignedView.setTransforms('eo', { + eo: IDENTITY, + ir: [[1, 0, 100], [0, 1, 0], [0, 0, 1]], + }); + alignedView.setEnabled(true); + await nextTick(); + + // No pan/zoom event fired: activation itself aligned the panes. + expect(ir.center()).toEqual({ x: 250, y: 150 }); + expect(ir.zoom()).toBeCloseTo(Math.log2(8 / 0.5), 6); + }); + it('links panes by IDENTITY center in the shared reference space', async () => { const { eo, ir, alignedView } = makeHarness(); // ir's imagery renders warped through ir->eo (translation +100), so its diff --git a/client/src/components/annotators/useAlignedNavigation.ts b/client/src/components/annotators/useAlignedNavigation.ts index bbe2e61f5..8982aef40 100644 --- a/client/src/components/annotators/useAlignedNavigation.ts +++ b/client/src/components/annotators/useAlignedNavigation.ts @@ -126,6 +126,14 @@ export default function useAlignedNavigation( viewer.geoOn(geo.event.zoom, handler); attached.push({ viewer, handler }); }); + // Snap immediately: run one link pass from the reference pane so hitting + // the Align button lines every pane up right away, instead of waiting for + // the first pan/zoom event to trigger the link. (Re-running on resize / + // camera changes is a no-op when panes are already synced.) + const reference = alignedView.reference.value; + if (reference && cameras.value.includes(reference)) { + link(reference)(); + } } watch( From 019cb70880160b6104b703808f35d37e60b77231 Mon Sep 17 00:00:00 2001 From: romleiaj Date: Mon, 6 Jul 2026 16:16:35 -0400 Subject: [PATCH 34/79] Reset panes on aligned-view exit; right-click recenter while aligned Toggling the aligned view off left every pane stranded at reference-space centers/zooms while the content reverted to native coordinates; deactivation now resets each pane to its full native view. Right-click recenter previously only existed inside the calibration picking flow, so it did nothing during normal aligned review. The aligned image layer now handles it whenever the aligned view is active: center the clicked pane on the clicked point and let the aligned link recenter every other pane on the same reference-space location. Co-Authored-By: Claude Fable 5 --- client/src/components/LayerManager.vue | 1 + .../annotators/useAlignedNavigation.spec.ts | 22 ++++++++++--- .../annotators/useAlignedNavigation.ts | 16 ++++++++++ client/src/layers/AlignedImageLayer.ts | 32 +++++++++++++++++++ 4 files changed, 67 insertions(+), 4 deletions(-) diff --git a/client/src/components/LayerManager.vue b/client/src/components/LayerManager.vue index 615d2a493..30e922e46 100644 --- a/client/src/components/LayerManager.vue +++ b/client/src/components/LayerManager.vue @@ -198,6 +198,7 @@ export default defineComponent({ } }, getTransform: () => alignedDisplayTransform.value, + getActive: () => alignedViewActive.value, }); const rectAnnotationLayer = new RectangleLayer({ diff --git a/client/src/components/annotators/useAlignedNavigation.spec.ts b/client/src/components/annotators/useAlignedNavigation.spec.ts index 1968f49b7..062dfba2a 100644 --- a/client/src/components/annotators/useAlignedNavigation.spec.ts +++ b/client/src/components/annotators/useAlignedNavigation.spec.ts @@ -48,9 +48,10 @@ function makeHarness() { // EO: high-res pane (fine zoom-0 baseline); IR: low-res pane. const eo = fakeViewer(1); const ir = fakeViewer(8); - const controllers: Record }> = { - eo: { geoViewerRef: ref(eo) }, - ir: { geoViewerRef: ref(ir) }, + const resetZoom = vi.fn(); + const controllers: Record; resetZoom: () => void }> = { + eo: { geoViewerRef: ref(eo), resetZoom }, + ir: { geoViewerRef: ref(ir), resetZoom }, }; const cameraSync = ref(false); // shallowRef: a plain ref would deep-unwrap the nested cameraSync / @@ -64,7 +65,7 @@ function makeHarness() { const alignedView = new AlignedViewStore(); useAlignedNavigation(aggregate, alignedView, cameras); return { - eo, ir, cameraSync, alignedView, + eo, ir, cameraSync, alignedView, resetZoom, }; } @@ -122,6 +123,19 @@ describe('useAlignedNavigation', () => { expect(eo.center()).toEqual({ x: 40, y: 60 }); }); + it('resets every pane to its native view when the aligned view deactivates', async () => { + const { alignedView, resetZoom } = makeHarness(); + alignedView.setTransforms('eo', { eo: IDENTITY, ir: IDENTITY }); + alignedView.setEnabled(true); + await nextTick(); + expect(resetZoom).not.toHaveBeenCalled(); + + alignedView.setEnabled(false); + await nextTick(); + // Once per pane (eo + ir). + expect(resetZoom).toHaveBeenCalledTimes(2); + }); + it('stands down while inactive, suspended, or raw camera sync is on', async () => { const { eo, ir, cameraSync, alignedView, diff --git a/client/src/components/annotators/useAlignedNavigation.ts b/client/src/components/annotators/useAlignedNavigation.ts index 8982aef40..cd1e994e2 100644 --- a/client/src/components/annotators/useAlignedNavigation.ts +++ b/client/src/components/annotators/useAlignedNavigation.ts @@ -146,5 +146,21 @@ export default function useAlignedNavigation( setup, ); + // Leaving the aligned view (toggle off, or suspension by point picking) + // strands every pane at reference-space centers/zooms while the content + // reverts to native coordinates -- reset each pane to its own full native + // view so the imagery is on-screen again. + watch(alignedView.active, (active, wasActive) => { + if (!active && wasActive) { + cameras.value.forEach((camera) => { + try { + aggregateController.value.getController(camera).resetZoom(); + } catch { + // A pane may already be torn down during dataset unload. + } + }); + } + }); + onBeforeUnmount(teardown); } diff --git a/client/src/layers/AlignedImageLayer.ts b/client/src/layers/AlignedImageLayer.ts index 530f72733..a0f59ab30 100644 --- a/client/src/layers/AlignedImageLayer.ts +++ b/client/src/layers/AlignedImageLayer.ts @@ -1,3 +1,4 @@ +import geo from 'geojs'; import type { MediaController } from '../components/annotators/mediaControllerType'; import { Matrix3, subdivideWarpQuads, warpGridSize } from '../homography'; import type { CameraImage } from './AnnotationLayers/CalibrationKeypointLayer'; @@ -16,6 +17,12 @@ interface AlignedImageLayerParams { getImage: () => CameraImage | null; /** Current native->reference display transform, or null when unwarped. */ getTransform: () => Matrix3 | null; + /** + * Whether the aligned view is active for the dataset (true for the + * reference pane too, whose own transform is null/identity). Gates the + * right-click recenter below. + */ + getActive: () => boolean; } /** @@ -39,6 +46,8 @@ export default class AlignedImageLayer { private getTransform: () => Matrix3 | null; + private getActive: () => boolean; + // eslint-disable-next-line @typescript-eslint/no-explicit-any private quadLayer: any; @@ -60,12 +69,35 @@ export default class AlignedImageLayer { this.annotator = params.annotator; this.getImage = params.getImage; this.getTransform = params.getTransform; + this.getActive = params.getActive; this.quadLayer = this.annotator.geoViewerRef.value.createLayer('feature', { features: ['quad'], autoshareRenderer: false, renderer: 'canvas', }); this.quadFeature = this.quadLayer.createFeature('quad'); + // Right-click recenter while the aligned view is active: center this pane + // on the clicked location; the aligned pan/zoom link then recenters every + // other pane on the same reference-space point. (While calibration point + // picking is active the aligned view is suspended -- getActive() false -- + // and CalibrationKeypointLayer owns the right-click instead.) + this.annotator.geoViewerRef.value.geoOn( + geo.event.mouseclick, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (e: any) => this.handleClick(e), + ); + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + private handleClick(e: any) { + if (!this.getActive() || !e.geo) { + return; + } + const buttonsDown = e.buttonsDown || (e.mouse && e.mouse.buttonsDown); + if (!buttonsDown || !buttonsDown.right) { + return; + } + this.annotator.geoViewerRef.value.center({ x: e.geo.x, y: e.geo.y }); } /** From 56cc56d94b5eb982dfed85c1915349bdb16a4865 Mon Sep 17 00:00:00 2001 From: romleiaj Date: Mon, 6 Jul 2026 16:32:43 -0400 Subject: [PATCH 35/79] Right-click recenter everywhere; picking-only markers; one link knob Right-click now recenters the clicked pane in every view mode (it only worked under the aligned view / picking flows), staying out of the way while annotation geometry is being created or edited, where right-click already means "remove last point". Picked-point markers are authoring UI: they now show only while Pick points is on (the correspondences themselves persist in the store and the panel table either way). The panel's separate "Link pan/zoom" switch is gone -- the pair link is simply active whenever Pick points is (it starts matching as soon as a transform is fitted), leaving Align as the only navigation-linking control outside authoring. Co-Authored-By: Claude Fable 5 --- .../CameraCalibration/CalibrationTools.vue | 13 ++------- client/src/CameraCalibrationStore.spec.ts | 16 ----------- client/src/CameraCalibrationStore.ts | 14 ++-------- client/src/components/LayerManager.vue | 4 ++- .../annotators/useCalibrationNavigation.ts | 28 +++++++++++-------- client/src/layers/AlignedImageLayer.ts | 25 +++++++++-------- 6 files changed, 36 insertions(+), 64 deletions(-) diff --git a/client/dive-common/components/CameraCalibration/CalibrationTools.vue b/client/dive-common/components/CameraCalibration/CalibrationTools.vue index 7e20e824a..c97b5c6a9 100644 --- a/client/dive-common/components/CameraCalibration/CalibrationTools.vue +++ b/client/dive-common/components/CameraCalibration/CalibrationTools.vue @@ -225,7 +225,6 @@ export default defineComponent({ pickingEnabled: calibration.pickingEnabled, alignment: calibration.alignment, fitError: calibration.fitError, - linkedNav: calibration.linkedNav, cursorReadout, correspondences, transformType, @@ -332,18 +331,10 @@ export default defineComponent({ /> Click a point in one camera, then the matching point in the other. - Right-click to recenter both cameras on that point (requires a fitted - transform). + Once a transform is fitted, pan/zoom is linked between the pair and + right-click recenters both cameras on the clicked point. - -
{ }); describe('linked navigation', () => { - it('setLinkedNav toggles the flag', () => { - const store = new CameraCalibrationStore(); - expect(store.linkedNav.value).toBe(false); - store.setLinkedNav(true); - expect(store.linkedNav.value).toBe(true); - store.setLinkedNav(false); - expect(store.linkedNav.value).toBe(false); - }); - it('linkedPoint maps a point from camA to camB and back, via the fitted homography', () => { const store = new CameraCalibrationStore(); store.setActivePair('left', 'right'); @@ -542,13 +533,6 @@ describe('CameraCalibrationStore', () => { addFourTranslationPairs(store); expect(store.linkedPoint('other', [1, 1])).toBeNull(); }); - - it('resets linkedNav on hydrate', () => { - const store = new CameraCalibrationStore(); - store.setLinkedNav(true); - store.hydrate(); - expect(store.linkedNav.value).toBe(false); - }); }); describe('cursor coordinate readout', () => { diff --git a/client/src/CameraCalibrationStore.ts b/client/src/CameraCalibrationStore.ts index 5c902de4a..006081a93 100644 --- a/client/src/CameraCalibrationStore.ts +++ b/client/src/CameraCalibrationStore.ts @@ -123,9 +123,6 @@ export default class CameraCalibrationStore { alignment: Ref; - /** Whether pan/zoom recentering is linked between the active pair's two cameras. */ - linkedNav: Ref; - /** Native-pixel coordinate under the cursor, for the calibration panel's live readout. */ cursorCoord: Ref<{ camera: string; coord: Point } | null>; @@ -169,7 +166,6 @@ export default class CameraCalibrationStore { this.homographies = ref({}); this.transformTypes = ref({}); this.alignment = ref({ mode: 'original', opacity: 0.5, pickTarget: 'native' }); - this.linkedNav = ref(false); this.cursorCoord = ref(null); this.recenterRequest = ref(null); this.fitError = ref(null); @@ -466,11 +462,6 @@ export default class CameraCalibrationStore { this.alignment.value = { ...this.alignment.value, pickTarget }; } - /** Enable or disable linked pan/zoom recentering between the active pair's cameras. */ - setLinkedNav(enabled: boolean) { - this.linkedNav.value = enabled; - } - /** * Map `coord` (native pixel space of `camera`) to the corresponding point in * the *other* camera of the active pair, via the fitted homography. Returns @@ -507,8 +498,8 @@ export default class CameraCalibrationStore { * a fitted homography, the other camera of the active pair (via * {@link linkedPoint}) recenter their views on this location. Consumed by * {@link useCalibrationNavigation}; a no-op if `camera` isn't part of the - * active pair. Works independent of {@link linkedNav} -- this is a one-shot - * "snap to this feature" action, not the continuous pan/zoom link. + * active pair. A one-shot "snap to this feature" action, distinct from the + * continuous pan/zoom link that is active while picking. */ requestRecenter(camera: string, coord: Point) { const pair = this.activePair.value; @@ -724,7 +715,6 @@ export default class CameraCalibrationStore { this.pendingPoint.value = null; this.pickingEnabled.value = false; this.alignment.value = { mode: 'original', opacity: 0.5, pickTarget: 'native' }; - this.linkedNav.value = false; this.cursorCoord.value = null; this.recenterRequest.value = null; this.fitError.value = null; diff --git a/client/src/components/LayerManager.vue b/client/src/components/LayerManager.vue index 30e922e46..ed2e54a63 100644 --- a/client/src/components/LayerManager.vue +++ b/client/src/components/LayerManager.vue @@ -198,7 +198,9 @@ export default defineComponent({ } }, getTransform: () => alignedDisplayTransform.value, - getActive: () => alignedViewActive.value, + // Right-click means "remove last point" while creating/editing + // geometry; recenter everywhere else. + getRecenterEnabled: () => !editingModeRef.value, }); const rectAnnotationLayer = new RectangleLayer({ diff --git a/client/src/components/annotators/useCalibrationNavigation.ts b/client/src/components/annotators/useCalibrationNavigation.ts index b87a746de..9fdc58967 100644 --- a/client/src/components/annotators/useCalibrationNavigation.ts +++ b/client/src/components/annotators/useCalibrationNavigation.ts @@ -17,11 +17,12 @@ import type { Point } from '../../homography'; * deltas/zoom levels and assumes identical pixel scale between panes -- not * true for cross-modality (e.g. EO/IR) rigs with differing resolutions. * - * The homography mapping assumes UNWARPED panes displaying native - * coordinates, so this link stands down while the aligned view is actively - * warping displays into reference space (useAlignedNavigation links all - * panes by identity there); it takes over whenever picking suspends the - * aligned view during authoring. + * Active whenever point picking is (no separate toggle): during authoring + * the panes are unwarped and this link maps through the pair homography; + * outside picking the Align button's link (useAlignedNavigation) owns + * navigation instead. The homography mapping assumes UNWARPED panes + * displaying native coordinates, so this link also stands down defensively + * if the aligned view is somehow active. */ export default function useCalibrationNavigation( aggregateController: Ref, @@ -100,7 +101,10 @@ export default function useCalibrationNavigation( function setup() { teardown(); const pair = calibration.activePair.value; - if (!calibration.linkedNav.value || !pair) { + // The pair link is authoring UI: it is simply active whenever point + // picking is (once a fit exists, linkedPoint starts returning matches). + // Outside picking, the Align button's link owns navigation. + if (!calibration.pickingEnabled.value || !pair) { return; } // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -132,7 +136,7 @@ export default function useCalibrationNavigation( watch( [ - calibration.linkedNav, + calibration.pickingEnabled, calibration.activePair, calibration.homographies, aggregateController.value.resizeTrigger, @@ -142,11 +146,11 @@ export default function useCalibrationNavigation( ); /** - * One-shot recenter (right-click): center the clicked camera on the clicked - * point, and -- when the pair has a fitted homography -- center the other - * camera on the corresponding point. Independent of {@link - * CameraCalibrationStore.linkedNav}; the `guard` flag still applies so this - * doesn't loop back through the continuous pan/zoom link above when it's on. + * One-shot recenter (right-click while picking): center the clicked camera + * on the clicked point, and -- when the pair has a fitted homography -- + * center the other camera on the corresponding point. The `guard` flag + * still applies so this doesn't loop back through the continuous pan/zoom + * link above. */ function handleRecenterRequest( request: { camera: string; coord: Point; id: number } | null, diff --git a/client/src/layers/AlignedImageLayer.ts b/client/src/layers/AlignedImageLayer.ts index a0f59ab30..cc4033572 100644 --- a/client/src/layers/AlignedImageLayer.ts +++ b/client/src/layers/AlignedImageLayer.ts @@ -18,11 +18,12 @@ interface AlignedImageLayerParams { /** Current native->reference display transform, or null when unwarped. */ getTransform: () => Matrix3 | null; /** - * Whether the aligned view is active for the dataset (true for the - * reference pane too, whose own transform is null/identity). Gates the - * right-click recenter below. + * Whether the right-click recenter is currently allowed. Right-click + * recenters in EVERY view mode (aligned or not, multicam or single); the + * caller disables it while annotation geometry is being created/edited, + * where right-click already means "remove last point". */ - getActive: () => boolean; + getRecenterEnabled: () => boolean; } /** @@ -46,7 +47,7 @@ export default class AlignedImageLayer { private getTransform: () => Matrix3 | null; - private getActive: () => boolean; + private getRecenterEnabled: () => boolean; // eslint-disable-next-line @typescript-eslint/no-explicit-any private quadLayer: any; @@ -69,18 +70,18 @@ export default class AlignedImageLayer { this.annotator = params.annotator; this.getImage = params.getImage; this.getTransform = params.getTransform; - this.getActive = params.getActive; + this.getRecenterEnabled = params.getRecenterEnabled; this.quadLayer = this.annotator.geoViewerRef.value.createLayer('feature', { features: ['quad'], autoshareRenderer: false, renderer: 'canvas', }); this.quadFeature = this.quadLayer.createFeature('quad'); - // Right-click recenter while the aligned view is active: center this pane - // on the clicked location; the aligned pan/zoom link then recenters every - // other pane on the same reference-space point. (While calibration point - // picking is active the aligned view is suspended -- getActive() false -- - // and CalibrationKeypointLayer owns the right-click instead.) + // Right-click recenter: center this pane on the clicked location, in any + // view mode. While the aligned view is active, the aligned pan/zoom link + // then recenters every other pane on the same reference-space point; + // during calibration picking, CalibrationKeypointLayer additionally maps + // the recenter across the active pair. this.annotator.geoViewerRef.value.geoOn( geo.event.mouseclick, // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -90,7 +91,7 @@ export default class AlignedImageLayer { // eslint-disable-next-line @typescript-eslint/no-explicit-any private handleClick(e: any) { - if (!this.getActive() || !e.geo) { + if (!this.getRecenterEnabled() || !e.geo) { return; } const buttonsDown = e.buttonsDown || (e.mouse && e.mouse.buttonsDown); From 7f9b5e2e691cbd756d8a16bfdf594aee3e66e383 Mon Sep 17 00:00:00 2001 From: romleiaj Date: Mon, 6 Jul 2026 16:39:27 -0400 Subject: [PATCH 36/79] Simplify the Align button tooltip to "Align View (requires calibration)" Co-Authored-By: Claude Fable 5 --- client/dive-common/components/Viewer.vue | 35 ++++-------------------- 1 file changed, 5 insertions(+), 30 deletions(-) diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index c60cf0cc7..f32b50cff 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -22,7 +22,7 @@ import { AlignedViewStore, StyleManager, TrackFilterControls, GroupFilterControls, } from 'vue-media-annotator/index'; -import { resolveToReferenceTransforms, unresolvedCameras } from 'vue-media-annotator/alignedView'; +import { resolveToReferenceTransforms } from 'vue-media-annotator/alignedView'; import { provideAnnotator, LassoModeSymbol } from 'vue-media-annotator/provides'; import { @@ -390,31 +390,7 @@ export default defineComponent({ const toggleAlignedView = () => { alignedView.setEnabled(!alignedView.enabled.value); }; - /** - * Why the Align button is disabled/inert right now, or the normal usage - * text: the button stays visible for every multicam dataset so an - * incomplete calibration or the picking suspension is explained rather - * than silently hiding it. - */ - const alignedViewTooltip = computed(() => { - if (!alignedViewAvailable.value) { - const missing = unresolvedCameras( - multiCamList.value, - multiCamList.value[0], - cameraCalibration.homographies.value, - ); - const cameraText = missing.length ? missing.join(', ') : 'some cameras'; - return `Aligned view unavailable: no transform maps ${cameraText} onto ` - + `${multiCamList.value[0]}. Pick points (or load a calibration) for the missing pair(s), ` - + 'then Save calibration.'; - } - if (alignedViewEnabled.value && !alignedViewActive.value) { - return 'Aligned view is on but suspended while "Pick points" is active in the ' - + 'calibration panel. Turn off point picking to apply it.'; - } - return `Aligned view: warp cameras into ${alignedViewReference.value}'s space and ` - + 'link pan/zoom. Editing is disabled while on.'; - }); + const alignedViewTooltip = 'Align View (requires calibration)'; // This context for removal const removeGroups = (id: AnnotationId) => { cameraStore.removeGroups(id); @@ -1719,14 +1695,13 @@ export default defineComponent({ Aligned view (SEAL-TK features 2 + 3): warp every camera's display into the reference camera's space and link pan/zoom across all panes. Usable when a transform exists for every non-reference - camera; shown (disabled, with the reason in the tooltip) for every - multicam dataset so an incomplete calibration is explained rather - than the button silently disappearing. + camera; shown (disabled) for every multicam dataset so an + incomplete calibration reads as "needs calibration" rather than + the button silently disappearing. --> {{ alignedViewTooltip }} - - - Aligned view is on: displays are warped into {{ alignedViewReference }}'s space and editing is disabled - From 267e34427f8ad24238691bc81316d8e6828c5179 Mon Sep 17 00:00:00 2001 From: romleiaj Date: Mon, 6 Jul 2026 17:04:45 -0400 Subject: [PATCH 38/79] Show only the active pair while picking; click-select and delete markers With 3+ cameras, picking now hides panes other than the Camera A / B selection (v-show, so hidden viewers keep their state; maps resize when panes hide/show). All panes return when picking turns off. Picked markers are selectable: grabbing one selects its correspondence, highlighted in BOTH cameras' panes and in the panel table (whose rows also click-select). Delete/Backspace -- or the existing per-row button -- removes the correspondence, i.e. the linked point in both cameras. Clicking empty space clears the selection and places a point as before; selection is authoring state and never persists. Co-Authored-By: Claude Fable 5 --- .../CameraCalibration/CalibrationTools.vue | 25 +++++++- client/dive-common/components/Viewer.vue | 26 ++++++++ client/src/CameraCalibrationStore.spec.ts | 59 ++----------------- client/src/CameraCalibrationStore.ts | 43 +++++++++++++- .../CalibrationKeypointLayer.ts | 54 ++++------------- 5 files changed, 109 insertions(+), 98 deletions(-) diff --git a/client/dive-common/components/CameraCalibration/CalibrationTools.vue b/client/dive-common/components/CameraCalibration/CalibrationTools.vue index c97b5c6a9..7a5a10930 100644 --- a/client/dive-common/components/CameraCalibration/CalibrationTools.vue +++ b/client/dive-common/components/CameraCalibration/CalibrationTools.vue @@ -47,6 +47,11 @@ export default defineComponent({ ); const minPoints = computed(() => minPointsForTransform(transformType.value)); const canFit = computed(() => correspondences.value.length >= minPoints.value); + const selectedCorrespondenceId = computed(() => calibration.selectedCorrespondenceId.value); + /** Delete the selected correspondence (both cameras' points). Bound to Del/Backspace. */ + function deleteSelectedCorrespondence() { + calibration.removeSelectedCorrespondence(); + } const canClearLast = computed( () => calibration.pendingPoint.value !== null || correspondences.value.length > 0, ); @@ -225,6 +230,8 @@ export default defineComponent({ pickingEnabled: calibration.pickingEnabled, alignment: calibration.alignment, fitError: calibration.fitError, + selectedCorrespondenceId, + deleteSelectedCorrespondence, cursorReadout, correspondences, transformType, @@ -253,7 +260,13 @@ export default defineComponent({