From c0b4e200ec6ca8786f61a3b971dc7fc9d2441b17 Mon Sep 17 00:00:00 2001 From: Matt Dawkins Date: Wed, 29 Apr 2026 05:48:39 -0400 Subject: [PATCH 01/10] Add interactive segmentation, text query, and stereo (desktop) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Brings in the SAM2/SAM3-based interactive segmentation feature, the SAM3 text-query workflow, and the desktop interactive stereo mode. Web-girder paths are intentionally untouched for now — web support will come in a follow-up. - New segmentation point-click recipe + EditorMenu wiring; SAM2/SAM3 models loaded via VIAME install configs. - Desktop backend: viame_segmentation_service-backed IPC handlers and matching frontend API for segmentationInitialize/Predict/SetImage/ ClearImage/Shutdown/IsReady, textQuery/refineDetections/ runTextQueryPipeline, and stereoEnable/Disable/SetFrame/GetStatus/ TransferLine/TransferPoints/SetCalibration/IsEnabled, plus disparity ready/error event hooks. - EditAnnotationLayer: track shift-key state and right-click for Point mode, propagate background flag for negative SAM points. - Sidebar / ViewerLoader / Viewer: stereo annotation mode UI, error dialog when seg or text-query model fails to load, dot-only-on-source -frame fix. - useModeManager / EditAnnotationLayer / recipes: keep existing geometry type when current editing mode already matches; right-click in Point creation finalises and deselects. --- client/bundle.css | 477 ++++++++++ client/dive-common/apispec.ts | 116 +++ .../dive-common/components/DeleteControls.vue | 3 + client/dive-common/components/EditorMenu.vue | 243 ++++- client/dive-common/components/Sidebar.vue | 73 ++ .../components/TrackSettingsPanel.vue | 86 ++ client/dive-common/components/Viewer.vue | 61 +- client/dive-common/recipes/headtail.ts | 55 +- .../recipes/segmentationpointclick.ts | 759 ++++++++++++++++ client/dive-common/store/settings.ts | 21 +- client/dive-common/use/useModeManager.ts | 402 ++++++++- client/platform/desktop/backend/ipcService.ts | 173 ++++ .../desktop/backend/native/segmentation.ts | 535 +++++++++++ .../platform/desktop/backend/native/stereo.ts | 668 ++++++++++++++ client/platform/desktop/frontend/api.ts | 207 +++++ .../frontend/components/ViewerLoader.vue | 843 ++++++++++++++++-- client/src/components/LayerManager.vue | 164 +++- .../src/layers/AnnotationLayers/LineLayer.ts | 19 +- .../SegmentationPointsLayer.ts | 76 ++ client/src/layers/EditAnnotationLayer.ts | 180 +++- client/src/provides.ts | 23 + client/src/recipe.ts | 4 + 22 files changed, 5104 insertions(+), 84 deletions(-) create mode 100644 client/bundle.css create mode 100644 client/dive-common/recipes/segmentationpointclick.ts create mode 100644 client/platform/desktop/backend/native/segmentation.ts create mode 100644 client/platform/desktop/backend/native/stereo.ts create mode 100644 client/src/layers/AnnotationLayers/SegmentationPointsLayer.ts diff --git a/client/bundle.css b/client/bundle.css new file mode 100644 index 000000000..b2f0abd65 --- /dev/null +++ b/client/bundle.css @@ -0,0 +1,477 @@ +.event-chart { + position: relative; + height: calc(100% - 10px); + margin: 5px 0; + overflow-y: auto; + overflow-x: hidden; +} +.event-chart .tooltip { + position: absolute; + background: black; + border: 1px solid white; + padding: 0px 5px; + font-size: 14px; + z-index: 2; +} + +.controls { + position: absolute; +} + +.video-annotator { + position: relative; + left: 0; + right: 0; + top: 0; + bottom: 0; + z-index: 0; + display: flex; + flex-direction: column; +} +.video-annotator .geojs-map { + margin: 2px; +} +.video-annotator .geojs-map.geojs-map:focus { + outline: none; +} +.video-annotator .playback-container { + flex: 1; +} +.video-annotator .loadingSpinnerContainer { + z-index: 20; + margin: 0; + position: absolute; + top: 50%; + left: 50%; + -ms-transform: translate(-50%, -50%); + transform: translate(-50%, -50%); +} +.video-annotator .geojs-map.annotation-input { + cursor: inherit; +} + +.selected-camera { + box-sizing: content-box; +} +.selected-camera .geojs-map { + outline: 3px cyan dashed; +} +.selected-camera .geojs-map.geojs-map:focus { + outline: 3px cyan dashed; +} + +.imageCursor { + z-index: 10; + position: fixed; + backface-visibility: hidden; + top: 0; + left: 0; + pointer-events: none; +} + +.controls { + bottom: 0; +} + +.controls { + position: absolute; +} + +.video-annotator { + position: relative; + left: 0; + right: 0; + top: 0; + bottom: 0; + z-index: 0; + display: flex; + flex-direction: column; +} +.video-annotator .geojs-map { + margin: 2px; +} +.video-annotator .geojs-map.geojs-map:focus { + outline: none; +} +.video-annotator .playback-container { + flex: 1; +} +.video-annotator .loadingSpinnerContainer { + z-index: 20; + margin: 0; + position: absolute; + top: 50%; + left: 50%; + -ms-transform: translate(-50%, -50%); + transform: translate(-50%, -50%); +} +.video-annotator .geojs-map.annotation-input { + cursor: inherit; +} + +.selected-camera { + box-sizing: content-box; +} +.selected-camera .geojs-map { + outline: 3px cyan dashed; +} +.selected-camera .geojs-map.geojs-map:focus { + outline: 3px cyan dashed; +} + +.imageCursor { + z-index: 10; + position: fixed; + backface-visibility: hidden; + top: 0; + left: 0; + pointer-events: none; +} + +.controls { + bottom: 0; +} + +.border-radius[data-v-77dee125] { + border: 1px solid #888888; + padding: 2px 5px; + border-radius: 5px; +} + +.line-chart { + height: 100%; +} +.line-chart .line { + fill: none; + stroke-width: 1.5px; +} +.line-chart .axis-y { + font-size: 12px; +} +.line-chart .axis-y g:first-of-type, +.line-chart .axis-y g:last-of-type { + display: none; +} +.line-chart .tooltip { + position: absolute; + background: black; + border: 1px solid white; + padding: 0px 5px; + font-size: 14px; +} + +.timeline .tick { + shape-rendering: crispEdges; + font-size: 12px; + stroke-opacity: 0.5; + stroke-dasharray: 2, 2; +} + +.timeline[data-v-0d0fe2ba] { + min-height: 175px; + position: relative; + display: flex; + flex-direction: column; +} +.timeline[data-v-0d0fe2ba] .work-area[data-v-0d0fe2ba] { + flex: 1; + position: relative; + overflow: hidden; +} +.timeline[data-v-0d0fe2ba] .work-area[data-v-0d0fe2ba] .hand[data-v-0d0fe2ba] { + position: absolute; + top: 0; + width: 0; + height: 100%; + border-left: 1px solid #299be3; + z-index: 10; +} +.timeline[data-v-0d0fe2ba] .work-area[data-v-0d0fe2ba] .time-filter-line[data-v-0d0fe2ba] { + position: absolute; + top: 0; + width: 0; + height: 100%; + z-index: 2; + cursor: col-resize; + pointer-events: auto; +} +.timeline[data-v-0d0fe2ba] .work-area[data-v-0d0fe2ba] .time-filter-tooltip[data-v-0d0fe2ba] { + position: absolute; + top: 30px; + transform: translateX(-50%); + background-color: rgba(0, 0, 0, 0.8); + color: white; + padding: 4px 8px; + border-radius: 4px; + font-size: 12px; + white-space: nowrap; + pointer-events: none; + z-index: 20; +} +.timeline[data-v-0d0fe2ba] .work-area[data-v-0d0fe2ba] .time-filter-start-line[data-v-0d0fe2ba] { + border-left: 3px solid #4caf50; +} +.timeline[data-v-0d0fe2ba] .work-area[data-v-0d0fe2ba] .time-filter-end-line[data-v-0d0fe2ba] { + border-left: 3px solid #f44336; +} +.timeline[data-v-0d0fe2ba] .work-area[data-v-0d0fe2ba] .time-filter-dimming[data-v-0d0fe2ba] { + position: absolute; + top: 0; + height: 100%; + background-color: rgba(0, 0, 0, 0.3); + pointer-events: none; + z-index: 1; +} +.timeline[data-v-0d0fe2ba] .work-area[data-v-0d0fe2ba] .child[data-v-0d0fe2ba] { + position: absolute; + top: 0; + bottom: 17px; + left: 0; + right: 0; + z-index: 0; +} +.timeline[data-v-0d0fe2ba] .minimap[data-v-0d0fe2ba] { + height: 10px; +} +.timeline[data-v-0d0fe2ba] .minimap[data-v-0d0fe2ba] .fill[data-v-0d0fe2ba] { + position: relative; + height: 100%; + background-color: #80c6e8; +} + +.controls { + position: absolute; +} + +.video-annotator { + position: relative; + left: 0; + right: 0; + top: 0; + bottom: 0; + z-index: 0; + display: flex; + flex-direction: column; +} +.video-annotator .geojs-map { + margin: 2px; +} +.video-annotator .geojs-map.geojs-map:focus { + outline: none; +} +.video-annotator .playback-container { + flex: 1; +} +.video-annotator .loadingSpinnerContainer { + z-index: 20; + margin: 0; + position: absolute; + top: 50%; + left: 50%; + -ms-transform: translate(-50%, -50%); + transform: translate(-50%, -50%); +} +.video-annotator .geojs-map.annotation-input { + cursor: inherit; +} + +.selected-camera { + box-sizing: content-box; +} +.selected-camera .geojs-map { + outline: 3px cyan dashed; +} +.selected-camera .geojs-map.geojs-map:focus { + outline: 3px cyan dashed; +} + +.imageCursor { + z-index: 10; + position: fixed; + backface-visibility: hidden; + top: 0; + left: 0; + pointer-events: none; +} + +.controls { + bottom: 0; +} + +.input-box { + border: 1px solid rgba(255, 255, 255, 0.15); + border-radius: 4px; + padding: 0 6px; + color: white; +} + +.trackNumber { + font-family: monospace; + max-width: 80px; + overflow: hidden; + white-space: nowrap; + text-overflow: ellipsis; +} +.trackNumber:hover { + cursor: pointer; + font-weight: bolder; + text-decoration: underline; +} + +.border-highlight[data-v-0d46f934] { + border-bottom: 1px solid gray; +} + +.type-checkbox[data-v-0d46f934] { + max-width: 80%; + overflow-wrap: anywhere; +} + +.hover-show-parent[data-v-0d46f934] .hover-show-child[data-v-0d46f934] { + display: none; +} +.hover-show-parent[data-v-0d46f934][data-v-0d46f934]:hover .hover-show-child[data-v-0d46f934] { + display: inherit; +} + +.outlined[data-v-0d46f934] { + background-color: gray; + color: #222; + font-weight: 600; + border-radius: 6px; + padding: 0 5px; + font-size: 12px; +} + +.input-box { + border: 1px solid rgba(255, 255, 255, 0.15); + border-radius: 4px; + padding: 0 6px; + color: white; +} + +.trackNumber { + font-family: monospace; + max-width: 80px; + overflow: hidden; + white-space: nowrap; + text-overflow: ellipsis; +} +.trackNumber:hover { + cursor: pointer; + font-weight: bolder; + text-decoration: underline; +} + +.freeform-input[data-v-d679c59c] { + width: 150px; +} + +.groups[data-v-c26ed586] { + overflow-y: auto; + overflow-x: hidden; +} + +.input-box { + border: 1px solid rgba(255, 255, 255, 0.15); + border-radius: 4px; + padding: 0 6px; + color: white; +} + +.trackNumber { + font-family: monospace; + max-width: 80px; + overflow: hidden; + white-space: nowrap; + text-overflow: ellipsis; +} +.trackNumber:hover { + cursor: pointer; + font-weight: bolder; + text-decoration: underline; +} + +.track-item[data-v-7a688bfe] { + border-radius: inherit; +} +.track-item[data-v-7a688bfe] .item-row[data-v-7a688bfe] { + width: 100%; +} +.track-item[data-v-7a688bfe] .type-color-box[data-v-7a688bfe] { + margin: 7px; + margin-top: 4px; + min-width: 15px; + max-width: 15px; + min-height: 15px; + max-height: 15px; +} + +.strcoller { + height: 100%; +} + +.trackHeader { + height: auto; +} + +.tracks { + overflow-y: auto; + overflow-x: hidden; +} +.tracks .v-input--checkbox label { + white-space: pre-wrap; +} + +.nowrap[data-v-a4da19c6] { + white-space: nowrap; + overflow: hidden; + max-width: var(--content-width); + text-overflow: ellipsis; +} + +.hover-show-parent[data-v-a4da19c6] .hover-show-child[data-v-a4da19c6] { + display: none; +} +.hover-show-parent[data-v-a4da19c6][data-v-a4da19c6]:hover .hover-show-child[data-v-a4da19c6] { + display: inherit; +} + +.outlined[data-v-a4da19c6] { + background-color: gray; + color: #222; + font-weight: 600; + border-radius: 6px; + padding: 0 5px; + font-size: 12px; +} + +.input-box { + border: 1px solid rgba(255, 255, 255, 0.15); + border-radius: 4px; + padding: 0 6px; + color: white; +} + +.trackNumber { + font-family: monospace; + max-width: 80px; + overflow: hidden; + white-space: nowrap; + text-overflow: ellipsis; +} +.trackNumber:hover { + cursor: pointer; + font-weight: bolder; + text-decoration: underline; +} + +.freeform-input[data-v-07a75698] { + width: 135px; +} + +.select-input[data-v-07a75698] { + width: 120px; + background-color: #1e1e1e; + appearance: menulist; +} \ No newline at end of file diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index e6e412077..d9ef2e455 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -266,6 +266,122 @@ function useApi() { return use>(ApiSymbol); } +/** + * Interactive Segmentation Types + */ +export interface SegmentationPredictRequest { + /** Path to the image file */ + imagePath: string; + /** Point coordinates as [x, y] pairs */ + points: [number, number][]; + /** Point labels: 1 for foreground, 0 for background */ + pointLabels: number[]; + /** Optional low-res mask from previous prediction for refinement */ + maskInput?: number[][]; + /** Whether to return multiple mask options */ + multimaskOutput?: boolean; +} + +export interface SegmentationPredictResponse { + /** Whether the prediction succeeded */ + success: boolean; + /** Error message if failed */ + error?: string; + /** Polygon coordinates as [x, y] pairs */ + polygon?: [number, number][]; + /** Bounding box [x_min, y_min, x_max, y_max] */ + bounds?: [number, number, number, number]; + /** Quality score from segmentation model */ + score?: number; + /** Low-res mask for subsequent refinement */ + lowResMask?: number[][]; + /** Mask dimensions [height, width] */ + maskShape?: [number, number]; + /** RLE-encoded full-resolution mask for display: [[value, count], ...] */ + rleMask?: [number, number][]; +} + +export interface SegmentationStatusResponse { + /** Whether segmentation is available */ + available: boolean; + /** Whether the model is currently loaded */ + loaded?: boolean; + /** Whether the service is ready for predictions */ + ready?: boolean; +} + +/** + * Text Query Types for open-vocabulary detection/segmentation + */ + +/** A single detection returned from a text query */ +export interface TextQueryDetection { + /** Bounding box [x1, y1, x2, y2] */ + box: [number, number, number, number]; + /** Polygon coordinates as [x, y] pairs */ + polygon?: [number, number][]; + /** Confidence score */ + score: number; + /** Label/class name (often the query text) */ + label: string; + /** Low-res mask for refinement (optional) */ + lowResMask?: number[][]; +} + +export interface TextQueryRequest { + /** Path to the image file */ + imagePath: string; + /** Text query describing what to find (e.g., "fish", "person swimming") */ + text: string; + /** Confidence threshold for detections (default: 0.3) */ + boxThreshold?: number; + /** Maximum number of detections to return (default: 10) */ + maxDetections?: number; + /** Optional boxes to refine [x1, y1, x2, y2][] */ + boxes?: [number, number, number, number][]; + /** Optional keypoints for refinement [x, y][] */ + points?: [number, number][]; + /** Labels for points: 1 for foreground, 0 for background */ + pointLabels?: number[]; + /** Optional masks to refine */ + masks?: number[][][]; +} + +export interface TextQueryResponse { + /** Whether the query succeeded */ + success: boolean; + /** Error message if failed */ + error?: string; + /** List of detections found */ + detections?: TextQueryDetection[]; + /** The original query text */ + query?: string; + /** Whether fallback method was used (no native text support) */ + fallback?: boolean; +} + +export interface RefineDetectionsRequest { + /** Path to the image file */ + imagePath: string; + /** Detections to refine */ + detections: TextQueryDetection[]; + /** Optional additional keypoints for refinement [x, y][] */ + points?: [number, number][]; + /** Labels for additional points: 1 for foreground, 0 for background */ + pointLabels?: number[]; + /** Whether to include refined masks in response */ + refineMasks?: boolean; +} + +export interface RefineDetectionsResponse { + /** Whether the refinement succeeded */ + success: boolean; + /** Error message if failed */ + error?: string; + /** Refined detections */ + detections?: TextQueryDetection[]; +} + export { provideApi, useApi, diff --git a/client/dive-common/components/DeleteControls.vue b/client/dive-common/components/DeleteControls.vue index 50ecdd08a..f6022ac8a 100644 --- a/client/dive-common/components/DeleteControls.vue +++ b/client/dive-common/components/DeleteControls.vue @@ -24,6 +24,9 @@ export default Vue.extend({ if (this.editingMode === 'rectangle') { return true; // deleting rectangle is unsupported } + if (this.editingMode === 'Point') { + return true; // Point mode uses reset instead of delete + } return false; }, }, diff --git a/client/dive-common/components/EditorMenu.vue b/client/dive-common/components/EditorMenu.vue index 2a02fdb9e..f29e6a6f7 100644 --- a/client/dive-common/components/EditorMenu.vue +++ b/client/dive-common/components/EditorMenu.vue @@ -11,6 +11,7 @@ import { flatten } from 'lodash'; import { Mousetrap } from 'vue-media-annotator/types'; import { EditAnnotationTypes, VisibleAnnotationTypes } from 'vue-media-annotator/layers'; import Recipe from 'vue-media-annotator/recipe'; +import SegmentationPointClick from 'dive-common/recipes/segmentationpointclick'; import AnnotationVisibilityMenu from './AnnotationVisibilityMenu.vue'; @@ -19,6 +20,7 @@ interface ButtonData { icon: string; type?: VisibleAnnotationTypes; active: boolean; + loading?: boolean; mousetrap?: Mousetrap[]; description: string; click: () => void; @@ -75,7 +77,14 @@ export default defineComponent({ default: true, }, }, - emits: ['set-annotation-state', 'update:tail-settings', 'update:show-user-created-icon'], + emits: [ + 'set-annotation-state', + 'update:tail-settings', + 'update:show-user-created-icon', + 'text-query-init', + 'text-query', + 'text-query-all-frames', + ], setup(props, { emit }) { const toolTimeTimeout = ref(null); const STORAGE_KEY = 'editorMenu.editButtonsExpanded'; @@ -93,6 +102,59 @@ export default defineComponent({ localStorage.setItem(STORAGE_KEY, String(value)); }); + // Text query state + const textQueryDialogOpen = ref(false); + const textQueryInput = ref(''); + const textQueryLoading = ref(false); + const textQueryThreshold = ref(0.3); + const textQueryInitializing = ref(false); + const textQueryServiceError = ref(''); + const textQueryAllFrames = ref(false); + + const openTextQueryDialog = () => { + textQueryDialogOpen.value = true; + textQueryInput.value = ''; + textQueryServiceError.value = ''; + textQueryAllFrames.value = false; + textQueryInitializing.value = true; + emit('text-query-init'); + }; + + const closeTextQueryDialog = () => { + textQueryDialogOpen.value = false; + textQueryInput.value = ''; + textQueryServiceError.value = ''; + textQueryInitializing.value = false; + textQueryAllFrames.value = false; + }; + + const onTextQueryServiceReady = (success: boolean, error?: string) => { + textQueryInitializing.value = false; + if (!success) { + textQueryServiceError.value = error || 'Text query service is not available'; + } + }; + + const submitTextQuery = () => { + if (!textQueryInput.value.trim()) { + return; + } + textQueryLoading.value = true; + if (textQueryAllFrames.value) { + emit('text-query-all-frames', { + text: textQueryInput.value.trim(), + boxThreshold: textQueryThreshold.value, + }); + } else { + emit('text-query', { + text: textQueryInput.value.trim(), + boxThreshold: textQueryThreshold.value, + }); + } + closeTextQueryDialog(); + textQueryLoading.value = false; + }; + const modeToolTips = { Creating: { rectangle: 'Drag to draw rectangle. Press ESC to exit.', @@ -129,6 +191,7 @@ export default defineComponent({ id: r.name, icon: r.icon.value || 'mdi-pencil', active: props.editingTrack && r.active.value, + loading: r.loading?.value ?? false, description: r.name, click: () => r.activate(), mousetrap: [ @@ -142,7 +205,13 @@ export default defineComponent({ ]; }); - const mousetrap = computed((): Mousetrap[] => flatten(editButtons.value.map((b) => b.mousetrap || []))); + const mousetrap = computed((): Mousetrap[] => [ + ...flatten(editButtons.value.map((b) => b.mousetrap || [])), + { + bind: 't', + handler: () => openTextQueryDialog(), + }, + ]); const activeEditButton = computed(() => editButtons.value.find((b) => b.active) || editButtons.value[0]); @@ -175,6 +244,13 @@ export default defineComponent({ return { text: 'Not editing', icon: 'mdi-pencil-off-outline', color: '' }; }); + const activeSegmentationRecipe = computed((): SegmentationPointClick | null => { + const segRecipe = props.recipes.find( + (r) => r instanceof SegmentationPointClick && r.active.value, + ) as SegmentationPointClick | undefined; + return segRecipe || null; + }); + const editingTooltip = computed(() => { if (props.editingDetails === 'disabled' || !props.editingMode || typeof props.editingMode !== 'string') { return ''; @@ -208,6 +284,19 @@ export default defineComponent({ toggleEditButtonsExpanded, activeEditButton, editButtonsMenuKey, + activeSegmentationRecipe, + // Text query + textQueryDialogOpen, + textQueryInput, + textQueryLoading, + textQueryThreshold, + textQueryInitializing, + textQueryServiceError, + textQueryAllFrames, + openTextQueryDialog, + closeTextQueryDialog, + onTextQueryServiceReady, + submitTextQuery, }; }, }); @@ -265,7 +354,7 @@ export default defineComponent({ + + +
T:
+ mdi-text-search +
+ + - + + @@ -357,6 +485,103 @@ export default defineComponent({ @update:show-user-created-icon="$emit('update:show-user-created-icon', $event)" /> + + + + + + + mdi-text-search + + Text Query + + + +
+ +

+ Loading text query model... +

+
+ +
+ + mdi-alert-circle + +

+ {{ textQueryServiceError }} +

+
+ + +
+ + + + {{ textQueryServiceError ? 'Close' : 'Cancel' }} + + + Search + + +
+
diff --git a/client/dive-common/components/Sidebar.vue b/client/dive-common/components/Sidebar.vue index aeeb45802..aa28551ba 100644 --- a/client/dive-common/components/Sidebar.vue +++ b/client/dive-common/components/Sidebar.vue @@ -49,6 +49,10 @@ export default defineComponent({ type: Boolean, default: false, }, + isStereoDataset: { + type: Boolean, + default: false, + }, }, setup() { @@ -230,6 +234,7 @@ export default defineComponent({ @@ -313,6 +318,7 @@ export default defineComponent({ diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index 4b60ed02d..49de2c3fc 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -1,6 +1,6 @@ - + + diff --git a/client/src/components/LayerManager.vue b/client/src/components/LayerManager.vue index 3679efcae..da3c57b53 100644 --- a/client/src/components/LayerManager.vue +++ b/client/src/components/LayerManager.vue @@ -41,7 +41,9 @@ import { useAttributes, useComparisonSets, useLassoModeContext, + useSegmentationPoints, } from '../provides'; +import SegmentationPointsLayer from '../layers/AnnotationLayers/SegmentationPointsLayer'; /** LayerManager is a component intended to be used as a child of an Annotator. * It provides logic for switching which layers are visible, but more importantly @@ -170,6 +172,20 @@ export default defineComponent({ setLassoDrawing, ); + // Segmentation points layer for displaying prompt points during point-click segmentation + const segmentationPointsRef = useSegmentationPoints(); + const segmentationPointsLayer = new SegmentationPointsLayer(annotator); + + // Watch for segmentation points updates - only show points for current frame + watch([segmentationPointsRef, frameNumberRef], ([newPoints, currentFrame]) => { + // Only display points if they belong to the current frame + if (newPoints.points.length > 0 && newPoints.frameNum === currentFrame) { + segmentationPointsLayer.updatePoints(newPoints.points, newPoints.labels); + } else { + segmentationPointsLayer.clear(); + } + }, { deep: true }); + const updateAttributes = () => { const newList = attributes.value.filter((item) => item.render).sort((a, b) => { if (a.render && b.render) { @@ -404,6 +420,7 @@ export default defineComponent({ typeStylingRef, toRef(props, 'colorBy'), selectedCamera, + selectedKeyRef, ], () => { updateLayers( @@ -476,20 +493,108 @@ export default defineComponent({ //So we only want to pass the click whjen not in creation mode or editing mode for features if (editAnnotationLayer.getMode() !== 'creation') { editAnnotationLayer.disable(); - handler.trackSelect(trackId, editing, modifiers); + // When entering editing mode (right-click), use trackEdit so the + // geometry type is auto-detected (e.g. LineString vs rectangle). + if (editing && trackId !== null) { + handler.trackEdit(trackId); + } else { + handler.trackSelect(trackId, editing, modifiers); + } + } else if (editing && trackId !== null) { + // Right-click on another detection while in creation mode: + // cancel creation and switch to editing the clicked detection + editAnnotationLayer.disable(); + handler.trackEdit(trackId); } }; //Sync of internal geoJS state with the application - editAnnotationLayer.bus.$on('editing-annotation-sync', (editing: boolean) => { - handler.trackSelect(selectedTrackIdRef.value, editing); + editAnnotationLayer.bus.$on('editing-annotation-sync', (editing: boolean, deselect?: boolean) => { + if (deselect) { + handler.trackSelect(null, false); + } else { + handler.trackSelect(selectedTrackIdRef.value, editing); + } + }); + // Handle right-click to confirm/lock annotation in Point mode (segmentation) + editAnnotationLayer.bus.$on('confirm-annotation', () => { + handler.confirmRecipe(); }); rectAnnotationLayer.bus.$on('annotation-clicked', Clicked); rectAnnotationLayer.bus.$on('annotation-right-clicked', Clicked); rectAnnotationLayer.bus.$on('annotation-ctrl-clicked', Clicked); polyAnnotationLayer.bus.$on('annotation-clicked', Clicked); polyAnnotationLayer.bus.$on('annotation-right-clicked', Clicked); + // Handle right-click polygon selection for multi-polygon support + polyAnnotationLayer.bus.$on('polygon-right-clicked', (_trackId: number, polygonKey: string) => { + // If in creation mode, cancel it first so we can select the polygon + if (editAnnotationLayer.getMode() === 'creation') { + handler.cancelCreation(); + } + // Set the polygon key for the right-clicked polygon + handler.selectFeatureHandle(-1, polygonKey); + // Force layer update to load the selected polygon + // This is especially important when already editing the same track + // since annotation-right-clicked won't be emitted in that case + window.setTimeout(() => { + updateLayers( + frameNumberRef.value, + editingModeRef.value, + selectedTrackIdRef.value, + multiSeletListRef.value, + enabledTracksRef.value, + visibleModesRef.value, + selectedKeyRef.value, + props.colorBy, + ); + }, 0); + }); polyAnnotationLayer.bus.$on('annotation-ctrl-clicked', Clicked); + lineLayer.bus.$on('annotation-clicked', Clicked); + lineLayer.bus.$on('annotation-right-clicked', Clicked); + // Handle polygon selection for multi-polygon support + polyAnnotationLayer.bus.$on('polygon-clicked', (_trackId: number, polygonKey: string) => { + // If in creation mode, don't interrupt - let the edit layer handle clicks for placing points + // This is important for hole drawing where left-clicks place hole vertices + if (editAnnotationLayer.getMode() === 'creation') { + return; + } + handler.selectFeatureHandle(-1, polygonKey); + // Force layer update to load the newly selected polygon + // Use nextTick to ensure the selectedKey ref has been updated + window.setTimeout(() => { + updateLayers( + frameNumberRef.value, + editingModeRef.value, + selectedTrackIdRef.value, + multiSeletListRef.value, + enabledTracksRef.value, + visibleModesRef.value, + selectedKeyRef.value, + props.colorBy, + ); + }, 0); + }); + // Handle right-click outside polygons to finalize/cancel creation + polyAnnotationLayer.bus.$on('polygon-right-clicked-outside', () => { + if (editAnnotationLayer.getMode() === 'creation') { + // Cancel creation and go back to editing the default polygon + handler.cancelCreation(); + handler.selectFeatureHandle(-1, ''); + window.setTimeout(() => { + updateLayers( + frameNumberRef.value, + editingModeRef.value, + selectedTrackIdRef.value, + multiSeletListRef.value, + enabledTracksRef.value, + visibleModesRef.value, + selectedKeyRef.value, + props.colorBy, + ); + }, 0); + } + }); editAnnotationLayer.bus.$on('update:geojson', ( mode: 'in-progress' | 'editing', geometryCompleteEvent: boolean, @@ -525,8 +630,59 @@ export default defineComponent({ }); editAnnotationLayer.bus.$on( 'update:selectedIndex', - (index: number, _type: EditAnnotationTypes, key = '') => handler.selectFeatureHandle(index, key), + (index: number, _type: EditAnnotationTypes, key?: string) => { + // When deselecting (index -1), don't change the key - it may have been + // set by polygon-right-clicked/polygon-clicked for multi-polygon selection + if (index >= 0 && key !== undefined) { + handler.selectFeatureHandle(index, key); + } else { + // Just update the handle index, preserve the current key + handler.selectFeatureHandle(index, selectedKeyRef.value); + } + }, ); + // Handle clicks outside the edit polygon to allow selecting other polygons + editAnnotationLayer.bus.$on('click-outside-edit', (geo: { x: number; y: number }) => { + // Check which polygon was clicked by iterating through formatted data + const point: [number, number] = [geo.x, geo.y]; + const polygonData = polyAnnotationLayer.formattedData; + + // Find the polygon that contains the click point + const clickedPolygon = polygonData.find((data) => { + const coords = data.polygon.coordinates[0] as [number, number][]; + // Ray casting algorithm + let inside = false; + for (let i = 0, j = coords.length - 1; i < coords.length; j = i, i += 1) { + const xi = coords[i][0]; + const yi = coords[i][1]; + const xj = coords[j][0]; + const yj = coords[j][1]; + const intersect = ((yi > point[1]) !== (yj > point[1])) + && (point[0] < ((xj - xi) * (point[1] - yi)) / (yj - yi) + xi); + if (intersect) inside = !inside; + } + return inside; + }); + + if (clickedPolygon) { + const polygonKey = clickedPolygon.polygonKey || ''; + // Select the clicked polygon + handler.selectFeatureHandle(-1, polygonKey); + // Force layer update to load the newly selected polygon + window.setTimeout(() => { + updateLayers( + frameNumberRef.value, + editingModeRef.value, + selectedTrackIdRef.value, + multiSeletListRef.value, + enabledTracksRef.value, + visibleModesRef.value, + selectedKeyRef.value, + props.colorBy, + ); + }, 0); + } + }); const annotationHoverTooltip = ( found: { styleType: [string, number]; diff --git a/client/src/layers/AnnotationLayers/LineLayer.ts b/client/src/layers/AnnotationLayers/LineLayer.ts index d4c2516fa..8f814003b 100644 --- a/client/src/layers/AnnotationLayers/LineLayer.ts +++ b/client/src/layers/AnnotationLayers/LineLayer.ts @@ -1,4 +1,5 @@ /* eslint-disable class-methods-use-this */ +import geo, { GeoEvent } from 'geojs'; import { cloneDeep } from 'lodash'; import BaseLayer, { LayerStyle, BaseLayerParams } from '../BaseLayer'; @@ -24,7 +25,23 @@ export default class LineLayer extends BaseLayer { const layer = this.annotator.geoViewerRef.value.createLayer('feature', { features: ['point', 'line'], }); - this.featureLayer = layer.createFeature('line'); + this.featureLayer = layer + .createFeature('line', { selectionAPI: true }) + .geoOn(geo.event.feature.mouseclick, (e: GeoEvent) => { + if (e.mouse.buttonsDown.left) { + if (!e.data.editing || (e.data.editing && !e.data.selected)) { + this.bus.$emit('annotation-clicked', e.data.trackId, false); + } + } else if (e.mouse.buttonsDown.right) { + if (!e.data.editing || (e.data.editing && !e.data.selected)) { + this.bus.$emit('annotation-right-clicked', e.data.trackId, true); + } + } + }); + this.featureLayer.geoOn( + geo.event.feature.mouseclick_order, + this.featureLayer.mouseOverOrderClosestBorder, + ); super.initialize(); } diff --git a/client/src/layers/AnnotationLayers/SegmentationPointsLayer.ts b/client/src/layers/AnnotationLayers/SegmentationPointsLayer.ts new file mode 100644 index 000000000..09db4c186 --- /dev/null +++ b/client/src/layers/AnnotationLayers/SegmentationPointsLayer.ts @@ -0,0 +1,76 @@ +import { MediaController } from '../../components/annotators/mediaControllerType'; + +interface SegmentationPointData { + x: number; + y: number; + label: number; // 1=foreground, 0=background +} + +/** + * Layer for displaying segmentation prompt points (green=foreground, red=background) + * This is a simple layer that doesn't follow the BaseLayer pattern since it's + * not tied to track data - it's UI feedback during the segmentation process. + */ +export default class SegmentationPointsLayer { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + private featureLayer: any; + + private annotator: MediaController; + + private points: SegmentationPointData[] = []; + + constructor(annotator: MediaController) { + this.annotator = annotator; + this.initialize(); + } + + private initialize() { + const layer = this.annotator.geoViewerRef.value.createLayer('feature', { + features: ['point'], + }); + this.featureLayer = layer.createFeature('point'); + this.featureLayer.style({ + radius: 8, + strokeWidth: 2, + strokeColor: (data: SegmentationPointData) => (data.label === 1 ? '#00FF00' : '#FF0000'), + fillColor: (data: SegmentationPointData) => (data.label === 1 ? '#00FF00' : '#FF0000'), + fillOpacity: 0.6, + strokeOpacity: 1, + }); + this.featureLayer.position((data: SegmentationPointData) => ({ + x: data.x, + y: data.y, + })); + } + + /** + * Update the displayed points + * @param points - Array of [x, y] coordinates + * @param labels - Array of labels (1=foreground, 0=background) + */ + updatePoints(points: [number, number][], labels: number[]) { + this.points = points.map((p, i) => ({ + x: p[0], + y: p[1], + label: labels[i] ?? 1, + })); + this.redraw(); + } + + /** + * Clear all displayed points + */ + clear() { + this.points = []; + this.redraw(); + } + + private redraw() { + this.featureLayer.data(this.points).draw(); + } + + disable() { + this.points = []; + this.featureLayer.data([]).draw(); + } +} diff --git a/client/src/layers/EditAnnotationLayer.ts b/client/src/layers/EditAnnotationLayer.ts index 4c1204a1a..17d7f9ece 100644 --- a/client/src/layers/EditAnnotationLayer.ts +++ b/client/src/layers/EditAnnotationLayer.ts @@ -85,6 +85,17 @@ export default class EditAnnotationLayer extends BaseLayer { unrotatedGeoJSONCoords: GeoJSON.Position[] | null; + /* Track if the last click was a right-click or shift-click for Point mode */ + lastClickWasBackground: boolean; + + /* Track shift key state from native DOM events (more reliable than GeoJS events) */ + lastShiftKeyState: boolean; + + /* Bound event handlers for cleanup */ + private boundTrackShiftKey: ((e: MouseEvent) => void) | null = null; + + private boundHandleContextMenu: ((e: MouseEvent) => void) | null = null; + constructor(params: BaseLayerParams & EditAnnotationLayerParams) { super(params); this.skipNextExternalUpdate = false; @@ -97,11 +108,83 @@ export default class EditAnnotationLayer extends BaseLayer { this.disableModeSync = false; this.leftButtonCheckTimeout = -1; this.unrotatedGeoJSONCoords = null; + this.lastClickWasBackground = false; + this.lastShiftKeyState = false; + + // Bind event handlers once (listeners are added/removed dynamically based on type) + this.boundTrackShiftKey = this.trackShiftKey.bind(this); + this.boundHandleContextMenu = this.handleContextMenu.bind(this); + + // Add listeners if starting in Point mode + if (this.type === 'Point') { + this.addPointModeListeners(); + } //Only initialize once, prevents recreating Layer each edit this.initialize(); } + /** + * Add event listeners needed for Point mode (segmentation). + */ + private addPointModeListeners() { + if (this.boundTrackShiftKey) { + document.addEventListener('mousedown', this.boundTrackShiftKey, true); + } + if (this.boundHandleContextMenu) { + document.addEventListener('contextmenu', this.boundHandleContextMenu, true); + } + } + + /** + * Remove event listeners used for Point mode. + */ + private removePointModeListeners() { + if (this.boundTrackShiftKey) { + document.removeEventListener('mousedown', this.boundTrackShiftKey, true); + } + if (this.boundHandleContextMenu) { + document.removeEventListener('contextmenu', this.boundHandleContextMenu, true); + } + } + + /** + * Track shift key state from native DOM mousedown events. + * This is more reliable than GeoJS events for detecting shift+click. + */ + trackShiftKey(e: MouseEvent) { + this.lastShiftKeyState = e.shiftKey; + // Also track middle-click (button 1) from native events for background points + if (e.button === 1 && this.type === 'Point' && this.getMode() === 'creation') { + this.lastClickWasBackground = true; + } + } + + /** + * Handle right-click context menu in Point mode. + * In segmentation mode, right-click confirms/locks the annotation. + * Prevents browser context menu. + */ + handleContextMenu(e: MouseEvent) { + if (this.type === 'Point' && this.getMode() === 'creation') { + // Prevent the browser context menu + e.preventDefault(); + e.stopPropagation(); + + // Emit confirm event to lock the annotation (like other edit modes) + this.bus.$emit('confirm-annotation'); + } + } + + /** + * Clean up event listeners when the layer is destroyed. + */ + destroy() { + this.removePointModeListeners(); + this.boundTrackShiftKey = null; + this.boundHandleContextMenu = null; + } + /** * Initialization of the layer should only be done once for edit layers * Handlers for edit_action and state which will emit data when necessary @@ -130,6 +213,13 @@ export default class EditAnnotationLayer extends BaseLayer { (e: GeoEvent) => this.hoverEditHandle(e), ); this.featureLayer.geoOn(geo.event.mouseclick, (e: GeoEvent) => { + // Right-click in creation mode (non-Point): cancel and fully deselect. + // Point mode has its own right-click handler (handleContextMenu). + if (e.buttonsDown.right && this.getMode() === 'creation' && this.type !== 'Point') { + this.shapeInProgress = null; + this.bus.$emit('editing-annotation-sync', false, true); + return; + } //Used to sync clicks that kick out of editing mode with application //This prevents that pseudo Edit state when left clicking on a object in edit mode if (!this.disableModeSync && (e.buttonsDown.left) @@ -184,10 +274,56 @@ export default class EditAnnotationLayer extends BaseLayer { * shape that GeoJS is keeps internally. Emit the shape as update:in-progress-geojson */ setShapeInProgress(e: GeoEvent) { - // Allow middle click movement when placing points - if (e.mouse.buttons.middle && !e.propogated) { + // Allow middle click movement when placing points (except in Point mode where it creates background points) + if (e.mouse.buttons.middle && !e.propogated && this.type !== 'Point') { return; } + // Right-click should never add vertices - cancel/confirm is handled by + // mouseclick (line/polygon cancel) and contextmenu (Point confirm) + if (e.mouse.buttons.right) { + return; + } + + // Track if this is a background point (shift+click or middle-click) for Point mode + // Check both GeoJS event modifiers and our native DOM event tracking for reliability + // Preserve the value if it was already set to true by trackShiftKey (native event) + if (this.type === 'Point' && this.getMode() === 'creation') { + this.lastClickWasBackground = this.lastClickWasBackground + || e.mouse.buttons.middle + || e.mouse.modifiers.shift + || this.lastShiftKeyState; + } + + // Handle middle-click in Point mode - GeoJS doesn't create points on middle-click, + // so we need to manually create the point and emit the event + if (this.type === 'Point' && this.getMode() === 'creation' && e.mouse.buttons.middle) { + const pointGeojson: GeoJSON.Feature = { + type: 'Feature', + geometry: { + type: 'Point', + coordinates: [Math.round(e.mouse.geo.x), Math.round(e.mouse.geo.y)], + }, + properties: { + background: true, + }, + }; + + // Emit the point creation event directly + this.bus.$emit( + 'update:geojson', + 'editing', + true, // geometryCompleteEvent - point is complete + pointGeojson, + this.type, + this.selectedKey, + this.skipNextFunc(), + ); + + // Reset background flag for next point + this.lastClickWasBackground = false; + return; + } + if (this.getMode() === 'creation' && ['LineString', 'Polygon'].includes(this.type)) { if (this.shapeInProgress === null) { // Initialize a new in-progress shape @@ -205,6 +341,25 @@ export default class EditAnnotationLayer extends BaseLayer { } else { const coords = this.shapeInProgress?.coordinates as GeoJSON.Position[]; coords.push(newPoint); + // Auto-complete LineString after 2 points (simple line with 2 endpoints) + if (coords.length >= 2) { + const feature: GeoJSON.Feature = { + type: 'Feature', + geometry: this.shapeInProgress!, + properties: {}, + }; + this.shapeInProgress = null; + this.disableModeSync = true; + this.bus.$emit( + 'update:geojson', + 'editing', + true, // geometryCompleteEvent - line is complete + feature, + this.type, + this.selectedKey, + ); + return; + } } this.bus.$emit( 'update:geojson', @@ -281,9 +436,19 @@ export default class EditAnnotationLayer extends BaseLayer { /** * Set the current Editing type for switching between editing polygons or rects. - * */ + * Also manages event listeners that are only needed for Point mode (segmentation). + */ setType(type: EditAnnotationTypes) { + const wasPoint = this.type === 'Point'; + const isPoint = type === 'Point'; this.type = type; + + // Add or remove Point mode listeners based on type change + if (!wasPoint && isPoint) { + this.addPointModeListeners(); + } else if (wasPoint && !isPoint) { + this.removePointModeListeners(); + } } setKey(key: string) { @@ -479,6 +644,15 @@ export default class EditAnnotationLayer extends BaseLayer { const geoJSONData = [e.annotation.geojson()]; this.unrotatedGeoJSONCoords = geoJSONData[0].geometry.coordinates[0] as GeoJSON.Position[]; + + // For Point mode, add background property if it was a right-click or shift-click + if (this.type === 'Point' && this.lastClickWasBackground) { + geoJSONData[0].properties = { + ...geoJSONData[0].properties, + background: true, + }; + this.lastClickWasBackground = false; // Reset for next point + } this.formattedData = geoJSONData; // The new annotation is in a state without styling, so apply local stypes this.applyStylesToAnnotations(); diff --git a/client/src/provides.ts b/client/src/provides.ts index 1ee7e1a7d..cb039a60f 100644 --- a/client/src/provides.ts +++ b/client/src/provides.ts @@ -8,6 +8,7 @@ import type { EditAnnotationTypes } from './layers/EditAnnotationLayer'; import type { AnnotationId, StringKeyObject } from './BaseAnnotation'; import type { VisibleAnnotationTypes } from './layers'; import type { RectBounds } from './utils'; +import type { TrackSupportedFeature } from './track'; import type { Attribute, AttributeFilter, @@ -54,6 +55,9 @@ type EditingModeType = Readonly>; const MultiSelectSymbol = Symbol('multiSelect'); type MultiSelectType = Readonly>; +const SegmentationPointsSymbol = Symbol('segmentationPoints'); +type SegmentationPointsType = Readonly>; + const PendingSaveCountSymbol = Symbol('pendingSaveCount'); type pendingSaveCountType = Readonly>; @@ -126,6 +130,8 @@ export interface Handler { seekFrame(frame: number): void; /* Toggle editing mode for track */ trackEdit(AnnotationId: AnnotationId): void; + /* Confirm/lock the current annotation for active recipes */ + confirmRecipe(): void; /* toggle selection mode for track */ trackSelect(AnnotationId: AnnotationId | null, edit: boolean, modifiers?: { ctrl: boolean }): void; /* select tracks enclosed by a lasso polygon */ @@ -143,6 +149,13 @@ export interface Handler { bounds: RectBounds, rotation?: number, ): void; + /* Set a feature on the selected track with proper interpolation handling */ + setTrackFeature( + frameNum: number, + bounds: RectBounds, + geometry: GeoJSON.Feature[], + runAfterLogic?: boolean, + ): void; /* update geojson for track */ updateGeoJSON( eventType: 'in-progress' | 'editing', @@ -209,12 +222,14 @@ function dummyHandler(handle: (name: string, args: unknown[]) => void): Handler trackSeek(...args) { handle('trackSeek', args); }, seekFrame(...args) { handle('seekFrame', args); }, trackEdit(...args) { handle('trackEdit', args); }, + confirmRecipe(...args) { handle('confirmRecipe', args); }, trackSelect(...args) { handle('trackSelect', args); }, lassoSelect(...args) { handle('lassoSelect', args); }, trackSelectNext(...args) { handle('trackSelectNext', args); }, trackSplit(...args) { handle('trackSplit', args); }, trackAdd(...args) { handle('trackAdd', args); return 0; }, updateRectBounds(...args) { handle('updateRectBounds', args); }, + setTrackFeature(...args) { handle('setTrackFeature', args); }, updateGeoJSON(...args) { handle('updateGeoJSON', args); }, removeTrack(...args) { handle('removeTrack', args); }, removeGroup(...args) { handle('removeGroup', args); }, @@ -262,6 +277,7 @@ export interface State { annotationSet: AnnotationSetType; annotationSets: AnnotationSetsType; comparisonSets: ComparisonSetsType; + segmentationPoints: SegmentationPointsType; selectedCamera: SelectedCameraType; selectedKey: SelectedKeyType; selectedTrackId: SelectedTrackIdType; @@ -328,6 +344,7 @@ function dummyState(): State { comparisonSets: ref([]), groupFilters: groupFilterControls, groupStyleManager: new StyleManager({ markChangesPending }), + segmentationPoints: ref({ points: [], labels: [], frameNum: -1 }), selectedCamera: ref('singleCam'), selectedKey: ref(''), selectedTrackId: ref(null), @@ -377,6 +394,7 @@ function provideAnnotator(state: State, handler: Handler, attributesFilters: Att provide(AnnotationSetSymbol, state.annotationSet); provide(AnnotationSetsSymbol, state.annotationSets); provide(ComparisonSetsSymbol, state.comparisonSets); + provide(SegmentationPointsSymbol, state.segmentationPoints); provide(TrackFilterControlsSymbol, state.trackFilters); provide(TrackStyleManagerSymbol, state.trackStyleManager); provide(SelectedCameraSymbol, state.selectedCamera); @@ -513,6 +531,10 @@ function useImageEnhancements() { return use(ImageEnhancementsSymbol); } +function useSegmentationPoints() { + return use(SegmentationPointsSymbol); +} + export { LassoModeSymbol, dummyHandler, @@ -547,4 +569,5 @@ export { useReadOnlyMode, useImageEnhancements, useAttributesFilters, + useSegmentationPoints, }; diff --git a/client/src/recipe.ts b/client/src/recipe.ts index efd5635b7..bd259fa08 100644 --- a/client/src/recipe.ts +++ b/client/src/recipe.ts @@ -28,6 +28,8 @@ interface Recipe { icon: Ref; active: Ref; toggleable: Ref; + /** Whether the recipe is currently loading (e.g., initializing models) */ + loading?: Ref; bus: Vue; update: ( mode: 'in-progress' | 'editing', @@ -52,6 +54,8 @@ interface Recipe { activate: () => unknown; mousetrap: () => Mousetrap[]; deactivate: () => void; + /** Optional method to confirm/lock the current annotation (e.g., for segmentation) */ + confirm?: () => void; } export default Recipe; From b6c7561eb7c240386133f814df5f32274578ffb9 Mon Sep 17 00:00:00 2001 From: Matt Dawkins Date: Wed, 29 Apr 2026 05:50:12 -0400 Subject: [PATCH 02/10] Multi-polygon support with holes A track-frame's polygon now expands to a list of polygons each with their own keys, and each polygon supports holes. - Server CSV (de)serializer: emit polygon-key column per polygon, support holes in the geoJSON FeatureCollection; auto_key path to append a new polygon to an existing track frame. - Client recipes / useModeManager: handleAddHole / handleAddPolygon / handleCancelCreation; PolygonLayer emits polygon-clicked. - Hole drawing reuses the polygon edit pipeline (left-click places a hole vertex without exiting creation mode). - Test fixtures cover multi-polygon and polygons-with-holes round-trip. --- .../dive-common/components/DeleteControls.vue | 122 +++++++++-- client/dive-common/recipes/polygonbase.ts | 201 +++++++++++++++++- client/dive-common/use/useModeManager.ts | 79 ++++++- .../desktop/backend/native/segmentation.ts | 4 +- .../platform/desktop/backend/native/stereo.ts | 4 +- .../desktop/backend/serializers/viame.ts | 88 +++++++- .../layers/AnnotationLayers/PolygonLayer.ts | 132 ++++++++++-- client/src/layers/EditAnnotationLayer.ts | 179 ++++++++++++++-- client/src/provides.ts | 9 + client/src/track.ts | 123 ++++++++++- server/dive_utils/models.py | 3 +- server/dive_utils/serializers/viame.py | 105 +++++++-- server/tests/test_serialize_viame_csv.py | 123 +++++++++++ testutils/viame.spec.json | 197 +++++++++++++++++ 14 files changed, 1263 insertions(+), 106 deletions(-) diff --git a/client/dive-common/components/DeleteControls.vue b/client/dive-common/components/DeleteControls.vue index f6022ac8a..3d10226e5 100644 --- a/client/dive-common/components/DeleteControls.vue +++ b/client/dive-common/components/DeleteControls.vue @@ -29,6 +29,15 @@ export default Vue.extend({ } return false; }, + isPolygonMode(): boolean { + return this.editingMode === 'Polygon'; + }, + editModeIcon(): string { + if (this.editingMode === 'Polygon') return 'mdi-vector-polygon'; + if (this.editingMode === 'LineString') return 'mdi-vector-line'; + if (this.editingMode === 'rectangle') return 'mdi-vector-square'; + return 'mdi-shape'; + }, }, methods: { @@ -42,33 +51,104 @@ export default Vue.extend({ this.$emit('delete-annotation'); } }, + addHole() { + this.$emit('add-hole'); + }, + addPolygon() { + this.$emit('add-polygon'); + }, }, }); diff --git a/client/dive-common/recipes/polygonbase.ts b/client/dive-common/recipes/polygonbase.ts index 1caae557a..8fe53cea3 100644 --- a/client/dive-common/recipes/polygonbase.ts +++ b/client/dive-common/recipes/polygonbase.ts @@ -8,6 +8,49 @@ import { EditAnnotationTypes } from 'vue-media-annotator/layers'; const EmptyResponse = { data: {}, union: [], unionWithoutBounds: [] }; +/** + * Check if a point is inside a polygon using ray casting algorithm + * @param point [x, y] coordinates + * @param polygon array of [x, y] coordinates forming the polygon (outer ring only) + * @returns true if point is inside polygon + */ +function isPointInsidePolygon(point: [number, number], polygon: GeoJSON.Position[]): boolean { + const [x, y] = point; + let inside = false; + + for (let i = 0, j = polygon.length - 1; i < polygon.length; j = i, i += 1) { + const xi = polygon[i][0]; + const yi = polygon[i][1]; + const xj = polygon[j][0]; + const yj = polygon[j][1]; + + const intersect = ((yi > y) !== (yj > y)) + && (x < ((xj - xi) * (y - yi)) / (yj - yi) + xi); + + if (intersect) { + inside = !inside; + } + } + + return inside; +} + +/** + * Check if all vertices of polygon P are inside polygon E + * @param innerPolygon polygon to check if it's inside + * @param outerPolygon polygon to check if it contains the inner polygon + * @returns true if all vertices of innerPolygon are inside outerPolygon + */ +function isPolygonInsidePolygon( + innerPolygon: GeoJSON.Position[], + outerPolygon: GeoJSON.Position[], +): boolean { + // Check if all vertices of the inner polygon are inside the outer polygon + return innerPolygon.every( + (vertex) => isPointInsidePolygon([vertex[0], vertex[1]], outerPolygon), + ); +} + export default class PolygonBoundsExpand implements Recipe { active: Ref; @@ -19,12 +62,41 @@ export default class PolygonBoundsExpand implements Recipe { bus: Vue; + // Mode for adding polygons: 'normal', 'hole', or 'newPolygon' + addingMode: Ref<'normal' | 'hole' | 'newPolygon'>; + constructor() { this.bus = new Vue(); this.active = ref(false); this.name = 'PolygonBase'; this.toggleable = ref(true); this.icon = ref('mdi-vector-polygon'); + this.addingMode = ref('normal'); + } + + setAddingHole() { + this.addingMode.value = 'hole'; + // Emit activate event with special key to trigger creation mode + // The special key ensures no geometry matches, forcing creation mode + this.bus.$emit('activate', { + editing: 'Polygon' as EditAnnotationTypes, + key: '__adding_hole__', + recipeName: this.name, + }); + } + + setAddingPolygon(newKey: string) { + this.addingMode.value = 'newPolygon'; + // Emit activate event with new key to trigger creation mode + this.bus.$emit('activate', { + editing: 'Polygon' as EditAnnotationTypes, + key: newKey, + recipeName: this.name, + }); + } + + resetAddingMode() { + this.addingMode.value = 'normal'; } update( @@ -37,12 +109,89 @@ export default class PolygonBoundsExpand implements Recipe { if (data.length === 1 && mode === 'editing' && this.active.value) { const poly = data[0].geometry; if (poly.type === 'Polygon') { + const newPolyCoords = poly.coordinates[0] as GeoJSON.Position[]; + const currentMode = this.addingMode.value; + + // Reset adding mode after processing + this.resetAddingMode(); + + if (currentMode === 'hole' || key === '__adding_hole__') { + // Adding a hole - find the first polygon and add hole to it + const existingPolygons = track.getPolygonFeatures(frameNum); + if (existingPolygons.length > 0) { + // Add as hole to the first (default) polygon + const targetPoly = existingPolygons[0]; + // Create updated polygon geometry with the hole added + const updatedCoordinates = [ + ...targetPoly.geometry.coordinates, + newPolyCoords, + ]; + const updatedPolygon: GeoJSON.Polygon = { + type: 'Polygon', + coordinates: updatedCoordinates, + }; + const updatedFeature: GeoJSON.Feature = { + type: 'Feature', + properties: { key: targetPoly.key }, + geometry: updatedPolygon, + }; + // Return data like add polygon mode so right-click behavior is consistent + return { + data: { + [targetPoly.key]: [updatedFeature], + }, + union: [], + done: true, + unionWithoutBounds: [], + newSelectedKey: targetPoly.key, // Set to target polygon's key for proper mode transition + }; + } + // No existing polygon, treat as normal (create first polygon) + return { + data: { + '': data, + }, + union: [], + done: true, + unionWithoutBounds: [poly], + newSelectedKey: '', + }; + } + + if (currentMode === 'newPolygon') { + // Adding a new separate polygon - key should already be set to new value + const useKey = key || track.getNextPolygonKey(frameNum); + const newFeature: GeoJSON.Feature = { + type: 'Feature', + properties: { key: useKey }, + geometry: poly, + }; + return { + data: { + [useKey]: [newFeature], + }, + union: [poly], // Use union to EXPAND bounds, not replace them + done: true, + unionWithoutBounds: [], + newSelectedKey: '', // Reset to default polygon for future edits + }; + } + + // Standard case: save polygon with the given key + // Calculate bounds from ALL polygons in the detection, not just the edited one + const allPolygons = track.getPolygonFeatures(frameNum); + const otherPolygons: GeoJSON.Polygon[] = allPolygons + .filter((p) => p.key !== (key || '')) + .map((p) => p.geometry); + return { data: { [key || '']: data, }, - union: [], + // Use union with other polygons to ensure bounds encompass all + union: otherPolygons, done: true, + // The edited polygon replaces the base bounds unionWithoutBounds: [poly], }; } @@ -50,19 +199,61 @@ export default class PolygonBoundsExpand implements Recipe { return EmptyResponse; } + /** + * Add a polygon as a hole to an existing polygon, or as a new separate polygon. + * Call this method explicitly when auto-hole detection is desired. + */ + // eslint-disable-next-line class-methods-use-this + addPolygonWithHoleDetection( + frameNum: number, + track: Track, + poly: GeoJSON.Polygon, + key?: string, + ) { + const newPolyCoords = poly.coordinates[0] as GeoJSON.Position[]; + + // Get existing polygons for this frame + const existingPolygons = track.getPolygonFeatures(frameNum); + + // Check if this is an edit to an existing polygon (key matches) + const isExistingEdit = existingPolygons.some((p) => p.key === (key || '')); + + if (!isExistingEdit && existingPolygons.length > 0) { + // This is a new polygon - check if it should be a hole in an existing polygon + const containingPoly = existingPolygons.find((existingPoly) => { + const outerRing = existingPoly.geometry.coordinates[0] as GeoJSON.Position[]; + return isPolygonInsidePolygon(newPolyCoords, outerRing); + }); + + if (containingPoly) { + // New polygon is inside existing polygon - add as hole + track.addHoleToPolygon(frameNum, containingPoly.key, newPolyCoords); + return { isHole: true, key: containingPoly.key }; + } + + // Not inside any existing polygon - add as new separate polygon with auto-key + const newKey = track.getNextPolygonKey(frameNum); + return { isHole: false, key: newKey }; + } + + // Standard case: use provided key or default + return { isHole: false, key: key || '' }; + } + // eslint-disable-next-line class-methods-use-this delete(frame: number, track: Track, key: string, type: EditAnnotationTypes) { - if (key === '' && type === 'Polygon') { - track.removeFeatureGeometry(frame, { key: '', type: 'Polygon' }); + if (type === 'Polygon') { + // Remove polygon with the specified key (supports multiple polygons) + track.removeFeatureGeometry(frame, { key, type: 'Polygon' }); } } // eslint-disable-next-line class-methods-use-this deletePoint(frame: number, track: Track, idx: number, key: string, type: EditAnnotationTypes) { - if (key === '' && type === 'Polygon') { + if (type === 'Polygon') { const geoJsonFeatures = track.getFeatureGeometry(frame, { type: 'Polygon', - key: '', + key, }); if (geoJsonFeatures.length === 0) return; const clone = cloneDeep(geoJsonFeatures[0]); diff --git a/client/dive-common/use/useModeManager.ts b/client/dive-common/use/useModeManager.ts index f27a3866e..3669fd68c 100644 --- a/client/dive-common/use/useModeManager.ts +++ b/client/dive-common/use/useModeManager.ts @@ -186,7 +186,12 @@ export default function useModeManager({ } if (annotationModes.editing === 'rectangle') { return 'Editing'; } - return (feature.geometry?.features.filter((item) => item.geometry.type === annotationModes.editing).length ? 'Editing' : 'Creating'); + // Check if there's a geometry matching both the type AND the selectedKey + const matchingGeometry = feature.geometry?.features.filter( + (item) => item.geometry.type === annotationModes.editing + && item.properties?.key === selectedKey.value, + ); + return (matchingGeometry?.length ? 'Editing' : 'Creating'); } return 'Creating'; } @@ -606,16 +611,18 @@ export default function useModeManager({ // If a drawable changed, but we aren't changing modes // prevent an interrupt within EditAnnotationLayer + // Use === undefined to distinguish "no key change" from "change to empty key" if ( somethingChanged - && !update.newSelectedKey + && update.newSelectedKey === undefined && !update.newType && preventInterrupt ) { preventInterrupt(); } else { // Otherwise, one of these state changes will trigger an interrupt. - if (update.newSelectedKey) { + // Use !== undefined to allow setting key to empty string + if (update.newSelectedKey !== undefined) { selectedKey.value = update.newSelectedKey; } if (update.newType) { @@ -730,11 +737,32 @@ export default function useModeManager({ const track = cameraStore.getPossibleTrack(selectedTrackId.value, selectedCamera.value); if (track) { const { frame } = aggregateController.value; + const frameNum = frame.value; recipes.forEach((r) => { if (r.active.value) { - r.delete(frame.value, track, selectedKey.value, annotationModes.editing); + r.delete(frameNum, track, selectedKey.value, annotationModes.editing); } }); + + // After deleting a polygon, recalculate bounds from remaining polygons + if (annotationModes.editing === 'Polygon') { + const remainingPolygons = track.getPolygonFeatures(frameNum); + if (remainingPolygons.length > 0) { + // Recalculate bounds from remaining polygons + const polygonGeometries = remainingPolygons.map((p) => p.geometry); + const newBounds = updateBounds(undefined, [], polygonGeometries); + + // Get current feature and update with new bounds + const [currentFeature] = track.getFeature(frameNum); + if (currentFeature && newBounds) { + track.setFeature({ + ...currentFeature, + bounds: newBounds, + }); + } + } + } + _nudgeEditingCanary(); } } @@ -1234,6 +1262,46 @@ export default function useModeManager({ _nudgeEditingCanary(); } + /** + * Set up polygon recipe for adding a hole to an existing polygon. + * The recipe emits an activate event that triggers creation mode. + */ + function handleAddHole() { + if (selectedTrackId.value === null) return; + + const polygonRecipe = recipes.find((r) => r.name === 'PolygonBase'); + if (polygonRecipe && 'setAddingHole' in polygonRecipe) { + (polygonRecipe as { setAddingHole: () => void }).setAddingHole(); + } + } + + /** + * Set up polygon recipe for adding a new separate polygon. + */ + function handleAddPolygon() { + if (selectedTrackId.value === null) return; + + const polygonRecipe = recipes.find((r) => r.name === 'PolygonBase'); + if (polygonRecipe && 'setAddingPolygon' in polygonRecipe) { + const track = cameraStore.getPossibleTrack(selectedTrackId.value, selectedCamera.value); + if (track) { + const { frame } = aggregateController.value; + const newKey = track.getNextPolygonKey(frame.value); + (polygonRecipe as { setAddingPolygon: (key: string) => void }).setAddingPolygon(newKey); + } + } + } + + /** + * Cancel any in-progress creation mode (hole or polygon addition). + */ + function handleCancelCreation() { + const polygonRecipe = recipes.find((r) => r.name === 'PolygonBase'); + if (polygonRecipe && 'resetAddingMode' in polygonRecipe) { + (polygonRecipe as { resetAddingMode: () => void }).resetAddingMode(); + } + } + /* Subscribe to recipe activation events */ recipes.forEach((r) => r.bus.$on('activate', handleSetAnnotationState)); @@ -1309,6 +1377,9 @@ export default function useModeManager({ startLinking: handleStartLinking, stopLinking: handleStopLinking, seekFrame, + addHole: handleAddHole, + addPolygon: handleAddPolygon, + cancelCreation: handleCancelCreation, }, }; } diff --git a/client/platform/desktop/backend/native/segmentation.ts b/client/platform/desktop/backend/native/segmentation.ts index eaaf6ac96..f171e7134 100644 --- a/client/platform/desktop/backend/native/segmentation.ts +++ b/client/platform/desktop/backend/native/segmentation.ts @@ -118,9 +118,7 @@ export class SegmentationServiceManager extends EventEmitter { return new Promise((resolve, reject) => { const viameSetup = npath.join(settings.viamePath, 'setup_viame.sh'); - const configPath = npath.join( - settings.viamePath, 'configs', 'pipelines', 'interactive_segmenter_default.conf', - ); + const configPath = npath.join(settings.viamePath, 'configs', 'pipelines', 'interactive_segmenter_default.conf'); // Build the command to run the interactive segmentation service const command = [ diff --git a/client/platform/desktop/backend/native/stereo.ts b/client/platform/desktop/backend/native/stereo.ts index a81b3fce7..95b0d8bdd 100644 --- a/client/platform/desktop/backend/native/stereo.ts +++ b/client/platform/desktop/backend/native/stereo.ts @@ -170,9 +170,7 @@ export class StereoServiceManager extends EventEmitter { return new Promise((resolve, reject) => { const viameSetup = npath.join(settings.viamePath, 'setup_viame.sh'); - const configPath = npath.join( - settings.viamePath, 'configs', 'pipelines', 'interactive_stereo_default.conf', - ); + const configPath = npath.join(settings.viamePath, 'configs', 'pipelines', 'interactive_stereo_default.conf'); // Build the command to run the interactive stereo service const command = [ diff --git a/client/platform/desktop/backend/serializers/viame.ts b/client/platform/desktop/backend/serializers/viame.ts index 5f3201aab..a02d1bab9 100644 --- a/client/platform/desktop/backend/serializers/viame.ts +++ b/client/platform/desktop/backend/serializers/viame.ts @@ -26,7 +26,10 @@ const HeadRegex = /^\(kp\) head (-?[0-9]+\.*-?[0-9]*) (-?[0-9]+\.*-?[0-9]*)/g; const TailRegex = /^\(kp\) tail (-?[0-9]+\.*-?[0-9]*) (-?[0-9]+\.*-?[0-9]*)/g; const AttrRegex = /^\(atr\) (.*?)\s(.+)/g; const TrackAttrRegex = /^\(trk-atr\) (.*?)\s(.+)/g; -const PolyRegex = /^(\(poly\)) ((?:-?[0-9]+\.*-?[0-9]*\s*)+)/g; +// Polygon format: (poly) coordinates +const PolyRegex = /^\(poly\)\s*((?:-?[0-9]+\.*-?[0-9]*\s*)+)/g; +// Hole format: (hole) coordinates +const HoleRegex = /^\(hole\)\s*((?:-?[0-9]+\.*-?[0-9]*\s*)+)/g; const NoteRegex = /^\(note\)\s*(.+)/g; const FpsRegex = /fps:\s*(\d+(\.\d+)?)/ig; const ExecTimeRegEx = /exec_time:\s*(\d+(\.\d+)?)/ig; @@ -154,6 +157,18 @@ function _deduceType(value: string): boolean | number | string { return value; } +/** + * Get the next available polygon key for a feature collection. + */ +function _getNextPolygonKey( + geoFeatureCollection: GeoJSON.FeatureCollection, +): string { + const polygonCount = geoFeatureCollection.features.filter( + (f) => f.geometry.type === 'Polygon', + ).length; + return polygonCount > 0 ? String(polygonCount) : ''; +} + /** * Simplified from python variant. Does not handle duplicate type/key pairs * within a single feature. @@ -185,6 +200,28 @@ function _createGeoJsonFeature( return geoFeature; } +/** + * Find an existing polygon feature by key and add a hole to it. + * @param geoFeatureCollection the feature collection to search + * @param coords hole coordinates + * @param key polygon key to find + */ +function _addHoleToPolygon( + geoFeatureCollection: GeoJSON.FeatureCollection, + coords: number[][], + key = '', +) { + const matchingFeature = geoFeatureCollection.features.find( + (feature) => feature.geometry.type === 'Polygon' && feature.properties?.key === key, + ); + if (matchingFeature) { + // Add hole as additional ring to the polygon coordinates + (matchingFeature.geometry.coordinates as number[][][]).push(coords); + return true; + } + return false; +} + function _parseRow(row: string[]) { // Create empty feature collection const geoFeatureCollection: @@ -241,11 +278,13 @@ function _parseRow(row: string[]) { trackAttributes[trackattr[1]] = _deduceType(trackattr[2]); } - /* Polygon */ + /* Polygon - format: (poly) coordinates + * Multiple (poly) entries create separate polygons with auto-generated keys */ const poly = getCaptureGroups(PolyRegex, value); if (poly !== null) { + const coordString = poly[1]; const coords: number[][] = []; - const polyList = poly[2].split(' '); + const polyList = coordString.split(' '); polyList.forEach((coord, j) => { if (j % 2 === 0) { // Filter out ODDs @@ -254,7 +293,32 @@ function _parseRow(row: string[]) { } } }); - geoFeatureCollection.features.push(_createGeoJsonFeature('Polygon', coords)); + // Create new polygon with auto-generated key + const newKey = _getNextPolygonKey(geoFeatureCollection); + geoFeatureCollection.features.push(_createGeoJsonFeature('Polygon', coords, newKey)); + } + + /* Hole - format: (hole) coordinates + * Added to the most recent polygon */ + const hole = getCaptureGroups(HoleRegex, value); + if (hole !== null) { + const coordString = hole[1]; + const coords: number[][] = []; + const polyList = coordString.split(' '); + polyList.forEach((coord, j) => { + if (j % 2 === 0) { + // Filter out ODDs + if (polyList[j + 1]) { + coords.push([parseFloat(coord), parseFloat(polyList[j + 1])]); + } + } + }); + // Add as hole to the most recent polygon + const polygons = geoFeatureCollection.features.filter((f) => f.geometry.type === 'Polygon'); + if (polygons.length > 0) { + const lastPolyKey = polygons[polygons.length - 1].properties?.key || ''; + _addHoleToPolygon(geoFeatureCollection, coords, lastPolyKey); + } } /* Note */ @@ -661,8 +725,19 @@ async function serialize( if (feature.geometry && feature.geometry.type === 'FeatureCollection') { feature.geometry.features.forEach((geoJSONFeature) => { if (geoJSONFeature.geometry.type === 'Polygon') { - const coordinates = flattenDeep(geoJSONFeature.geometry.coordinates[0]); - row.push(`${PolyToken} ${coordinates.map(Math.round).join(' ')}`); + const allRings = geoJSONFeature.geometry.coordinates as number[][][]; + + // Write outer ring (first ring) + if (allRings.length > 0) { + const outerCoords = flattenDeep(allRings[0]); + row.push(`${PolyToken} ${outerCoords.map(Math.round).join(' ')}`); + + // Write holes (additional rings) + for (let holeIdx = 0; holeIdx < allRings.length - 1; holeIdx += 1) { + const holeCoords = flattenDeep(allRings[holeIdx + 1]); + row.push(`(hole) ${holeCoords.map(Math.round).join(' ')}`); + } + } } else if (geoJSONFeature.geometry.type === 'Point') { if (geoJSONFeature.properties) { const kpname = geoJSONFeature.properties.key; @@ -672,7 +747,6 @@ async function serialize( ); } } - /* TODO support for multiple GeoJSON Objects of the same type */ }); } diff --git a/client/src/layers/AnnotationLayers/PolygonLayer.ts b/client/src/layers/AnnotationLayers/PolygonLayer.ts index 302c030b7..79fe0043a 100644 --- a/client/src/layers/AnnotationLayers/PolygonLayer.ts +++ b/client/src/layers/AnnotationLayers/PolygonLayer.ts @@ -10,7 +10,28 @@ interface PolyGeoJSData{ editing: boolean | string; styleType: [string, number] | null; polygon: GeoJSON.Polygon; + polygonKey: string; set?: string; + isHole?: boolean; // True if this is a hole polygon (for styling) +} + +/** + * Darken a hex color by a given factor (0-1, where 0 = black, 1 = original) + */ +function darkenColor(color: string, factor: number): string { + // Handle hex colors + if (color.startsWith('#')) { + const hex = color.slice(1); + const r = parseInt(hex.slice(0, 2), 16); + const g = parseInt(hex.slice(2, 4), 16); + const b = parseInt(hex.slice(4, 6), 16); + const newR = Math.round(r * factor); + const newG = Math.round(g * factor); + const newB = Math.round(b * factor); + return `#${newR.toString(16).padStart(2, '0')}${newG.toString(16).padStart(2, '0')}${newB.toString(16).padStart(2, '0')}`; + } + // For non-hex colors, return as-is (could extend to support rgb(), etc.) + return color; } export default class PolygonLayer extends BaseLayer { @@ -35,19 +56,36 @@ export default class PolygonLayer extends BaseLayer { .geoOn(geo.event.feature.mouseclick, (e: GeoEvent) => { /** * Handle clicking on individual annotations, if DrawingOther is true we use the - * Rectangle type if only the polygon is visible we use the polygon bounds + * Rectangle type for track selection. However, polygon key events are always + * emitted so that multi-polygon selection works regardless of drawingOther. * */ - if (e.mouse.buttonsDown.left && !this.drawingOther) { - if (!e.data.editing || (e.data.editing && !e.data.selected)) { - if (e.mouse.modifiers.ctrl) { - this.bus.$emit('annotation-ctrl-clicked', e.data.trackId, false, { ctrl: true }); - } else { - this.bus.$emit('annotation-clicked', e.data.trackId, false); + if (e.mouse.buttonsDown.left) { + // Always emit polygon-clicked for multi-polygon support, regardless of drawingOther + const polygonKey = e.data.polygonKey || ''; + if (e.data.selected) { + // Already selected track - user may be selecting a different polygon + this.bus.$emit('polygon-clicked', e.data.trackId, polygonKey); + } + // Track-level events only when not drawingOther (rectangle layer handles those) + if (!this.drawingOther) { + if (!e.data.editing || (e.data.editing && !e.data.selected)) { + if (e.mouse.modifiers.ctrl) { + this.bus.$emit('annotation-ctrl-clicked', e.data.trackId, false, { ctrl: true }); + } else { + this.bus.$emit('polygon-clicked', e.data.trackId, polygonKey); + this.bus.$emit('annotation-clicked', e.data.trackId, false); + } } } - } else if (e.mouse.buttonsDown.right && !this.drawingOther) { - if (!e.data.editing || (e.data.editing && !e.data.selected)) { - this.bus.$emit('annotation-right-clicked', e.data.trackId, true); + } else if (e.mouse.buttonsDown.right) { + // Always emit polygon key for right-click so the correct polygon can be selected + const polygonKey = e.data.polygonKey || ''; + this.bus.$emit('polygon-right-clicked', e.data.trackId, polygonKey); + // Track-level events only when not drawingOther + if (!this.drawingOther) { + if (!e.data.editing || (e.data.editing && !e.data.selected)) { + this.bus.$emit('annotation-right-clicked', e.data.trackId, true); + } } } }); @@ -58,7 +96,12 @@ export default class PolygonLayer extends BaseLayer { this.featureLayer.geoOn(geo.event.mouseclick, (e: GeoEvent) => { // If we aren't clicking on an annotation we can deselect the current track if (this.featureLayer.pointSearch(e.geo).found.length === 0 && !this.drawingOther) { - this.bus.$emit('annotation-clicked', null, false); + if (e.mouse.buttonsDown.left) { + this.bus.$emit('annotation-clicked', null, false); + } else if (e.mouse.buttonsDown.right) { + // Right-click outside polygons - emit event to finalize/cancel creation + this.bus.$emit('polygon-right-clicked-outside'); + } } }); super.initialize(); @@ -106,15 +149,41 @@ export default class PolygonLayer extends BaseLayer { frameData.features.geometry.features.forEach((feature) => { if (feature.geometry && feature.geometry.type === 'Polygon') { const polygon = feature.geometry; + const polygonKey = feature.properties?.key || ''; const annotation: PolyGeoJSData = { trackId: frameData.track.id, selected: frameData.selected, editing: frameData.editing, styleType: frameData.styleType, polygon, + polygonKey, set: frameData.set, + isHole: false, }; arr.push(annotation); + + // Also add holes as separate polygon entries for distinct styling + const coords = polygon.coordinates as GeoJSON.Position[][]; + if (coords.length > 1) { + // coords[0] is outer ring, coords[1..n] are holes + for (let i = 1; i < coords.length; i += 1) { + const holePolygon: GeoJSON.Polygon = { + type: 'Polygon', + coordinates: [coords[i]], // Hole as its own polygon + }; + const holeAnnotation: PolyGeoJSData = { + trackId: frameData.track.id, + selected: frameData.selected, + editing: frameData.editing, + styleType: frameData.styleType, + polygon: holePolygon, + polygonKey, // Same key as parent polygon + set: frameData.set, + isHole: true, + }; + arr.push(holeAnnotation); + } + } } }); } @@ -126,7 +195,17 @@ export default class PolygonLayer extends BaseLayer { redraw() { this.featureLayer .data(this.formattedData) - .polygon((d: PolyGeoJSData) => d.polygon.coordinates[0]) + .polygon((d: PolyGeoJSData) => { + // GeoJS expects outer ring as array of points for simple polygons + // For polygons with holes, return object with outer/inner properties + if (d.polygon.coordinates.length > 1) { + return { + outer: d.polygon.coordinates[0], + inner: d.polygon.coordinates.slice(1), + }; + } + return d.polygon.coordinates[0]; + }) .draw(); } @@ -142,15 +221,22 @@ export default class PolygonLayer extends BaseLayer { // Style conversion to get array objects to work in geoJS position: (point) => ({ x: point[0], y: point[1] }), strokeColor: (_point, _index, data) => { + let color: string; if (data.selected) { - return this.stateStyling.selected.color; - } - if (data.styleType) { - return this.typeStyling.value.color(data.styleType[0]); + color = this.stateStyling.selected.color; + } else if (data.styleType) { + color = this.typeStyling.value.color(data.styleType[0]); + } else { + color = this.typeStyling.value.color(''); } - return this.typeStyling.value.color(''); + // Darken color for holes + return data.isHole ? darkenColor(color, 0.5) : color; }, fill: (data) => { + // Holes should always be filled to show the darker color + if (data.isHole) { + return true; + } if (data.set) { return this.typeStyling.value.fill(data.set); } @@ -160,12 +246,20 @@ export default class PolygonLayer extends BaseLayer { return this.stateStyling.standard.fill; }, fillColor: (_point, _index, data) => { + let color: string; if (data.styleType) { - return this.typeStyling.value.color(data.styleType[0]); + color = this.typeStyling.value.color(data.styleType[0]); + } else { + color = this.typeStyling.value.color(''); } - return this.typeStyling.value.color(''); + // Darken color for holes + return data.isHole ? darkenColor(color, 0.5) : color; }, fillOpacity: (_point, _index, data) => { + // Holes get higher opacity to stand out + if (data.isHole) { + return 0.5; + } if (data.set) { return this.typeStyling.value.opacity(data.set); } diff --git a/client/src/layers/EditAnnotationLayer.ts b/client/src/layers/EditAnnotationLayer.ts index 17d7f9ece..e12de9f37 100644 --- a/client/src/layers/EditAnnotationLayer.ts +++ b/client/src/layers/EditAnnotationLayer.ts @@ -69,6 +69,8 @@ export default class EditAnnotationLayer extends BaseLayer { selectedKey?: string; + selectedPolygonIndex: number; + selectedHandleIndex: number; hoverHandleIndex: number; @@ -80,11 +82,6 @@ export default class EditAnnotationLayer extends BaseLayer { /* in-progress events only emitted for lines and polygons */ shapeInProgress: GeoJSON.LineString | GeoJSON.Polygon | null; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - arrowFeatureLayer: any; - - unrotatedGeoJSONCoords: GeoJSON.Position[] | null; - /* Track if the last click was a right-click or shift-click for Point mode */ lastClickWasBackground: boolean; @@ -96,20 +93,26 @@ export default class EditAnnotationLayer extends BaseLayer { private boundHandleContextMenu: ((e: MouseEvent) => void) | null = null; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + arrowFeatureLayer: any; + + unrotatedGeoJSONCoords: GeoJSON.Position[] | null; + constructor(params: BaseLayerParams & EditAnnotationLayerParams) { super(params); this.skipNextExternalUpdate = false; this._mode = 'editing'; this.selectedKey = ''; + this.selectedPolygonIndex = 0; this.type = params.type; this.selectedHandleIndex = -1; this.hoverHandleIndex = -1; this.shapeInProgress = null; this.disableModeSync = false; this.leftButtonCheckTimeout = -1; - this.unrotatedGeoJSONCoords = null; this.lastClickWasBackground = false; this.lastShiftKeyState = false; + this.unrotatedGeoJSONCoords = null; // Bind event handlers once (listeners are added/removed dynamically based on type) this.boundTrackShiftKey = this.trackShiftKey.bind(this); @@ -167,12 +170,14 @@ export default class EditAnnotationLayer extends BaseLayer { */ handleContextMenu(e: MouseEvent) { if (this.type === 'Point' && this.getMode() === 'creation') { - // Prevent the browser context menu e.preventDefault(); e.stopPropagation(); - // Emit confirm event to lock the annotation (like other edit modes) - this.bus.$emit('confirm-annotation'); + // Emit right-click event with screen coordinates so LayerManager can + // check if an annotation is under the cursor and select it. + // On Windows/Electron, GeoJS mouseclick may not fire for right-button, + // so this provides a reliable fallback for annotation selection. + this.bus.$emit('right-click-point-mode', { x: e.clientX, y: e.clientY }); } } @@ -227,6 +232,35 @@ export default class EditAnnotationLayer extends BaseLayer { this.bus.$emit('editing-annotation-sync', false); } else if (e.buttonsDown.left) { const newIndex = this.hoverHandleIndex; + // If not hovering over an edit handle and not on the edit polygon, + // emit event so other layers can handle the click (e.g., selecting different polygon) + if (newIndex < 0 && this.type === 'Polygon') { + const annotations = this.featureLayer.annotations(); + if (annotations.length > 0) { + const annotation = annotations[0]; + const geojson = annotation.geojson(); + if (geojson && geojson.geometry && geojson.geometry.type === 'Polygon') { + const coords = geojson.geometry.coordinates[0] as [number, number][]; + const point: [number, number] = [e.geo.x, e.geo.y]; + // Ray casting algorithm to check if point is inside polygon + let inside = false; + for (let i = 0, j = coords.length - 1; i < coords.length; j = i, i += 1) { + const xi = coords[i][0]; + const yi = coords[i][1]; + const xj = coords[j][0]; + const yj = coords[j][1]; + const intersect = ((yi > point[1]) !== (yj > point[1])) + && (point[0] < ((xj - xi) * (point[1] - yi)) / (yj - yi) + xi); + if (intersect) inside = !inside; + } + if (!inside) { + // Click is outside the current edit polygon - emit passthrough event + this.bus.$emit('click-outside-edit', e.geo); + return; + } + } + } + } // Click features like a toggle: unselect if it's clicked twice. if (newIndex === this.selectedHandleIndex) { this.selectedHandleIndex = -1; @@ -500,6 +534,82 @@ export default class EditAnnotationLayer extends BaseLayer { } } + /** + * Attempt to finalize any in-progress annotation before switching tracks. + * Handles: + * - Polygon with 3+ vertices tracked in shapeInProgress + * - GeoJS-managed annotations (rectangles) in creation/done state + * - Discards invalid partial shapes (polygon < 3 vertices, line with 1 point) + * Returns true if a shape was finalized. + */ + finalizeInProgress(): boolean { + // Handle shapeInProgress (polygon/line tracked manually) + if (this.shapeInProgress && this.getMode() === 'creation') { + if (this.shapeInProgress.type === 'Polygon') { + const coords = this.shapeInProgress.coordinates as GeoJSON.Position[][]; + if (coords[0] && coords[0].length >= 3) { + const ring = coords[0]; + const first = ring[0]; + const last = ring[ring.length - 1]; + if (first[0] !== last[0] || first[1] !== last[1]) { + ring.push([...first]); + } + const feature: GeoJSON.Feature = { + type: 'Feature', + geometry: this.shapeInProgress, + properties: {}, + }; + this.disableModeSync = true; + this.bus.$emit( + 'update:geojson', + 'editing', + true, + feature, + this.type, + this.selectedKey, + this.skipNextFunc(), + ); + this.shapeInProgress = null; + return true; + } + } + // Discard invalid partial shapes (polygon < 3 vertices, line with 1 point) + this.shapeInProgress = null; + return false; + } + + // Handle GeoJS-managed annotations (rectangles, completed shapes) + // Skip Point mode — segmentation manages its own polygon via the recipe, + // not through the edit layer's GeoJS annotation. + if (this.featureLayer && this.type !== 'Point') { + const annotations = this.featureLayer.annotations(); + if (annotations.length > 0) { + const annotation = annotations[0]; + const geoJSONData = annotation.geojson(); + if (geoJSONData && geoJSONData.geometry) { + if (this.type === 'rectangle') { + geoJSONData.geometry.coordinates[0] = reOrdergeoJSON( + geoJSONData.geometry.coordinates[0] as GeoJSON.Position[], + ); + } + this.disableModeSync = true; + this.bus.$emit( + 'update:geojson', + 'editing', + true, + geoJSONData, + this.type, + this.selectedKey, + this.skipNextFunc(), + ); + return true; + } + } + } + + return false; + } + /** * Removes the current annotation and resets the mode when completed editing */ @@ -524,25 +634,52 @@ export default class EditAnnotationLayer extends BaseLayer { /** * retrieves geoJSON data based on the key and type - * @param frameData + * @param track + * @param polygonIndex optional index to get a specific polygon when multiple exist */ - getGeoJSONData(track: FrameDataTrack) { - let geoJSONData; + getGeoJSONData( + track: FrameDataTrack, + polygonIndex?: number, + ): GeoJSON.Point | GeoJSON.Polygon | GeoJSON.LineString | undefined { + let geoJSONData: GeoJSON.Point | GeoJSON.Polygon | GeoJSON.LineString | undefined; if (track && track.features && track.features.geometry) { + const matchingFeatures: (GeoJSON.Point | GeoJSON.Polygon | GeoJSON.LineString)[] = []; track.features.geometry.features.forEach((feature) => { if (feature.geometry && feature.geometry.type.toLowerCase() === this.type.toLowerCase()) { - if (feature.properties && feature.properties.key !== 'undefined') { - if (feature.properties.key === this.selectedKey) { - geoJSONData = feature.geometry; - } + // Get the feature key, defaulting to '' for undefined/null keys + const featureKey = feature.properties?.key ?? ''; + if (featureKey === this.selectedKey) { + matchingFeatures.push( + feature.geometry as GeoJSON.Point | GeoJSON.Polygon | GeoJSON.LineString, + ); } } }); + // If polygonIndex is specified and valid, use it; otherwise use first match + if (polygonIndex !== undefined && polygonIndex >= 0 && polygonIndex < matchingFeatures.length) { + geoJSONData = matchingFeatures[polygonIndex]; + } else if (matchingFeatures.length > 0) { + [geoJSONData] = matchingFeatures; + } } return geoJSONData; } + /** + * Set which polygon index to edit when multiple polygons exist + */ + setPolygonIndex(index: number) { + this.selectedPolygonIndex = index; + } + + /** + * Get the currently selected polygon index + */ + getPolygonIndex() { + return this.selectedPolygonIndex; + } + /** overrides default function to disable and clear anotations before drawing again */ async changeData(frameData: FrameDataTrack[]) { if (this.skipNextExternalUpdate === false) { @@ -642,9 +779,11 @@ export default class EditAnnotationLayer extends BaseLayer { // Only calls this once on completion of an annotation if (e.annotation.state() === 'done' && this.getMode() === 'creation') { const geoJSONData = [e.annotation.geojson()]; - - this.unrotatedGeoJSONCoords = geoJSONData[0].geometry.coordinates[0] as GeoJSON.Position[]; - + if (this.type === 'rectangle') { + geoJSONData[0].geometry.coordinates[0] = reOrdergeoJSON( + geoJSONData[0].geometry.coordinates[0] as GeoJSON.Position[], + ); + } // For Point mode, add background property if it was a right-click or shift-click if (this.type === 'Point' && this.lastClickWasBackground) { geoJSONData[0].properties = { @@ -653,6 +792,8 @@ export default class EditAnnotationLayer extends BaseLayer { }; this.lastClickWasBackground = false; // Reset for next point } + + this.unrotatedGeoJSONCoords = geoJSONData[0].geometry.coordinates[0] as GeoJSON.Position[]; this.formattedData = geoJSONData; // The new annotation is in a state without styling, so apply local stypes this.applyStylesToAnnotations(); diff --git a/client/src/provides.ts b/client/src/provides.ts index cb039a60f..9b0dc18ee 100644 --- a/client/src/provides.ts +++ b/client/src/provides.ts @@ -207,6 +207,12 @@ export interface Handler { startLinking(camera: string): void; stopLinking(): void; setChange(set: string): void; + /* Add a hole to the current polygon */ + addHole(): void; + /* Add a new separate polygon */ + addPolygon(): void; + /* Cancel any in-progress creation mode (hole or polygon addition) */ + cancelCreation(): void; } const HandlerSymbol = Symbol('handler'); @@ -252,6 +258,9 @@ function dummyHandler(handle: (name: string, args: unknown[]) => void): Handler startLinking(...args) { handle('startLinking', args); }, stopLinking(...args) { handle('stopLinking', args); }, setChange(...args) { handle('setChange', args); }, + addHole(...args) { handle('addHole', args); }, + addPolygon(...args) { handle('addPolygon', args); }, + cancelCreation(...args) { handle('cancelCreation', args); }, }; } diff --git a/client/src/track.ts b/client/src/track.ts index e28d63516..413f4dc3d 100644 --- a/client/src/track.ts +++ b/client/src/track.ts @@ -311,7 +311,10 @@ export default class Track extends BaseAnnotation { geometry.forEach((geo) => { const i = fg.features .findIndex((item) => { - const keyMatch = !geo.properties?.key || item.properties?.key === geo.properties?.key; + // Compare keys directly, treating undefined/null as empty string + const geoKey = geo.properties?.key ?? ''; + const itemKey = item.properties?.key ?? ''; + const keyMatch = geoKey === itemKey; const typeMatch = item.geometry.type === geo.geometry.type; return keyMatch && typeMatch; }); @@ -349,7 +352,9 @@ export default class Track extends BaseAnnotation { return []; } return feature.geometry.features.filter((item) => { - const matchesKey = !key || item.properties?.key === key; + // Check key match: undefined means match all, otherwise compare (treating undefined/null as '') + const matchesKey = key === undefined + || (item.properties?.key ?? '') === key; const matchesType = !type || item.geometry.type === type; return matchesKey && matchesType; }); @@ -362,7 +367,9 @@ export default class Track extends BaseAnnotation { return false; } const index = feature.geometry.features.findIndex((item) => { - const matchesKey = !key || item.properties?.key === key; + // Check key match: undefined means match all, otherwise compare (treating undefined/null as '') + const matchesKey = key === undefined + || (item.properties?.key ?? '') === key; const matchesType = !type || item.geometry.type === type; return matchesKey && matchesType; }); @@ -391,6 +398,116 @@ export default class Track extends BaseAnnotation { } } + /** + * Get all polygon features for a frame + * @returns Array of polygon GeoJSON features with their keys + */ + getPolygonFeatures(frame: number): Array<{ + key: string; + geometry: GeoJSON.Polygon; + hasHoles: boolean; + holeCount: number; + }> { + const feature = this.features[frame]; + if (!feature?.geometry) { + return []; + } + const polygons: Array<{ + key: string; + geometry: GeoJSON.Polygon; + hasHoles: boolean; + holeCount: number; + }> = []; + feature.geometry.features.forEach((item) => { + if (item.geometry.type === 'Polygon') { + const coords = item.geometry.coordinates as GeoJSON.Position[][]; + polygons.push({ + key: item.properties?.key || '', + geometry: item.geometry, + hasHoles: coords.length > 1, + holeCount: Math.max(0, coords.length - 1), + }); + } + }); + return polygons; + } + + /** + * Add a hole to an existing polygon + * @param frame frame number + * @param key polygon key to add hole to + * @param holeCoords coordinates of the hole (array of [x,y] positions) + * @returns true if hole was added successfully + */ + addHoleToPolygon(frame: number, key: string, holeCoords: GeoJSON.Position[]): boolean { + const feature = this.features[frame]; + if (!feature?.geometry) { + return false; + } + const polygonFeature = feature.geometry.features.find( + (item) => item.geometry.type === 'Polygon' && item.properties?.key === key, + ); + if (polygonFeature && polygonFeature.geometry.type === 'Polygon') { + (polygonFeature.geometry.coordinates as GeoJSON.Position[][]).push(holeCoords); + this.notify('feature', feature); + return true; + } + return false; + } + + /** + * Remove a hole from a polygon + * @param frame frame number + * @param key polygon key + * @param holeIndex index of the hole to remove (0 = first hole, which is coordinates[1]) + * @returns true if hole was removed successfully + */ + removeHoleFromPolygon(frame: number, key: string, holeIndex: number): boolean { + const feature = this.features[frame]; + if (!feature?.geometry) { + return false; + } + const polygonFeature = feature.geometry.features.find( + (item) => item.geometry.type === 'Polygon' && item.properties?.key === key, + ); + if (polygonFeature && polygonFeature.geometry.type === 'Polygon') { + const coords = polygonFeature.geometry.coordinates as GeoJSON.Position[][]; + // holeIndex 0 corresponds to coords[1], holeIndex 1 to coords[2], etc. + const actualIndex = holeIndex + 1; + if (actualIndex > 0 && actualIndex < coords.length) { + coords.splice(actualIndex, 1); + this.notify('feature', feature); + return true; + } + } + return false; + } + + /** + * Get the next available polygon key for this frame + * @param frame frame number + * @returns next available key (e.g., "1", "2", etc.) + */ + getNextPolygonKey(frame: number): string { + const polygons = this.getPolygonFeatures(frame); + if (polygons.length === 0) { + return ''; + } + // Find the highest numeric key and increment + let maxKey = 0; + polygons.forEach((p) => { + if (p.key === '') { + maxKey = Math.max(maxKey, 0); + } else { + const numKey = parseInt(p.key, 10); + if (!Number.isNaN(numKey)) { + maxKey = Math.max(maxKey, numKey); + } + } + }); + return String(maxKey + 1); + } + setFeatureAttribute(frame: number, name: string, value: unknown, user: null | string = null) { if (this.features[frame]) { if (user !== null) { diff --git a/server/dive_utils/models.py b/server/dive_utils/models.py index 7f3e93377..d82e68066 100644 --- a/server/dive_utils/models.py +++ b/server/dive_utils/models.py @@ -29,7 +29,8 @@ class GeoJSONGeometry(BaseModel): class GeoJSONFeature(BaseModel): type: str geometry: GeoJSONGeometry - properties: Dict[str, Union[bool, float, str]] + # str first in the Union to keep numeric strings like "1" from coercing to bool/float + properties: Dict[str, Union[str, float, bool]] class GeoJSONFeatureCollection(BaseModel): diff --git a/server/dive_utils/serializers/viame.py b/server/dive_utils/serializers/viame.py index bcd3a6889..5173f9240 100644 --- a/server/dive_utils/serializers/viame.py +++ b/server/dive_utils/serializers/viame.py @@ -80,11 +80,28 @@ def _deduceType(value: Any) -> Union[bool, float, str, None]: return value -def create_geoJSONFeature(features: Dict[str, Any], type: str, coords: List[Any], key=''): +def get_next_polygon_key(features: Dict[str, Any]) -> str: + """Get the next available polygon key for a feature.""" + if "geometry" not in features or not features["geometry"]["features"]: + return '' + # Count existing polygons to determine the next key + polygon_count = sum( + 1 for f in features["geometry"]["features"] + if f["geometry"]["type"] == "Polygon" + ) + return str(polygon_count) if polygon_count > 0 else '' + + +def create_geoJSONFeature(features: Dict[str, Any], type: str, coords: List[Any], key='', auto_key=False): feature = {} if "geometry" not in features: features["geometry"] = {"type": "FeatureCollection", "features": []} - else: # check for existing type/key pairs + + # For polygons with auto_key, always create a new feature with a unique key + if type == 'Polygon' and auto_key: + key = get_next_polygon_key(features) + elif not auto_key: + # Check for existing type/key pairs (for non-polygon or explicit key) if features["geometry"]["features"]: for subfeature in features["geometry"]["features"]: if ( @@ -93,18 +110,34 @@ def create_geoJSONFeature(features: Dict[str, Any], type: str, coords: List[Any] ): feature = subfeature break + if "geometry" not in feature: feature = { "type": "Feature", "properties": {"key": key}, "geometry": {"type": type}, } + features['geometry']['features'].append(feature) if type == 'Polygon': feature["geometry"]['coordinates'] = [coords] elif type in ["LineString", "Point"]: feature['geometry']['coordinates'] = coords - features['geometry']['features'].append(feature) + return key # Return the key used (useful for auto-generated keys) + + +def add_hole_to_polygon(features: Dict[str, Any], coords: List[Any], key=''): + """Add a hole to an existing polygon feature with the given key.""" + if "geometry" not in features or not features["geometry"]["features"]: + return + for subfeature in features["geometry"]["features"]: + if ( + subfeature["geometry"]["type"] == 'Polygon' + and subfeature["properties"]["key"] == key + ): + # Add hole as additional ring to the polygon coordinates + subfeature["geometry"]["coordinates"].append(coords) + break def _parse_row(row: List[str]) -> Tuple[Dict, Dict, Dict, List, List]: @@ -149,12 +182,32 @@ def _parse_row(row: List[str]) -> Tuple[Dict, Dict, Dict, List, List]: if trk_regex: track_attributes[trk_regex[1]] = _deduceType(trk_regex[2]) - # (poly) x1 y1 x2 y2 ... - poly_regex = re.match(r"^(\(poly\)) ((?:-?[0-9]+\.*-?[0-9]*\s*)+)", row[j]) + # (poly) x1 y1 x2 y2 ... - polygon (multiple allowed, auto-keyed internally) + # (hole) x1 y1 x2 y2 ... - hole in the most recent polygon + poly_regex = re.match( + r"^\(poly\)\s*((?:-?[0-9]+\.*-?[0-9]*\s*)+)", + row[j] + ) if poly_regex: - temp = [float(x) for x in poly_regex[2].split()] - coords = list(zip(temp[::2], temp[1::2])) - create_geoJSONFeature(features, 'Polygon', coords) + temp = [float(x) for x in poly_regex.group(1).split()] + coords = [[temp[i], temp[i + 1]] for i in range(0, len(temp), 2)] + # Create new polygon with auto-generated key + create_geoJSONFeature(features, 'Polygon', coords, auto_key=True) + + # (hole) x1 y1 x2 y2 ... - hole in the most recent polygon + hole_regex = re.match( + r"^\(hole\)\s*((?:-?[0-9]+\.*-?[0-9]*\s*)+)", + row[j] + ) + if hole_regex: + temp = [float(x) for x in hole_regex.group(1).split()] + coords = [[temp[i], temp[i + 1]] for i in range(0, len(temp), 2)] + # Add hole to the most recent polygon (last one added) + if "geometry" in features and features["geometry"]["features"]: + polygons = [f for f in features["geometry"]["features"] if f["geometry"]["type"] == "Polygon"] + if polygons: + last_poly_key = polygons[-1]["properties"]["key"] + add_hole_to_polygon(features, coords, last_poly_key) # (note) text note_regex = re.match(r"^\(note\)\s*(.+)", row[j]) @@ -594,25 +647,35 @@ def export_tracks_as_csv( if feature.geometry and "FeatureCollection" == feature.geometry.type: for geoJSONFeature in feature.geometry.features: if 'Polygon' == geoJSONFeature.geometry.type: - # Coordinates need to be flattened out from their list of tuples - coordinates = [ - item - for sublist in geoJSONFeature.geometry.coordinates[ - 0 - ] # type: ignore - for item in sublist # type: ignore - ] - columns.append( - f"(poly) {' '.join(map(lambda x: str(round(x)), coordinates))}" - ) + all_rings = geoJSONFeature.geometry.coordinates # type: ignore + + # Write outer ring (first ring) + if len(all_rings) > 0: + outer_coords = [ + item + for sublist in all_rings[0] + for item in sublist # type: ignore + ] + columns.append( + f"(poly) {' '.join(map(lambda x: str(round(x)), outer_coords))}" + ) + + # Write holes (additional rings) + for hole_ring in all_rings[1:]: + hole_coords = [ + item + for sublist in hole_ring + for item in sublist # type: ignore + ] + columns.append( + f"(hole) {' '.join(map(lambda x: str(round(x)), hole_coords))}" + ) if 'Point' == geoJSONFeature.geometry.type: coordinates = geoJSONFeature.geometry.coordinates # type: ignore columns.append( f"(kp) {geoJSONFeature.properties['key']} " f"{round(coordinates[0])} {round(coordinates[1])}" ) - # TODO: support for multiple GeoJSON Objects of the same type - # once the CSV supports it writer.writerow(columns) yield csvFile.getvalue() diff --git a/server/tests/test_serialize_viame_csv.py b/server/tests/test_serialize_viame_csv.py index 7b8b584ed..ea647f0ec 100644 --- a/server/tests/test_serialize_viame_csv.py +++ b/server/tests/test_serialize_viame_csv.py @@ -195,6 +195,129 @@ ], [], ), + # Testing multi-polygon with different keys + ( + { + "0": { + "id": 0, + "attributes": {}, + "confidencePairs": [["fish", 1.0]], + "features": [ + { + "frame": 0, + "bounds": [100, 100, 500, 500], + "geometry": { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": {"key": ""}, + "geometry": { + "type": "Polygon", + "coordinates": [[[100, 100], [200, 100], [200, 200], [100, 200]]], + }, + }, + { + "type": "Feature", + "properties": {"key": "1"}, + "geometry": { + "type": "Polygon", + "coordinates": [[[300, 300], [400, 300], [400, 400], [300, 400]]], + }, + }, + ], + }, + }, + ], + "begin": 0, + "end": 0, + }, + }, + [ + "0,1.png,0,100,100,500,500,1.0,-1,fish,1.0,(poly) 100 100 200 100 200 200 100 200,(poly) 300 300 400 300 400 400 300 400", + "", + ], + [], + ), + # Testing polygon with hole + ( + { + "0": { + "id": 0, + "attributes": {}, + "confidencePairs": [["object", 1.0]], + "features": [ + { + "frame": 0, + "bounds": [100, 100, 500, 500], + "geometry": { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": {"key": ""}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [[100, 100], [500, 100], [500, 500], [100, 500]], + [[200, 200], [400, 200], [400, 400], [200, 400]], + ], + }, + }, + ], + }, + }, + ], + "begin": 0, + "end": 0, + }, + }, + [ + "0,1.png,0,100,100,500,500,1.0,-1,object,1.0,(poly) 100 100 500 100 500 500 100 500,(hole) 200 200 400 200 400 400 200 400", + "", + ], + [], + ), + # Testing keyed polygon with hole + ( + { + "0": { + "id": 0, + "attributes": {}, + "confidencePairs": [["region", 1.0]], + "features": [ + { + "frame": 0, + "bounds": [0, 0, 1000, 1000], + "geometry": { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": {"key": "2"}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [[0, 0], [1000, 0], [1000, 1000], [0, 1000]], + [[100, 100], [200, 100], [200, 200], [100, 200]], + [[300, 300], [400, 300], [400, 400], [300, 400]], + ], + }, + }, + ], + }, + }, + ], + "begin": 0, + "end": 0, + }, + }, + [ + "0,1.png,0,0,0,1000,1000,1.0,-1,region,1.0,(poly) 0 0 1000 0 1000 1000 0 1000,(hole) 100 100 200 100 200 200 100 200,(hole) 300 300 400 300 400 400 300 400", + "", + ], + [], + ), # Testing type filter ( { diff --git a/testutils/viame.spec.json b/testutils/viame.spec.json index 2a71ee623..571582389 100644 --- a/testutils/viame.spec.json +++ b/testutils/viame.spec.json @@ -675,5 +675,202 @@ } }, {} + ], + [ + [ + "0,1.png,0,100,100,500,500,1.0,-1,fish,1.0,(poly) 100 100 200 100 200 200 100 200,(poly) 300 300 400 300 400 400 300 400", + "" + ], + { + "0": { + "id": 0, + "attributes": {}, + "confidencePairs": [ + [ + "fish", + 1.0 + ] + ], + "features": [ + { + "frame": 0, + "bounds": [ + 100, + 100, + 500, + 500 + ], + "geometry": { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": { + "key": "" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [100.0, 100.0], + [200.0, 100.0], + [200.0, 200.0], + [100.0, 200.0] + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "key": "1" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [300.0, 300.0], + [400.0, 300.0], + [400.0, 400.0], + [300.0, 400.0] + ] + ] + } + } + ] + } + } + ], + "begin": 0, + "end": 0 + } + }, + {} + ], + [ + [ + "0,1.png,0,100,100,500,500,1.0,-1,object,1.0,(poly) 100 100 500 100 500 500 100 500,(hole) 200 200 400 200 400 400 200 400", + "" + ], + { + "0": { + "id": 0, + "attributes": {}, + "confidencePairs": [ + [ + "object", + 1.0 + ] + ], + "features": [ + { + "frame": 0, + "bounds": [ + 100, + 100, + 500, + 500 + ], + "geometry": { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": { + "key": "" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [100.0, 100.0], + [500.0, 100.0], + [500.0, 500.0], + [100.0, 500.0] + ], + [ + [200.0, 200.0], + [400.0, 200.0], + [400.0, 400.0], + [200.0, 400.0] + ] + ] + } + } + ] + } + } + ], + "begin": 0, + "end": 0 + } + }, + {} + ], + [ + [ + "0,1.png,0,0,0,1000,1000,1.0,-1,region,1.0,(poly) 0 0 1000 0 1000 1000 0 1000,(hole) 100 100 200 100 200 200 100 200,(hole) 300 300 400 300 400 400 300 400", + "" + ], + { + "0": { + "id": 0, + "attributes": {}, + "confidencePairs": [ + [ + "region", + 1.0 + ] + ], + "features": [ + { + "frame": 0, + "bounds": [ + 0, + 0, + 1000, + 1000 + ], + "geometry": { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": { + "key": "" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [0.0, 0.0], + [1000.0, 0.0], + [1000.0, 1000.0], + [0.0, 1000.0] + ], + [ + [100.0, 100.0], + [200.0, 100.0], + [200.0, 200.0], + [100.0, 200.0] + ], + [ + [300.0, 300.0], + [400.0, 300.0], + [400.0, 400.0], + [300.0, 400.0] + ] + ] + } + } + ] + } + } + ], + "begin": 0, + "end": 0 + } + }, + {} ] ] From d17b9bad3f9ba8c41cca3c0b6f94daa843753768 Mon Sep 17 00:00:00 2001 From: Matt Dawkins Date: Wed, 7 Jan 2026 19:45:16 -0500 Subject: [PATCH 03/10] Fix runtime error (cherry picked from commit c2f3cd043e05ee70a2b67d62cd9ecdf08b2b7856) --- client/platform/desktop/backend/native/common.spec.ts | 1 + client/platform/desktop/backend/native/multiCamImport.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/client/platform/desktop/backend/native/common.spec.ts b/client/platform/desktop/backend/native/common.spec.ts index 4598d323c..fcaca395f 100644 --- a/client/platform/desktop/backend/native/common.spec.ts +++ b/client/platform/desktop/backend/native/common.spec.ts @@ -101,6 +101,7 @@ const settings: Settings = { dataPath: '/home/user/viamedata', viamePath: '/opt/viame', readonlyMode: false, + nativeVideoPlayback: false, overrides: {}, }; const urlMapper = (a: string) => `http://localhost:8888/api/media?path=${a}`; diff --git a/client/platform/desktop/backend/native/multiCamImport.ts b/client/platform/desktop/backend/native/multiCamImport.ts index 5a7b160d2..2d1678f62 100644 --- a/client/platform/desktop/backend/native/multiCamImport.ts +++ b/client/platform/desktop/backend/native/multiCamImport.ts @@ -218,6 +218,7 @@ async function beginMultiCamImport(args: MultiCamImportArgs): Promise Date: Sun, 24 May 2026 20:20:53 -0400 Subject: [PATCH 04/10] Re-compute lengths when stereo lines changed (cherry picked from commit 3db19951d0b29cbb9a854123bf5353213dcd14f3) --- .../components/TrackSettingsPanel.vue | 2 +- client/dive-common/components/Viewer.vue | 8 + client/platform/desktop/backend/ipcService.ts | 14 + .../platform/desktop/backend/native/stereo.ts | 141 ++++++++++ .../desktop/backend/serializers/viame.ts | 23 +- client/platform/desktop/frontend/api.ts | 46 ++++ .../frontend/components/ViewerLoader.vue | 256 ++++++++++++++++++ 7 files changed, 487 insertions(+), 3 deletions(-) diff --git a/client/dive-common/components/TrackSettingsPanel.vue b/client/dive-common/components/TrackSettingsPanel.vue index ba8672680..1ae98bbd1 100644 --- a/client/dive-common/components/TrackSettingsPanel.vue +++ b/client/dive-common/components/TrackSettingsPanel.vue @@ -37,7 +37,7 @@ export default defineComponent({ filterTracksByFrame: 'Filter the track list by those with detections in the current frame', autoZoom: 'Automatically zoom to the track when selected', showMultiCamToolbar: 'Show multi-camera tools in the top toolbar when a track is selected', - stereoInteractiveMode: 'When enabled, annotations created on one camera are automatically warped to the other camera using stereo disparity', + stereoInteractiveMode: 'When enabled, annotations created on one camera are automatically warped to the other camera using stereo disparity. Line annotations also get a stereo measurement (length, midpoint, range, RMS) that is recomputed whenever the line is drawn, edited, or linked across cameras.', }); const modes = ref(['Track', 'Detection']); // Add unknown as the default type to the typeList diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index 49de2c3fc..e935efd3a 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -521,6 +521,14 @@ export default defineComponent({ trackStore.insert(newTrack, { imported: false }); } handler.trackSelect(newTrack.id); + + // In interactive stereo mode, a freshly linked pair should get its stereo + // measurement (length, midpoint, range, RMS) computed for every frame + // where both cameras now have a line. The desktop loader owns the stereo + // service, so delegate via an event. + if (clientSettings.stereoSettings.interactiveModeEnabled) { + emit('stereo-track-linked', baseTrack); + } } watch(linkingTrack, () => { if (linkingTrack.value !== null && selectedTrackId.value !== null) { diff --git a/client/platform/desktop/backend/ipcService.ts b/client/platform/desktop/backend/ipcService.ts index 50c77f50f..54aa66fee 100644 --- a/client/platform/desktop/backend/ipcService.ts +++ b/client/platform/desktop/backend/ipcService.ts @@ -36,6 +36,8 @@ import { StereoSetFrameRequest, StereoTransferLineRequest, StereoTransferPointsRequest, + StereoMeasureLineRequest, + StereoAggregateLengthsRequest, } from './native/stereo'; // defaults to linux if win32 doesn't exist @@ -400,6 +402,18 @@ export default function register() { return result; }); + ipcMain.handle('stereo-measure-line', async (_, args: StereoMeasureLineRequest) => { + const stereoService = getStereoServiceManager(); + const result = await stereoService.measureLine(args); + return result; + }); + + ipcMain.handle('stereo-aggregate-lengths', async (_, args: StereoAggregateLengthsRequest) => { + const stereoService = getStereoServiceManager(); + const result = await stereoService.aggregateLengths(args); + return result; + }); + ipcMain.handle('stereo-set-calibration', async (_, args: { calibration: StereoCalibration }) => { const stereoService = getStereoServiceManager(); await stereoService.setCalibration(args.calibration); diff --git a/client/platform/desktop/backend/native/stereo.ts b/client/platform/desktop/backend/native/stereo.ts index 95b0d8bdd..3b2a64e85 100644 --- a/client/platform/desktop/backend/native/stereo.ts +++ b/client/platform/desktop/backend/native/stereo.ts @@ -45,6 +45,19 @@ export interface StereoTransferLineRequest { line: [[number, number], [number, number]]; } +/** + * Full stereo measurement for a line, mirroring VIAME's compute_stereo_measurement. + * All values are in calibration units (e.g. mm). Keys match VIAME's attribute names. + */ +export interface StereoMeasurement { + length: number; + midpoint_x: number; + midpoint_y: number; + midpoint_z: number; + midpoint_range: number; + stereo_rms: number; +} + /** Response from transfer line request */ export interface StereoTransferLineResponse { id: string; @@ -52,6 +65,10 @@ export interface StereoTransferLineResponse { error?: string; transferredLine?: [[number, number], [number, number]]; originalLine?: [[number, number], [number, number]]; + /** Triangulated 3D length of the line in calibration units (e.g. mm). */ + length?: number; + /** Full stereo measurement (length, midpoint, range, RMS). */ + measurement?: StereoMeasurement; depthInfo?: { depthPoint1: number | null; depthPoint2: number | null; @@ -60,6 +77,39 @@ export interface StereoTransferLineResponse { }; } +/** Request to triangulate the length of a line already corresponded on both images */ +export interface StereoMeasureLineRequest { + leftLine: [[number, number], [number, number]]; + rightLine: [[number, number], [number, number]]; +} + +/** Response from a measure-line request */ +export interface StereoMeasureLineResponse { + id: string; + success: boolean; + error?: string; + /** Triangulated 3D length in calibration units (e.g. mm). */ + length?: number; + /** Full stereo measurement (length, midpoint, range, RMS). */ + measurement?: StereoMeasurement; +} + +/** Request to aggregate per-detection lengths along a track into a single value */ +export interface StereoAggregateLengthsRequest { + lengths: number[]; + /** 'average' (default), 'average_iqr', or 'median'. */ + method?: string; +} + +/** Response from an aggregate-lengths request */ +export interface StereoAggregateLengthsResponse { + id: string; + success: boolean; + error?: string; + /** Aggregated length in calibration units (e.g. mm). */ + avgLength?: number; +} + /** Request to transfer multiple points */ export interface StereoTransferPointsRequest { points: [number, number][]; @@ -518,6 +568,8 @@ export class StereoServiceManager extends EventEmitter { error: response.error, transferredLine: response.transferred_line, originalLine: response.original_line, + length: response.length, + measurement: response.measurement, depthInfo: response.depth_info ? { depthPoint1: response.depth_info.depth_point1, depthPoint2: response.depth_info.depth_point2, @@ -534,6 +586,95 @@ export class StereoServiceManager extends EventEmitter { }); } + /** + * Triangulate the 3D length of a line whose endpoints are already known on + * both images (no matching performed). Used to recompute length when a stereo + * line annotation that exists on both cameras is edited. + */ + async measureLine(request: StereoMeasureLineRequest): Promise { + if (!this.isEnabled()) { + return { + id: '', + success: false, + error: 'Stereo service is not enabled', + }; + } + + const id = this.generateRequestId(); + const fullRequest = { + id, + command: 'measure_line', + left_line: request.leftLine, + right_line: request.rightLine, + }; + + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + this.pendingRequests.delete(id); + reject(new Error('measure_line request timed out')); + }, this.requestTimeoutMs); + + this.pendingRequests.set(id, { + resolve: (response) => resolve({ + id: response.id, + success: response.success, + error: response.error, + length: response.length, + measurement: response.measurement, + }), + reject, + timeout, + }); + + const requestLine = `${JSON.stringify(fullRequest)}\n`; + this.process!.stdin!.write(requestLine); + }); + } + + /** + * Aggregate per-detection lengths along a track into a single value + * (mean / IQR-trimmed mean / median) via the shared C++ helper. Needs only + * the list of lengths gathered from the track. + */ + async aggregateLengths(request: StereoAggregateLengthsRequest): Promise { + if (!this.isEnabled()) { + return { + id: '', + success: false, + error: 'Stereo service is not enabled', + }; + } + + const id = this.generateRequestId(); + const fullRequest = { + id, + command: 'aggregate_lengths', + lengths: request.lengths, + method: request.method || 'average', + }; + + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + this.pendingRequests.delete(id); + reject(new Error('aggregate_lengths request timed out')); + }, this.requestTimeoutMs); + + this.pendingRequests.set(id, { + resolve: (response) => resolve({ + id: response.id, + success: response.success, + error: response.error, + avgLength: response.avg_length, + }), + reject, + timeout, + }); + + const requestLine = `${JSON.stringify(fullRequest)}\n`; + this.process!.stdin!.write(requestLine); + }); + } + /** * Transfer multiple points from left image to right image */ diff --git a/client/platform/desktop/backend/serializers/viame.ts b/client/platform/desktop/backend/serializers/viame.ts index a02d1bab9..5d33cadd3 100644 --- a/client/platform/desktop/backend/serializers/viame.ts +++ b/client/platform/desktop/backend/serializers/viame.ts @@ -352,12 +352,19 @@ function _parseFeature(row: string[]) { frame: rowInfo.frame, bounds: rowInfo.bounds, }; - if (rowInfo.fishLength !== -1) { + if (rowInfo.fishLength !== -1 && Number.isFinite(rowInfo.fishLength)) { feature.fishLength = rowInfo.fishLength; } if (rowData.attributes) { feature.attributes = rowData.attributes; } + // Surface the VIAME length column (col 8) as a 'length' detection attribute so + // stereo length measurements are visible/editable in the Attributes panel. + // The attribute is the canonical source on export (see serialize), so this + // round-trips through the length column rather than an (atr) column. + if (feature.fishLength !== undefined) { + feature.attributes = { ...(feature.attributes || {}), length: feature.fishLength }; + } if (rowData.geoFeatureCollection.features.length > 0) { feature.geometry = rowData.geoFeatureCollection; } @@ -702,18 +709,30 @@ async function serialize( column2 = moment.utc((feature.frame / meta.fps) * 1000).format('HH:mm:ss.SSSSSS'); } + // The 'length' detection attribute is the editable source of truth + // for the VIAME length column (col 8); fall back to fishLength. + const lengthAttr = feature.attributes?.length; + const lengthValue = (lengthAttr !== undefined && lengthAttr !== null) + ? Number(lengthAttr) + : feature.fishLength; + const row = [ track.id, column2, feature.frame, ...(feature.bounds as number[]), sortedPairs[0][1], // always take highest confidence to be track confidence - feature.fishLength || -1, + (lengthValue !== undefined && Number.isFinite(lengthValue)) ? lengthValue : -1, ...flattenDeep(sortedPairs), ]; /* Feature Attributes */ Object.entries(feature.attributes || {}).forEach(([key, val]) => { + // 'length' is written to the dedicated length column above; don't + // duplicate it as an (atr) column. + if (key === 'length') { + return; + } row.push(`${AtrToken} ${key} ${val}`); }); /* Track Attributes */ diff --git a/client/platform/desktop/frontend/api.ts b/client/platform/desktop/frontend/api.ts index 6f2b5eb29..a52d3e8d3 100644 --- a/client/platform/desktop/frontend/api.ts +++ b/client/platform/desktop/frontend/api.ts @@ -376,12 +376,23 @@ interface StereoTransferLineRequest { line: [[number, number], [number, number]]; } +interface StereoMeasurement { + length: number; + midpoint_x: number; + midpoint_y: number; + midpoint_z: number; + midpoint_range: number; + stereo_rms: number; +} + interface StereoTransferLineResponse { id: string; success: boolean; error?: string; transferredLine?: [[number, number], [number, number]]; originalLine?: [[number, number], [number, number]]; + length?: number; + measurement?: StereoMeasurement; depthInfo?: { depthPoint1: number | null; depthPoint2: number | null; @@ -390,6 +401,31 @@ interface StereoTransferLineResponse { }; } +interface StereoMeasureLineRequest { + leftLine: [[number, number], [number, number]]; + rightLine: [[number, number], [number, number]]; +} + +interface StereoMeasureLineResponse { + id: string; + success: boolean; + error?: string; + length?: number; + measurement?: StereoMeasurement; +} + +interface StereoAggregateLengthsRequest { + lengths: number[]; + method?: string; +} + +interface StereoAggregateLengthsResponse { + id: string; + success: boolean; + error?: string; + avgLength?: number; +} + interface StereoTransferPointsRequest { points: [number, number][]; } @@ -427,6 +463,14 @@ async function stereoTransferPoints(request: StereoTransferPointsRequest): Promi return ipcRenderer.invoke('stereo-transfer-points', request); } +async function stereoMeasureLine(request: StereoMeasureLineRequest): Promise { + return ipcRenderer.invoke('stereo-measure-line', request); +} + +async function stereoAggregateLengths(request: StereoAggregateLengthsRequest): Promise { + return ipcRenderer.invoke('stereo-aggregate-lengths', request); +} + async function stereoSetCalibration(calibration: StereoCalibration): Promise<{ success: boolean }> { return ipcRenderer.invoke('stereo-set-calibration', { calibration }); } @@ -589,6 +633,8 @@ export { stereoGetStatus, stereoTransferLine, stereoTransferPoints, + stereoMeasureLine, + stereoAggregateLengths, stereoSetCalibration, stereoIsEnabled, onStereoDisparityReady, diff --git a/client/platform/desktop/frontend/components/ViewerLoader.vue b/client/platform/desktop/frontend/components/ViewerLoader.vue index 38a33411a..c6d1e6060 100644 --- a/client/platform/desktop/frontend/components/ViewerLoader.vue +++ b/client/platform/desktop/frontend/components/ViewerLoader.vue @@ -16,6 +16,7 @@ import { segmentationPredict, segmentationInitialize, segmentationIsReady, loadMetadata, textQuery, runTextQueryPipeline, stereoEnable, stereoDisable, stereoSetFrame, stereoTransferLine, stereoTransferPoints, + stereoMeasureLine, stereoAggregateLengths, onStereoDisparityReady, onStereoDisparityError, } from 'platform/desktop/frontend/api'; import Export from './Export.vue'; @@ -410,6 +411,9 @@ export default defineComponent({ const stereoLoadingMessage = ref('Loading stereo model...'); const stereoLoadingError = ref(''); const stereoEnabled = ref(false); + // Transient notification reporting the latest computed stereo length + const stereoLengthSnackbar = ref(false); + const stereoLengthMessage = ref(''); // Cache image path getters per camera for stereo frame setting const stereoImagePathGetters = ref({} as Record string>); @@ -562,6 +566,225 @@ export default defineComponent({ return track; } + /** + * Extract the two endpoints of a 2-point LineString from a track's feature + * at the given frame. Returns null if there is no such line. + */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + function getStereoLineEndpoints(track: any, frameNum: number) + : [[number, number], [number, number]] | null { + if (!track) return null; + const [feature] = track.getFeature(frameNum); + if (!feature || !feature.geometry) return null; + const lineFeat = feature.geometry.features.find( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (f: any) => f.geometry.type === 'LineString' && f.geometry.coordinates.length === 2, + ); + if (!lineFeat) return null; + const c = lineFeat.geometry.coordinates as [number, number][]; + return [c[0], c[1]]; + } + + // Detection attribute names for the standard VIAME stereo measurement, + // matching the keys produced by the interactive stereo service. 'length' is + // additionally stored in the VIAME CSV length column (feature.fishLength). + const STEREO_MEASUREMENT_ATTRS = [ + 'length', 'midpoint_x', 'midpoint_y', 'midpoint_z', 'midpoint_range', 'stereo_rms', + ]; + + /** + * Ensure the standard stereo measurement attributes are defined as numeric + * detection attributes so they're visible in the Attributes panel. + */ + function ensureMeasurementAttributes() { + const viewer = viewerRef.value; + if (!viewer || !viewer.handler || !viewer.handler.setAttribute) return; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const existing = (viewer.attributes || []) as any[]; + STEREO_MEASUREMENT_ATTRS.forEach((name) => { + if (!existing.find((a) => a.name === name && a.belongs === 'detection')) { + viewer.handler.setAttribute({ + data: { + belongs: 'detection', + datatype: 'number', + name, + key: `detection_${name}`, + }, + }); + } + }); + // Track-level average length (mean of the per-frame lengths along the track). + if (!existing.find((a) => a.name === 'avg_length' && a.belongs === 'track')) { + viewer.handler.setAttribute({ + data: { + belongs: 'track', + datatype: 'number', + name: 'avg_length', + key: 'track_avg_length', + }, + }); + } + } + + /** + * Recompute the average stereo length along a whole track and store it as + * the track-level 'avg_length' attribute. The per-frame lengths are gathered + * from the track (needs nothing but the track); the aggregation itself is + * done in VIAME by viame::core::aggregate_lengths (the same helper the + * pair_stereo_tracks pipeline uses), reached via the stereo service. + */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + async function updateTrackAverageLength(track: any) { + if (!track) return; + const frames = track.featureIndex || []; + const lengths: number[] = []; + for (let i = 0; i < frames.length; i += 1) { + const feature = track.features[frames[i]]; + if (feature && Number.isFinite(feature.fishLength)) { + lengths.push(feature.fishLength); + } + } + if (lengths.length === 0) return; + + const response = await stereoAggregateLengths({ lengths }); + if (response.success && Number.isFinite(response.avgLength)) { + track.setAttribute('avg_length', Math.round((response.avgLength as number) * 100) / 100); + } + } + + /** + * Recompute the track-level average length for both camera tracks of a + * linked pair. Called once per measurement event (after per-frame lengths + * have been written). + */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + async function updateStereoTrackAverages(cameraStore: any, trackId: number) { + const cameras = Object.keys(stereoImagePathGetters.value); + if (cameras.length < 2) return; + const [leftCamera, rightCamera] = cameras; + await updateTrackAverageLength(cameraStore.getPossibleTrack(trackId, leftCamera)); + await updateTrackAverageLength(cameraStore.getPossibleTrack(trackId, rightCamera)); + } + + /** + * Store a full stereo measurement on a track feature. length is written to + * the canonical fishLength (VIAME CSV length column) and to a 'length' + * attribute; the remaining standard measurements (midpoint x/y/z, range, + * RMS) are stored as detection attributes. + */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + function applyStereoMeasurement(track: any, frameNum: number, measurement: any) { + if (!track || !measurement) return; + const round2 = (v: number) => Math.round(v * 100) / 100; + const { length } = measurement; + if (length !== undefined && Number.isFinite(length)) { + const [feature] = track.getFeature(frameNum); + if (feature && feature.keyframe) { + // Merge fishLength without disturbing existing geometry/bounds + track.setFeature({ frame: frameNum, fishLength: round2(length) }); + } + } + STEREO_MEASUREMENT_ATTRS.forEach((name) => { + const v = measurement[name]; + if (v !== undefined && Number.isFinite(v)) { + track.setFeatureAttribute(frameNum, name, round2(v)); + } + }); + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + function reportStereoMeasurement(measurement: any) { + if (!measurement || !Number.isFinite(measurement.length)) return; + const round2 = (v: number) => Math.round(v * 100) / 100; + const parts = [`Stereo length: ${round2(measurement.length)}`]; + if (Number.isFinite(measurement.midpoint_range)) { + parts.push(`range: ${round2(measurement.midpoint_range)}`); + } + stereoLengthMessage.value = parts.join(' • '); + stereoLengthSnackbar.value = true; + } + + /** + * Triangulate and store the stereo measurement for one frame of a track that + * has a 2-point line on both cameras. Returns the measurement, or null if + * either side lacks a line (or the service fails). + */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + async function measureStereoLineAtFrame(cameraStore: any, trackId: number, frameNum: number) { + const cameras = Object.keys(stereoImagePathGetters.value); + if (cameras.length < 2) return null; + const [leftCamera, rightCamera] = cameras; + const leftTrack = cameraStore.getPossibleTrack(trackId, leftCamera); + const rightTrack = cameraStore.getPossibleTrack(trackId, rightCamera); + const leftLine = getStereoLineEndpoints(leftTrack, frameNum); + const rightLine = getStereoLineEndpoints(rightTrack, frameNum); + if (!leftLine || !rightLine) return null; + + const response = await stereoMeasureLine({ leftLine, rightLine }); + if (response.success && response.measurement) { + ensureMeasurementAttributes(); + applyStereoMeasurement(leftTrack, frameNum, response.measurement); + applyStereoMeasurement(rightTrack, frameNum, response.measurement); + return response.measurement; + } + return null; + } + + /** + * Recompute the stereo measurement for a line annotation that exists on both + * cameras (e.g. after the user edited or drew one side) and update both tracks. + */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + async function autoUpdateStereoLength(cameraStore: any, trackId: number, frameNum: number) { + const measurement = await measureStereoLineAtFrame(cameraStore, trackId, frameNum); + if (measurement) { + reportStereoMeasurement(measurement); + await updateStereoTrackAverages(cameraStore, trackId); + } + } + + /** + * Handle two detections being linked across cameras (multicam link tool). + * Recompute the stereo measurement for every frame where both the left and + * right tracks now have a 2-point line. + */ + async function handleStereoTrackLinked(trackId: number) { + if (!stereoEnabled.value) return; + const viewer = viewerRef.value; + if (!viewer) return; + const { cameraStore, multiCamList } = viewer; + if (multiCamList.length < 2) return; + const cameras = Object.keys(stereoImagePathGetters.value); + if (cameras.length < 2) return; + + const [leftCamera, rightCamera] = cameras; + const leftTrack = cameraStore.getPossibleTrack(trackId, leftCamera); + const rightTrack = cameraStore.getPossibleTrack(trackId, rightCamera); + if (!leftTrack || !rightTrack) return; + + // Frames present in either track (each frame's line presence is re-checked + // inside measureStereoLineAtFrame). + const frames = Array.from(new Set([ + ...(leftTrack.featureIndex || []), + ...(rightTrack.featureIndex || []), + ])); + + let lastMeasurement = null; + for (let i = 0; i < frames.length; i += 1) { + try { + // eslint-disable-next-line no-await-in-loop + const measurement = await measureStereoLineAtFrame(cameraStore, trackId, frames[i]); + if (measurement) lastMeasurement = measurement; + } catch (err) { + console.warn('[Stereo] Link measurement failed:', err); + } + } + if (lastMeasurement) { + reportStereoMeasurement(lastMeasurement); + await updateStereoTrackAverages(cameraStore, trackId); + } + } + /** * Handle stereo annotation complete event from Viewer * Warps annotation from source camera to the other camera @@ -586,6 +809,17 @@ export default defineComponent({ if (existingTrack) { const [feature] = existingTrack.getFeature(params.frameNum); if (feature !== null) { + // Both cameras already have this annotation. Recompute the stereo + // measurement from the current (possibly edited) left+right lines + // instead of re-warping, preserving the user's manual corrections on + // both sides. Enabled whenever interactive stereo mode is on. + if (params.type === 'line') { + try { + await autoUpdateStereoLength(cameraStore, params.trackId, params.frameNum); + } catch (err) { + console.warn('[Stereo] Measurement update failed:', err); + } + } return; } } @@ -629,6 +863,16 @@ export default defineComponent({ keyframe: true, interpolate: false, }, lineGeometry); + + // Report and store the full stereo measurement on both cameras + if (response.measurement) { + ensureMeasurementAttributes(); + const sourceTrack = cameraStore.getPossibleTrack(params.trackId, params.camera); + applyStereoMeasurement(sourceTrack, params.frameNum, response.measurement); + applyStereoMeasurement(track, params.frameNum, response.measurement); + reportStereoMeasurement(response.measurement); + await updateStereoTrackAverages(cameraStore, params.trackId); + } } } else if (params.type === 'box') { // Convert box bounds to 4 corner points @@ -800,8 +1044,11 @@ export default defineComponent({ stereoLoadingDialog, stereoLoadingMessage, stereoLoadingError, + stereoLengthSnackbar, + stereoLengthMessage, closeStereoLoadingDialog, handleStereoAnnotationComplete, + handleStereoTrackLinked, }; }, }); @@ -818,6 +1065,7 @@ export default defineComponent({ @text-query-init="handleTextQueryInit" @text-query-all-frames="handleTextQueryAllFrames" @stereo-annotation-complete="handleStereoAnnotationComplete" + @stereo-track-linked="handleStereoTrackLinked" > From 1e0b103485bf6d9403e16b154138af1e4b79b1a1 Mon Sep 17 00:00:00 2001 From: Matt Dawkins Date: Mon, 25 May 2026 23:02:31 -0400 Subject: [PATCH 05/10] Consolidate stereo and segmentation interactives into same process (cherry picked from commit b7d4fa333fd0a4182c80388e412617f468dff88b) --- client/dive-common/apispec.ts | 45 ++ .../recipes/segmentationpointclick.ts | 8 + client/dive-common/use/useModeManager.ts | 32 +- client/platform/desktop/backend/ipcService.ts | 86 ++- .../desktop/backend/native/interactive.ts | 591 +++++++++++++++ .../desktop/backend/native/segmentation.ts | 532 ++------------ .../platform/desktop/backend/native/stereo.ts | 686 +----------------- client/platform/desktop/frontend/api.ts | 57 +- .../frontend/components/ViewerLoader.vue | 325 ++++++--- 9 files changed, 1048 insertions(+), 1314 deletions(-) create mode 100644 client/platform/desktop/backend/native/interactive.ts diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index d9ef2e455..dc8649663 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -301,6 +301,51 @@ export interface SegmentationPredictResponse { rleMask?: [number, number][]; } +/** + * Stereo point-segmentation. The segmentation service warps the seed to the + * other camera (configured stereo backend), segments there, and -- when enabled + * -- derives head/tail lines + the measurement. + */ +export interface SegmentationStereoSegmentRequest { + /** The already-segmented source-camera polygon (sampling + measurement). */ + polygon?: [number, number][]; + /** Source-camera click points and labels. */ + points: [number, number][]; + pointLabels: number[]; + /** Source (clicked) and other camera image/video paths. */ + sourceImagePath: string; + otherImagePath: string; + /** Calibration file path, read by the embedded stereo warper. */ + calibrationFile?: string; + /** Time in seconds when the paths are video files. */ + frameTime?: number; +} + +export interface SegmentationStereoSegmentResponse { + success: boolean; + error?: string; + /** Other-camera polygon from SAM. */ + polygon?: [number, number][]; + bounds?: [number, number, number, number]; + score?: number; + /** Seed point(s) used on the other camera (median of warped samples). */ + seedPoints?: [number, number][]; + seedLabels?: number[]; + /** Optional head/tail lines: source = clicked camera, other = warped. */ + generateLine?: boolean; + lineSource?: [[number, number], [number, number]]; + lineOther?: [[number, number], [number, number]]; + /** Stereo measurement for the derived line (calibration units, e.g. mm). */ + measurement?: { + length: number; + midpoint_x: number; + midpoint_y: number; + midpoint_z: number; + midpoint_range: number; + stereo_rms: number; + }; +} + export interface SegmentationStatusResponse { /** Whether segmentation is available */ available: boolean; diff --git a/client/dive-common/recipes/segmentationpointclick.ts b/client/dive-common/recipes/segmentationpointclick.ts index 805794f86..4b158ba6f 100644 --- a/client/dive-common/recipes/segmentationpointclick.ts +++ b/client/dive-common/recipes/segmentationpointclick.ts @@ -329,6 +329,10 @@ export default class SegmentationPointClick implements Recipe { // Emit event to notify that prediction is ready // Include frameNum so listeners can update the correct frame // Includes mask data for display during editing + // controlPoints carries the click points/labels so interactive-stereo + // mode can auto-generate the other-camera annotation immediately on a + // fresh prediction (the frame-change re-emit above omits them, so + // merely navigating does not re-trigger stereo work). this.bus.$emit('prediction-ready', { polygon: response.polygon, bounds: response.bounds, @@ -336,6 +340,10 @@ export default class SegmentationPointClick implements Recipe { frameNum, rleMask: response.rleMask, maskShape: response.maskShape, + controlPoints: this.points.length > 0 ? { + points: [...this.points], + labels: [...this.pointLabels], + } : undefined, } as SegmentationPredictionResult & { score?: number }); } else { // Prediction returned an error - handle point rejection diff --git a/client/dive-common/use/useModeManager.ts b/client/dive-common/use/useModeManager.ts index 3669fd68c..91f54e778 100644 --- a/client/dive-common/use/useModeManager.ts +++ b/client/dive-common/use/useModeManager.ts @@ -1134,6 +1134,23 @@ export default function useModeManager({ }, polygonGeometry); _nudgeEditingCanary(); + + // Interactive stereo: as soon as the left polygon is predicted, generate + // the other-camera polygon + head/tail lines + measurement automatically, + // without waiting for the user to finalize the detection. Only fired for + // fresh predictions (controlPoints present), so navigating frames or + // restoring a pending preview does not re-trigger stereo work. + if (onStereoAnnotationComplete && clientSettings.stereoSettings.interactiveModeEnabled + && selectedTrackId.value !== null && result.controlPoints) { + onStereoAnnotationComplete({ + type: 'segmentation', + camera: selectedCamera.value, + trackId: selectedTrackId.value as number, + frameNum: targetFrame, + points: result.controlPoints.points, + labels: result.controlPoints.labels, + }); + } } } @@ -1196,18 +1213,9 @@ export default function useModeManager({ interpolate, }, polygonGeometry); - // Stereo: emit segmentation annotation complete for each frame - if (onStereoAnnotationComplete && clientSettings.stereoSettings.interactiveModeEnabled - && selectedTrackId.value !== null && frameResult.controlPoints) { - onStereoAnnotationComplete({ - type: 'segmentation', - camera: selectedCamera.value, - trackId: selectedTrackId.value as number, - frameNum, - points: frameResult.controlPoints.points, - labels: frameResult.controlPoints.labels, - }); - } + // Note: the other-camera (stereo) annotation is generated earlier, on + // each fresh prediction (handleSegmentationPredictionReady), so there is + // no need to regenerate it on confirm. } }); diff --git a/client/platform/desktop/backend/ipcService.ts b/client/platform/desktop/backend/ipcService.ts index 54aa66fee..10566a38e 100644 --- a/client/platform/desktop/backend/ipcService.ts +++ b/client/platform/desktop/backend/ipcService.ts @@ -25,13 +25,14 @@ import beginMultiCamImport from './native/multiCamImport'; import settings from './state/settings'; import { listen } from './server'; import { - getSegmentationServiceManager, - shutdownSegmentationService, + getInteractiveServiceManager, + shutdownInteractiveService, +} from './native/interactive'; +import { SegmentationPredictRequest, + SegmentationStereoSegmentRequest, } from './native/segmentation'; import { - getStereoServiceManager, - shutdownStereoService, StereoCalibration, StereoSetFrameRequest, StereoTransferLineRequest, @@ -263,13 +264,26 @@ export default function register() { */ ipcMain.handle('segmentation-initialize', async () => { - const segService = getSegmentationServiceManager(); - await segService.initialize(settings.get()); - return { success: true }; + const currentSettings = settings.get(); + const pipelinesDir = path.join(currentSettings.viamePath, 'configs', 'pipelines'); + const hasSam2 = fs.existsSync(path.join(pipelinesDir, 'interactive_segmenter_sam2.conf')); + const hasSam3 = fs.existsSync(path.join(pipelinesDir, 'interactive_segmenter_sam3.conf')); + const noSamInstalled = !hasSam2 && !hasSam3; + + // Show a one-time warning if neither SAM pack is installed, + // but still proceed with initialization (VIAME has a default GrabCut fallback) + const showWarning = noSamInstalled && !samWarningShown; + if (showWarning) { + samWarningShown = true; + } + + const segService = getInteractiveServiceManager(); + await segService.initialize(currentSettings); + return { success: true, noSamInstalled: showWarning }; }); ipcMain.handle('segmentation-predict', async (_, args: SegmentationPredictRequest) => { - const segService = getSegmentationServiceManager(); + const segService = getInteractiveServiceManager(); // Auto-initialize if not ready if (!segService.isReady()) { @@ -280,8 +294,20 @@ export default function register() { return response; }); + ipcMain.handle('segmentation-stereo-segment', async (_, args: SegmentationStereoSegmentRequest) => { + const segService = getInteractiveServiceManager(); + + // Auto-initialize if not ready + if (!segService.isReady()) { + await segService.initialize(settings.get()); + } + + const response = await segService.stereoSegment(args); + return response; + }); + ipcMain.handle('segmentation-set-image', async (_, imagePath: string) => { - const segService = getSegmentationServiceManager(); + const segService = getInteractiveServiceManager(); if (!segService.isReady()) { await segService.initialize(settings.get()); @@ -292,7 +318,7 @@ export default function register() { }); ipcMain.handle('segmentation-clear-image', async () => { - const segService = getSegmentationServiceManager(); + const segService = getInteractiveServiceManager(); if (segService.isReady()) { await segService.clearImage(); @@ -301,12 +327,12 @@ export default function register() { }); ipcMain.handle('segmentation-shutdown', async () => { - await shutdownSegmentationService(); + await shutdownInteractiveService(); return { success: true }; }); ipcMain.handle('segmentation-is-ready', () => { - const segService = getSegmentationServiceManager(); + const segService = getInteractiveServiceManager(); return { ready: segService.isReady() }; }); @@ -319,7 +345,7 @@ export default function register() { points?: [number, number][]; pointLabels?: number[]; }) => { - const segService = getSegmentationServiceManager(); + const segService = getInteractiveServiceManager(); // Auto-initialize if not ready if (!segService.isReady()) { @@ -342,7 +368,7 @@ export default function register() { pointLabels?: number[]; refineMasks?: boolean; }) => { - const segService = getSegmentationServiceManager(); + const segService = getInteractiveServiceManager(); // Auto-initialize if not ready if (!segService.isReady()) { @@ -357,10 +383,16 @@ export default function register() { * Interactive Stereo Service */ - ipcMain.handle('stereo-enable', async (event, args?: { calibration?: StereoCalibration }) => { - const stereoService = getStereoServiceManager(); + ipcMain.handle('stereo-enable', async ( + event, args?: { calibration?: StereoCalibration; calibrationFile?: string }, + ) => { + const stereoService = getInteractiveServiceManager(); - // Forward async disparity events to the renderer + // Forward async disparity events to the renderer. The manager is a + // long-lived singleton, so clear any prior forwarders before re-adding to + // avoid accumulating listeners across enable cycles. + stereoService.removeAllListeners('disparity_ready'); + stereoService.removeAllListeners('disparity_error'); stereoService.on('disparity_ready', (data) => { event.sender.send('stereo-disparity-ready', data); }); @@ -373,60 +405,60 @@ export default function register() { }); ipcMain.handle('stereo-disable', async () => { - const stereoService = getStereoServiceManager(); + const stereoService = getInteractiveServiceManager(); const result = await stereoService.disable(); return result; }); ipcMain.handle('stereo-set-frame', async (_, args: StereoSetFrameRequest) => { - const stereoService = getStereoServiceManager(); + const stereoService = getInteractiveServiceManager(); const result = await stereoService.setFrame(args); return result; }); ipcMain.handle('stereo-get-status', async () => { - const stereoService = getStereoServiceManager(); + const stereoService = getInteractiveServiceManager(); const result = await stereoService.getStatus(); return result; }); ipcMain.handle('stereo-transfer-line', async (_, args: StereoTransferLineRequest) => { - const stereoService = getStereoServiceManager(); + const stereoService = getInteractiveServiceManager(); const result = await stereoService.transferLine(args); return result; }); ipcMain.handle('stereo-transfer-points', async (_, args: StereoTransferPointsRequest) => { - const stereoService = getStereoServiceManager(); + const stereoService = getInteractiveServiceManager(); const result = await stereoService.transferPoints(args); return result; }); ipcMain.handle('stereo-measure-line', async (_, args: StereoMeasureLineRequest) => { - const stereoService = getStereoServiceManager(); + const stereoService = getInteractiveServiceManager(); const result = await stereoService.measureLine(args); return result; }); ipcMain.handle('stereo-aggregate-lengths', async (_, args: StereoAggregateLengthsRequest) => { - const stereoService = getStereoServiceManager(); + const stereoService = getInteractiveServiceManager(); const result = await stereoService.aggregateLengths(args); return result; }); ipcMain.handle('stereo-set-calibration', async (_, args: { calibration: StereoCalibration }) => { - const stereoService = getStereoServiceManager(); + const stereoService = getInteractiveServiceManager(); await stereoService.setCalibration(args.calibration); return { success: true }; }); ipcMain.handle('stereo-shutdown', async () => { - await shutdownStereoService(); + await shutdownInteractiveService(); return { success: true }; }); ipcMain.handle('stereo-is-enabled', () => { - const stereoService = getStereoServiceManager(); + const stereoService = getInteractiveServiceManager(); return { enabled: stereoService.isEnabled() }; }); } diff --git a/client/platform/desktop/backend/native/interactive.ts b/client/platform/desktop/backend/native/interactive.ts new file mode 100644 index 000000000..f8e350a25 --- /dev/null +++ b/client/platform/desktop/backend/native/interactive.ts @@ -0,0 +1,591 @@ +/** + * Interactive Service Manager for Desktop + * + * Manages ONE persistent Python subprocess (viame.core.interactive_service) that + * hosts both interactive segmentation (point/SAM, text query, stereo point + * segmentation) and interactive stereo (enable, transfer, measure, dense + * disparity). Merging them into a single process lets stereo point-segmentation + * reuse the same stereo backend that interactive-stereo mode loads, instead of + * loading the stereo model twice. + * + * The process is spawned lazily on first use of either feature, and each + * feature's models load lazily on first relevant request (segmentation on the + * first predict, stereo on the first enable), so nothing heavy loads + * unnecessarily. Separate, unmodified VIAME config files drive each feature. + */ + +import OS from 'os'; +import { spawn, ChildProcess } from 'child_process'; +import npath from 'path'; +import readline from 'readline'; +import { EventEmitter } from 'events'; +import { Settings } from 'platform/desktop/constants'; +import { observeChild } from './processManager'; +import { + SegmentationStereoSegmentRequest, + SegmentationStereoSegmentResponse, + SegmentationPredictRequest, + SegmentationPredictResponse, +} from './segmentation'; +import { + StereoCalibration, + StereoSetFrameRequest, + StereoSetFrameResponse, + StereoTransferLineRequest, + StereoTransferLineResponse, + StereoMeasureLineRequest, + StereoMeasureLineResponse, + StereoAggregateLengthsRequest, + StereoAggregateLengthsResponse, + StereoTransferPointsRequest, + StereoTransferPointsResponse, + StereoStatusResponse, + StereoMeasurement, +} from './stereo'; + +/** Error message shown to users when the interactive service fails to load. */ +export const INTERACTIVE_LOAD_ERROR_MESSAGE = 'Unable to load the interactive service.'; + +/** Loose shape of a JSON response line from the Python service. */ +interface ServiceResponse { + id: string; + success?: boolean; + error?: string; + type?: string; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + [key: string]: any; +} + +interface PendingRequest { + resolve: (response: ServiceResponse) => void; + reject: (error: Error) => void; + timeout: NodeJS.Timeout; +} + +export class InteractiveServiceManager extends EventEmitter { + private process: ChildProcess | null = null; + + private readline: readline.Interface | null = null; + + private pendingRequests: Map = new Map(); + + private isStarting = false; + + private startPromise: Promise | null = null; + + private settings: Settings | null = null; + + private requestCounter = 0; + + /** Whether interactive stereo has been enabled in the running process. */ + private stereoEnabled = false; + + // Generous timeout: the first request of each feature lazily loads its model. + private readonly requestTimeoutMs = 300000; + + /** Is the shared process running? */ + isReady(): boolean { + return this.process !== null && this.process.exitCode === null; + } + + /** Is interactive stereo currently enabled? */ + isEnabled(): boolean { + return this.stereoEnabled && this.isReady(); + } + + private generateRequestId(): string { + this.requestCounter += 1; + return `req_${Date.now()}_${this.requestCounter}`; + } + + /** + * Ensure the shared Python process is running. Idempotent and de-duplicates + * concurrent callers. Spawning loads plugins (the slow, feature-agnostic step) + * but no models; models load lazily on first relevant request. + */ + async ensureStarted(settings: Settings): Promise { + if (this.isReady()) { + return; + } + if (this.isStarting && this.startPromise) { + await this.startPromise; + return; + } + this.isStarting = true; + this.settings = settings; + this.startPromise = this.doStart(settings); + try { + await this.startPromise; + } finally { + this.isStarting = false; + } + } + + private async doStart(settings: Settings): Promise { + // Clean up any defunct process first. + await this.shutdown(); + + return new Promise((resolve, reject) => { + const isWin32 = OS.platform() === 'win32'; + const viameSetup = npath.join( + settings.viamePath, + isWin32 ? 'setup_viame.bat' : 'setup_viame.sh', + ); + const pipelines = npath.join(settings.viamePath, 'configs', 'pipelines'); + const segConfig = npath.join(pipelines, 'interactive_segmenter_default.conf'); + const stereoConfig = npath.join(pipelines, 'interactive_stereo_default.conf'); + + const pyCommand = [ + 'python -m viame.core.interactive_service', + `--segmentation-config "${segConfig}"`, + `--stereo-config "${stereoConfig}"`, + ].join(' '); + + let command: string; + let shellOption: string | boolean; + if (isWin32) { + command = [`"${viameSetup}"`, '&&', pyCommand].join(' '); + shellOption = true; + } else { + command = [`. "${viameSetup}"`, '&&', pyCommand].join(' '); + shellOption = '/bin/bash'; + } + + console.log('[Interactive] Starting interactive service...'); + console.log(`[Interactive] Command: ${command}`); + + this.process = observeChild(spawn(command, { + shell: shellOption, + stdio: ['pipe', 'pipe', 'pipe'], + })); + + if (this.process.stdout) { + this.readline = readline.createInterface({ + input: this.process.stdout, + crlfDelay: Infinity, + }); + this.readline.on('line', (line) => this.handleResponse(line)); + } + + if (this.process.stderr) { + this.process.stderr.on('data', (data: Buffer) => { + const message = data.toString().trim(); + if (message) { + console.log(`[Interactive] ${message}`); + // Ready once plugins are loaded and the stdin loop is live (before + // any model loads — those happen lazily on first request). + if (message.includes('Service started, waiting for requests')) { + resolve(); + } + } + }); + } + + this.process.on('exit', (code, signal) => { + console.log(`[Interactive] Process exited with code ${code}, signal ${signal}`); + this.cleanup(); + if (this.isStarting) { + reject(new Error(INTERACTIVE_LOAD_ERROR_MESSAGE)); + } + }); + + this.process.on('error', (err) => { + console.error('[Interactive] Process error:', err); + this.cleanup(); + if (this.isStarting) { + reject(new Error(INTERACTIVE_LOAD_ERROR_MESSAGE)); + } + }); + + // Startup covers plugin loading only (~tens of seconds), not models. + setTimeout(() => { + if (this.isStarting) { + reject(new Error(INTERACTIVE_LOAD_ERROR_MESSAGE)); + } + }, 300000); + }); + } + + /** Route an incoming JSON line to its pending request, or emit async events. */ + private handleResponse(line: string): void { + let response: ServiceResponse; + try { + response = JSON.parse(line) as ServiceResponse; + } catch (err) { + console.error('[Interactive] Failed to parse response:', line, err); + return; + } + const pending = this.pendingRequests.get(response.id); + if (pending) { + clearTimeout(pending.timeout); + this.pendingRequests.delete(response.id); + pending.resolve(response); + } else if (response.type === 'disparity_ready') { + this.emit('disparity_ready', response); + } else if (response.type === 'disparity_error') { + this.emit('disparity_error', response); + } else { + console.warn(`[Interactive] Received response for unknown request: ${response.id}`); + } + } + + /** Write a command to the process and resolve with the matching response. */ + private sendRequest(payload: Record, timeoutLabel: string): Promise { + if (!this.isReady() || !this.process?.stdin) { + return Promise.reject(new Error('Interactive service is not running')); + } + const id = this.generateRequestId(); + const fullRequest = { ...payload, id }; + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + this.pendingRequests.delete(id); + reject(new Error(`${timeoutLabel} request timed out after ${this.requestTimeoutMs}ms`)); + }, this.requestTimeoutMs); + + this.pendingRequests.set(id, { resolve, reject, timeout }); + + const requestLine = `${JSON.stringify(fullRequest)}\n`; + this.process!.stdin!.write(requestLine, (err) => { + if (err) { + clearTimeout(timeout); + this.pendingRequests.delete(id); + reject(err); + } + }); + }); + } + + // ----------------------------------------------------- segmentation API + + /** Start (or no-op if running) the shared process. */ + async initialize(settings: Settings): Promise { + await this.ensureStarted(settings); + } + + async predict(request: SegmentationPredictRequest): Promise { + if (!request.imagePath) { + throw new Error('imagePath is required for segmentation prediction'); + } + const response = await this.sendRequest({ + command: 'predict', + image_path: request.imagePath, + points: request.points, + point_labels: request.pointLabels, + mask_input: request.maskInput, + multimask_output: request.multimaskOutput ?? false, + frame_time: request.frameTime, + }, 'Segmentation predict'); + return response as unknown as SegmentationPredictResponse; + } + + async stereoSegment( + request: SegmentationStereoSegmentRequest, + ): Promise { + const r = await this.sendRequest({ + command: 'stereo_segment', + polygon: request.polygon, + points: request.points, + point_labels: request.pointLabels, + source_image_path: request.sourceImagePath, + other_image_path: request.otherImagePath, + calibration_file: request.calibrationFile, + frame_time: request.frameTime, + }, 'Segmentation stereo_segment'); + return { + id: r.id, + success: r.success, + error: r.error, + polygon: r.polygon, + bounds: r.bounds, + score: r.score, + seedPoints: r.seed_points as [number, number][] | undefined, + seedLabels: r.seed_labels as number[] | undefined, + generateLine: r.generate_line as boolean | undefined, + lineSource: r.line_source as [[number, number], [number, number]] | undefined, + lineOther: r.line_other as [[number, number], [number, number]] | undefined, + measurement: r.measurement as StereoMeasurement | undefined, + }; + } + + async setImage(imagePath: string): Promise { + await this.sendRequest({ command: 'set_image', image_path: imagePath }, 'set_image'); + } + + async clearImage(): Promise { + if (!this.isReady()) { + return; + } + await this.sendRequest({ command: 'clear_image' }, 'clear_image'); + } + + async textQuery(request: { + imagePath: string; + text: string; + boxThreshold?: number; + maxDetections?: number; + boxes?: [number, number, number, number][]; + points?: [number, number][]; + pointLabels?: number[]; + frameTime?: number; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + }): Promise { + return this.sendRequest({ + command: 'text_query', + image_path: request.imagePath, + text: request.text, + box_threshold: request.boxThreshold ?? 0.3, + max_detections: request.maxDetections ?? 10, + boxes: request.boxes, + points: request.points, + point_labels: request.pointLabels, + frame_time: request.frameTime, + }, 'Text query'); + } + + async refineDetections(request: { + imagePath: string; + detections: { + box: [number, number, number, number]; + polygon?: [number, number][]; + score: number; + label: string; + }[]; + points?: [number, number][]; + pointLabels?: number[]; + refineMasks?: boolean; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + }): Promise { + return this.sendRequest({ + command: 'refine', + image_path: request.imagePath, + detections: request.detections, + points: request.points, + point_labels: request.pointLabels, + refine_masks: request.refineMasks ?? true, + }, 'Refine'); + } + + // ----------------------------------------------------------- stereo API + + async enable( + settings: Settings, + calibration?: StereoCalibration, + calibrationFile?: string, + ): Promise<{ success: boolean; error?: string }> { + try { + await this.ensureStarted(settings); + } catch (err) { + return { success: false, error: err instanceof Error ? err.message : String(err) }; + } + try { + const response = await this.sendRequest({ + command: 'enable', + calibration, + calibration_file: calibrationFile, + }, 'Stereo enable'); + if (response.success) { + this.stereoEnabled = true; + return { success: true }; + } + return { success: false, error: response.error || 'Failed to enable stereo service' }; + } catch (err) { + return { success: false, error: err instanceof Error ? err.message : String(err) }; + } + } + + /** + * Disable interactive stereo. Unlike the old standalone manager this does NOT + * kill the process (segmentation may still be in use); it just tells the + * backend to disable and marks stereo disabled. + */ + async disable(): Promise<{ success: boolean }> { + if (!this.stereoEnabled || !this.isReady()) { + this.stereoEnabled = false; + return { success: true }; + } + try { + await this.sendRequest({ command: 'disable' }, 'Stereo disable'); + } catch { + // ignore — disabling is best-effort + } + this.stereoEnabled = false; + return { success: true }; + } + + async setCalibration(calibration: StereoCalibration): Promise { + await this.sendRequest({ command: 'set_calibration', calibration }, 'set_calibration'); + } + + async setFrame(request: StereoSetFrameRequest): Promise { + if (!this.isEnabled()) { + return { + id: '', success: false, error: 'Stereo service is not enabled', disparityReady: false, + }; + } + const r = await this.sendRequest({ + command: 'set_frame', + left_image_path: request.leftImagePath, + right_image_path: request.rightImagePath, + frame_time: request.frameTime, + }, 'set_frame'); + return { + id: r.id, + success: r.success, + error: r.error, + disparityReady: r.disparity_ready || false, + message: r.message, + }; + } + + async getStatus(): Promise { + if (!this.isReady()) { + return { + id: '', success: true, enabled: false, disparityReady: false, hasCalibration: false, + }; + } + const r = await this.sendRequest({ command: 'get_status' }, 'get_status'); + return { + id: r.id, + success: r.success, + enabled: r.enabled || false, + disparityReady: r.disparity_ready || false, + computing: r.computing, + currentLeftPath: r.current_left_path, + currentRightPath: r.current_right_path, + hasCalibration: r.has_calibration || false, + }; + } + + async transferLine(request: StereoTransferLineRequest): Promise { + if (!this.isEnabled()) { + return { id: '', success: false, error: 'Stereo service is not enabled' }; + } + const r = await this.sendRequest({ command: 'transfer_line', line: request.line }, 'transfer_line'); + return { + id: r.id, + success: r.success, + error: r.error, + transferredLine: r.transferred_line, + originalLine: r.original_line, + length: r.length, + measurement: r.measurement, + depthInfo: r.depth_info ? { + depthPoint1: r.depth_info.depth_point1, + depthPoint2: r.depth_info.depth_point2, + disparityPoint1: r.depth_info.disparity_point1, + disparityPoint2: r.depth_info.disparity_point2, + } : undefined, + }; + } + + async transferPoints(request: StereoTransferPointsRequest): Promise { + if (!this.isEnabled()) { + return { id: '', success: false, error: 'Stereo service is not enabled' }; + } + const r = await this.sendRequest({ command: 'transfer_points', points: request.points }, 'transfer_points'); + return { + id: r.id, + success: r.success, + error: r.error, + transferredPoints: r.transferred_points, + originalPoints: r.original_points, + disparityValues: r.disparity_values, + }; + } + + async measureLine(request: StereoMeasureLineRequest): Promise { + if (!this.isEnabled()) { + return { id: '', success: false, error: 'Stereo service is not enabled' }; + } + const r = await this.sendRequest({ + command: 'measure_line', + left_line: request.leftLine, + right_line: request.rightLine, + }, 'measure_line'); + return { + id: r.id, success: r.success, error: r.error, length: r.length, measurement: r.measurement, + }; + } + + async aggregateLengths(request: StereoAggregateLengthsRequest): Promise { + if (!this.isEnabled()) { + return { id: '', success: false, error: 'Stereo service is not enabled' }; + } + const r = await this.sendRequest({ + command: 'aggregate_lengths', + lengths: request.lengths, + method: request.method || 'average', + }, 'aggregate_lengths'); + return { + id: r.id, success: r.success, error: r.error, avgLength: r.avg_length, + }; + } + + // ------------------------------------------------------------ lifecycle + + private cleanup(): void { + this.pendingRequests.forEach((pending) => { + clearTimeout(pending.timeout); + pending.reject(new Error('Interactive service terminated')); + }); + this.pendingRequests.clear(); + + if (this.readline) { + this.readline.close(); + this.readline = null; + } + + this.process = null; + this.stereoEnabled = false; + this.emit('shutdown'); + } + + async shutdown(): Promise { + if (!this.process) { + return; + } + console.log('[Interactive] Shutting down interactive service...'); + await new Promise((resolve) => { + const reqId = this.generateRequestId(); + const request = { id: reqId, command: 'shutdown' }; + if (this.process?.stdin?.writable) { + this.process.stdin.write(`${JSON.stringify(request)}\n`); + } + const timeoutId = setTimeout(() => { + if (this.process) { + console.log('[Interactive] Force killing interactive service...'); + this.process.kill('SIGTERM'); + } + this.cleanup(); + resolve(); + }, 5000); + if (this.process) { + this.process.once('exit', () => { + clearTimeout(timeoutId); + this.cleanup(); + resolve(); + }); + } else { + clearTimeout(timeoutId); + resolve(); + } + }); + } +} + +// Singleton instance shared by all IPC handlers. +let serviceManager: InteractiveServiceManager | null = null; + +export function getInteractiveServiceManager(): InteractiveServiceManager { + if (!serviceManager) { + serviceManager = new InteractiveServiceManager(); + } + return serviceManager; +} + +export async function shutdownInteractiveService(): Promise { + if (serviceManager) { + await serviceManager.shutdown(); + serviceManager = null; + } +} diff --git a/client/platform/desktop/backend/native/segmentation.ts b/client/platform/desktop/backend/native/segmentation.ts index f171e7134..e71ad0d1e 100644 --- a/client/platform/desktop/backend/native/segmentation.ts +++ b/client/platform/desktop/backend/native/segmentation.ts @@ -1,25 +1,21 @@ /** - * Interactive Segmentation Service Manager for Desktop + * Interactive Segmentation types. * - * Manages a persistent Python subprocess that keeps the segmentation model loaded in memory - * for fast interactive segmentation from point clicks. + * The runtime service manager was merged into the unified InteractiveServiceManager + * (see ./interactive). This module now only holds the segmentation request/response + * contracts shared between that manager and the IPC layer. */ -import { spawn, ChildProcess } from 'child_process'; -import npath from 'path'; -import readline from 'readline'; -import { EventEmitter } from 'events'; -import { Settings } from 'platform/desktop/constants'; -import { observeChild } from './processManager'; +import { StereoMeasurement } from './stereo'; -/** Error message shown to users when segmentation fails to load */ -export const SEGMENTATION_LOAD_ERROR_MESSAGE = 'Unable to load segmentation module'; +/** Error message shown to users when segmentation model process fails to load */ +export const SEGMENTATION_LOAD_ERROR_MESSAGE = "Model failed to load. If you haven't downloaded the SAM2 model pack from the VIAME Add-On wiki, please do so."; /** Request to the segmentation service */ export interface SegmentationInternalPredictRequest { /** Unique request ID for correlation */ id: string; - /** Path to the image file */ + /** Path to the image file (or video file if frame is specified) */ imagePath: string; /** Point coordinates as [x, y] pairs */ points: [number, number][]; @@ -29,6 +25,8 @@ export interface SegmentationInternalPredictRequest { maskInput?: number[][]; /** Whether to return multiple mask options */ multimaskOutput?: boolean; + /** Time in seconds when imagePath is a video file */ + frameTime?: number; } /** Response from the segmentation service */ @@ -39,8 +37,10 @@ export interface SegmentationInternalPredictResponse { success: boolean; /** Error message if failed */ error?: string; - /** Polygon coordinates as [x, y] pairs */ + /** Polygon coordinates as [x, y] pairs (largest exterior, for backward compat) */ polygon?: [number, number][]; + /** Multi-polygon data with holes support */ + polygons?: Array<{ exterior: [number, number][]; holes: [number, number][][] }>; /** Bounding box [x_min, y_min, x_max, y_max] */ bounds?: [number, number, number, number]; /** Quality score from segmentation model */ @@ -51,483 +51,45 @@ export interface SegmentationInternalPredictResponse { maskShape?: [number, number]; } -interface PendingRequest { - resolve: (response: SegmentationInternalPredictResponse) => void; - reject: (error: Error) => void; - timeout: NodeJS.Timeout; -} - -/** - * Segmentation Service Manager - * - * Manages a persistent Python subprocess for interactive segmentation inference. - * The service is started on-demand and kept alive for the session. - */ -export class SegmentationServiceManager extends EventEmitter { - private process: ChildProcess | null = null; - - private readline: readline.Interface | null = null; - - private pendingRequests: Map = new Map(); - - private isInitializing = false; - - private initPromise: Promise | null = null; - - private settings: Settings | null = null; - - private requestCounter = 0; - - private readonly requestTimeoutMs = 30000; // 30 second timeout - - /** - * Initialize the segmentation service with the given settings. - * This spawns the Python process and loads the segmentation model. - * The model stays loaded for the entire session to avoid reload delays. - */ - async initialize(settings: Settings): Promise { - // If already initialized and running, return immediately - // This keeps the model loaded between activations - if (this.isReady()) { - console.log('[Segmentation] Service already running, skipping initialization'); - return undefined; - } - - // If currently initializing, wait for it - if (this.isInitializing && this.initPromise) { - await this.initPromise; - return undefined; - } - - this.isInitializing = true; - this.settings = settings; - - this.initPromise = this._doInitialize(settings); - try { - await this.initPromise; - } finally { - this.isInitializing = false; - } - return undefined; - } - - private async _doInitialize(settings: Settings): Promise { - // Clean up any existing process - await this.shutdown(); - - return new Promise((resolve, reject) => { - const viameSetup = npath.join(settings.viamePath, 'setup_viame.sh'); - - const configPath = npath.join(settings.viamePath, 'configs', 'pipelines', 'interactive_segmenter_default.conf'); - - // Build the command to run the interactive segmentation service - const command = [ - `. "${viameSetup}"`, - '&&', - 'python -m viame.core.interactive_segmentation', - `--config "${configPath}"`, - ].join(' '); - - console.log('[Segmentation] Starting interactive segmentation service...'); - console.log(`[Segmentation] Command: ${command}`); - - this.process = observeChild(spawn(command, { - shell: '/bin/bash', - stdio: ['pipe', 'pipe', 'pipe'], - })); - - // Set up readline for stdout (JSON responses) - if (this.process.stdout) { - this.readline = readline.createInterface({ - input: this.process.stdout, - crlfDelay: Infinity, - }); - - this.readline.on('line', (line) => { - this.handleResponse(line); - }); - } - - // Log stderr (diagnostic messages) - if (this.process.stderr) { - this.process.stderr.on('data', (data: Buffer) => { - const message = data.toString().trim(); - if (message) { - console.log(`[Segmentation] ${message}`); - // Detect successful initialization - if (message.includes('model initialized successfully')) { - resolve(); - } - } - }); - } - - // Handle process exit - this.process.on('exit', (code, signal) => { - console.log(`[Segmentation] Process exited with code ${code}, signal ${signal}`); - this.cleanup(); - if (this.isInitializing) { - reject(new Error(SEGMENTATION_LOAD_ERROR_MESSAGE)); - } - }); - - this.process.on('error', (err) => { - console.error('[Segmentation] Process error:', err); - this.cleanup(); - if (this.isInitializing) { - reject(new Error(SEGMENTATION_LOAD_ERROR_MESSAGE)); - } - }); - - // Timeout for initialization (60 seconds for model loading) - setTimeout(() => { - if (this.isInitializing) { - reject(new Error(SEGMENTATION_LOAD_ERROR_MESSAGE)); - } - }, 60000); - }); - } - - /** - * Check if the service is ready for requests - */ - isReady(): boolean { - return this.process !== null && this.process.exitCode === null; - } - - /** - * Generate a unique request ID - */ - private generateRequestId(): string { - this.requestCounter += 1; - return `req_${Date.now()}_${this.requestCounter}`; - } - - /** - * Handle a response line from the segmentation service - */ - private handleResponse(line: string): void { - try { - const response = JSON.parse(line) as SegmentationInternalPredictResponse; - const pending = this.pendingRequests.get(response.id); - - if (pending) { - clearTimeout(pending.timeout); - this.pendingRequests.delete(response.id); - pending.resolve(response); - } else { - console.warn(`[Segmentation] Received response for unknown request: ${response.id}`); - } - } catch (err) { - console.error('[Segmentation] Failed to parse response:', line, err); - } - } - - /** - * Send a predict request to the segmentation service - */ - async predict(request: Omit): Promise { - if (!this.isReady()) { - throw new Error('Segmentation service is not ready. Call initialize() first.'); - } - - if (!this.process?.stdin) { - throw new Error('Segmentation service stdin is not available'); - } - - if (!request.imagePath) { - throw new Error('imagePath is required for segmentation prediction'); - } - - const id = this.generateRequestId(); - const fullRequest = { - id, - command: 'predict', - image_path: request.imagePath, - points: request.points, - point_labels: request.pointLabels, - mask_input: request.maskInput, - multimask_output: request.multimaskOutput ?? false, - }; - - return new Promise((resolve, reject) => { - const timeout = setTimeout(() => { - this.pendingRequests.delete(id); - reject(new Error(`Segmentation predict request timed out after ${this.requestTimeoutMs}ms`)); - }, this.requestTimeoutMs); - - this.pendingRequests.set(id, { resolve, reject, timeout }); - - // Send the request as JSON line - const requestLine = `${JSON.stringify(fullRequest)}\n`; - this.process!.stdin!.write(requestLine, (err) => { - if (err) { - clearTimeout(timeout); - this.pendingRequests.delete(id); - reject(err); - } - }); - }); - } - - /** - * Pre-load an image for multiple predictions (optional optimization) - */ - async setImage(imagePath: string): Promise { - if (!this.isReady()) { - throw new Error('Segmentation service is not ready'); - } - - const id = this.generateRequestId(); - const request = { - id, - command: 'set_image', - image_path: imagePath, - }; - - return new Promise((resolve, reject) => { - const timeout = setTimeout(() => { - this.pendingRequests.delete(id); - reject(new Error('set_image request timed out')); - }, this.requestTimeoutMs); - - this.pendingRequests.set(id, { - resolve: () => resolve(), - reject, - timeout, - }); - - const requestLine = `${JSON.stringify(request)}\n`; - this.process!.stdin!.write(requestLine); - }); - } - - /** - * Clear the cached image - */ - async clearImage(): Promise { - if (!this.isReady()) { - return undefined; // Nothing to clear - } - - const id = this.generateRequestId(); - const request = { - id, - command: 'clear_image', - }; - - await new Promise((resolve, reject) => { - const timeout = setTimeout(() => { - this.pendingRequests.delete(id); - reject(new Error('clear_image request timed out')); - }, this.requestTimeoutMs); - - this.pendingRequests.set(id, { - resolve: () => resolve(), - reject, - timeout, - }); - - const requestLine = `${JSON.stringify(request)}\n`; - this.process!.stdin!.write(requestLine); - }); - return undefined; - } - - /** - * Send a text query request for open-vocabulary detection/segmentation - */ - async textQuery(request: { - imagePath: string; - text: string; - boxThreshold?: number; - maxDetections?: number; - boxes?: [number, number, number, number][]; - points?: [number, number][]; - pointLabels?: number[]; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - }): Promise { - if (!this.isReady()) { - throw new Error('Segmentation service is not ready. Call initialize() first.'); - } - - if (!this.process?.stdin) { - throw new Error('Segmentation service stdin is not available'); - } - - const id = this.generateRequestId(); - const fullRequest = { - id, - command: 'text_query', - image_path: request.imagePath, - text: request.text, - box_threshold: request.boxThreshold ?? 0.3, - max_detections: request.maxDetections ?? 10, - boxes: request.boxes, - points: request.points, - point_labels: request.pointLabels, - }; - - return new Promise((resolve, reject) => { - const timeout = setTimeout(() => { - this.pendingRequests.delete(id); - reject(new Error(`Text query request timed out after ${this.requestTimeoutMs}ms`)); - }, this.requestTimeoutMs); - - this.pendingRequests.set(id, { resolve, reject, timeout }); - - const requestLine = `${JSON.stringify(fullRequest)}\n`; - this.process!.stdin!.write(requestLine, (err) => { - if (err) { - clearTimeout(timeout); - this.pendingRequests.delete(id); - reject(err); - } - }); - }); - } - - /** - * Refine existing detections with additional prompts - */ - async refineDetections(request: { - imagePath: string; - detections: { - box: [number, number, number, number]; - polygon?: [number, number][]; - score: number; - label: string; - }[]; - points?: [number, number][]; - pointLabels?: number[]; - refineMasks?: boolean; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - }): Promise { - if (!this.isReady()) { - throw new Error('Segmentation service is not ready. Call initialize() first.'); - } - - if (!this.process?.stdin) { - throw new Error('Segmentation service stdin is not available'); - } - - const id = this.generateRequestId(); - const fullRequest = { - id, - command: 'refine', - image_path: request.imagePath, - detections: request.detections, - points: request.points, - point_labels: request.pointLabels, - refine_masks: request.refineMasks ?? true, - }; - - return new Promise((resolve, reject) => { - const timeout = setTimeout(() => { - this.pendingRequests.delete(id); - reject(new Error(`Refine request timed out after ${this.requestTimeoutMs}ms`)); - }, this.requestTimeoutMs); - - this.pendingRequests.set(id, { resolve, reject, timeout }); - - const requestLine = `${JSON.stringify(fullRequest)}\n`; - this.process!.stdin!.write(requestLine, (err) => { - if (err) { - clearTimeout(timeout); - this.pendingRequests.delete(id); - reject(err); - } - }); - }); - } - - /** - * Clean up internal state after process exits - */ - private cleanup(): void { - // Reject all pending requests - this.pendingRequests.forEach((pending) => { - clearTimeout(pending.timeout); - pending.reject(new Error('Segmentation service terminated')); - }); - this.pendingRequests.clear(); - - if (this.readline) { - this.readline.close(); - this.readline = null; - } - - this.process = null; - this.emit('shutdown'); - } - - /** - * Gracefully shutdown the segmentation service - */ - async shutdown(): Promise { - if (!this.process) { - return undefined; - } - - console.log('[Segmentation] Shutting down segmentation service...'); - - await new Promise((resolve) => { - // Send shutdown command - const reqId = this.generateRequestId(); - const request = { id: reqId, command: 'shutdown' }; - - if (this.process?.stdin?.writable) { - this.process.stdin.write(`${JSON.stringify(request)}\n`); - } - - // Wait for process to exit or timeout - const timeoutId = setTimeout(() => { - if (this.process) { - console.log('[Segmentation] Force killing segmentation service...'); - this.process.kill('SIGTERM'); - } - this.cleanup(); - resolve(); - }, 5000); - - if (this.process) { - this.process.once('exit', () => { - clearTimeout(timeoutId); - this.cleanup(); - resolve(); - }); - } else { - clearTimeout(timeoutId); - resolve(); - } - }); - return undefined; - } -} - -// Singleton instance -let serviceManager: SegmentationServiceManager | null = null; - /** - * Get the segmentation service manager singleton + * Request for stereo point-segmentation. The segmentation service orchestrates + * everything: it warps a seed to the other camera (reusing the configured + * interactive-stereo backend), segments there, and -- when enabled -- derives + * head/tail lines and the stereo measurement. */ -export function getSegmentationServiceManager(): SegmentationServiceManager { - if (!serviceManager) { - serviceManager = new SegmentationServiceManager(); - } - return serviceManager; +export interface SegmentationStereoSegmentRequest { + /** The already-segmented source-camera polygon (for sampling + measurement). */ + polygon?: [number, number][]; + /** Source-camera click points and labels. */ + points: [number, number][]; + pointLabels: number[]; + /** Source (clicked) and other camera image/video paths. */ + sourceImagePath: string; + otherImagePath: string; + /** Calibration file path, read by the embedded stereo warper. */ + calibrationFile?: string; + /** Time in seconds when the paths are video files. */ + frameTime?: number; } -/** - * Shutdown the segmentation service (call on app close) - */ -export async function shutdownSegmentationService(): Promise { - if (serviceManager) { - await serviceManager.shutdown(); - serviceManager = null; - } +/** Response: the other-camera polygon plus optional lines + measurement. */ +export interface SegmentationStereoSegmentResponse { + id: string; + success: boolean; + error?: string; + /** Other-camera polygon from SAM. */ + polygon?: [number, number][]; + bounds?: [number, number, number, number]; + score?: number; + /** Seed point(s) used on the other camera (median of warped samples). */ + seedPoints?: [number, number][]; + seedLabels?: number[]; + /** Optional head/tail lines: source = clicked camera, other = warped. */ + generateLine?: boolean; + lineSource?: [[number, number], [number, number]]; + lineOther?: [[number, number], [number, number]]; + measurement?: StereoMeasurement; } -// Export type aliases for generic naming export type SegmentationPredictRequest = Omit; export type SegmentationPredictResponse = SegmentationInternalPredictResponse; diff --git a/client/platform/desktop/backend/native/stereo.ts b/client/platform/desktop/backend/native/stereo.ts index 3b2a64e85..4e07a1fec 100644 --- a/client/platform/desktop/backend/native/stereo.ts +++ b/client/platform/desktop/backend/native/stereo.ts @@ -1,18 +1,11 @@ /** - * Interactive Stereo Service Manager for Desktop + * Interactive Stereo types. * - * Manages a persistent Python subprocess that keeps the interactive stereo model loaded - * for fast interactive stereo annotation. When enabled, the service proactively computes - * disparity maps when the user navigates to new frames, so annotation transfers are instant. + * The runtime service manager was merged into the unified InteractiveServiceManager + * (see ./interactive). This module now only holds the stereo request/response + * contracts shared between that manager and the IPC layer. */ -import { spawn, ChildProcess } from 'child_process'; -import npath from 'path'; -import readline from 'readline'; -import { EventEmitter } from 'events'; -import { Settings } from 'platform/desktop/constants'; -import { observeChild } from './processManager'; - /** Error message shown to users when stereo service fails to load */ export const STEREO_LOAD_ERROR_MESSAGE = 'Unable to load stereo service'; @@ -29,6 +22,8 @@ export interface StereoCalibration { export interface StereoSetFrameRequest { leftImagePath: string; rightImagePath: string; + /** Time in seconds when paths are video files */ + frameTime?: number; } /** Response from set frame request */ @@ -136,672 +131,3 @@ export interface StereoStatusResponse { currentRightPath?: string; hasCalibration: boolean; } - -// Generic response type for internal use -interface StereoResponse { - id: string; - success: boolean; - error?: string; - message?: string; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - [key: string]: any; -} - -interface PendingRequest { - resolve: (response: StereoResponse) => void; - reject: (error: Error) => void; - timeout: NodeJS.Timeout; -} - -/** - * Stereo Service Manager - * - * Manages a persistent Python subprocess for interactive stereo disparity computation. - * The service is started when enabled and kept alive until disabled. - */ -export class StereoServiceManager extends EventEmitter { - private process: ChildProcess | null = null; - - private readline: readline.Interface | null = null; - - private pendingRequests: Map = new Map(); - - private isInitializing = false; - - private initPromise: Promise | null = null; - - private settings: Settings | null = null; - - private requestCounter = 0; - - private enabled = false; - - private readonly requestTimeoutMs = 60000; // 60 second timeout (disparity can take time) - - /** - * Enable the stereo service with the given settings and calibration. - * This spawns the Python process and loads the interactive stereo model. - */ - async enable(settings: Settings, calibration?: StereoCalibration): Promise<{ success: boolean; error?: string }> { - // If already enabled and running, just update calibration if provided - if (this.enabled && this.isReady()) { - console.log('[Stereo] Service already running'); - if (calibration) { - await this.setCalibration(calibration); - } - return { success: true }; - } - - // If currently initializing, wait for it - if (this.isInitializing && this.initPromise) { - await this.initPromise; - return { success: true }; - } - - this.isInitializing = true; - this.settings = settings; - - try { - this.initPromise = this._doInitialize(settings, calibration); - await this.initPromise; - this.enabled = true; - return { success: true }; - } catch (err) { - const errorMessage = err instanceof Error ? err.message : String(err); - return { success: false, error: errorMessage }; - } finally { - this.isInitializing = false; - } - } - - private async _doInitialize(settings: Settings, calibration?: StereoCalibration): Promise { - // Clean up any existing process - await this.shutdown(); - - return new Promise((resolve, reject) => { - const viameSetup = npath.join(settings.viamePath, 'setup_viame.sh'); - const configPath = npath.join(settings.viamePath, 'configs', 'pipelines', 'interactive_stereo_default.conf'); - - // Build the command to run the interactive stereo service - const command = [ - `. "${viameSetup}"`, - '&&', - 'python -m viame.core.interactive_stereo', - `--config "${configPath}"`, - ].join(' '); - - console.log('[Stereo] Starting interactive stereo service...'); - console.log(`[Stereo] Command: ${command}`); - - this.process = observeChild(spawn(command, { - shell: '/bin/bash', - stdio: ['pipe', 'pipe', 'pipe'], - })); - - // Set up readline for stdout (JSON responses) - if (this.process.stdout) { - this.readline = readline.createInterface({ - input: this.process.stdout, - crlfDelay: Infinity, - }); - - this.readline.on('line', (line) => { - this.handleResponse(line); - }); - } - - let initialized = false; - - // Log stderr (diagnostic messages) - if (this.process.stderr) { - this.process.stderr.on('data', (data: Buffer) => { - const message = data.toString().trim(); - if (message) { - console.log(`[Stereo] ${message}`); - // Detect successful startup (service is waiting for requests) - if (message.includes('Service started, waiting for requests')) { - // Now send the enable command with calibration - this.sendEnableCommand(calibration) - .then(() => { - initialized = true; - resolve(); - }) - .catch(reject); - } - } - }); - } - - // Handle process exit - this.process.on('exit', (code, signal) => { - console.log(`[Stereo] Process exited with code ${code}, signal ${signal}`); - this.cleanup(); - if (this.isInitializing && !initialized) { - reject(new Error(STEREO_LOAD_ERROR_MESSAGE)); - } - }); - - this.process.on('error', (err) => { - console.error('[Stereo] Process error:', err); - this.cleanup(); - if (this.isInitializing) { - reject(new Error(STEREO_LOAD_ERROR_MESSAGE)); - } - }); - - // Timeout for initialization (90 seconds for model loading) - setTimeout(() => { - if (this.isInitializing && !initialized) { - reject(new Error(STEREO_LOAD_ERROR_MESSAGE)); - } - }, 90000); - }); - } - - /** - * Send the enable command to the Python service - */ - private async sendEnableCommand(calibration?: StereoCalibration): Promise { - const id = this.generateRequestId(); - const request = { - id, - command: 'enable', - calibration, - }; - - return new Promise((resolve, reject) => { - const timeout = setTimeout(() => { - this.pendingRequests.delete(id); - reject(new Error('Enable command timed out')); - }, this.requestTimeoutMs); - - this.pendingRequests.set(id, { - resolve: (response) => { - if (response.success) { - resolve(); - } else { - reject(new Error(response.error || 'Failed to enable stereo service')); - } - }, - reject, - timeout, - }); - - const requestLine = `${JSON.stringify(request)}\n`; - this.process!.stdin!.write(requestLine); - }); - } - - /** - * Disable the stereo service and unload the model - */ - async disable(): Promise<{ success: boolean }> { - if (!this.enabled || !this.isReady()) { - this.enabled = false; - return { success: true }; - } - - try { - const id = this.generateRequestId(); - const request = { id, command: 'disable' }; - - await new Promise((resolve) => { - const timeout = setTimeout(() => { - this.pendingRequests.delete(id); - resolve(); - }, 5000); - - this.pendingRequests.set(id, { - resolve: () => resolve(), - reject: () => resolve(), - timeout, - }); - - const requestLine = `${JSON.stringify(request)}\n`; - this.process!.stdin!.write(requestLine); - }); - - await this.shutdown(); - this.enabled = false; - return { success: true }; - } catch { - this.enabled = false; - return { success: true }; - } - } - - /** - * Check if the service is ready for requests - */ - isReady(): boolean { - return this.process !== null && this.process.exitCode === null; - } - - /** - * Check if the service is enabled - */ - isEnabled(): boolean { - return this.enabled && this.isReady(); - } - - /** - * Generate a unique request ID - */ - private generateRequestId(): string { - this.requestCounter += 1; - return `req_${Date.now()}_${this.requestCounter}`; - } - - /** - * Handle a response line from the stereo service - */ - private handleResponse(line: string): void { - try { - const response = JSON.parse(line) as StereoResponse; - const pending = this.pendingRequests.get(response.id); - - if (pending) { - clearTimeout(pending.timeout); - this.pendingRequests.delete(response.id); - pending.resolve(response); - } else if (response.type === 'disparity_ready') { - // Async notification that disparity is ready - this.emit('disparity_ready', response); - } else if (response.type === 'disparity_error') { - this.emit('disparity_error', response); - } else { - console.warn(`[Stereo] Received response for unknown request: ${response.id}`); - } - } catch (err) { - console.error('[Stereo] Failed to parse response:', line, err); - } - } - - /** - * Set calibration parameters - */ - async setCalibration(calibration: StereoCalibration): Promise { - if (!this.isReady()) { - throw new Error('Stereo service is not ready'); - } - - const id = this.generateRequestId(); - const request = { - id, - command: 'set_calibration', - calibration, - }; - - return new Promise((resolve, reject) => { - const timeout = setTimeout(() => { - this.pendingRequests.delete(id); - reject(new Error('set_calibration request timed out')); - }, this.requestTimeoutMs); - - this.pendingRequests.set(id, { - resolve: () => resolve(), - reject, - timeout, - }); - - const requestLine = `${JSON.stringify(request)}\n`; - this.process!.stdin!.write(requestLine); - }); - } - - /** - * Set the current frame and start computing disparity proactively - */ - async setFrame(request: StereoSetFrameRequest): Promise { - if (!this.isEnabled()) { - return { - id: '', - success: false, - error: 'Stereo service is not enabled', - disparityReady: false, - }; - } - - const id = this.generateRequestId(); - const fullRequest = { - id, - command: 'set_frame', - left_image_path: request.leftImagePath, - right_image_path: request.rightImagePath, - }; - - return new Promise((resolve, reject) => { - const timeout = setTimeout(() => { - this.pendingRequests.delete(id); - reject(new Error('set_frame request timed out')); - }, this.requestTimeoutMs); - - this.pendingRequests.set(id, { - resolve: (response) => resolve({ - id: response.id, - success: response.success, - error: response.error, - disparityReady: response.disparity_ready || false, - message: response.message, - }), - reject, - timeout, - }); - - const requestLine = `${JSON.stringify(fullRequest)}\n`; - this.process!.stdin!.write(requestLine); - }); - } - - /** - * Get the current status of the stereo service - */ - async getStatus(): Promise { - if (!this.isReady()) { - return { - id: '', - success: true, - enabled: false, - disparityReady: false, - hasCalibration: false, - }; - } - - const id = this.generateRequestId(); - const request = { id, command: 'get_status' }; - - return new Promise((resolve, reject) => { - const timeout = setTimeout(() => { - this.pendingRequests.delete(id); - reject(new Error('get_status request timed out')); - }, this.requestTimeoutMs); - - this.pendingRequests.set(id, { - resolve: (response) => resolve({ - id: response.id, - success: response.success, - enabled: response.enabled || false, - disparityReady: response.disparity_ready || false, - computing: response.computing, - currentLeftPath: response.current_left_path, - currentRightPath: response.current_right_path, - hasCalibration: response.has_calibration || false, - }), - reject, - timeout, - }); - - const requestLine = `${JSON.stringify(request)}\n`; - this.process!.stdin!.write(requestLine); - }); - } - - /** - * Transfer a line from left image to right image using disparity - */ - async transferLine(request: StereoTransferLineRequest): Promise { - if (!this.isEnabled()) { - return { - id: '', - success: false, - error: 'Stereo service is not enabled', - }; - } - - const id = this.generateRequestId(); - const fullRequest = { - id, - command: 'transfer_line', - line: request.line, - }; - - return new Promise((resolve, reject) => { - const timeout = setTimeout(() => { - this.pendingRequests.delete(id); - reject(new Error('transfer_line request timed out')); - }, this.requestTimeoutMs); - - this.pendingRequests.set(id, { - resolve: (response) => resolve({ - id: response.id, - success: response.success, - error: response.error, - transferredLine: response.transferred_line, - originalLine: response.original_line, - length: response.length, - measurement: response.measurement, - depthInfo: response.depth_info ? { - depthPoint1: response.depth_info.depth_point1, - depthPoint2: response.depth_info.depth_point2, - disparityPoint1: response.depth_info.disparity_point1, - disparityPoint2: response.depth_info.disparity_point2, - } : undefined, - }), - reject, - timeout, - }); - - const requestLine = `${JSON.stringify(fullRequest)}\n`; - this.process!.stdin!.write(requestLine); - }); - } - - /** - * Triangulate the 3D length of a line whose endpoints are already known on - * both images (no matching performed). Used to recompute length when a stereo - * line annotation that exists on both cameras is edited. - */ - async measureLine(request: StereoMeasureLineRequest): Promise { - if (!this.isEnabled()) { - return { - id: '', - success: false, - error: 'Stereo service is not enabled', - }; - } - - const id = this.generateRequestId(); - const fullRequest = { - id, - command: 'measure_line', - left_line: request.leftLine, - right_line: request.rightLine, - }; - - return new Promise((resolve, reject) => { - const timeout = setTimeout(() => { - this.pendingRequests.delete(id); - reject(new Error('measure_line request timed out')); - }, this.requestTimeoutMs); - - this.pendingRequests.set(id, { - resolve: (response) => resolve({ - id: response.id, - success: response.success, - error: response.error, - length: response.length, - measurement: response.measurement, - }), - reject, - timeout, - }); - - const requestLine = `${JSON.stringify(fullRequest)}\n`; - this.process!.stdin!.write(requestLine); - }); - } - - /** - * Aggregate per-detection lengths along a track into a single value - * (mean / IQR-trimmed mean / median) via the shared C++ helper. Needs only - * the list of lengths gathered from the track. - */ - async aggregateLengths(request: StereoAggregateLengthsRequest): Promise { - if (!this.isEnabled()) { - return { - id: '', - success: false, - error: 'Stereo service is not enabled', - }; - } - - const id = this.generateRequestId(); - const fullRequest = { - id, - command: 'aggregate_lengths', - lengths: request.lengths, - method: request.method || 'average', - }; - - return new Promise((resolve, reject) => { - const timeout = setTimeout(() => { - this.pendingRequests.delete(id); - reject(new Error('aggregate_lengths request timed out')); - }, this.requestTimeoutMs); - - this.pendingRequests.set(id, { - resolve: (response) => resolve({ - id: response.id, - success: response.success, - error: response.error, - avgLength: response.avg_length, - }), - reject, - timeout, - }); - - const requestLine = `${JSON.stringify(fullRequest)}\n`; - this.process!.stdin!.write(requestLine); - }); - } - - /** - * Transfer multiple points from left image to right image - */ - async transferPoints(request: StereoTransferPointsRequest): Promise { - if (!this.isEnabled()) { - return { - id: '', - success: false, - error: 'Stereo service is not enabled', - }; - } - - const id = this.generateRequestId(); - const fullRequest = { - id, - command: 'transfer_points', - points: request.points, - }; - - return new Promise((resolve, reject) => { - const timeout = setTimeout(() => { - this.pendingRequests.delete(id); - reject(new Error('transfer_points request timed out')); - }, this.requestTimeoutMs); - - this.pendingRequests.set(id, { - resolve: (response) => resolve({ - id: response.id, - success: response.success, - error: response.error, - transferredPoints: response.transferred_points, - originalPoints: response.original_points, - disparityValues: response.disparity_values, - }), - reject, - timeout, - }); - - const requestLine = `${JSON.stringify(fullRequest)}\n`; - this.process!.stdin!.write(requestLine); - }); - } - - /** - * Clean up internal state after process exits - */ - private cleanup(): void { - // Reject all pending requests - this.pendingRequests.forEach((pending) => { - clearTimeout(pending.timeout); - pending.reject(new Error('Stereo service terminated')); - }); - this.pendingRequests.clear(); - - if (this.readline) { - this.readline.close(); - this.readline = null; - } - - this.process = null; - this.enabled = false; - this.emit('shutdown'); - } - - /** - * Gracefully shutdown the stereo service - */ - async shutdown(): Promise { - if (!this.process) { - return undefined; - } - - console.log('[Stereo] Shutting down stereo service...'); - - await new Promise((resolve) => { - // Send shutdown command - const reqId = this.generateRequestId(); - const request = { id: reqId, command: 'shutdown' }; - - if (this.process?.stdin?.writable) { - this.process.stdin.write(`${JSON.stringify(request)}\n`); - } - - // Wait for process to exit or timeout - const timeoutId = setTimeout(() => { - if (this.process) { - console.log('[Stereo] Force killing stereo service...'); - this.process.kill('SIGTERM'); - } - this.cleanup(); - resolve(); - }, 5000); - - if (this.process) { - this.process.once('exit', () => { - clearTimeout(timeoutId); - this.cleanup(); - resolve(); - }); - } else { - clearTimeout(timeoutId); - resolve(); - } - }); - return undefined; - } -} - -// Singleton instance -let serviceManager: StereoServiceManager | null = null; - -/** - * Get the stereo service manager singleton - */ -export function getStereoServiceManager(): StereoServiceManager { - if (!serviceManager) { - serviceManager = new StereoServiceManager(); - } - return serviceManager; -} - -/** - * Shutdown the stereo service (call on app close) - */ -export async function shutdownStereoService(): Promise { - if (serviceManager) { - await serviceManager.shutdown(); - serviceManager = null; - } -} diff --git a/client/platform/desktop/frontend/api.ts b/client/platform/desktop/frontend/api.ts index a52d3e8d3..50bd91934 100644 --- a/client/platform/desktop/frontend/api.ts +++ b/client/platform/desktop/frontend/api.ts @@ -5,6 +5,7 @@ import type { Pipe, Pipelines, PipelineParams, SaveAttributeArgs, SaveAttributeTrackFilterArgs, SaveDetectionsArgs, TrainingConfigs, SegmentationPredictRequest, SegmentationPredictResponse, SegmentationStatusResponse, + SegmentationStereoSegmentRequest, SegmentationStereoSegmentResponse, TextQueryRequest, TextQueryResponse, RefineDetectionsRequest, RefineDetectionsResponse, } from 'dive-common/apispec'; @@ -269,27 +270,33 @@ async function cancelJob(job: DesktopJob): Promise { */ async function segmentationInitialize(): Promise<{ success: boolean }> { - return ipcRenderer.invoke('segmentation-initialize'); + return window.diveDesktop.invoke('segmentation-initialize'); } async function segmentationPredict(request: SegmentationPredictRequest): Promise { - return ipcRenderer.invoke('segmentation-predict', request); + return window.diveDesktop.invoke('segmentation-predict', request); +} + +async function segmentationStereoSegment( + request: SegmentationStereoSegmentRequest, +): Promise { + return window.diveDesktop.invoke('segmentation-stereo-segment', request); } async function segmentationSetImage(imagePath: string): Promise<{ success: boolean }> { - return ipcRenderer.invoke('segmentation-set-image', imagePath); + return window.diveDesktop.invoke('segmentation-set-image', imagePath); } async function segmentationClearImage(): Promise<{ success: boolean }> { - return ipcRenderer.invoke('segmentation-clear-image'); + return window.diveDesktop.invoke('segmentation-clear-image'); } async function segmentationShutdown(): Promise<{ success: boolean }> { - return ipcRenderer.invoke('segmentation-shutdown'); + return window.diveDesktop.invoke('segmentation-shutdown'); } async function segmentationIsReady(): Promise { - return ipcRenderer.invoke('segmentation-is-ready'); + return window.diveDesktop.invoke('segmentation-is-ready'); } /** @@ -298,11 +305,11 @@ async function segmentationIsReady(): Promise { */ async function textQuery(request: TextQueryRequest): Promise { - return ipcRenderer.invoke('segmentation-text-query', request); + return window.diveDesktop.invoke('segmentation-text-query', request); } async function refineDetections(request: RefineDetectionsRequest): Promise { - return ipcRenderer.invoke('segmentation-refine', request); + return window.diveDesktop.invoke('segmentation-refine', request); } /** @@ -439,56 +446,55 @@ interface StereoTransferPointsResponse { disparityValues?: number[]; } -async function stereoEnable(calibration?: StereoCalibration): Promise<{ success: boolean; error?: string }> { - return ipcRenderer.invoke('stereo-enable', { calibration }); +async function stereoEnable( + calibration?: StereoCalibration, + calibrationFile?: string, +): Promise<{ success: boolean; error?: string }> { + return window.diveDesktop.invoke('stereo-enable', { calibration, calibrationFile }); } async function stereoDisable(): Promise<{ success: boolean }> { - return ipcRenderer.invoke('stereo-disable'); + return window.diveDesktop.invoke('stereo-disable'); } async function stereoSetFrame(request: StereoSetFrameRequest): Promise { - return ipcRenderer.invoke('stereo-set-frame', request); + return window.diveDesktop.invoke('stereo-set-frame', request); } async function stereoGetStatus(): Promise { - return ipcRenderer.invoke('stereo-get-status'); + return window.diveDesktop.invoke('stereo-get-status'); } async function stereoTransferLine(request: StereoTransferLineRequest): Promise { - return ipcRenderer.invoke('stereo-transfer-line', request); + return window.diveDesktop.invoke('stereo-transfer-line', request); } async function stereoTransferPoints(request: StereoTransferPointsRequest): Promise { - return ipcRenderer.invoke('stereo-transfer-points', request); + return window.diveDesktop.invoke('stereo-transfer-points', request); } async function stereoMeasureLine(request: StereoMeasureLineRequest): Promise { - return ipcRenderer.invoke('stereo-measure-line', request); + return window.diveDesktop.invoke('stereo-measure-line', request); } async function stereoAggregateLengths(request: StereoAggregateLengthsRequest): Promise { - return ipcRenderer.invoke('stereo-aggregate-lengths', request); + return window.diveDesktop.invoke('stereo-aggregate-lengths', request); } async function stereoSetCalibration(calibration: StereoCalibration): Promise<{ success: boolean }> { - return ipcRenderer.invoke('stereo-set-calibration', { calibration }); + return window.diveDesktop.invoke('stereo-set-calibration', { calibration }); } async function stereoIsEnabled(): Promise<{ enabled: boolean }> { - return ipcRenderer.invoke('stereo-is-enabled'); + return window.diveDesktop.invoke('stereo-is-enabled'); } function onStereoDisparityReady(callback: (data: unknown) => void): () => void { - const handler = (_event: unknown, data: unknown) => callback(data); - ipcRenderer.on('stereo-disparity-ready', handler); - return () => ipcRenderer.removeListener('stereo-disparity-ready', handler); + return window.diveDesktop.on('stereo-disparity-ready', (data: unknown) => callback(data)); } function onStereoDisparityError(callback: (data: unknown) => void): () => void { - const handler = (_event: unknown, data: unknown) => callback(data); - ipcRenderer.on('stereo-disparity-error', handler); - return () => ipcRenderer.removeListener('stereo-disparity-error', handler); + return window.diveDesktop.on('stereo-disparity-error', (data: unknown) => callback(data)); } /** @@ -618,6 +624,7 @@ export { /* Segmentation APIs */ segmentationInitialize, segmentationPredict, + segmentationStereoSegment, segmentationSetImage, segmentationClearImage, segmentationShutdown, diff --git a/client/platform/desktop/frontend/components/ViewerLoader.vue b/client/platform/desktop/frontend/components/ViewerLoader.vue index c6d1e6060..8ca602070 100644 --- a/client/platform/desktop/frontend/components/ViewerLoader.vue +++ b/client/platform/desktop/frontend/components/ViewerLoader.vue @@ -1,5 +1,4 @@ +