From 5e39ec5ee8a822227f70b56d480c4d20d57b46c3 Mon Sep 17 00:00:00 2001 From: naglepuff Date: Wed, 12 Aug 2026 17:59:35 -0400 Subject: [PATCH] Enable mask images and batbot metadata in NABat --- bats_ai/core/views/nabat/nabat_recording.py | 82 +++++++++++++++++++ client/src/api/NABatApi.ts | 20 +++++ client/src/components/PulseMetadataButton.vue | 12 ++- client/src/components/SpectrogramViewer.vue | 36 +++++--- client/src/components/ThumbnailViewer.vue | 42 ++++++---- client/src/components/geoJS/LayerManager.vue | 19 ++++- client/src/use/usePulseMetadata.ts | 14 ++++ client/src/use/useState.ts | 44 +++++++--- client/src/views/NABat/NABatSpectrogram.vue | 63 ++++++++++++++ client/src/views/Spectrogram.vue | 16 +++- 10 files changed, 306 insertions(+), 42 deletions(-) diff --git a/bats_ai/core/views/nabat/nabat_recording.py b/bats_ai/core/views/nabat/nabat_recording.py index d6198263..33e132b9 100644 --- a/bats_ai/core/views/nabat/nabat_recording.py +++ b/bats_ai/core/views/nabat/nabat_recording.py @@ -3,6 +3,7 @@ import base64 import json import logging +from typing import TYPE_CHECKING, Any from django.conf import settings from django.db import transaction @@ -17,12 +18,17 @@ from bats_ai.core.models import ProcessingTask, ProcessingTaskType, Species from bats_ai.core.models.nabat import ( NABatCompressedSpectrogram, + NABatPulseMetadata, NABatRecording, NABatRecordingAnnotation, ) from bats_ai.core.tasks.nabat.nabat_data_retrieval import nabat_recording_initialize from bats_ai.core.views.species import SpeciesSchema +if TYPE_CHECKING: + from bats_ai.core.views.recording import PulseMetadataSlopesSchema + + logger = logging.getLogger(__name__) router = RouterPaginated() @@ -590,3 +596,79 @@ def delete_recording_annotation( # Check permission annotation.delete() return "Recording annotation deleted successfully." + + +class NABatPulseContourSchema(Schema): + id: int | None + index: int + bounding_box: Any + contours: list + + @classmethod + def from_orm(cls, obj: NABatPulseMetadata): + return cls( + id=obj.id, + index=obj.index, + contours=obj.contours if obj.contours is not None else [], + bounding_box=json.loads(obj.bounding_box.geojson), + ) + + +class NABatPulseMetadataSchema(Schema): + id: int | None + index: int + curve: list[list[float]] | None = None + char_freq: list[float] | None = None + knee: list[float] | None = None + heel: list[float] | None = None + slopes: PulseMetadataSlopesSchema | None = None + + @classmethod + def from_orm(cls, obj: NABatPulseMetadata): + def point_to_list(pt): + if pt is None: + return None + return [pt.x, pt.y] + + def linestring_to_list(ls): + if ls is None: + return None + return [[c[0], c[1]] for c in ls.coords] + + return cls( + id=obj.id, + index=obj.index, + curve=linestring_to_list(obj.curve), + char_freq=point_to_list(obj.char_freq), + knee=point_to_list(obj.knee), + heel=point_to_list(obj.heel), + slopes=obj.slopes, + ) + + +@router.get("/{pk}/pulse_contours", auth=None) +def get_pulse_contours(request: HttpRequest, pk: int, api_token: str): + recording = get_object_or_404(NABatRecording, pk=pk) + + email_or_response = get_email_if_authorized(request, api_token, recording.recording_id) + if isinstance(email_or_response, JsonResponse): + return email_or_response + + computed_pulse_annotation_qs = NABatPulseMetadata.objects.filter( + nabat_recording=recording + ).order_by("index") + return [NABatPulseContourSchema.from_orm(pulse) for pulse in computed_pulse_annotation_qs] + + +@router.get("/{pk}/pulse_metadata", auth=None) +def get_pulse_data(request: HttpRequest, pk: int, api_token: str): + recording = get_object_or_404(NABatRecording, pk=pk) + + email_or_response = get_email_if_authorized(request, api_token, recording.recording_id) + if isinstance(email_or_response, JsonResponse): + return email_or_response + + computed_pulse_annotation_qs = NABatPulseMetadata.objects.filter( + nabat_recording=recording + ).order_by("index") + return [NABatPulseMetadataSchema.from_orm(pulse) for pulse in computed_pulse_annotation_qs] diff --git a/client/src/api/NABatApi.ts b/client/src/api/NABatApi.ts index 81f65835..ba6de069 100644 --- a/client/src/api/NABatApi.ts +++ b/client/src/api/NABatApi.ts @@ -6,6 +6,8 @@ import { type Spectrogram, type UpdateFileAnnotation, type Species, + type ComputedPulseContour, + type PulseMetadata, } from "./api"; export interface NABatRecordingCompleteResponse { @@ -275,6 +277,22 @@ async function exportNABatAnnotations( return response.data; } +async function getNabatPulseContours(recordingId: string, apiToken: string) { + const result = await axiosInstance.get( + `nabat/recording/${recordingId}/pulse_contours`, + { params: { api_token: apiToken } }, + ); + return result.data; +} + +async function getNabatPulseMetadata(recordingId: string, apiToken: string) { + const result = await axiosInstance.get( + `nabat/recording/${recordingId}/pulse_metadata`, + { params: { api_token: apiToken } }, + ); + return result.data; +} + export { postNABatRecording, getNABatSpectrogram, @@ -292,4 +310,6 @@ export { getNABatConfigurationRecordings, exportNABatAnnotations, adminNaBatUpdateSpecies, + getNabatPulseContours, + getNabatPulseMetadata, }; diff --git a/client/src/components/PulseMetadataButton.vue b/client/src/components/PulseMetadataButton.vue index 4f1112bd..51897b56 100644 --- a/client/src/components/PulseMetadataButton.vue +++ b/client/src/components/PulseMetadataButton.vue @@ -3,6 +3,7 @@ import { defineComponent, ref } from "vue"; import usePulseMetadata, { PULSE_METADATA_LABELS_OPTIONS, } from "@use/usePulseMetadata"; +import useState from "@use/useState"; export default defineComponent({ name: "PulseMetadataButton", @@ -22,6 +23,7 @@ export default defineComponent({ viewPulseMetadataLayer, toggleViewPulseMetadataLayer, loadPulseMetadata, + loadNabatPulseMetadata, pulseMetadataList, pulseMetadataLoading, pulseMetadataLineColor, @@ -35,10 +37,18 @@ export default defineComponent({ pulseMetadataLabels, pulseMetadataDurationFreqLineColor, } = usePulseMetadata(); + const { isNaBat, nabatApiToken } = useState(); const togglePulseMetadata = async () => { if (pulseMetadataList.value.length === 0 && props.recordingId != null) { - await loadPulseMetadata(Number(props.recordingId)); + if (isNaBat()) { + await loadNabatPulseMetadata( + String(props.recordingId), + nabatApiToken.value, + ); + } else { + await loadPulseMetadata(Number(props.recordingId)); + } } toggleViewPulseMetadataLayer(); }; diff --git a/client/src/components/SpectrogramViewer.vue b/client/src/components/SpectrogramViewer.vue index 3d56f33c..be21e765 100644 --- a/client/src/components/SpectrogramViewer.vue +++ b/client/src/components/SpectrogramViewer.vue @@ -41,6 +41,10 @@ export default defineComponent({ type: Array as PropType, default: () => [], }, + maskLoaded: { + type: Boolean, + default: false, + }, waveplotImages: { type: Array as PropType, default: () => [], @@ -496,18 +500,26 @@ export default defineComponent({ } }); - watch([viewMaskOverlay, maskOverlayOpacity, () => props.maskImages], () => { - if (viewMaskOverlay.value && props.maskImages.length) { - geoJS.drawMaskImages( - props.maskImages, - scaledWidth.value, - scaledHeight.value, - maskOverlayOpacity.value, - ); - } else { - geoJS.clearMaskQuadFeatures(true); - } - }); + watch( + [ + viewMaskOverlay, + maskOverlayOpacity, + () => props.maskImages, + () => props.maskLoaded, + ], + () => { + if (viewMaskOverlay.value && props.maskImages.length) { + geoJS.drawMaskImages( + props.maskImages, + scaledWidth.value, + scaledHeight.value, + maskOverlayOpacity.value, + ); + } else { + geoJS.clearMaskQuadFeatures(true); + } + }, + ); watch([showWaveplot], () => { resetViewerBounds(false); diff --git a/client/src/components/ThumbnailViewer.vue b/client/src/components/ThumbnailViewer.vue index 6bf04fdf..8171bbb8 100644 --- a/client/src/components/ThumbnailViewer.vue +++ b/client/src/components/ThumbnailViewer.vue @@ -28,6 +28,10 @@ export default defineComponent({ type: Array as PropType, default: () => [], }, + maskLoaded: { + type: Boolean, + default: false, + }, waveplotImages: { type: Array as PropType, default: () => [], @@ -259,21 +263,29 @@ export default defineComponent({ drawWaveplotIfEnabled(finalWidth, finalHeight); }); - watch([viewMaskOverlay, maskOverlayOpacity, () => props.maskImages], () => { - const { width, height } = getImageDimensions(props.images); - const finalWidth = scaledWidth.value || width; - const finalHeight = scaledHeight.value || height; - if (viewMaskOverlay.value && props.maskImages.length) { - geoJS.drawMaskImages( - props.maskImages, - finalWidth, - finalHeight, - maskOverlayOpacity.value, - ); - } else { - geoJS.clearMaskQuadFeatures(true); - } - }); + watch( + [ + viewMaskOverlay, + maskOverlayOpacity, + () => props.maskImages, + () => props.maskLoaded, + ], + () => { + const { width, height } = getImageDimensions(props.images); + const finalWidth = scaledWidth.value || width; + const finalHeight = scaledHeight.value || height; + if (viewMaskOverlay.value && props.maskImages.length) { + geoJS.drawMaskImages( + props.maskImages, + finalWidth, + finalHeight, + maskOverlayOpacity.value, + ); + } else { + geoJS.clearMaskQuadFeatures(true); + } + }, + ); watch(viewWaveplot, () => { const { width, height } = getImageDimensions(props.images); diff --git a/client/src/components/geoJS/LayerManager.vue b/client/src/components/geoJS/LayerManager.vue index b2d3861d..1d0210dd 100644 --- a/client/src/components/geoJS/LayerManager.vue +++ b/client/src/components/geoJS/LayerManager.vue @@ -101,6 +101,9 @@ export default defineComponent({ contoursEnabled, contourOpacity, loadContours, + loadNabatContours, + isNaBat, + nabatApiToken, computedPulseContours, transparencyThreshold, } = useState(); @@ -108,6 +111,7 @@ export default defineComponent({ viewPulseMetadataLayer, pulseMetadataList, loadPulseMetadata, + loadNabatPulseMetadata, clearPulseMetadata, pulseMetadataLineColor, pulseMetadataLineSize, @@ -595,7 +599,11 @@ export default defineComponent({ return; } if (computedPulseContours.value.length === 0) { - await loadContours(new Number(props.recordingId) as number); + if (isNaBat()) { + await loadNabatContours(props.recordingId); + } else { + await loadContours(new Number(props.recordingId) as number); + } } if (!contourLayer) { contourLayer = new ContourLayer( @@ -636,7 +644,14 @@ export default defineComponent({ if (!props.recordingId || !props.spectroInfo?.compressedWidth) return; if (viewPulseMetadataLayer.value) { if (pulseMetadataList.value.length === 0) { - await loadPulseMetadata(Number(props.recordingId)); + if (isNaBat()) { + await loadNabatPulseMetadata( + props.recordingId, + nabatApiToken.value, + ); + } else { + await loadPulseMetadata(Number(props.recordingId)); + } } if (!pulseMetadataLayer) { pulseMetadataLayer = new PulseMetadataLayer( diff --git a/client/src/use/usePulseMetadata.ts b/client/src/use/usePulseMetadata.ts index 3f0ce14a..e88dd325 100644 --- a/client/src/use/usePulseMetadata.ts +++ b/client/src/use/usePulseMetadata.ts @@ -1,5 +1,6 @@ import { ref, type Ref, watch } from "vue"; import { getPulseMetadata, type PulseMetadata } from "../api/api"; +import { getNabatPulseMetadata } from "@/api/NABatApi"; const STORAGE_KEY = "pulseMetadata"; @@ -79,6 +80,18 @@ async function loadPulseMetadata(recordingId: number) { } } +async function loadNabatPulseMetadata(recordingId: string, apiToken: string) { + pulseMetadataLoading.value = true; + try { + pulseMetadataList.value = await getNabatPulseMetadata( + recordingId, + apiToken, + ); + } finally { + pulseMetadataLoading.value = false; + } +} + function clearPulseMetadata() { pulseMetadataList.value = []; } @@ -144,6 +157,7 @@ export default function usePulseMetadata() { pulseMetadataList, pulseMetadataLoading, loadPulseMetadata, + loadNabatPulseMetadata, clearPulseMetadata, viewPulseMetadataLayer, toggleViewPulseMetadataLayer, diff --git a/client/src/use/useState.ts b/client/src/use/useState.ts index 3e8909c5..75a095ed 100644 --- a/client/src/use/useState.ts +++ b/client/src/use/useState.ts @@ -22,6 +22,7 @@ import { interpolatePlasma, interpolateTurbo, } from "d3-scale-chromatic"; +import { getNabatPulseContours } from "@/api/NABatApi"; const annotationState: Ref = ref(""); const creationType: Ref<"pulse" | "sequence"> = ref("pulse"); @@ -114,6 +115,20 @@ async function loadContours(recordingId: number) { computedPulseContours.value = await getComputedPulseContour(recordingId); contoursLoading.value = false; } + +const nabatApiToken = ref(""); +async function loadNabatContours(recordingId: string) { + contoursLoading.value = true; + try { + computedPulseContours.value = await getNabatPulseContours( + recordingId, + nabatApiToken.value, + ); + } finally { + contoursLoading.value = false; + } +} + function clearContours() { computedPulseContours.value = []; } @@ -203,17 +218,6 @@ export default function useState() { currentUserId.value = userInfo.id; } - /** - * Function used to determine whether or not we are currently looking - * at an NABat-specific view. - * - * returns `true` if looking at an NABat view, `false` otherwise - */ - function isNaBat(): boolean { - const router = useRouter(); - return router.currentRoute.value.fullPath.includes("nabat"); - } - // Server filters by exclude_submitted when "Show submitted" is unchecked; we refetch on toggle. const myRecordingsDisplay = computed(() => recordingList.value); const sharedRecordingsDisplay = computed(() => sharedList.value); @@ -281,6 +285,21 @@ export default function useState() { } } + /** + * Function used to determine whether or not we are currently looking + * at an NABat-specific view. + * + * @returns `true` if looking at an NABat view, `false` otherwise + */ + function isNaBat(): boolean { + const router = useRouter(); + return router.currentRoute.value.fullPath.includes("nabat"); + } + + function setNabatApiToken(apiToken: string) { + nabatApiToken.value = apiToken; + } + return { annotationState, creationType, @@ -331,6 +350,7 @@ export default function useState() { contoursLoading, setContoursEnabled, loadContours, + loadNabatContours, clearContours, computedPulseContours, showSubmittedRecordings, @@ -350,5 +370,7 @@ export default function useState() { clearMapFilterBounds, loadMapFilterBounds, spectrogramFilename, + setNabatApiToken, + nabatApiToken, }; } diff --git a/client/src/views/NABat/NABatSpectrogram.vue b/client/src/views/NABat/NABatSpectrogram.vue index 101e549a..ae915c28 100644 --- a/client/src/views/NABat/NABatSpectrogram.vue +++ b/client/src/views/NABat/NABatSpectrogram.vue @@ -10,8 +10,11 @@ import SpectrogramViewer from "@components/SpectrogramViewer.vue"; import { spectroXToTime, type SpectroInfo } from "@components/geoJS/geoJSUtils"; import ThumbnailViewer from "@components/ThumbnailViewer.vue"; import useState from "@use/useState"; +import usePulseMetadata from "@/use/usePulseMetadata"; import ColorSchemeDialog from "@components/ColorSchemeDialog.vue"; import TransparencyFilterControl from "@/components/TransparencyFilterControl.vue"; +import PulseMetadataButton from "@/components/PulseMetadataButton.vue"; +import SpectrogramImageContentMenu from "@/components/SpectrogramImageContentMenu.vue"; import RecordingInfoDialog from "@components/RecordingInfoDialog.vue"; import RecordingAnnotations from "@components/RecordingAnnotations.vue"; import { usePrompt } from "@use/prompt-service"; @@ -26,6 +29,8 @@ export default defineComponent({ RecordingAnnotations, ColorSchemeDialog, TransparencyFilterControl, + PulseMetadataButton, + SpectrogramImageContentMenu, }, props: { id: { @@ -56,7 +61,14 @@ export default defineComponent({ toggleDrawingBoundingBox, fixedAxes, toggleFixedAxes, + setNabatApiToken, } = useState(); + const { + clearPulseMetadata, + viewPulseMetadataLayer, + loadNabatPulseMetadata, + pulseMetadataList, + } = usePulseMetadata(); const secondsWarning = 60; const { prompt } = usePrompt(); const { shouldWarn } = useJWTToken({ @@ -64,11 +76,14 @@ export default defineComponent({ warningSeconds: secondsWarning, }); const images: Ref = ref([]); + const maskImages: Ref = ref([]); const spectroInfo: Ref = ref(); const selectedUsers: Ref = ref([]); const speciesList: Ref = ref([]); const loadedImage = ref(false); const allImagesLoaded: Ref = ref([]); + const maskImagesLoaded: Ref = ref([]); + const maskLoaded = ref(false); const compressed = ref( configuration.value.spectrogram_view === "compressed", ); @@ -87,7 +102,11 @@ export default defineComponent({ ]); const loadData = async () => { loadedImage.value = false; + clearPulseMetadata(); + setNabatApiToken(props.apiToken); try { + const tempViewPulseMetadataLayer = viewPulseMetadataLayer.value; + viewPulseMetadataLayer.value = false; const response = compressed.value ? await getNABatSpectrogramCompressed(props.id, props.apiToken) : await getNABatSpectrogram(props.id, props.apiToken); @@ -110,10 +129,30 @@ export default defineComponent({ } }; }); + if (tempViewPulseMetadataLayer) { + viewPulseMetadataLayer.value = true; + } } else { // TODO Error Out if there is no URL console.error("No URL found for the spectrogram"); } + maskImages.value = []; + maskImagesLoaded.value = []; + maskLoaded.value = false; + if (response.data.mask_urls?.length) { + response.data.mask_urls.forEach((url, index) => { + maskImagesLoaded.value.push(false); + const image = new Image(); + image.src = url; + maskImages.value.push(image); + image.onload = () => { + maskImagesLoaded.value[index] = true; + if (maskImagesLoaded.value.every((item) => item)) { + maskLoaded.value = true; + } + }; + }); + } spectroInfo.value = response.data["spectroInfo"]; if (response.data["compressed"] && spectroInfo.value) { spectroInfo.value.start_times = response.data.compressed.start_times; @@ -130,6 +169,12 @@ export default defineComponent({ index === self.findIndex((t) => t.species_code === value.species_code), ); + if ( + viewPulseMetadataLayer.value && + pulseMetadataList.value.length === 0 + ) { + await loadNabatPulseMetadata(props.id, props.apiToken); + } // eslint-disable-next-line @typescript-eslint/no-explicit-any } catch (error: any) { errorMessage.value = `Failed fetch Spectrogram: ${error.message}:`; @@ -233,6 +278,8 @@ export default defineComponent({ pendingCenterTimeMs, loadedImage, images, + maskImages, + maskLoaded, spectroInfo, selectedId, selectedType, @@ -457,6 +504,18 @@ export default defineComponent({ Highlight Compressed Areas +
+ +
+
+ +
@@ -503,6 +562,8 @@ export default defineComponent({ = ref([]); const loadedImage = ref(false); const allImagesLoaded: Ref = ref([]); + const maskLoaded = ref(false); + const maskImagesLoaded: Ref = ref([]); const gridEnabled = ref(false); const recordingInfo = ref(false); const recordingMap = ref(false); @@ -204,11 +206,20 @@ export default defineComponent({ console.error("No URL found for the spectrogram"); } maskImages.value = []; + maskImagesLoaded.value = []; + maskLoaded.value = false; if (spectrogramData.value.mask_urls?.length) { - spectrogramData.value.mask_urls.forEach((url) => { + spectrogramData.value.mask_urls.forEach((url, index) => { const image = new Image(); + maskImagesLoaded.value.push(false); image.src = url; maskImages.value.push(image); + image.onload = () => { + maskImagesLoaded.value[index] = true; + if (maskImagesLoaded.value.every((item) => item)) { + maskLoaded.value = true; + } + }; }); } waveplotImages.value = []; @@ -435,6 +446,7 @@ export default defineComponent({ loading, images, maskImages, + maskLoaded, waveplotImages, spectroInfo, annotations, @@ -804,6 +816,7 @@ export default defineComponent({ v-if="loadedImage && spectroInfo" :images="images" :mask-images="maskImages" + :mask-loaded="maskLoaded" :waveplot-images="waveplotImages" :spectro-info="spectroInfo" :recording-id="id" @@ -821,6 +834,7 @@ export default defineComponent({ v-if="loadedImage && parentGeoViewerRef" :images="images" :mask-images="maskImages" + :mask-loaded="maskLoaded" :waveplot-images="waveplotImages" :spectro-info="spectroInfo" :recording-id="id"