feat: normalize datetimes with deployment time zones

Author: wietzesuijker

ami/main/api/serializers.py

@@ -185,6 +185,7 @@ class Meta:
             "updated_at",
             "latitude",
             "longitude",
+            "time_zone",
             "first_date",
             "last_date",
             "device",
@@ -234,6 +235,7 @@ class Meta:
             "id",
             "name",
             "details",
+            "time_zone",
         ]
 
 
@@ -247,6 +249,7 @@ class Meta:
             "details",
             "latitude",
             "longitude",
+            "time_zone",
             "events_count",
             # "captures_count",
             # "detections_count",

ami/main/migrations/0079_deployment_time_zone.py

@@ -0,0 +1,21 @@
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("main", "0078_classification_applied_to"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="deployment",
+            name="time_zone",
+            field=models.CharField(
+                default=settings.TIME_ZONE,
+                help_text="IANA time zone for this deployment. Naive datetimes are interpreted in this zone before being stored as UTC.",
+                max_length=64,
+            ),
+        ),
+    ]

ami/main/models.py

@@ -8,6 +8,7 @@
 import urllib.parse
 from io import BytesIO
 from typing import Final, final  # noqa: F401
+from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
 import PIL.Image
 import pydantic
@@ -606,6 +607,11 @@ class Deployment(BaseModel):
     latitude = models.FloatField(null=True, blank=True)
     longitude = models.FloatField(null=True, blank=True)
     image = models.ImageField(upload_to="deployments", blank=True, null=True)
+    time_zone = models.CharField(
+        max_length=64,
+        default=settings.TIME_ZONE,
+        help_text="IANA time zone for this deployment. Naive datetimes are interpreted in this zone before being stored as UTC.",
+    )
 
     project = models.ForeignKey(Project, on_delete=models.SET_NULL, null=True, related_name="deployments")
 
@@ -662,6 +668,14 @@ class Deployment(BaseModel):
     class Meta:
         ordering = ["name"]
 
+    def clean(self):
+        super().clean()
+        if self.time_zone:
+            try:
+                ZoneInfo(self.time_zone)
+            except ZoneInfoNotFoundError as exc:
+                raise ValidationError({"time_zone": f"Invalid IANA time zone '{self.time_zone}': {exc}"}) from exc
+
     def taxa(self) -> models.QuerySet["Taxon"]:
         return Taxon.objects.filter(Q(occurrences__deployment=self)).distinct()
 

ami/main/signals.py

@@ -1,13 +1,16 @@
 import logging
+from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
 from django.contrib.auth.models import Group
 from django.db.models.signals import m2m_changed, post_save, pre_delete, pre_save
 from django.dispatch import receiver
+from django.conf import settings
+from django.utils import timezone
 from guardian.shortcuts import assign_perm
 
 from ami.users.roles import BasicMember, ProjectManager, create_roles_for_project
 
-from .models import Project, User
+from .models import Classification, Detection, Event, Project, SourceImage, User
 
 logger = logging.getLogger(__name__)
 
@@ -110,3 +113,52 @@ def delete_project_groups(sender, instance, **kwargs):
     prefix = f"{instance.pk}_"
     # Find and delete all groups that start with {project_id}_
     Group.objects.filter(name__startswith=prefix).delete()
+
+
+_TZ_FIELDS = {
+    Event: ("start", "end", "calculated_fields_updated_at"),
+    SourceImage: ("timestamp", "last_modified"),
+    Detection: ("timestamp", "detection_time"),
+    Classification: ("timestamp",),
+}
+
+
+def _resolve_deployment(instance):
+    deployment = getattr(instance, "deployment", None)
+    if deployment:
+        return deployment
+
+    source_image = getattr(instance, "source_image", None)
+    if source_image:
+        return getattr(source_image, "deployment", None)
+
+    detection = getattr(instance, "detection", None)
+    if detection:
+        src = getattr(detection, "source_image", None)
+        return getattr(src, "deployment", None) if src else None
+
+    return None
+
+
+def _normalize_datetimes(sender, instance, **_kwargs):
+    fields = _TZ_FIELDS.get(sender)
+    if not fields or not settings.USE_TZ:
+        return
+
+    deployment = _resolve_deployment(instance)
+    try:
+        tz = ZoneInfo(getattr(deployment, "time_zone", None) or timezone.get_default_timezone_name())
+    except (ZoneInfoNotFoundError, TypeError):
+        tz = timezone.get_default_timezone()
+
+    for field in fields:
+        value = getattr(instance, field, None)
+        if value is None:
+            continue
+        if timezone.is_naive(value):
+            value = timezone.make_aware(value, tz)
+        setattr(instance, field, value.astimezone(timezone.utc))
+
+
+for model in _TZ_FIELDS:
+    pre_save.connect(_normalize_datetimes, sender=model, dispatch_uid=f"tznorm-{model.__name__}", weak=False)

ami/main/tests.py

@@ -4,17 +4,20 @@
 from io import BytesIO
 
 from django.contrib.auth.models import AnonymousUser
+from django.core.exceptions import ValidationError
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.db import connection, models
 from django.test import TestCase, override_settings
 from guardian.shortcuts import assign_perm, get_perms, remove_perm
 from PIL import Image
 from rest_framework import status
 from rest_framework.test import APIRequestFactory, APITestCase
+from django.utils import timezone
 from rich import print
 
 from ami.exports.models import DataExport
 from ami.jobs.models import VALID_JOB_TYPES, Job
+from ami.main.api.serializers import DeploymentSerializer
 from ami.main.models import (
     Classification,
     Deployment,
@@ -46,6 +49,42 @@
 logger = logging.getLogger(__name__)
 
 
+class TestTimeZoneNormalization(TestCase):
+    def test_deployment_invalid_time_zone_raises(self):
+        project = Project.objects.create(name="TZ Project", create_defaults=False)
+        deployment = Deployment(project=project, name="D1", time_zone="Mars/Phobos")
+        with self.assertRaises(ValidationError):
+            deployment.full_clean()
+
+    def test_deployment_serializer_exposes_time_zone(self):
+        project = Project.objects.create(name="TZ Project", create_defaults=False)
+        deployment = Deployment.objects.create(project=project, name="D1", time_zone="UTC")
+
+        class MinimalDeploymentSerializer(DeploymentSerializer):
+            class Meta(DeploymentSerializer.Meta):
+                fields = ("id", "time_zone")
+
+        request = APIRequestFactory().get("/")
+        request.user = AnonymousUser()
+
+        data = MinimalDeploymentSerializer(deployment, context={"request": request}).data
+        self.assertEqual(data["time_zone"], "UTC")
+
+    @override_settings(USE_TZ=True)
+    def test_event_presave_normalizes_with_deployment_tz(self):
+        project = Project.objects.create(name="TZ Project", create_defaults=False)
+        deployment = Deployment.objects.create(project=project, name="D1", time_zone="America/New_York")
+        start = datetime.datetime(2020, 1, 1, 0, 0, 0)  # naive
+        end = datetime.datetime(2020, 1, 1, 1, 0, 0)
+        event = Event.objects.create(project=project, deployment=deployment, start=start, end=end, group_by="g1")
+        self.assertEqual(event.start.tzinfo, timezone.utc)
+        self.assertEqual(event.end.tzinfo, timezone.utc)
+        expected_start = datetime.datetime(2020, 1, 1, 5, 0, 0, tzinfo=timezone.utc)
+        expected_end = datetime.datetime(2020, 1, 1, 6, 0, 0, tzinfo=timezone.utc)
+        self.assertEqual(event.start, expected_start)  # America/New_York (Eastern Standard Time, UTC-5) -> UTC
+        self.assertEqual(event.end, expected_end)
+
+
 class TestProjectSetup(TestCase):
     def test_project_creation(self):
         project = Project.objects.create(name="New Project with Defaults", create_defaults=True)