2026 March release

api/management/commands/import-admin1-data.py

@@ -57,7 +57,7 @@ def handle(self, *args, **options):
 
         # loop through each feature in the shapefile
         for feature in data[0]:
-            code = feature.get("ADMIN01COD")
+            code = feature.get("admin1_pco")
             name = feature.get("ADMIN01NAM")
 
             geom_wkt = feature.geom.wkt
@@ -142,7 +142,7 @@ def handle(self, *args, **options):
         print("done!")
 
     def add_district(self, options, import_missing, feature, geom, centroid, bbox):
-        code = feature.get("ADMIN01COD") or "N.A"
+        code = feature.get("admin1_pco") or "N.A"
         name = feature.get("ADMIN01NAM")
         district = District()
         district.code = code

api/management/commands/index_and_notify.py

@@ -24,12 +24,20 @@
 )
 from deployments.models import ERU, Personnel, PersonnelDeployment
 from main.sentry import SentryMonitor
+from main.utils import logger_context
 from notifications.hello import get_hello
-from notifications.models import RecordType, Subscription, SubscriptionType, SurgeAlert
+from notifications.models import (
+    NotificationGUID,
+    RecordType,
+    Subscription,
+    SubscriptionType,
+    SurgeAlert,
+)
 from notifications.notification import send_notification
 from utils.elasticsearch import construct_es_data
 
-time_5_minutes = timedelta(minutes=5)
+# avoid skipping new entities that were created during 2 consecutive runs, scheduled to every 5 minutes:
+time_9_minutes = timedelta(minutes=9)
 time_1_day = timedelta(days=1)
 time_1_week = timedelta(days=7)  # for digest mode
 digest_time = int(10314)  # weekday - hour - min for digest timing (5 minutes once a week, Monday dawn)
@@ -72,23 +80,23 @@ class Command(BaseCommand):
 
     # Digest mode duration is 5 minutes once a week
     def is_digest_mode(self):
-        today = datetime.utcnow().replace(tzinfo=timezone.utc)
+        today = datetime.now(timezone.utc)
         weekdayhourmin = int(today.strftime("%w%H%M"))
         return digest_time <= weekdayhourmin and weekdayhourmin < digest_time + 5
 
     def is_daily_checkup_time(self):
-        today = datetime.utcnow().replace(tzinfo=timezone.utc)
+        today = datetime.now(timezone.utc)
         hourmin = int(today.strftime("%H%M"))
         return daily_retro <= hourmin and hourmin < daily_retro + 5
 
-    def diff_5_minutes(self):
-        return datetime.utcnow().replace(tzinfo=timezone.utc) - time_5_minutes
+    def diff_9_minutes(self):
+        return datetime.now(timezone.utc) - time_9_minutes
 
     def diff_1_day(self):
-        return datetime.utcnow().replace(tzinfo=timezone.utc) - time_1_day
+        return datetime.now(timezone.utc) - time_1_day
 
     def diff_1_week(self):
-        return datetime.utcnow().replace(tzinfo=timezone.utc) - time_1_week
+        return datetime.now(timezone.utc) - time_1_week
 
     def gather_country_and_region(self, records):
         # Appeals only, since these have a single country/region
@@ -121,7 +129,7 @@ def gather_event_countries_and_regions(self, records):
         # many-to-many relationship to countries and regions through event table
         countries = []
         for record in records:
-            if record.event.countries is not None:
+            if record.event is not None and record.event.countries is not None:
                 countries += [country.id for country in record.event.countries.all()]
         countries = list(set(countries))
         qs = Country.objects.filter(pk__in=countries)
@@ -231,24 +239,24 @@ def get_template(self, rtype=99):
     # Get the front-end url of the resource
     def get_resource_uri(self, record, rtype):
         # Determine the front-end URL
-        resource_uri = settings.FRONTEND_URL
+        resource_uri = settings.GO_WEB_URL
         if (
             rtype == RecordType.SURGE_ALERT or rtype == RecordType.FIELD_REPORT
         ):  # Pointing to event instead of field report %s/%s/%s - Munu asked - ¤
             belonging_event = (
                 record.event.id if record.event is not None else 57
             )  # Very rare – giving a non-existent | manually created surge – no event
-            resource_uri = "%s/emergencies/%s" % (settings.FRONTEND_URL, belonging_event)
+            resource_uri = "%s/emergencies/%s" % (settings.GO_WEB_URL, belonging_event)
         elif rtype == RecordType.SURGE_DEPLOYMENT_MESSAGES:
-            resource_uri = "%s/%s" % (settings.FRONTEND_URL, "surge/overview")  # could be further sophisticated:
+            resource_uri = "%s/%s" % (settings.GO_WEB_URL, "surge/overview")  # could be further sophisticated:
             # e.g. emergencies/6700/surge, where 6700 is the related emergency ID
         elif rtype == RecordType.APPEAL and (record.event is not None and not record.needs_confirmation):
             # Appeals with confirmed emergencies link to that emergency
-            resource_uri = "%s/emergencies/%s" % (settings.FRONTEND_URL, record.event.id)
+            resource_uri = "%s/emergencies/%s" % (settings.GO_WEB_URL, record.event.id)
         elif rtype != RecordType.APPEAL:
             # One-by-one followed or globally subscribed emergencies
             resource_uri = "%s/%s/%s" % (
-                settings.FRONTEND_URL,
+                settings.GO_WEB_URL,
                 # this else never occurs, see ¤
                 "emergencies" if rtype == RecordType.EVENT or rtype == RecordType.FOLLOWED_EVENT else "field-reports",
                 record.id,
@@ -337,7 +345,7 @@ def get_record_display(self, rtype, count):
         return display
 
     def get_weekly_digest_data(self, field):
-        today = datetime.utcnow().replace(tzinfo=timezone.utc)
+        today = datetime.now(timezone.utc)
         if field == "dref":
             return Appeal.objects.filter(end_date__gt=today, atype=0).count()
         elif field == "ea":
@@ -394,8 +402,8 @@ def get_weekly_digest_latest_deployments(self):
         #             'type': 'Alert',
         #             'operation': alert.operation,
         #            'event_url': (
-        #                '{}/emergencies/{}'.format(settings.FRONTEND_URL, event.id) if event else
-        #                settings.FRONTEND_URL,
+        #                '{}/emergencies/{}'.format(settings.GO_WEB_URL, event.id) if event else
+        #                settings.GO_WEB_URL,
         #            ),
         #             'society_from': '',
         #             'deployed_to': '',
@@ -413,7 +421,7 @@ def get_weekly_digest_latest_deployments(self):
             country_from = Country.objects.get(id=pers.country_from_id) if pers.country_from_id is not None else None
             dep_to_add = {
                 "operation": event.name if event else "",
-                "event_url": ("{}/emergencies/{}".format(settings.FRONTEND_URL, event.id) if event else settings.FRONTEND_URL),
+                "event_url": ("{}/emergencies/{}".format(settings.GO_WEB_URL, event.id) if event else settings.GO_WEB_URL),
                 "society_from": country_from.society_name if country_from else "",
                 "name": pers.name,
                 "role": pers.role,
@@ -655,10 +663,10 @@ def construct_template_record(self, rtype, record):
                     volunteers += int(f.num_volunteers or 0)
                     delegates += int(f.num_expats_delegates or 0)
             resource_uri, follow_url = self.get_resource_uri(record, rtype), None
-            if resource_uri != settings.FRONTEND_URL:
+            if resource_uri != settings.GO_WEB_URL:
                 follow_url = resource_uri + "/follow"
             rec_obj = {
-                "frontend_url": settings.FRONTEND_URL,
+                "frontend_url": settings.GO_WEB_URL,
                 "resource_uri": resource_uri,
                 "follow_url": follow_url,
                 "admin_uri": self.get_admin_uri(record, rtype),
@@ -682,7 +690,7 @@ def construct_template_record(self, rtype, record):
             }
         elif rtype == RecordType.WEEKLY_DIGEST:
             rec_obj = {
-                "resource_uri": settings.FRONTEND_URL,
+                "resource_uri": settings.GO_WEB_URL,
                 "active_dref": self.get_weekly_digest_data("dref"),
                 "active_ea": self.get_weekly_digest_data("ea"),
                 "funding_coverage": self.get_weekly_digest_data("fund"),
@@ -709,6 +717,11 @@ def construct_template_record(self, rtype, record):
             }
         return rec_obj
 
+    def has_recent_notificationguid_entry(self, mailtypes, since):
+        if not mailtypes:
+            return False
+        return NotificationGUID.objects.filter(created_at__gte=since, email_type__in=mailtypes).exists()
+
     def notify(self, records, rtype, stype, uid=None):
         record_count = 0
         if records:
@@ -781,6 +794,21 @@ def notify(self, records, rtype, stype, uid=None):
         if self.is_daily_checkup_time():
             subject += " [daily followup]"
 
+        mailtype_prefixes = [f"{RTYPE_NAMES[rtype]} notification"]
+        if uid is None and rtype == RecordType.FIELD_REPORT:
+            mailtype_prefixes = [
+                f"{RTYPE_NAMES[rtype]} notification (non_ifrc)",
+                f"{RTYPE_NAMES[rtype]} notification (ifrc)",
+            ]
+
+        mailtype_variants = [f"{prefix} - {subject}" for prefix in mailtype_prefixes]
+        if record_count == 1 and rtype != RecordType.WEEKLY_DIGEST:
+            mailtype_variants.extend([f"{prefix} - {subject}: {record_entries[0]['title']}" for prefix in mailtype_prefixes])
+
+        if self.has_recent_notificationguid_entry(mailtype_variants, self.diff_9_minutes()):
+            logger.info("Skipping notification – already sent recently.")
+            return
+
         template_path = self.get_template()
         if rtype == RecordType.FIELD_REPORT or rtype == RecordType.APPEAL or rtype == RecordType.WEEKLY_DIGEST:
             template_path = self.get_template(rtype)
@@ -794,6 +822,7 @@ def notify(self, records, rtype, stype, uid=None):
                 "is_staff": True if uid is None else is_staff,  # TODO: fork the sending to "is_staff / not ~" groups
                 "subject": subject,
                 "hide_preferences": False,
+                "frontend_url": settings.GO_WEB_URL,
             },
         )
         recipients = emails
@@ -830,6 +859,7 @@ def notify(self, records, rtype, stype, uid=None):
                                 "is_staff": True if uid is None else is_staff,
                                 "subject": subject,
                                 "hide_preferences": False,
+                                "frontend_url": settings.GO_WEB_URL,
                             },
                         )
                         send_notification(
@@ -864,6 +894,7 @@ def notify(self, records, rtype, stype, uid=None):
                         "is_staff": True if uid is None else is_staff,
                         "subject": subject,
                         "hide_preferences": False,
+                        "frontend_url": settings.GO_WEB_URL,
                     },
                 )
 
@@ -933,12 +964,16 @@ def index_records(self, records, to_create=True):
     def bulk(self, actions):
         try:
             created, errors = bulk(client=ES_CLIENT, actions=actions)
-            if len(errors):
-                logger.error("Produced the following errors:")
-                logger.error("[%s]" % ", ".join(map(str, errors)))
-        except Exception as e:
-            logger.error("Could not index records")
-            logger.error("%s..." % str(e)[:512])
+            if errors:
+                logger.error(
+                    "(index_and_notify:bulk): produced errors:",
+                    extra=logger_context(dict(errors=errors)),
+                )
+        except Exception:
+            logger.error(
+                "(index_and_notify:bulk): could not index records",
+                exc_info=True,
+            )
 
     # Remove items in a queryset where updated_at == created_at.
     # This leaves us with only ones that have been modified.
@@ -992,7 +1027,7 @@ def handle(self, *args, **options):
         if self.is_digest_mode():
             time_diff = self.diff_1_week()  # in digest mode (once a week, for new_entities only) we use a bigger interval
         else:
-            time_diff = self.diff_5_minutes()
+            time_diff = self.diff_9_minutes()
         time_diff_1_day = self.diff_1_day()
 
         cond1 = Q(created_at__gte=time_diff)

api/management/commands/ingest_appeals.py

@@ -99,26 +99,28 @@ def get_new_or_modified_appeals(self):
         else:
             # get latest BILATERALS
             logger.info("Querying appeals API for new appeals data (bilateral)")
-            url = "http://go-api.ifrc.org/api/appealbilaterals"
+            url = "https://go-api.ifrc.org/api/appealbilaterals"
             auth = (settings.APPEALS_USER, settings.APPEALS_PASS)
 
             adapter = HTTPAdapter(max_retries=settings.RETRY_STRATEGY)
             sess = Session()
-            sess.mount("http://", adapter)
+            sess.mount("https://", adapter)
+            # IFRC App Gateway doesn't like python-requests/2... as User-Agent.
+            headers = {"User-Agent": "go-requests/2.32.4"}
 
             # try 3 times to reach the API
             try:
-                response = sess.get(url, auth=auth)
+                response = sess.get(url, auth=auth, headers=headers)  # Bilaterals API doesn't need narrowing by date
             except reqexc.HTTPError as ex:
                 log_text = f"Error querying AppealBilaterals API: {ex}"
                 logger.error(log_text)
                 create_cron_record(CRON_NAME, log_text, CronJobStatus.ERRONEOUS)
-                return
+                return None, None, None
             except Exception as ex:
                 log_text = f"Error querying AppealBilaterals API at {url}: {str(ex)}"
                 logger.error(log_text)
                 create_cron_record(CRON_NAME, log_text, CronJobStatus.ERRONEOUS)
-                return
+                return None, None, None
             records = response.json()
             bilaterals = self.create_bilaterals_dict(records)
 
@@ -128,20 +130,21 @@ def get_new_or_modified_appeals(self):
 
             # get latest APPEALS
             logger.info("Querying appeals API for new appeals data")
-            url = "http://go-api.ifrc.org/api/appeals"  # DEBUG: can append filter ?app_code=MDRDJ003
+            url = "https://go-api.ifrc.org/api/appeals"  # DEBUG: can append filter &app_code=MDRDJ003
+            params = {"App_startDate": "2020-01-01|2100-01-01"}
             # try 3 times to reach the API
             try:
-                response = sess.get(url, auth=auth)
+                response = sess.get(url, auth=auth, headers=headers, params=params)
             except reqexc.HTTPError as ex:
                 log_text = f"Error querying Appeals API: {ex}"
                 logger.error(log_text)
                 create_cron_record(CRON_NAME, log_text, CronJobStatus.ERRONEOUS)
-                return
+                return None, None, None
             except Exception as ex:
                 log_text = f"Error querying Appeals API at {url}: {str(ex)}"
                 logger.error(log_text)
                 create_cron_record(CRON_NAME, log_text, CronJobStatus.ERRONEOUS)
-                return
+                return None, None, None
 
             records = response.json()
 
@@ -300,6 +303,9 @@ def handle(self, *args, **options):
         except Exception as ex:
             logger.error(f"Getting Appeals and AppealBilaterals failed: {str(ex)}")
             return
+        if new is None or modified is None or bilaterals is None:
+            logger.error("Appeals ingest aborted due to upstream API errors.")
+            return
         logger.info(f"{start_appeals_count} current appeals")
         logger.info(f"Creating {len(new)} new appeals")
         logger.info(f"Updating {len(modified)} existing appeals that MIGHT have been modified")

api/management/commands/ingest_ns_contact.py

@@ -22,7 +22,8 @@ class Command(BaseCommand):
     def handle(self, *args, **kwargs):
         logger.info("Starting NS Contacts")
         url = "https://go-api.ifrc.org/"
-        headers = {"accept": "application/xml;q=0.9, */*;q=0.8"}
+        # IFRC App Gateway doesn't like python-requests/2... as User-Agent, so let's fix it via the first one:
+        headers = {"User-Agent": "go-requests/2.32.4", "accept": "application/xml;q=0.9, */*;q=0.8"}
         response = requests.get(
             "https://go-api.ifrc.org/api/NationalSocieties",
             auth=HTTPBasicAuth(settings.NS_CONTACT_USERNAME, settings.NS_CONTACT_PASSWORD),

api/management/commands/ingest_ns_directory.py

@@ -24,7 +24,8 @@ def postprocessor(path, key, value):
 
         logger.info("Starting NS Contacts")
         url = "https://go-api.ifrc.org/"
-        headers = {"accept": "application/xml;q=0.9, */*;q=0.8"}
+        # IFRC App Gateway doesn't like python-requests/2... as User-Agent, so let's fix it via the first one:
+        headers = {"User-Agent": "go-requests/2.32.4", "accept": "application/xml;q=0.9, */*;q=0.8"}
         response = requests.get(
             "https://go-api.ifrc.org/api/NationalSocietiesContacts/",
             auth=HTTPBasicAuth(settings.NS_CONTACT_USERNAME, settings.NS_CONTACT_PASSWORD),

api/management/commands/ingest_ns_document.py

@@ -23,7 +23,8 @@ def handle(self, *args, **kwargs):
 
         # Fetch country codes
         country_code_url = "https://go-api.ifrc.org/api/NationalSocietiesContacts/"
-        headers = {"Authorization": f"Basic {settings.NS_INITIATIVES_API_TOKEN}"}
+        # IFRC App Gateway doesn't like python-requests/2... as User-Agent, so let's fix it via the first one:
+        headers = {"User-Agent": "go-requests/2.32.4", "Authorization": f"Basic {settings.NS_INITIATIVES_API_TOKEN}"}
         country_code_response = requests.get(url=country_code_url, headers=headers)
 
         if country_code_response.status_code != 200:

api/management/commands/sync_appealdocs.py

@@ -48,7 +48,7 @@ def handle(self, *args, **options):
             print("Doing a full scan of all Appeals")
             qset = Appeal.objects.all()
         else:
-            # By default, only check appeals for the past 3 months where Appeal Documents is 0
+            # By default, only check appeals for the past 6 months where Appeal Documents is 0
             now = datetime.now().replace(tzinfo=timezone.utc)
             six_months_ago = now - relativedelta(months=6)
             # This was the original qset, but it wouldn't get newer docs for the same Appeals
@@ -59,7 +59,8 @@ def handle(self, *args, **options):
         # First get all Appeal Codes
         appeal_codes = [a.code for a in qset]
         auth = (settings.APPEALS_USER, settings.APPEALS_PASS)
-        headers = {"Accept": "application/json"}
+        # IFRC App Gateway doesn't like python-requests/2... as User-Agent, so let's fix it via the first one:
+        headers = {"User-Agent": "go-requests/2.32.4", "Accept": "application/json"}
         existing = []
         created = []