From 60d48162da9e6736ceedd7e038f08fe72c021f18 Mon Sep 17 00:00:00 2001
From: kirtimanmishrazipstack <kirtiman.mishra@zipstack.com>
Date: Fri, 24 Apr 2026 16:25:56 +0530
Subject: [PATCH 1/7] api deployment notification init

---
 backend/api_v2/notification.py                |  31 +-
 backend/notification_v2/enums.py              |  12 +
 .../migrations/0002_notification_notify_on.py |  30 ++
 backend/notification_v2/models.py             |  13 +-
 backend/notification_v2/serializers.py        |   7 +-
 backend/notification_v2/tests/__init__.py     |   0
 .../tests/test_notification_filter.py         | 321 ++++++++++++++++++
 backend/pipeline_v2/notification.py           |  31 +-
 .../notification-modal/CreateNotification.jsx |  13 +
 9 files changed, 448 insertions(+), 10 deletions(-)
 create mode 100644 backend/notification_v2/migrations/0002_notification_notify_on.py
 create mode 100644 backend/notification_v2/tests/__init__.py
 create mode 100644 backend/notification_v2/tests/test_notification_filter.py

diff --git a/backend/api_v2/notification.py b/backend/api_v2/notification.py
index 733084c637..a41ebc3000 100644
--- a/backend/api_v2/notification.py
+++ b/backend/api_v2/notification.py
@@ -1,8 +1,10 @@
 import logging
 
+from notification_v2.enums import NotificationTrigger
 from notification_v2.helper import NotificationHelper
 from notification_v2.models import Notification
 from pipeline_v2.dto import PipelineStatusPayload
+from workflow_manager.workflow_v2.enums import ExecutionStatus
 from workflow_manager.workflow_v2.models.execution import WorkflowExecution
 
 from api_v2.models import APIDeployment
@@ -16,11 +18,32 @@ def __init__(self, api: APIDeployment, workflow_execution: WorkflowExecution) ->
         self.api = api
         self.workflow_execution = workflow_execution
 
-    def send(self):
-        if not self.notifications.count():
-            logger.info(f"No notifications found for api {self.api}")
+    def send(self) -> None:
+        # Partition notifications by the run outcome so each row's notify_on
+        # preference is honored. STOPPED and any other non-terminal status
+        # fire only for ALL — explicit opt-ins to FAILURES/SUCCESS shouldn't.
+        status = self.workflow_execution.status
+        if status == ExecutionStatus.ERROR.value:
+            self.notifications = self.notifications.exclude(
+                notify_on=NotificationTrigger.SUCCESS_ONLY.value
+            )
+        elif status == ExecutionStatus.COMPLETED.value:
+            self.notifications = self.notifications.exclude(
+                notify_on=NotificationTrigger.FAILURES_ONLY.value
+            )
+        else:
+            self.notifications = self.notifications.filter(
+                notify_on=NotificationTrigger.ALL.value
+            )
+
+        if not self.notifications.exists():
+            logger.info(
+                "No notifications to dispatch for api %s (status=%s)",
+                self.api,
+                status,
+            )
             return
-        logger.info(f"Sending api status notification for api {self.api}")
+        logger.info("Sending api status notification for api %s", self.api)
 
         payload_dto = PipelineStatusPayload(
             type="API",
diff --git a/backend/notification_v2/enums.py b/backend/notification_v2/enums.py
index 991b08cac9..516b34074b 100644
--- a/backend/notification_v2/enums.py
+++ b/backend/notification_v2/enums.py
@@ -36,3 +36,15 @@ class PlatformType(Enum):
     @classmethod
     def choices(cls):
         return [(e.value, e.name.replace("_", " ").capitalize()) for e in cls]
+
+
+class NotificationTrigger(Enum):
+    """Controls which run outcomes fire a notification."""
+
+    ALL = "ALL"
+    FAILURES_ONLY = "FAILURES_ONLY"
+    SUCCESS_ONLY = "SUCCESS_ONLY"
+
+    @classmethod
+    def choices(cls):
+        return [(e.value, e.name.replace("_", " ").capitalize()) for e in cls]
diff --git a/backend/notification_v2/migrations/0002_notification_notify_on.py b/backend/notification_v2/migrations/0002_notification_notify_on.py
new file mode 100644
index 0000000000..53c1180126
--- /dev/null
+++ b/backend/notification_v2/migrations/0002_notification_notify_on.py
@@ -0,0 +1,30 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("notification_v2", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="notification",
+            name="notify_on",
+            field=models.CharField(
+                max_length=50,
+                choices=[
+                    ("ALL", "All"),
+                    ("FAILURES_ONLY", "Failures only"),
+                    ("SUCCESS_ONLY", "Success only"),
+                ],
+                default="ALL",
+                db_comment=(
+                    "Controls which run outcomes trigger this notification. ALL "
+                    "(default) preserves the historical 'notify on every "
+                    "completion' behavior; FAILURES_ONLY fires only on failed "
+                    "runs (ERROR for API deployments, FAILURE for ETL "
+                    "pipelines); SUCCESS_ONLY fires only on successful runs."
+                ),
+            ),
+        ),
+    ]
diff --git a/backend/notification_v2/models.py b/backend/notification_v2/models.py
index 489a8c827e..4fe21cd6c4 100644
--- a/backend/notification_v2/models.py
+++ b/backend/notification_v2/models.py
@@ -5,7 +5,7 @@
 from pipeline_v2.models import Pipeline
 from utils.models.base_model import BaseModel
 
-from .enums import AuthorizationType, NotificationType, PlatformType
+from .enums import AuthorizationType, NotificationTrigger, NotificationType, PlatformType
 
 NOTIFICATION_NAME_MAX_LENGTH = 255
 
@@ -47,6 +47,17 @@ class Notification(BaseModel):
         default=True,
         db_comment="Flag indicating whether the notification is active or not.",
     )
+    notify_on = models.CharField(
+        max_length=50,
+        choices=NotificationTrigger.choices(),
+        default=NotificationTrigger.ALL.value,
+        db_comment=(
+            "Controls which run outcomes trigger this notification. ALL (default) "
+            "preserves the historical 'notify on every completion' behavior; "
+            "FAILURES_ONLY fires only on failed runs (ERROR for API deployments, "
+            "FAILURE for ETL pipelines); SUCCESS_ONLY fires only on successful runs."
+        ),
+    )
     # Foreign keys to specific models
     pipeline = models.ForeignKey(
         Pipeline,
diff --git a/backend/notification_v2/serializers.py b/backend/notification_v2/serializers.py
index 115487c481..784ec75413 100644
--- a/backend/notification_v2/serializers.py
+++ b/backend/notification_v2/serializers.py
@@ -1,7 +1,7 @@
 from rest_framework import serializers
 from utils.input_sanitizer import validate_name_field
 
-from .enums import AuthorizationType, NotificationType, PlatformType
+from .enums import AuthorizationType, NotificationTrigger, NotificationType, PlatformType
 from .models import Notification
 
 
@@ -12,6 +12,11 @@ class NotificationSerializer(serializers.ModelSerializer):
     max_retries = serializers.IntegerField(
         max_value=4, min_value=0, default=0, required=False
     )
+    notify_on = serializers.ChoiceField(
+        choices=NotificationTrigger.choices(),
+        default=NotificationTrigger.ALL.value,
+        required=False,
+    )
 
     class Meta:
         model = Notification
diff --git a/backend/notification_v2/tests/__init__.py b/backend/notification_v2/tests/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/backend/notification_v2/tests/test_notification_filter.py b/backend/notification_v2/tests/test_notification_filter.py
new file mode 100644
index 0000000000..7f887be500
--- /dev/null
+++ b/backend/notification_v2/tests/test_notification_filter.py
@@ -0,0 +1,321 @@
+"""Unit tests for the ``notify_on`` dispatch partition on notifications.
+
+Covers both dispatch paths that fan out to ``NotificationHelper.send_notification``:
+
+* ``APINotification.send`` — keyed on ``ExecutionStatus`` (ERROR, COMPLETED, STOPPED)
+* ``PipelineNotification.send`` — keyed on ``Pipeline.PipelineStatus``
+  (FAILURE, SUCCESS, INPROGRESS)
+
+Follows the repo convention (see ``usage_v2/tests/test_helper.py``) of stubbing
+Django-heavy modules at import time so the tests run without a live DB.
+"""
+
+from __future__ import annotations
+
+import sys
+import types
+from unittest.mock import MagicMock, patch
+
+
+# ---------------------------------------------------------------------------
+# Module-level stubs — must be installed BEFORE importing the modules under
+# test so Django's ORM imports resolve to our MagicMock-backed fakes.
+# ---------------------------------------------------------------------------
+
+
+def _ensure_mod(name: str) -> types.ModuleType:
+    """Force-install a fresh stub module in ``sys.modules``."""
+    mod = types.ModuleType(name)
+    sys.modules[name] = mod
+    return mod
+
+
+def _install_stubs() -> None:
+    # Only stub leaf modules that pull in Django ORM. Parent packages
+    # (api_v2, pipeline_v2, notification_v2, workflow_manager*) load normally.
+
+    exec_enums = _ensure_mod("workflow_manager.workflow_v2.enums")
+
+    class _ExecStatusNS:
+        class ERROR:
+            value = "ERROR"
+
+        class COMPLETED:
+            value = "COMPLETED"
+
+        class STOPPED:
+            value = "STOPPED"
+
+    exec_enums.ExecutionStatus = _ExecStatusNS  # type: ignore[attr-defined]
+
+    exec_models = _ensure_mod("workflow_manager.workflow_v2.models.execution")
+    exec_models.WorkflowExecution = MagicMock(name="WorkflowExecution")  # type: ignore[attr-defined]
+
+    api_models = _ensure_mod("api_v2.models")
+    api_models.APIDeployment = MagicMock(name="APIDeployment")  # type: ignore[attr-defined]
+
+    # notification_v2.models.Notification with a patchable ``objects``.
+    notif_models = _ensure_mod("notification_v2.models")
+
+    class _FakeNotification:
+        objects = MagicMock(name="Notification.objects")
+
+    notif_models.Notification = _FakeNotification  # type: ignore[attr-defined]
+
+    # notification_v2.helper.NotificationHelper
+    notif_helper = _ensure_mod("notification_v2.helper")
+
+    class _FakeHelper:
+        send_notification = MagicMock(name="NotificationHelper.send_notification")
+
+    notif_helper.NotificationHelper = _FakeHelper  # type: ignore[attr-defined]
+
+    # pipeline_v2.dto.PipelineStatusPayload
+    pipeline_dto = _ensure_mod("pipeline_v2.dto")
+    pipeline_dto.PipelineStatusPayload = MagicMock(name="PipelineStatusPayload")  # type: ignore[attr-defined]
+
+    # pipeline_v2.models.Pipeline with a PipelineStatus text-choices surface.
+    pipeline_models = _ensure_mod("pipeline_v2.models")
+
+    class _PipelineStatus:
+        SUCCESS = "SUCCESS"
+        FAILURE = "FAILURE"
+        INPROGRESS = "INPROGRESS"
+
+    class _FakePipeline:
+        PipelineStatus = _PipelineStatus
+
+    pipeline_models.Pipeline = _FakePipeline  # type: ignore[attr-defined]
+
+
+_install_stubs()
+
+
+# Now safe to import the modules under test.
+from api_v2 import notification as api_notification_mod  # noqa: E402
+from notification_v2.enums import NotificationTrigger  # noqa: E402
+from notification_v2.helper import NotificationHelper  # noqa: E402
+from notification_v2.models import Notification  # noqa: E402
+from pipeline_v2 import notification as pipeline_notification_mod  # noqa: E402
+from pipeline_v2.models import Pipeline  # noqa: E402
+
+
+# ---------------------------------------------------------------------------
+# Test helpers
+# ---------------------------------------------------------------------------
+
+
+def _make_queryset(notifications: list[MagicMock]) -> MagicMock:
+    """Return a MagicMock that mimics the chained QuerySet surface we use.
+
+    Supports:
+        qs.filter(notify_on=<value>)   -> qs with matching rows
+        qs.exclude(notify_on=<value>)  -> qs with non-matching rows
+        qs.exists()                    -> bool based on contents
+        iter(qs)                       -> notifications
+    """
+    qs = MagicMock(name="qs")
+    qs.__iter__ = lambda self: iter(notifications)
+
+    def _filter(**kwargs):
+        if "notify_on" in kwargs:
+            target = kwargs["notify_on"]
+            kept = [n for n in notifications if n.notify_on == target]
+            return _make_queryset(kept)
+        return _make_queryset(notifications)
+
+    def _exclude(**kwargs):
+        if "notify_on" in kwargs:
+            target = kwargs["notify_on"]
+            kept = [n for n in notifications if n.notify_on != target]
+            return _make_queryset(kept)
+        return _make_queryset(notifications)
+
+    qs.filter.side_effect = _filter
+    qs.exclude.side_effect = _exclude
+    qs.exists.return_value = bool(notifications)
+    qs.count.return_value = len(notifications)
+    return qs
+
+
+def _make_notification(*, notify_on: str) -> MagicMock:
+    n = MagicMock(name="Notification")
+    n.notify_on = notify_on
+    return n
+
+
+# ---------------------------------------------------------------------------
+# APINotification — 3 modes × 3 statuses
+# ---------------------------------------------------------------------------
+
+
+class TestAPINotificationFilter:
+    def _setup(self, *, status: str, notifications: list[MagicMock]):
+        Notification.objects.filter.reset_mock()
+        Notification.objects.filter.side_effect = None
+        Notification.objects.filter.return_value = _make_queryset(notifications)
+        NotificationHelper.send_notification.reset_mock()
+
+        api = MagicMock(name="APIDeployment")
+        api.api_name = "test-api"
+        api.id = "api-uuid"
+
+        execution = MagicMock(name="WorkflowExecution")
+        execution.status = status
+        execution.id = "exec-uuid"
+        execution.error_message = "boom" if status == "ERROR" else None
+
+        return api_notification_mod.APINotification(api=api, workflow_execution=execution)
+
+    # --- ALL: fires on every status ---
+    def test_all_fires_on_completed(self):
+        n = _make_notification(notify_on=NotificationTrigger.ALL.value)
+        self._setup(status="COMPLETED", notifications=[n]).send()
+        assert NotificationHelper.send_notification.call_count == 1
+
+    def test_all_fires_on_error(self):
+        n = _make_notification(notify_on=NotificationTrigger.ALL.value)
+        self._setup(status="ERROR", notifications=[n]).send()
+        assert NotificationHelper.send_notification.call_count == 1
+
+    def test_all_fires_on_stopped(self):
+        n = _make_notification(notify_on=NotificationTrigger.ALL.value)
+        self._setup(status="STOPPED", notifications=[n]).send()
+        assert NotificationHelper.send_notification.call_count == 1
+
+    # --- FAILURES_ONLY: fires on ERROR only ---
+    def test_failures_only_suppressed_on_completed(self):
+        n = _make_notification(notify_on=NotificationTrigger.FAILURES_ONLY.value)
+        self._setup(status="COMPLETED", notifications=[n]).send()
+        NotificationHelper.send_notification.assert_not_called()
+
+    def test_failures_only_fires_on_error(self):
+        n = _make_notification(notify_on=NotificationTrigger.FAILURES_ONLY.value)
+        self._setup(status="ERROR", notifications=[n]).send()
+        assert NotificationHelper.send_notification.call_count == 1
+
+    def test_failures_only_suppressed_on_stopped(self):
+        n = _make_notification(notify_on=NotificationTrigger.FAILURES_ONLY.value)
+        self._setup(status="STOPPED", notifications=[n]).send()
+        NotificationHelper.send_notification.assert_not_called()
+
+    # --- SUCCESS_ONLY: fires on COMPLETED only ---
+    def test_success_only_fires_on_completed(self):
+        n = _make_notification(notify_on=NotificationTrigger.SUCCESS_ONLY.value)
+        self._setup(status="COMPLETED", notifications=[n]).send()
+        assert NotificationHelper.send_notification.call_count == 1
+
+    def test_success_only_suppressed_on_error(self):
+        n = _make_notification(notify_on=NotificationTrigger.SUCCESS_ONLY.value)
+        self._setup(status="ERROR", notifications=[n]).send()
+        NotificationHelper.send_notification.assert_not_called()
+
+    def test_success_only_suppressed_on_stopped(self):
+        n = _make_notification(notify_on=NotificationTrigger.SUCCESS_ONLY.value)
+        self._setup(status="STOPPED", notifications=[n]).send()
+        NotificationHelper.send_notification.assert_not_called()
+
+    # --- Mixed partition on a COMPLETED run: ALL + SUCCESS_ONLY fire, FAILURES_ONLY doesn't ---
+    def test_mixed_partition_on_completed(self):
+        all_mode = _make_notification(notify_on=NotificationTrigger.ALL.value)
+        failures_only = _make_notification(notify_on=NotificationTrigger.FAILURES_ONLY.value)
+        success_only = _make_notification(notify_on=NotificationTrigger.SUCCESS_ONLY.value)
+        notifier = self._setup(
+            status="COMPLETED", notifications=[all_mode, failures_only, success_only]
+        )
+        with patch.object(api_notification_mod, "PipelineStatusPayload") as payload_cls:
+            payload_cls.return_value.to_dict.return_value = {}
+            notifier.send()
+
+        assert NotificationHelper.send_notification.call_count == 1
+        kwargs = NotificationHelper.send_notification.call_args.kwargs
+        dispatched = sorted(n.notify_on for n in kwargs["notifications"])
+        assert dispatched == ["ALL", "SUCCESS_ONLY"]
+
+
+# ---------------------------------------------------------------------------
+# PipelineNotification — 3 modes × 3 statuses
+# ---------------------------------------------------------------------------
+
+
+class TestPipelineNotificationFilter:
+    def _setup(self, *, last_run_status: str, notifications: list[MagicMock]):
+        Notification.objects.filter.reset_mock()
+        Notification.objects.filter.side_effect = None
+        Notification.objects.filter.return_value = _make_queryset(notifications)
+        NotificationHelper.send_notification.reset_mock()
+
+        pipeline = MagicMock(name="Pipeline")
+        pipeline.id = "pipeline-uuid"
+        pipeline.pipeline_name = "test-pipeline"
+        pipeline.pipeline_type = "ETL"
+        pipeline.last_run_status = last_run_status
+
+        return pipeline_notification_mod.PipelineNotification(
+            pipeline=pipeline, execution_id="exec-uuid", error_message=None
+        )
+
+    # --- ALL ---
+    def test_all_fires_on_success(self):
+        n = _make_notification(notify_on=NotificationTrigger.ALL.value)
+        self._setup(
+            last_run_status=Pipeline.PipelineStatus.SUCCESS, notifications=[n]
+        ).send()
+        assert NotificationHelper.send_notification.call_count == 1
+
+    def test_all_fires_on_failure(self):
+        n = _make_notification(notify_on=NotificationTrigger.ALL.value)
+        self._setup(
+            last_run_status=Pipeline.PipelineStatus.FAILURE, notifications=[n]
+        ).send()
+        assert NotificationHelper.send_notification.call_count == 1
+
+    # --- FAILURES_ONLY ---
+    def test_failures_only_suppressed_on_success(self):
+        n = _make_notification(notify_on=NotificationTrigger.FAILURES_ONLY.value)
+        self._setup(
+            last_run_status=Pipeline.PipelineStatus.SUCCESS, notifications=[n]
+        ).send()
+        NotificationHelper.send_notification.assert_not_called()
+
+    def test_failures_only_fires_on_failure(self):
+        n = _make_notification(notify_on=NotificationTrigger.FAILURES_ONLY.value)
+        self._setup(
+            last_run_status=Pipeline.PipelineStatus.FAILURE, notifications=[n]
+        ).send()
+        assert NotificationHelper.send_notification.call_count == 1
+
+    # --- SUCCESS_ONLY ---
+    def test_success_only_fires_on_success(self):
+        n = _make_notification(notify_on=NotificationTrigger.SUCCESS_ONLY.value)
+        self._setup(
+            last_run_status=Pipeline.PipelineStatus.SUCCESS, notifications=[n]
+        ).send()
+        assert NotificationHelper.send_notification.call_count == 1
+
+    def test_success_only_suppressed_on_failure(self):
+        n = _make_notification(notify_on=NotificationTrigger.SUCCESS_ONLY.value)
+        self._setup(
+            last_run_status=Pipeline.PipelineStatus.FAILURE, notifications=[n]
+        ).send()
+        NotificationHelper.send_notification.assert_not_called()
+
+    # --- Mixed partition on a SUCCESS run ---
+    def test_mixed_partition_on_success(self):
+        all_mode = _make_notification(notify_on=NotificationTrigger.ALL.value)
+        failures_only = _make_notification(notify_on=NotificationTrigger.FAILURES_ONLY.value)
+        success_only = _make_notification(notify_on=NotificationTrigger.SUCCESS_ONLY.value)
+        notifier = self._setup(
+            last_run_status=Pipeline.PipelineStatus.SUCCESS,
+            notifications=[all_mode, failures_only, success_only],
+        )
+        with patch.object(
+            pipeline_notification_mod, "PipelineStatusPayload"
+        ) as payload_cls:
+            payload_cls.return_value.to_dict.return_value = {}
+            notifier.send()
+
+        assert NotificationHelper.send_notification.call_count == 1
+        kwargs = NotificationHelper.send_notification.call_args.kwargs
+        dispatched = sorted(n.notify_on for n in kwargs["notifications"])
+        assert dispatched == ["ALL", "SUCCESS_ONLY"]
diff --git a/backend/pipeline_v2/notification.py b/backend/pipeline_v2/notification.py
index dbfc0dea52..64df6f1126 100644
--- a/backend/pipeline_v2/notification.py
+++ b/backend/pipeline_v2/notification.py
@@ -1,5 +1,6 @@
 import logging
 
+from notification_v2.enums import NotificationTrigger
 from notification_v2.helper import NotificationHelper
 from notification_v2.models import Notification
 
@@ -23,11 +24,33 @@ def __init__(
         self.error_message = error_message
         self.execution_id = execution_id
 
-    def send(self):
-        if not self.notifications.count():
-            logger.info(f"No notifications found for pipeline {self.pipeline}")
+    def send(self) -> None:
+        # Partition notifications by the run outcome so each row's notify_on
+        # preference is honored. PipelineUtils.update_pipeline_status collapses
+        # both ERROR and STOPPED execution statuses into PipelineStatus.FAILURE,
+        # so FAILURES_ONLY subscribers get alerts for both on the pipeline side.
+        status = self.pipeline.last_run_status
+        if status == Pipeline.PipelineStatus.FAILURE:
+            self.notifications = self.notifications.exclude(
+                notify_on=NotificationTrigger.SUCCESS_ONLY.value
+            )
+        elif status == Pipeline.PipelineStatus.SUCCESS:
+            self.notifications = self.notifications.exclude(
+                notify_on=NotificationTrigger.FAILURES_ONLY.value
+            )
+        else:
+            self.notifications = self.notifications.filter(
+                notify_on=NotificationTrigger.ALL.value
+            )
+
+        if not self.notifications.exists():
+            logger.info(
+                "No notifications to dispatch for pipeline %s (status=%s)",
+                self.pipeline,
+                status,
+            )
             return
-        logger.info(f"Sending pipeline status notification for pipeline {self.pipeline}")
+        logger.info("Sending pipeline status notification for pipeline %s", self.pipeline)
         payload_dto = PipelineStatusPayload(
             type=self.pipeline.pipeline_type,
             pipeline_id=str(self.pipeline.id),
diff --git a/frontend/src/components/pipelines-or-deployments/notification-modal/CreateNotification.jsx b/frontend/src/components/pipelines-or-deployments/notification-modal/CreateNotification.jsx
index 9c207bd1a9..9302f9f1dc 100644
--- a/frontend/src/components/pipelines-or-deployments/notification-modal/CreateNotification.jsx
+++ b/frontend/src/components/pipelines-or-deployments/notification-modal/CreateNotification.jsx
@@ -12,6 +12,7 @@ const DEFAULT_FORM_DETAILS = {
   authorization_key: "",
   is_active: false,
   max_retries: 0,
+  notify_on: "ALL",
   pipeline: "",
   api: "",
   url: "",
@@ -54,6 +55,12 @@ const AUTHORIZATION_TYPES = [
   },
 ];
 
+const NOTIFY_ON_OPTIONS = [
+  { value: "ALL", label: "On every completion" },
+  { value: "FAILURES_ONLY", label: "On failures only" },
+  { value: "SUCCESS_ONLY", label: "On success only" },
+];
+
 function CreateNotification({
   setIsForm,
   type,
@@ -192,6 +199,12 @@ function CreateNotification({
       tooltip:
         "Specify the maximum number of times the notification should be retried if it fails.",
     },
+    {
+      label: "Notify on",
+      name: "notify_on",
+      component: <Select options={NOTIFY_ON_OPTIONS} />,
+      tooltip: "Choose which run outcomes should trigger this webhook.",
+    },
   ];
 
   return (

From 207277e008c25f8ed3ea86c320c087086579e3a4 Mon Sep 17 00:00:00 2001
From: Chandrasekharan M
 <117059509+chandrasekharan-zipstack@users.noreply.github.com>
Date: Fri, 24 Apr 2026 13:18:02 +0530
Subject: [PATCH 2/7] UN-3431 [FIX] Stream tool-run logs to workflow execution
 UI with markdown rendering (#1927)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* [FIX] Make tool-run logs visible in workflow execution UI

Two stacked gaps were keeping tool-level log lines (Processing prompt,
Running LLM completion, lookup calls, etc.) out of the workflow
execution logs UI and the execution_log DB table for API / workflow
runs:

1. Empty log_events_id.  structure_tool_task seeded LOG_EVENTS_ID in
   StateStore but never threaded it into pipeline_ctx / agentic_ctx.
   ExecutorToolShim.stream_log gated publishing on
   self.log_events_id, so every tool-level log was dropped before it
   ever reached the broker.

2. Wrong payload shape.  Even with the channel threaded,
   stream_log used LogPublisher.log_progress(...) whose payload omits
   execution_id / organization_id / file_execution_id.
   get_validated_log_data (log_utils.py) requires those IDs and
   LogType == LOG to persist to execution_log, so tool-level messages
   were silently filtered at the Redis->DB drain step — orchestration
   logs persisted, tool logs did not.

Fixes:
- ExecutionContext gains execution_id + file_execution_id, populated
  in structure_tool_task for both the legacy pipeline and agentic
  contexts.
- LegacyExecutor caches the three IDs on self during execute() and
  passes them into every ExecutorToolShim construction
  (~7 callsites).
- ExecutorToolShim.stream_log now dual-emits: PROGRESS (unchanged,
  drives the IDE prompt-card live progress pane) plus LOG carrying
  the workflow IDs (feeds the workflow execution logs UI and persists
  to execution_log via the existing drain). LOG emission is gated on
  execution_id + organization_id being present, so bare IDE test
  runs without a workflow still behave as before.

Rendering polish
- The LogModal and pipeline LogsModal now pipe log text through the
  existing CustomMarkdown renderer, so backticked identifiers render
  as inline-code pills and embedded newlines break lines. This lets
  multi-line structured events (e.g. the lookup pre-call trio)
  surface as a single row with readable inner formatting.
- Prompt-key mentions inside legacy_executor tool logs are wrapped
  in backticks for consistency with the rest of the log surface.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* [FIX] Wrap prompt_name in backticks in remaining stream_log calls

Completes the consistency pass on tool-run log formatting: the table-
and line-item-extraction success and error paths still emitted prompt
names without backticks, so the markdown-rendered logs UI showed them
as bare text instead of inline-code pills. Matches the pattern already
applied to the other 9 stream_log calls in this file.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* [FIX] Validate URL schemes in CustomMarkdown link renderer

Workflow logs rendered via CustomMarkdown can contain tool-generated or
user-derived content, so an untrusted \`[text](url)\` sequence could
inject a \`javascript:\` or \`data:\` scheme and get clickable through
antd \`Typography.Link\`. Allow-list the safe external schemes (http,
https, mailto, tel) before rendering as a link; everything else falls
back to plain text while still honouring the existing internal-path
branch used for in-app navigation.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* [FIX] Thread workflow IDs into remaining shim/context callsites

Addresses CodeRabbit review gaps so the log-plumbing fix is consistent
across every pre-dispatch and plugin-dispatch path:

- `table_ctx` / `line_item_ctx` in `legacy_executor.py` now carry
  `log_events_id`, `execution_id`, `file_execution_id` from context so
  downstream table/line-item plugins that build their own
  `ExecutorToolShim` pass the `execution_id + organization_id` gate
  and emit workflow LOG payloads.
- `structure_tool_task.py` threads the same IDs into the bare
  pre-dispatch shim, so `X2Text.process()` calls during agentic
  extraction reach the workflow logs UI.
- `LogsModal.jsx` stores the raw log string in row data and lets the
  column renderer wrap it in `CustomMarkdown` — the previous map
  stored a `<CustomMarkdown />` element that was then passed back into
  `CustomMarkdown.text`, producing `[object Object]` for multi-row
  lookups.
- Dropped `getattr(context, ...)` on `execution_id` /
  `file_execution_id` now that they are dataclass fields — matches the
  direct access used for `organization_id`.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* [REFACTOR] Trim overly specific comments in log-plumbing changes

Pass through the new comments added across this PR and either remove or
tighten the ones that restate what the code already shows. Keep only
the WHY lines that protect future readers from missing a non-obvious
constraint (XSS guard in CustomMarkdown, dual PROGRESS/LOG emission in
the shim, pre-dispatch shim needing workflow IDs so X2Text logs are
not silently dropped).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* [REFACTOR] Extract isSafeExternalUrl into shared helpers module

Moves the URL scheme allow-list check out of CustomMarkdown into
helpers/urlSafety.js so any future component that renders links from
user- or tool-derived content can reuse the same guard instead of
re-implementing it.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* [FIX] Tighten URL guard, split publish try/excepts, and extract shim builder

Addresses the must-fix and worth-doing comments from the PR review:

Security
- CustomMarkdown: treat protocol-relative URLs (`//host/...`) as external,
  not internal, so they can no longer skip the scheme guard via the
  `startsWith("/")` branch.
- `isSafeExternalUrl`: drop the `window.location.origin` base so bare
  strings ("javascript", "../foo") fail to parse instead of silently
  resolving to `https://<origin>/...` and passing the scheme check.

Silent failure + comment accuracy
- ExecutorToolShim.stream_log: split the PROGRESS and LOG publish paths
  into separate try/except blocks so a LogDataDTO validation failure on
  the LOG payload is no longer mis-attributed to "progress publish
  failed". Corrected the inline comments — the DB drop is driven by
  LogPublisher's `payload.type == 'LOG'` check, and only
  `execution_id` + `organization_id` are strictly required.

Refactor
- New `LegacyExecutor._build_shim()` helper — all seven
  ExecutorToolShim callsites now share one construction path so the
  workflow-ID plumbing can't drift out of sync across sites again.
- Thread `execution_id` / `file_execution_id` into the seven
  self-dispatched sub-`ExecutionContext`s alongside `log_events_id`,
  matching the table/line-item sites and keeping the context
  consistent for any downstream consumer that reads the IDs from the
  context rather than from the executor instance.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* [FIX] Address remaining type-design and silent-failure comments

- ExecutionContext: drop the BE-coupled inline comment, document the
  new IDs in the Attributes block, and enforce the invariant that
  execution_id implies organization_id via __post_init__.
- ExecutorToolShim: typed the three new IDs as str | None instead of
  str = "" so the signature matches the Optional semantics already
  enforced by the runtime guards.
- LegacyExecutor: move per-request state to __init__ so _log_component
  is no longer a class-level mutable default shared across instances;
  stop silently coercing None IDs to ""; add a one-shot warning when a
  tool-sourced run lands without workflow IDs so the silent-no-persist
  case is visible in GKE logs.
- structure_tool_task: emit the same warning when LOG_EVENTS_ID is
  absent from StateStore.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* [FIX] Surface first publish failure per shim at WARN

Both PROGRESS and LOG publish paths previously swallowed every broker
failure at DEBUG, so a misconfigured or down Redis broker meant every
tool-level log silently vanished with no operator-visible signal.

Track a per-shim _progress_publish_failed / _log_publish_failed flag
and log the first failure at WARNING (with traceback), then downgrade
subsequent failures on the same shim back to DEBUG. Preserves the
non-fatal semantics of the publish path while making broker outages
visible in GKE logs.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 .../custom-markdown/CustomMarkdown.jsx        |   8 +-
 .../components/logging/log-modal/LogModal.jsx |   2 +
 .../log-modal/LogsModal.jsx                   |   3 +-
 frontend/src/helpers/urlSafety.js             |  19 +++
 .../src/unstract/sdk1/execution/context.py    |  15 ++
 workers/executor/executor_tool_shim.py        |  90 ++++++++---
 workers/executor/executors/legacy_executor.py | 145 +++++++++++-------
 .../file_processing/structure_tool_task.py    |  26 +++-
 8 files changed, 232 insertions(+), 76 deletions(-)
 create mode 100644 frontend/src/helpers/urlSafety.js

diff --git a/frontend/src/components/helpers/custom-markdown/CustomMarkdown.jsx b/frontend/src/components/helpers/custom-markdown/CustomMarkdown.jsx
index 9e895674b5..458f319557 100644
--- a/frontend/src/components/helpers/custom-markdown/CustomMarkdown.jsx
+++ b/frontend/src/components/helpers/custom-markdown/CustomMarkdown.jsx
@@ -3,6 +3,7 @@ import PropTypes from "prop-types";
 import { useMemo } from "react";
 import { Link as RouterLink } from "react-router-dom";
 
+import { isSafeExternalUrl } from "../../../helpers/urlSafety";
 import { useSessionStore } from "../../../store/session-store";
 
 const { Text, Link, Paragraph } = Typography;
@@ -54,11 +55,16 @@ const CustomMarkdown = ({
           </Text>
         );
       case "link": {
-        const isInternal = url?.startsWith("/");
+        // Protocol-relative URLs (`//evil.com/...`) also start with `/`
+        // so exclude them from the internal-route branch.
+        const isInternal = url?.startsWith("/") && !url.startsWith("//");
         if (isInternal) {
           const resolvedUrl = orgName ? `/${orgName}${url}` : url;
           return <RouterLink to={resolvedUrl}>{content}</RouterLink>;
         }
+        if (!isSafeExternalUrl(url)) {
+          return content;
+        }
         return (
           <Link href={url} target="_blank" rel="noopener noreferrer">
             {content}
diff --git a/frontend/src/components/logging/log-modal/LogModal.jsx b/frontend/src/components/logging/log-modal/LogModal.jsx
index 2f21cb050b..d709780313 100644
--- a/frontend/src/components/logging/log-modal/LogModal.jsx
+++ b/frontend/src/components/logging/log-modal/LogModal.jsx
@@ -13,6 +13,7 @@ import { useCopyToClipboard } from "../../../hooks/useCopyToClipboard";
 import { useExceptionHandler } from "../../../hooks/useExceptionHandler";
 import useRequestUrl from "../../../hooks/useRequestUrl";
 import { useAlertStore } from "../../../store/alert-store";
+import CustomMarkdown from "../../helpers/custom-markdown/CustomMarkdown";
 import { FilterDropdown, FilterIcon } from "../filter-dropdown/FilterDropdown";
 
 function LogModal({
@@ -120,6 +121,7 @@ function LogModal({
       title: "Log",
       dataIndex: "log",
       key: "log",
+      render: (log) => <CustomMarkdown text={log || ""} />,
     },
   ];
 
diff --git a/frontend/src/components/pipelines-or-deployments/log-modal/LogsModal.jsx b/frontend/src/components/pipelines-or-deployments/log-modal/LogsModal.jsx
index 09dd7ac047..9ab7e69f3b 100644
--- a/frontend/src/components/pipelines-or-deployments/log-modal/LogsModal.jsx
+++ b/frontend/src/components/pipelines-or-deployments/log-modal/LogsModal.jsx
@@ -41,7 +41,7 @@ const LogsModal = ({
       .then((res) => {
         const logDetails = res?.data?.results?.map((item) => ({
           id: item?.id,
-          log: <CustomMarkdown text={item?.data?.log} />,
+          log: item?.data?.log,
           type: item?.data?.type,
           stage: item?.data?.stage,
           level: item?.data?.level,
@@ -111,6 +111,7 @@ const LogsModal = ({
       title: "Log",
       dataIndex: "log",
       key: "log",
+      render: (log) => <CustomMarkdown text={log || ""} />,
     },
   ];
 
diff --git a/frontend/src/helpers/urlSafety.js b/frontend/src/helpers/urlSafety.js
new file mode 100644
index 0000000000..b5ca23e795
--- /dev/null
+++ b/frontend/src/helpers/urlSafety.js
@@ -0,0 +1,19 @@
+const SAFE_URL_SCHEMES = ["http:", "https:", "mailto:", "tel:"];
+
+// Guards against unsafe schemes (e.g. `javascript:`, `data:`) when
+// rendering links built from user- or tool-derived content.
+const isSafeExternalUrl = (url) => {
+  if (typeof url !== "string" || url === "") {
+    return false;
+  }
+  // Parse without a base so bare strings (e.g. "javascript", "../foo") fail
+  // instead of silently resolving to `https://<origin>/...` and passing.
+  try {
+    const parsed = new URL(url);
+    return SAFE_URL_SCHEMES.includes(parsed.protocol);
+  } catch {
+    return false;
+  }
+};
+
+export { isSafeExternalUrl, SAFE_URL_SCHEMES };
diff --git a/unstract/sdk1/src/unstract/sdk1/execution/context.py b/unstract/sdk1/src/unstract/sdk1/execution/context.py
index a1efb4c3f8..83eb650a29 100644
--- a/unstract/sdk1/src/unstract/sdk1/execution/context.py
+++ b/unstract/sdk1/src/unstract/sdk1/execution/context.py
@@ -68,6 +68,11 @@ class ExecutionContext:
         log_events_id: Socket.IO channel ID for streaming progress
             logs to the frontend.  ``None`` when not in an IDE
             session (no logs published).
+        execution_id: Workflow execution identifier for log
+            correlation. When set, ``organization_id`` must also be
+            set (enforced in ``__post_init__``).
+        file_execution_id: Child identifier for per-file execution
+            within a workflow run. Optional.
     """
 
     executor_name: str
@@ -78,6 +83,8 @@ class ExecutionContext:
     executor_params: dict[str, Any] = field(default_factory=dict)
     request_id: str | None = None
     log_events_id: str | None = None
+    execution_id: str | None = None
+    file_execution_id: str | None = None
 
     def __post_init__(self) -> None:
         """Validate required fields after initialization."""
@@ -89,6 +96,10 @@ def __post_init__(self) -> None:
             raise ValueError("run_id is required")
         if not self.execution_source:
             raise ValueError("execution_source is required")
+        # When execution_id is set, organization_id must be too — they
+        # travel together for workflow-level log correlation.
+        if self.execution_id and not self.organization_id:
+            raise ValueError("organization_id is required when execution_id is set")
 
         # Normalize enum values to plain strings for serialization
         if isinstance(self.operation, Operation):
@@ -111,6 +122,8 @@ def to_dict(self) -> dict[str, Any]:
             "executor_params": self.executor_params,
             "request_id": self.request_id,
             "log_events_id": self.log_events_id,
+            "execution_id": self.execution_id,
+            "file_execution_id": self.file_execution_id,
         }
 
     @classmethod
@@ -125,4 +138,6 @@ def from_dict(cls, data: dict[str, Any]) -> "ExecutionContext":
             executor_params=data.get("executor_params", {}),
             request_id=data.get("request_id"),
             log_events_id=data.get("log_events_id"),
+            execution_id=data.get("execution_id"),
+            file_execution_id=data.get("file_execution_id"),
         )
diff --git a/workers/executor/executor_tool_shim.py b/workers/executor/executor_tool_shim.py
index 63f48dd253..828e641398 100644
--- a/workers/executor/executor_tool_shim.py
+++ b/workers/executor/executor_tool_shim.py
@@ -60,6 +60,9 @@ def __init__(
         platform_api_key: str = "",
         log_events_id: str = "",
         component: dict[str, str] | None = None,
+        execution_id: str | None = None,
+        organization_id: str | None = None,
+        file_execution_id: str | None = None,
     ) -> None:
         """Initialize the shim.
 
@@ -72,10 +75,24 @@ def __init__(
             component: Structured identifier dict for log correlation
                 (``tool_id``, ``run_id``, ``doc_name``, optionally
                 ``prompt_key``).
+            execution_id: Workflow execution id. When provided with
+                ``organization_id``, enables persistent log attribution
+                to the ``execution_log`` table.
+            organization_id: Tenant/org scope for log persistence.
+            file_execution_id: File execution id (child of workflow
+                execution). Optional — populated for per-file tool runs.
         """
         self.platform_api_key = platform_api_key
         self.log_events_id = log_events_id
         self.component = component or {}
+        self.execution_id = execution_id
+        self.organization_id = organization_id
+        self.file_execution_id = file_execution_id
+        # Track whether we've already surfaced a publish failure on this
+        # shim so that a down broker is flagged loudly once instead of
+        # silently swallowing every subsequent log line at DEBUG.
+        self._progress_publish_failed = False
+        self._log_publish_failed = False
         # Initialize StreamMixin.  EXECUTION_BY_TOOL is not set in
         # the worker environment, so _exec_by_tool will be False.
         super().__init__(log_level=LogLevel.INFO)
@@ -145,24 +162,61 @@ def stream_log(
             return
 
         # Publish progress to frontend via the log consumer queue.
-        if self.log_events_id:
-            try:
-                wf_level = _SDK_TO_WF_LEVEL.get(level, "INFO")
-                payload = LogPublisher.log_progress(
-                    component=self.component,
-                    level=wf_level,
-                    state=stage,
-                    message=log,
-                )
-                LogPublisher.publish(
-                    channel_id=self.log_events_id,
-                    payload=payload,
-                )
-            except Exception:
-                logger.debug(
-                    "Failed to publish progress log (non-fatal)",
-                    exc_info=True,
-                )
+        if not self.log_events_id:
+            return
+
+        wf_level = _SDK_TO_WF_LEVEL.get(level, "INFO")
+
+        # PROGRESS payload — IDE prompt-card live updates only. Dropped at
+        # the DB persist layer because LogPublisher.publish only stores
+        # payloads whose `type == "LOG"`.
+        try:
+            progress_payload = LogPublisher.log_progress(
+                component=self.component,
+                level=wf_level,
+                state=stage,
+                message=log,
+            )
+            LogPublisher.publish(
+                channel_id=self.log_events_id,
+                payload=progress_payload,
+            )
+        except Exception:
+            first_failure = not self._progress_publish_failed
+            self._progress_publish_failed = True
+            logger.log(
+                logging.WARNING if first_failure else logging.DEBUG,
+                "Failed to publish progress log (non-fatal)",
+                exc_info=first_failure,
+            )
+
+        # LOG payload — feeds workflow logs UI and persists to execution_log.
+        # LogDataDTO validation requires `execution_id` and `organization_id`;
+        # `file_execution_id` is optional.
+        if not (self.execution_id and self.organization_id):
+            return
+
+        try:
+            log_payload = LogPublisher.log_workflow(
+                stage=stage,
+                message=log,
+                level=wf_level,
+                execution_id=self.execution_id,
+                file_execution_id=self.file_execution_id or None,
+                organization_id=self.organization_id,
+            )
+            LogPublisher.publish(
+                channel_id=self.log_events_id,
+                payload=log_payload,
+            )
+        except Exception:
+            first_failure = not self._log_publish_failed
+            self._log_publish_failed = True
+            logger.log(
+                logging.WARNING if first_failure else logging.DEBUG,
+                "Failed to publish workflow log (non-fatal)",
+                exc_info=first_failure,
+            )
 
     def stream_error_and_exit(self, message: str, err: Exception | None = None) -> None:
         """Log error and raise SdkError.
diff --git a/workers/executor/executors/legacy_executor.py b/workers/executor/executors/legacy_executor.py
index 18e1f47749..2a7dfc160d 100644
--- a/workers/executor/executors/legacy_executor.py
+++ b/workers/executor/executors/legacy_executor.py
@@ -57,9 +57,13 @@ class LegacyExecutor(BaseExecutor):
         Operation.STRUCTURE_PIPELINE.value: "_handle_structure_pipeline",
     }
 
-    # Defaults for log streaming (overridden by execute()).
-    _log_events_id: str = ""
-    _log_component: dict[str, str] = {}
+    def __init__(self) -> None:
+        # Per-request state — overwritten on every ``execute()`` call.
+        self._log_events_id: str = ""
+        self._log_component: dict[str, str] = {}
+        self._execution_id: str | None = None
+        self._file_execution_id: str | None = None
+        self._organization_id: str | None = None
 
     @property
     def name(self) -> str:
@@ -77,8 +81,21 @@ def execute(self, context: ExecutionContext) -> ExecutionResult:
             NotImplementedError: From stub handlers (until 2D–2H).
         """
         # Extract log streaming info (set by tasks.py for IDE sessions).
-        self._log_events_id: str = context.log_events_id or ""
-        self._log_component: dict[str, str] = getattr(context, "_log_component", {})
+        self._log_events_id = context.log_events_id or ""
+        self._log_component = getattr(context, "_log_component", None) or {}
+        self._execution_id = context.execution_id
+        self._file_execution_id = context.file_execution_id
+        self._organization_id = context.organization_id
+        if (
+            context.execution_source == "tool"
+            and self._log_events_id
+            and not (self._execution_id and self._organization_id)
+        ):
+            logger.warning(
+                "Workflow IDs missing on tool context run_id=%s — tool-level "
+                "logs will reach the UI but won't persist to execution_log.",
+                context.run_id,
+            )
 
         handler_name = self._OPERATION_MAP.get(context.operation)
         if handler_name is None:
@@ -121,10 +138,7 @@ def execute(self, context: ExecutionContext) -> ExecutionResult:
             # Stream error to FE so the user sees the failure in real-time
             if self._log_events_id:
                 try:
-                    shim = ExecutorToolShim(
-                        log_events_id=self._log_events_id,
-                        component=self._log_component,
-                    )
+                    shim = self._build_shim()
                     shim.stream_log(
                         f"Error: {exc.message or type(exc).__name__}",
                         level=LogLevel.ERROR,
@@ -133,6 +147,26 @@ def execute(self, context: ExecutionContext) -> ExecutionResult:
                     pass  # Best-effort — don't mask the original error
             return ExecutionResult.failure(error=exc.message)
 
+    def _build_shim(
+        self,
+        *,
+        platform_api_key: str = "",
+        component: dict[str, str] | None = None,
+    ) -> ExecutorToolShim:
+        """Construct an ``ExecutorToolShim`` pre-populated with the
+        log-streaming and workflow-attribution fields captured in
+        ``execute()``. Callers override ``platform_api_key`` and
+        ``component`` as needed; everything else is shared.
+        """
+        return ExecutorToolShim(
+            platform_api_key=platform_api_key,
+            log_events_id=self._log_events_id,
+            component=self._log_component if component is None else component,
+            execution_id=self._execution_id,
+            organization_id=self._organization_id,
+            file_execution_id=self._file_execution_id,
+        )
+
     # ------------------------------------------------------------------
     # Phase 2B — Extract handler
     # ------------------------------------------------------------------
@@ -173,11 +207,7 @@ def _handle_extract(self, context: ExecutionContext) -> ExecutionResult:
         execution_data_dir: str | None = params.get(IKeys.EXECUTION_DATA_DIR)
 
         # Build adapter shim and X2Text
-        shim = ExecutorToolShim(
-            platform_api_key=platform_api_key,
-            log_events_id=self._log_events_id,
-            component=self._log_component,
-        )
+        shim = self._build_shim(platform_api_key=platform_api_key)
         x2text = X2Text(
             tool=shim,
             adapter_instance_id=x2text_instance_id,
@@ -352,6 +382,8 @@ def _run_summarize_step(
             organization_id=context.organization_id,
             request_id=context.request_id,
             log_events_id=context.log_events_id,
+            execution_id=context.execution_id,
+            file_execution_id=context.file_execution_id,
             executor_params={
                 "llm_adapter_instance_id": llm_adapter_id,
                 "summarize_prompt": summarize_prompt,
@@ -427,6 +459,8 @@ def _handle_ide_index(self, context: ExecutionContext) -> ExecutionResult:
                 executor_params=extract_params,
                 request_id=context.request_id,
                 log_events_id=context.log_events_id,
+                execution_id=context.execution_id,
+                file_execution_id=context.file_execution_id,
             )
             extract_result = self._handle_extract(extract_ctx)
             if not extract_result.success:
@@ -454,6 +488,8 @@ def _handle_ide_index(self, context: ExecutionContext) -> ExecutionResult:
             executor_params=index_params,
             request_id=context.request_id,
             log_events_id=context.log_events_id,
+            execution_id=context.execution_id,
+            file_execution_id=context.file_execution_id,
         )
         index_result = self._handle_index(index_ctx)
         if not index_result.success:
@@ -519,10 +555,8 @@ def _handle_structure_pipeline(self, context: ExecutionContext) -> ExecutionResu
         extracted_text = ""
         index_metrics: dict = {}
 
-        shim = ExecutorToolShim(
+        shim = self._build_shim(
             platform_api_key=extract_params.get("platform_api_key", ""),
-            log_events_id=self._log_events_id,
-            component=self._log_component,
         )
         step = 1
 
@@ -539,6 +573,8 @@ def _handle_structure_pipeline(self, context: ExecutionContext) -> ExecutionResu
                 executor_params=extract_params,
                 request_id=context.request_id,
                 log_events_id=context.log_events_id,
+                execution_id=context.execution_id,
+                file_execution_id=context.file_execution_id,
             )
             extract_result = self._handle_extract(extract_ctx)
             if not extract_result.success:
@@ -643,6 +679,8 @@ def _run_pipeline_answer_step(
             executor_params=answer_params,
             request_id=context.request_id,
             log_events_id=context.log_events_id,
+            execution_id=context.execution_id,
+            file_execution_id=context.file_execution_id,
         )
         if is_single_pass:
             return self._handle_single_pass_extraction(answer_ctx)
@@ -745,6 +783,8 @@ def _run_pipeline_summarize(
                 organization_id=context.organization_id,
                 request_id=context.request_id,
                 log_events_id=context.log_events_id,
+                execution_id=context.execution_id,
+                file_execution_id=context.file_execution_id,
                 executor_params={
                     "llm_adapter_instance_id": llm_adapter_id,
                     "summarize_prompt": summarize_prompt,
@@ -840,6 +880,8 @@ def _run_pipeline_index(
                     organization_id=context.organization_id,
                     request_id=context.request_id,
                     log_events_id=context.log_events_id,
+                    execution_id=context.execution_id,
+                    file_execution_id=context.file_execution_id,
                     executor_params={
                         "embedding_instance_id": embedding,
                         "vector_db_instance_id": vector_db,
@@ -954,11 +996,7 @@ def _handle_index(self, context: ExecutionContext) -> ExecutionResult:
             usage_kwargs=usage_kwargs,
         )
 
-        shim = ExecutorToolShim(
-            platform_api_key=platform_api_key,
-            log_events_id=self._log_events_id,
-            component=self._log_component,
-        )
+        shim = self._build_shim(platform_api_key=platform_api_key)
         fs_instance = FileUtils.get_fs_instance(execution_source=execution_source)
 
         logger.info(
@@ -1199,11 +1237,7 @@ def _handle_answer_prompt(self, context: ExecutionContext) -> ExecutionResult:
         process_text_fn = None
         enable_highlight = tool_settings.get(PSKeys.ENABLE_HIGHLIGHT, False)
         enable_word_confidence = tool_settings.get(PSKeys.ENABLE_WORD_CONFIDENCE, False)
-        pipeline_shim = ExecutorToolShim(
-            platform_api_key=platform_api_key,
-            log_events_id=self._log_events_id,
-            component=self._log_component,
-        )
+        pipeline_shim = self._build_shim(platform_api_key=platform_api_key)
         if enable_highlight:
             from executor.executors.plugins import ExecutorPluginLoader
 
@@ -1371,7 +1405,7 @@ def _run_challenge_if_enabled(
         challenge_llm_id = tool_settings.get(PSKeys.CHALLENGE_LLM)
         if not challenge_llm_id:
             return
-        shim.stream_log(f"Running challenge for: {prompt_name}")
+        shim.stream_log(f"Running challenge for: `{prompt_name}`")
         challenge_llm = llm_cls(
             adapter_instance_id=challenge_llm_id,
             tool=shim,
@@ -1390,7 +1424,7 @@ def _run_challenge_if_enabled(
             metadata=metadata,
         )
         challenger.run()
-        shim.stream_log(f"Challenge verification completed for: {prompt_name}")
+        shim.stream_log(f"Challenge verification completed for: `{prompt_name}`")
         logger.info("Challenge completed: prompt=%s", prompt_name)
 
     @staticmethod
@@ -1412,7 +1446,7 @@ def _run_evaluation_if_enabled(
         evaluator_cls = ExecutorPluginLoader.get("evaluation")
         if not evaluator_cls:
             return
-        shim.stream_log(f"Running evaluation for: {prompt_name}")
+        shim.stream_log(f"Running evaluation for: `{prompt_name}`")
         evaluator = evaluator_cls(
             query=output.get(PSKeys.COMBINED_PROMPT, ""),
             context="\n".join(context_list),
@@ -1476,12 +1510,11 @@ def _execute_single_prompt(
             output.get(PSKeys.TYPE, "TEXT"),
         )
 
-        shim = ExecutorToolShim(
+        shim = self._build_shim(
             platform_api_key=platform_api_key,
-            log_events_id=self._log_events_id,
             component={**self._log_component, "prompt_key": prompt_name},
         )
-        shim.stream_log(f"Processing prompt: {prompt_name}")
+        shim.stream_log(f"Processing prompt: `{prompt_name}`")
 
         if variable_replacement_svc.is_variables_present(prompt_text=prompt_text):
             prompt_text = variable_replacement_svc.replace_variables_in_prompt(
@@ -1494,7 +1527,7 @@ def _execute_single_prompt(
                 custom_data=custom_data,
                 is_ide=execution_source == "ide",
             )
-            shim.stream_log(f"Resolved template variables for: {prompt_name}")
+            shim.stream_log(f"Resolved template variables for: `{prompt_name}`")
 
         logger.info(
             "Executing prompt: tool_id=%s name=%s run_id=%s", tool_id, prompt_name, run_id
@@ -1574,7 +1607,9 @@ def _execute_single_prompt(
                     adapter_instance_id=output[PSKeys.VECTOR_DB],
                     embedding=embedding,
                 )
-            shim.stream_log(f"Initialized LLM and retrieval adapters for: {prompt_name}")
+            shim.stream_log(
+                f"Initialized LLM and retrieval adapters for: `{prompt_name}`"
+            )
         except Exception as e:
             msg = f"Couldn't fetch adapter. {e}"
             logger.error(msg)
@@ -1588,7 +1623,7 @@ def _execute_single_prompt(
             retrieval_strategy = output.get(PSKeys.RETRIEVAL_STRATEGY)
             valid_strategies = {s.value for s in RetrievalStrategy}
             if retrieval_strategy in valid_strategies:
-                shim.stream_log(f"Retrieving context for: {prompt_name}")
+                shim.stream_log(f"Retrieving context for: `{prompt_name}`")
                 logger.info(
                     "Performing retrieval: prompt=%s strategy=%s chunk_size=%d",
                     prompt_name,
@@ -1613,14 +1648,14 @@ def _execute_single_prompt(
                     )
                 metadata[PSKeys.CONTEXT][prompt_name] = context_list
                 shim.stream_log(
-                    f"Retrieved {len(context_list)} context chunks for: {prompt_name}"
+                    f"Retrieved {len(context_list)} context chunks for: `{prompt_name}`"
                 )
                 logger.debug(
                     "Retrieved %d context chunks for prompt: %s",
                     len(context_list),
                     prompt_name,
                 )
-                shim.stream_log(f"Running LLM completion for: {prompt_name}")
+                shim.stream_log(f"Running LLM completion for: `{prompt_name}`")
                 answer = answer_prompt_svc.construct_and_run_prompt(
                     tool_settings=tool_settings,
                     output=output,
@@ -1652,7 +1687,7 @@ def _execute_single_prompt(
                 tool_id=tool_id,
                 doc_name=doc_name,
             )
-            shim.stream_log(f"Applied type conversion for: {prompt_name}")
+            shim.stream_log(f"Applied type conversion for: `{prompt_name}`")
 
             self._run_challenge_if_enabled(
                 tool_settings=tool_settings,
@@ -1676,7 +1711,7 @@ def _execute_single_prompt(
                 shim=shim,
                 prompt_name=prompt_name,
             )
-            shim.stream_log(f"Completed prompt: {prompt_name}")
+            shim.stream_log(f"Completed prompt: `{prompt_name}`")
 
             val = structured_output.get(prompt_name)
             if isinstance(val, str):
@@ -1725,6 +1760,9 @@ def _run_table_extraction(
             execution_source=execution_source,
             organization_id=context.organization_id,
             request_id=context.request_id,
+            log_events_id=self._log_events_id,
+            execution_id=self._execution_id,
+            file_execution_id=self._file_execution_id,
             executor_params={
                 "llm_adapter_instance_id": output.get(PSKeys.LLM, ""),
                 "table_settings": output.get(PSKeys.TABLE_SETTINGS, {}),
@@ -1736,9 +1774,8 @@ def _run_table_extraction(
             },
         )
         table_ctx._log_component = self._log_component
-        table_ctx.log_events_id = self._log_events_id
 
-        shim.stream_log(f"Running table extraction for: {prompt_name}")
+        shim.stream_log(f"Running table extraction for: `{prompt_name}`")
         table_result = table_executor.execute(table_ctx)
 
         if table_result.success:
@@ -1747,9 +1784,9 @@ def _run_table_extraction(
             metrics.setdefault(prompt_name, {}).update(
                 {"table_extraction": table_metrics}
             )
-            shim.stream_log(f"Table extraction completed for: {prompt_name}")
+            shim.stream_log(f"Table extraction completed for: `{prompt_name}`")
             logger.info("TABLE extraction completed: prompt=%s", prompt_name)
-            shim.stream_log(f"Completed prompt: {prompt_name}")
+            shim.stream_log(f"Completed prompt: `{prompt_name}`")
         else:
             structured_output[prompt_name] = ""
             error_msg = table_result.error or "unknown error"
@@ -1759,7 +1796,7 @@ def _run_table_extraction(
                 error_msg,
             )
             shim.stream_log(
-                f"Table extraction failed for {prompt_name}: {error_msg}",
+                f"Table extraction failed for `{prompt_name}`: {error_msg}",
                 level=LogLevel.ERROR,
             )
 
@@ -1799,6 +1836,9 @@ def _run_line_item_extraction(
             execution_source=prompt_run_args["execution_source"],
             organization_id=context.organization_id,
             request_id=context.request_id,
+            log_events_id=self._log_events_id,
+            execution_id=self._execution_id,
+            file_execution_id=self._file_execution_id,
             executor_params={
                 "llm_adapter_instance_id": output.get(PSKeys.LLM, ""),
                 "tool_settings": prompt_run_args["tool_settings"],
@@ -1813,9 +1853,8 @@ def _run_line_item_extraction(
             },
         )
         line_item_ctx._log_component = self._log_component
-        line_item_ctx.log_events_id = self._log_events_id
 
-        shim.stream_log(f"Running line-item extraction for: {prompt_name}")
+        shim.stream_log(f"Running line-item extraction for: `{prompt_name}`")
         line_item_result = line_item_executor.execute(line_item_ctx)
 
         if line_item_result.success:
@@ -1828,9 +1867,9 @@ def _run_line_item_extraction(
             context_list = data.get("context")
             if context_list:
                 metadata[PSKeys.CONTEXT][prompt_name] = context_list
-            shim.stream_log(f"Line-item extraction completed for: {prompt_name}")
+            shim.stream_log(f"Line-item extraction completed for: `{prompt_name}`")
             logger.info("LINE_ITEM extraction completed: prompt=%s", prompt_name)
-            shim.stream_log(f"Completed prompt: {prompt_name}")
+            shim.stream_log(f"Completed prompt: `{prompt_name}`")
         else:
             structured_output[prompt_name] = ""
             error_msg = line_item_result.error or "unknown error"
@@ -1840,7 +1879,7 @@ def _run_line_item_extraction(
                 error_msg,
             )
             shim.stream_log(
-                f"Line-item extraction failed for {prompt_name}: {error_msg}",
+                f"Line-item extraction failed for `{prompt_name}`: {error_msg}",
                 level=LogLevel.ERROR,
             )
 
@@ -2032,11 +2071,7 @@ def _handle_summarize(self, context: ExecutionContext) -> ExecutionResult:
             f"Context:\n---------------\n{doc_context}\n-----------------\n\nSummary:"
         )
 
-        shim = ExecutorToolShim(
-            platform_api_key=platform_api_key,
-            log_events_id=self._log_events_id,
-            component=self._log_component,
-        )
+        shim = self._build_shim(platform_api_key=platform_api_key)
         usage_kwargs = {
             "run_id": context.run_id,
             PSKeys.LLM_USAGE_REASON: PSKeys.SUMMARIZE,
diff --git a/workers/file_processing/structure_tool_task.py b/workers/file_processing/structure_tool_task.py
index 8df8603855..87c9426376 100644
--- a/workers/file_processing/structure_tool_task.py
+++ b/workers/file_processing/structure_tool_task.py
@@ -24,6 +24,7 @@
 
 from file_processing.worker import app
 from shared.enums.task_enums import TaskName
+from shared.infrastructure.context import StateStore
 
 from unstract.sdk1.constants import ToolEnv, UsageKwargs
 from unstract.sdk1.execution.context import ExecutionContext
@@ -235,7 +236,22 @@ def _execute_structure_tool_impl(params: dict) -> dict:
     # ---- Step 1: Setup ----
     from executor.executor_tool_shim import ExecutorToolShim
 
-    shim = ExecutorToolShim(platform_api_key=platform_service_api_key)
+    # Workflow IDs on the pre-dispatch shim let X2Text/platform helper logs
+    # reach the workflow logs UI before the executor dispatch happens.
+    log_events_id = StateStore.get("LOG_EVENTS_ID") or ""
+    if not log_events_id:
+        logger.warning(
+            "LOG_EVENTS_ID missing from StateStore for execution_id=%s — "
+            "tool-level logs will not stream to the workflow logs UI.",
+            execution_id,
+        )
+    shim = ExecutorToolShim(
+        platform_api_key=platform_service_api_key,
+        log_events_id=log_events_id,
+        execution_id=execution_id,
+        file_execution_id=file_execution_id,
+        organization_id=organization_id,
+    )
 
     platform_helper = _create_platform_helper(shim, file_execution_id)
     dispatcher = ExecutionDispatcher(celery_app=app)
@@ -257,6 +273,7 @@ def _execute_structure_tool_impl(params: dict) -> dict:
             dispatcher=dispatcher,
             shim=shim,
             file_execution_id=file_execution_id,
+            execution_id=execution_id,
             organization_id=organization_id,
             source_file_name=source_file_name,
             fs=fs,
@@ -388,6 +405,9 @@ def _execute_structure_tool_impl(params: dict) -> dict:
         execution_source="tool",
         organization_id=organization_id,
         request_id=file_execution_id,
+        log_events_id=StateStore.get("LOG_EVENTS_ID") or "",
+        execution_id=execution_id,
+        file_execution_id=file_execution_id,
         executor_params={
             "extract_params": extract_params,
             "index_template": index_template,
@@ -533,6 +553,7 @@ def _run_agentic_extraction(
     dispatcher: ExecutionDispatcher,
     shim: Any,
     file_execution_id: str,
+    execution_id: str,
     organization_id: str,
     source_file_name: str,
     fs: Any,
@@ -587,6 +608,9 @@ def _run_agentic_extraction(
         execution_source="tool",
         organization_id=organization_id,
         request_id=file_execution_id,
+        log_events_id=StateStore.get("LOG_EVENTS_ID") or "",
+        execution_id=execution_id,
+        file_execution_id=file_execution_id,
         executor_params={
             "document_id": file_execution_id,
             "document_text": document_text,

From 29bdf64e55d46cd848ef6bc230a34040aabe901c Mon Sep 17 00:00:00 2001
From: Chandrasekharan M
 <117059509+chandrasekharan-zipstack@users.noreply.github.com>
Date: Fri, 24 Apr 2026 13:32:23 +0530
Subject: [PATCH 3/7] UN-3430 [FIX] Update modified_at field correctly for
 models (#1928)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* [FIX] Auto-bump modified_at on QuerySet.update() and bulk_update()

Django's auto_now=True only fires on Model.save(); QuerySet.update() and
bulk_update() bypass save(), so BaseModel.modified_at silently stayed at
the creation time for every bulk-path write. Audit trail drifted.

Introduce BaseModelQuerySet that injects modified_at=timezone.now() into
both paths, and expose it via BaseModelManager. Migrate all custom
managers on BaseModel subclasses to compose BaseModelManager so their
querysets inherit the overrides. Drop the ad-hoc modified_at=now() kwarg
in FileHistoryHelper now that the queryset handles it.

* [FIX] Materialize objs in BaseModelQuerySet.bulk_update to support generators

Addresses PR review: if callers pass a non-rewindable iterable (generator,
queryset iterator), the modified_at stamping loop would exhaust it before
super().bulk_update() saw it, silently updating zero rows. list(objs) up
front keeps generator callers working.

Also drop the mock-based unit test — it needed django.setup() at module
import which isn't viable without pytest-django, and proper DB-backed
coverage is tracked separately.

* [FIX] Auto-inject modified_at into BaseModel.save(update_fields=...)

Django only runs auto_now for fields listed in update_fields, so every
save(update_fields=["foo"]) on a BaseModel subclass silently drops the
modified_at bump — same family of bug as QuerySet.update/bulk_update.

Override BaseModel.save() to add modified_at to update_fields whenever
the caller supplies a restricted list without it. Also drop two dead
manual-assignment lines (execution.modified_at = timezone.now() before
save()) that were redundant with auto_now on a full save().

* [FIX] Auto-bump modified_at on upsert bulk_create and drop workarounds

QuerySet.bulk_create(update_conflicts=True, update_fields=[...]) runs an
UPDATE on conflict with only the listed fields — same auto_now-bypass as
save(update_fields=...) and QuerySet.update(). Patch BaseModelQuerySet's
bulk_create to inject modified_at into update_fields on upsert.

With that in place, the explicit "modified_at" entries in dashboard_metrics
upsert callers are redundant. Drop them.

* [REFACTOR] Tighten BaseModel auto-bump helpers and edge cases

- Extract `_with_modified_at` helper; single source of truth for the "inject
  modified_at into a partial field list" rule across `bulk_update`,
  `bulk_create` and `BaseModel.save`.
- Preserve Django's documented `save(update_fields=[])` no-op (signals-only
  save, no column writes) instead of rewriting it to `["modified_at"]`.
  Apply the same guard to `bulk_create(update_conflicts=True, update_fields=[])`.
- Match Django's positional `save()` signature (`force_insert`, `force_update`,
  `using`, `update_fields`) so callers passing flags positionally still hit
  the auto-bump override.
- Skip the per-obj `modified_at` stamp + `objs` materialization in
  `bulk_update` when the caller already listed `modified_at` — lets the
  opt-in path stay O(1) before the `super()` delegation.
- Docstring corrections: "previous save() timestamp" (not just creation
  time); manager-level convention note; precise `auto_now` semantics
  (attribute still updates in-memory, just isn't persisted without
  `update_fields` inclusion).
---
 backend/adapter_processor_v2/models.py        |   4 +-
 backend/api_v2/models.py                      |   4 +-
 backend/connector_v2/models.py                |   4 +-
 .../management/commands/backfill_metrics.py   |   6 +-
 backend/dashboard_metrics/models.py           |   8 +-
 backend/dashboard_metrics/tasks.py            |   6 +-
 backend/pipeline_v2/models.py                 |   4 +-
 .../prompt_studio_core_v2/models.py           |   4 +-
 .../prompt_studio_registry_v2/models.py       |   4 +-
 backend/tags/models.py                        |   4 +-
 backend/tool_instance_v2/models.py            |   4 +-
 backend/usage_v2/models.py                    |   4 +-
 backend/utils/models/base_model.py            | 113 ++++++++++++++++++
 backend/utils/models/org_aware_manager.py     |   4 +-
 .../workflow_manager/endpoint_v2/models.py    |   4 +-
 .../workflow_manager/file_execution/models.py |   4 +-
 backend/workflow_manager/internal_views.py    |   2 -
 .../workflow_v2/file_history_helper.py        |   1 -
 .../workflow_v2/models/execution.py           |   4 +-
 .../workflow_v2/models/workflow.py            |   4 +-
 backend/workflow_manager/workflow_v2/views.py |   2 -
 21 files changed, 151 insertions(+), 43 deletions(-)

diff --git a/backend/adapter_processor_v2/models.py b/backend/adapter_processor_v2/models.py
index 13be07cdc0..a6fab0c1f9 100644
--- a/backend/adapter_processor_v2/models.py
+++ b/backend/adapter_processor_v2/models.py
@@ -10,7 +10,7 @@
 from django.db.models import QuerySet
 from tenant_account_v2.models import OrganizationMember
 from utils.exceptions import InvalidEncryptionKey
-from utils.models.base_model import BaseModel
+from utils.models.base_model import BaseModel, BaseModelManager
 from utils.models.organization_mixin import (
     DefaultOrganizationManagerMixin,
     DefaultOrganizationMixin,
@@ -29,7 +29,7 @@
 logger = logging.getLogger(__name__)
 
 
-class AdapterInstanceModelManager(DefaultOrganizationManagerMixin, models.Manager):
+class AdapterInstanceModelManager(DefaultOrganizationManagerMixin, BaseModelManager):
     def get_queryset(self) -> QuerySet[Any]:
         return super().get_queryset()
 
diff --git a/backend/api_v2/models.py b/backend/api_v2/models.py
index 532b67d2dc..cc19902bde 100644
--- a/backend/api_v2/models.py
+++ b/backend/api_v2/models.py
@@ -7,7 +7,7 @@
 from django.db.models.signals import post_delete
 from django.dispatch import receiver
 from pipeline_v2.models import Pipeline
-from utils.models.base_model import BaseModel
+from utils.models.base_model import BaseModel, BaseModelManager
 from utils.models.organization_mixin import (
     DefaultOrganizationManagerMixin,
     DefaultOrganizationMixin,
@@ -24,7 +24,7 @@
 API_ENDPOINT_MAX_LENGTH = 255
 
 
-class APIDeploymentModelManager(DefaultOrganizationManagerMixin, models.Manager):
+class APIDeploymentModelManager(DefaultOrganizationManagerMixin, BaseModelManager):
     def for_user(self, user):
         """Filter API deployments that the user can access:
         - API deployments created by the user
diff --git a/backend/connector_v2/models.py b/backend/connector_v2/models.py
index c3d0e6108b..73ea38b57c 100644
--- a/backend/connector_v2/models.py
+++ b/backend/connector_v2/models.py
@@ -8,7 +8,7 @@
 from connector_processor.constants import ConnectorKeys
 from django.db import models
 from utils.fields import EncryptedBinaryField
-from utils.models.base_model import BaseModel
+from utils.models.base_model import BaseModel, BaseModelManager
 from utils.models.organization_mixin import (
     DefaultOrganizationManagerMixin,
     DefaultOrganizationMixin,
@@ -22,7 +22,7 @@
 logger = logging.getLogger(__name__)
 
 
-class ConnectorInstanceModelManager(DefaultOrganizationManagerMixin, models.Manager):
+class ConnectorInstanceModelManager(DefaultOrganizationManagerMixin, BaseModelManager):
     def get_queryset(self) -> models.QuerySet:
         return super().get_queryset()
 
diff --git a/backend/dashboard_metrics/management/commands/backfill_metrics.py b/backend/dashboard_metrics/management/commands/backfill_metrics.py
index daf0b3130e..9c4d82baca 100644
--- a/backend/dashboard_metrics/management/commands/backfill_metrics.py
+++ b/backend/dashboard_metrics/management/commands/backfill_metrics.py
@@ -424,7 +424,7 @@ def _bulk_upsert_hourly(self, aggregations: dict) -> int:
                 "project",
                 "tag",
             ],
-            update_fields=["metric_type", "metric_value", "metric_count", "modified_at"],
+            update_fields=["metric_type", "metric_value", "metric_count"],
         )
         return len(objects)
 
@@ -457,7 +457,7 @@ def _bulk_upsert_daily(self, aggregations: dict) -> int:
                 "project",
                 "tag",
             ],
-            update_fields=["metric_type", "metric_value", "metric_count", "modified_at"],
+            update_fields=["metric_type", "metric_value", "metric_count"],
         )
         return len(objects)
 
@@ -490,6 +490,6 @@ def _bulk_upsert_monthly(self, aggregations: dict) -> int:
                 "project",
                 "tag",
             ],
-            update_fields=["metric_type", "metric_value", "metric_count", "modified_at"],
+            update_fields=["metric_type", "metric_value", "metric_count"],
         )
         return len(objects)
diff --git a/backend/dashboard_metrics/models.py b/backend/dashboard_metrics/models.py
index 44fec5439a..2bc4baf6ac 100644
--- a/backend/dashboard_metrics/models.py
+++ b/backend/dashboard_metrics/models.py
@@ -3,7 +3,7 @@
 import uuid
 
 from django.db import models
-from utils.models.base_model import BaseModel
+from utils.models.base_model import BaseModel, BaseModelManager
 from utils.models.organization_mixin import (
     DefaultOrganizationManagerMixin,
     DefaultOrganizationMixin,
@@ -25,19 +25,19 @@ class MetricType(models.TextChoices):
     HISTOGRAM = "histogram", "Histogram"
 
 
-class EventMetricsHourlyManager(DefaultOrganizationManagerMixin):
+class EventMetricsHourlyManager(DefaultOrganizationManagerMixin, BaseModelManager):
     """Manager for EventMetricsHourly with organization filtering."""
 
     pass
 
 
-class EventMetricsDailyManager(DefaultOrganizationManagerMixin):
+class EventMetricsDailyManager(DefaultOrganizationManagerMixin, BaseModelManager):
     """Manager for EventMetricsDaily with organization filtering."""
 
     pass
 
 
-class EventMetricsMonthlyManager(DefaultOrganizationManagerMixin):
+class EventMetricsMonthlyManager(DefaultOrganizationManagerMixin, BaseModelManager):
     """Manager for EventMetricsMonthly with organization filtering."""
 
     pass
diff --git a/backend/dashboard_metrics/tasks.py b/backend/dashboard_metrics/tasks.py
index 3246e7e46e..181c985137 100644
--- a/backend/dashboard_metrics/tasks.py
+++ b/backend/dashboard_metrics/tasks.py
@@ -121,7 +121,7 @@ def _bulk_upsert_hourly(aggregations: dict) -> int:
         objects,
         update_conflicts=True,
         unique_fields=["organization", "timestamp", "metric_name", "project", "tag"],
-        update_fields=["metric_type", "metric_value", "metric_count", "modified_at"],
+        update_fields=["metric_type", "metric_value", "metric_count"],
     )
     return len(objects)
 
@@ -160,7 +160,7 @@ def _bulk_upsert_daily(aggregations: dict) -> int:
         objects,
         update_conflicts=True,
         unique_fields=["organization", "date", "metric_name", "project", "tag"],
-        update_fields=["metric_type", "metric_value", "metric_count", "modified_at"],
+        update_fields=["metric_type", "metric_value", "metric_count"],
     )
     return len(objects)
 
@@ -199,7 +199,7 @@ def _bulk_upsert_monthly(aggregations: dict) -> int:
         objects,
         update_conflicts=True,
         unique_fields=["organization", "month", "metric_name", "project", "tag"],
-        update_fields=["metric_type", "metric_value", "metric_count", "modified_at"],
+        update_fields=["metric_type", "metric_value", "metric_count"],
     )
     return len(objects)
 
diff --git a/backend/pipeline_v2/models.py b/backend/pipeline_v2/models.py
index 00b12c8483..65fb5257a8 100644
--- a/backend/pipeline_v2/models.py
+++ b/backend/pipeline_v2/models.py
@@ -4,7 +4,7 @@
 from django.conf import settings
 from django.db import models
 from django.db.models import Q
-from utils.models.base_model import BaseModel
+from utils.models.base_model import BaseModel, BaseModelManager
 from utils.models.organization_mixin import (
     DefaultOrganizationManagerMixin,
     DefaultOrganizationMixin,
@@ -18,7 +18,7 @@
 PIPELINE_NAME_LENGTH = 32
 
 
-class PipelineModelManager(DefaultOrganizationManagerMixin, models.Manager):
+class PipelineModelManager(DefaultOrganizationManagerMixin, BaseModelManager):
     def for_user(self, user):
         """Filter pipelines that the user can access:
         - Pipelines created by the user
diff --git a/backend/prompt_studio/prompt_studio_core_v2/models.py b/backend/prompt_studio/prompt_studio_core_v2/models.py
index 406c157efc..1e1802b776 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/models.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/models.py
@@ -8,7 +8,7 @@
 from django.db.models import QuerySet
 from utils.file_storage.constants import FileStorageKeys
 from utils.file_storage.helpers.prompt_studio_file_helper import PromptStudioFileHelper
-from utils.models.base_model import BaseModel
+from utils.models.base_model import BaseModel, BaseModelManager
 from utils.models.organization_mixin import (
     DefaultOrganizationManagerMixin,
     DefaultOrganizationMixin,
@@ -21,7 +21,7 @@
 logger = logging.getLogger(__name__)
 
 
-class CustomToolModelManager(DefaultOrganizationManagerMixin, models.Manager):
+class CustomToolModelManager(DefaultOrganizationManagerMixin, BaseModelManager):
     def for_user(self, user: User) -> QuerySet[Any]:
         if getattr(user, "is_service_account", False):
             return self.all()
diff --git a/backend/prompt_studio/prompt_studio_registry_v2/models.py b/backend/prompt_studio/prompt_studio_registry_v2/models.py
index c4cd54cc0f..34478c8e70 100644
--- a/backend/prompt_studio/prompt_studio_registry_v2/models.py
+++ b/backend/prompt_studio/prompt_studio_registry_v2/models.py
@@ -5,7 +5,7 @@
 from account_v2.models import User
 from django.db import models
 from django.db.models import QuerySet
-from utils.models.base_model import BaseModel
+from utils.models.base_model import BaseModel, BaseModelManager
 from utils.models.organization_mixin import (
     DefaultOrganizationManagerMixin,
     DefaultOrganizationMixin,
@@ -21,7 +21,7 @@
 logger = logging.getLogger(__name__)
 
 
-class PromptStudioRegistryModelManager(DefaultOrganizationManagerMixin, models.Manager):
+class PromptStudioRegistryModelManager(DefaultOrganizationManagerMixin, BaseModelManager):
     def get_queryset(self) -> QuerySet[Any]:
         return super().get_queryset()
 
diff --git a/backend/tags/models.py b/backend/tags/models.py
index 94c5c8cf54..7f366152b0 100644
--- a/backend/tags/models.py
+++ b/backend/tags/models.py
@@ -1,7 +1,7 @@
 import uuid
 
 from django.db import models
-from utils.models.base_model import BaseModel
+from utils.models.base_model import BaseModel, BaseModelManager
 from utils.models.organization_mixin import (
     DefaultOrganizationManagerMixin,
     DefaultOrganizationMixin,
@@ -9,7 +9,7 @@
 from utils.user_context import UserContext
 
 
-class TagModelManager(DefaultOrganizationManagerMixin, models.Manager):
+class TagModelManager(DefaultOrganizationManagerMixin, BaseModelManager):
     def get_or_create_tags(self, tag_names: list[str]) -> list["Tag"]:
         """Retrieves or creates tags based on a list of tag names.
 
diff --git a/backend/tool_instance_v2/models.py b/backend/tool_instance_v2/models.py
index 971f1249bf..d858da2323 100644
--- a/backend/tool_instance_v2/models.py
+++ b/backend/tool_instance_v2/models.py
@@ -3,7 +3,7 @@
 from account_v2.models import User
 from django.db import models
 from django.db.models import QuerySet
-from utils.models.base_model import BaseModel
+from utils.models.base_model import BaseModel, BaseModelManager
 from workflow_manager.workflow_v2.models.workflow import Workflow
 
 TOOL_ID_LENGTH = 64
@@ -11,7 +11,7 @@
 TOOL_STATUS_LENGTH = 32
 
 
-class ToolInstanceManager(models.Manager):
+class ToolInstanceManager(BaseModelManager):
     def get_instances_for_workflow(self, workflow: uuid.UUID) -> QuerySet["ToolInstance"]:
         return self.filter(workflow=workflow)
 
diff --git a/backend/usage_v2/models.py b/backend/usage_v2/models.py
index 8da3d751ba..57ae3d143d 100644
--- a/backend/usage_v2/models.py
+++ b/backend/usage_v2/models.py
@@ -1,7 +1,7 @@
 import uuid
 
 from django.db import models
-from utils.models.base_model import BaseModel
+from utils.models.base_model import BaseModel, BaseModelManager
 from utils.models.organization_mixin import (
     DefaultOrganizationManagerMixin,
     DefaultOrganizationMixin,
@@ -19,7 +19,7 @@ class LLMUsageReason(models.TextChoices):
     SUMMARIZE = "summarize", "Summarize"
 
 
-class UsageModelManager(DefaultOrganizationManagerMixin, models.Manager):
+class UsageModelManager(DefaultOrganizationManagerMixin, BaseModelManager):
     pass
 
 
diff --git a/backend/utils/models/base_model.py b/backend/utils/models/base_model.py
index b26f0f67d2..c2b288b9ef 100644
--- a/backend/utils/models/base_model.py
+++ b/backend/utils/models/base_model.py
@@ -1,9 +1,122 @@
 from django.db import models
+from django.utils import timezone
+
+_AUTO_NOW_FIELD = "modified_at"
+
+
+def _with_modified_at(fields):
+    """Return a new list containing ``fields`` plus ``modified_at`` if absent.
+
+    Centralises the "inject modified_at into a partial field list" rule so
+    ``bulk_update``, ``bulk_create`` and ``BaseModel.save`` apply it the same
+    way.
+    """
+    fields = list(fields)
+    if _AUTO_NOW_FIELD not in fields:
+        fields.append(_AUTO_NOW_FIELD)
+    return fields
+
+
+class BaseModelQuerySet(models.QuerySet):
+    """QuerySet that mirrors ``auto_now`` semantics for bulk update paths.
+
+    ``modified_at = models.DateTimeField(auto_now=True)`` only fires on
+    ``Model.save()``. ``QuerySet.update()`` and ``QuerySet.bulk_update()``
+    issue raw SQL and bypass ``save()``, leaving ``modified_at`` at whatever
+    value it had before the bulk path ran (creation time for never-saved
+    rows, the previous save() timestamp for others) — silently drifting the
+    audit trail. This QuerySet patches both paths so callers don't have to
+    remember.
+
+    Callers can still override by passing ``modified_at`` explicitly (or by
+    including ``modified_at`` in the ``fields`` list for ``bulk_update``).
+
+    Note: this is a manager-level convention, not a model-level guarantee.
+    Subclasses that reassign ``objects`` to a plain ``models.Manager``, raw
+    SQL, and migration-time models returned by ``apps.get_model()`` all
+    bypass these overrides.
+    """
+
+    def update(self, **kwargs):
+        kwargs.setdefault(_AUTO_NOW_FIELD, timezone.now())
+        return super().update(**kwargs)
+
+    def bulk_update(self, objs, fields, *args, **kwargs):
+        # Stamp modified_at on each obj only when the caller didn't list it;
+        # materialize objs first because we iterate the sequence twice (once
+        # to stamp, once via super()) and a generator would be exhausted.
+        if _AUTO_NOW_FIELD not in fields:
+            objs = list(objs)
+            now = timezone.now()
+            for obj in objs:
+                obj.modified_at = now
+            fields = _with_modified_at(fields)
+        return super().bulk_update(objs, fields, *args, **kwargs)
+
+    def bulk_create(
+        self, objs, *args, update_conflicts=False, update_fields=None, **kwargs
+    ):
+        # On upsert-on-conflict Django runs an UPDATE with only the listed
+        # fields, which skips auto_now the same way save(update_fields=...)
+        # does. Insert-only bulk_create already handles auto_now itself.
+        if update_conflicts and update_fields:
+            update_fields = _with_modified_at(update_fields)
+        return super().bulk_create(
+            objs,
+            *args,
+            update_conflicts=update_conflicts,
+            update_fields=update_fields,
+            **kwargs,
+        )
+
+
+BaseModelManager = models.Manager.from_queryset(BaseModelQuerySet)
 
 
 class BaseModel(models.Model):
+    """Abstract base with managed ``created_at`` / ``modified_at`` timestamps.
+
+    Subclasses inherit ``BaseModelManager`` as the default manager, which
+    auto-bumps ``modified_at`` on ``QuerySet.update()``, ``bulk_update()``
+    and upsert-mode ``bulk_create()``. The ``save()`` override below does
+    the same for partial ``save(update_fields=[...])`` calls.
+
+    Subclasses that need a custom manager should compose ``BaseModelManager``
+    (e.g. ``class FooManager(MyMixin, BaseModelManager)``) — otherwise the
+    auto-bump on bulk paths is silently lost.
+    """
+
     created_at = models.DateTimeField(auto_now_add=True)
     modified_at = models.DateTimeField(auto_now=True)
 
+    objects = BaseModelManager()
+
     class Meta:
         abstract = True
+
+    def save(
+        self,
+        force_insert=False,
+        force_update=False,
+        using=None,
+        update_fields=None,
+        **kwargs,
+    ):
+        # Django's save(update_fields=...) only writes the listed columns.
+        # auto_now still updates modified_at on the in-memory instance, but
+        # the new value is never persisted unless modified_at is in
+        # update_fields. Auto-include it so partial saves don't silently drop
+        # the bump. Preserve Django's documented no-op semantics for
+        # update_fields=[] (signals-only save, no column writes).
+        #
+        # Signature mirrors Django's positional order so callers passing
+        # force_insert/force_update positionally still hit this override.
+        if update_fields:
+            update_fields = _with_modified_at(update_fields)
+        return super().save(
+            force_insert=force_insert,
+            force_update=force_update,
+            using=using,
+            update_fields=update_fields,
+            **kwargs,
+        )
diff --git a/backend/utils/models/org_aware_manager.py b/backend/utils/models/org_aware_manager.py
index 3ec6e70bd7..ab40303dbb 100644
--- a/backend/utils/models/org_aware_manager.py
+++ b/backend/utils/models/org_aware_manager.py
@@ -3,15 +3,15 @@
 import logging
 
 from django.core.exceptions import ImproperlyConfigured
-from django.db import models
 from django.db.utils import OperationalError, ProgrammingError
+from utils.models.base_model import BaseModelManager
 from utils.models.org_path_discovery import get_org_path
 from utils.user_context import UserContext
 
 logger = logging.getLogger(__name__)
 
 
-class OrgAwareManager(models.Manager):
+class OrgAwareManager(BaseModelManager):
     """Manager that auto-discovers FK path to Organization and applies
     org filtering to all queries in request context.
 
diff --git a/backend/workflow_manager/endpoint_v2/models.py b/backend/workflow_manager/endpoint_v2/models.py
index e07737ba7a..44742dfc36 100644
--- a/backend/workflow_manager/endpoint_v2/models.py
+++ b/backend/workflow_manager/endpoint_v2/models.py
@@ -2,12 +2,12 @@
 
 from connector_v2.models import ConnectorInstance
 from django.db import models
-from utils.models.base_model import BaseModel
+from utils.models.base_model import BaseModel, BaseModelManager
 from utils.user_context import UserContext
 from workflow_manager.workflow_v2.models.workflow import Workflow
 
 
-class WorkflowEndpointModelManager(models.Manager):
+class WorkflowEndpointModelManager(BaseModelManager):
     def get_queryset(self):
         # Validating organization
         organization = UserContext.get_organization()
diff --git a/backend/workflow_manager/file_execution/models.py b/backend/workflow_manager/file_execution/models.py
index d5106855e0..105b3875a9 100644
--- a/backend/workflow_manager/file_execution/models.py
+++ b/backend/workflow_manager/file_execution/models.py
@@ -4,7 +4,7 @@
 
 from django.db import models
 from utils.common_utils import CommonUtils
-from utils.models.base_model import BaseModel
+from utils.models.base_model import BaseModel, BaseModelManager
 
 from workflow_manager.endpoint_v2.dto import FileHash
 from workflow_manager.workflow_v2.enums import ExecutionStatus
@@ -15,7 +15,7 @@
 MIME_TYPE_LENGTH = 128
 
 
-class WorkflowFileExecutionManager(models.Manager):
+class WorkflowFileExecutionManager(BaseModelManager):
     def get_or_create_file_execution(
         self,
         workflow_execution: Any,
diff --git a/backend/workflow_manager/internal_views.py b/backend/workflow_manager/internal_views.py
index d11ae56179..c822e5e7b5 100644
--- a/backend/workflow_manager/internal_views.py
+++ b/backend/workflow_manager/internal_views.py
@@ -7,7 +7,6 @@
 
 from django.db import transaction
 from django.shortcuts import get_object_or_404
-from django.utils import timezone
 from rest_framework import status, viewsets
 from rest_framework.decorators import action
 from rest_framework.response import Response
@@ -2009,7 +2008,6 @@ def post(self, request):
                         if update.get("execution_time") is not None:
                             execution.execution_time = update["execution_time"]
 
-                        execution.modified_at = timezone.now()
                         execution.save()
 
                         successful_updates.append(
diff --git a/backend/workflow_manager/workflow_v2/file_history_helper.py b/backend/workflow_manager/workflow_v2/file_history_helper.py
index 12b0a9c2c0..687f88dfe9 100644
--- a/backend/workflow_manager/workflow_v2/file_history_helper.py
+++ b/backend/workflow_manager/workflow_v2/file_history_helper.py
@@ -288,7 +288,6 @@ def _increment_file_history(
             result=str(result),
             metadata=FileHistoryHelper._safe_str(metadata),
             error=FileHistoryHelper._safe_str(error),
-            modified_at=timezone.now(),
         )
         # Refresh from DB to get updated values
         file_history.refresh_from_db()
diff --git a/backend/workflow_manager/workflow_v2/models/execution.py b/backend/workflow_manager/workflow_v2/models/execution.py
index 93623118d7..45886bd64e 100644
--- a/backend/workflow_manager/workflow_v2/models/execution.py
+++ b/backend/workflow_manager/workflow_v2/models/execution.py
@@ -12,7 +12,7 @@
 from usage_v2.helper import UsageHelper
 from usage_v2.models import Usage
 from utils.common_utils import CommonUtils
-from utils.models.base_model import BaseModel
+from utils.models.base_model import BaseModel, BaseModelManager
 
 from workflow_manager.execution.dto import ExecutionCache
 from workflow_manager.execution.execution_cache_utils import ExecutionCacheUtils
@@ -26,7 +26,7 @@
 EXECUTION_ERROR_LENGTH = 256
 
 
-class WorkflowExecutionManager(models.Manager):
+class WorkflowExecutionManager(BaseModelManager):
     """Custom manager for WorkflowExecution model to handle user-specific filtering."""
 
     def for_user(self, user) -> QuerySet:
diff --git a/backend/workflow_manager/workflow_v2/models/workflow.py b/backend/workflow_manager/workflow_v2/models/workflow.py
index dd945fc7f9..0029f95997 100644
--- a/backend/workflow_manager/workflow_v2/models/workflow.py
+++ b/backend/workflow_manager/workflow_v2/models/workflow.py
@@ -4,7 +4,7 @@
 from django.conf import settings
 from django.core.validators import MinValueValidator
 from django.db import models
-from utils.models.base_model import BaseModel
+from utils.models.base_model import BaseModel, BaseModelManager
 from utils.models.organization_mixin import (
     DefaultOrganizationManagerMixin,
     DefaultOrganizationMixin,
@@ -15,7 +15,7 @@
 WORKFLOW_NAME_SIZE = 128
 
 
-class WorkflowModelManager(DefaultOrganizationManagerMixin, models.Manager):
+class WorkflowModelManager(DefaultOrganizationManagerMixin, BaseModelManager):
     def for_user(self, user):
         """Filter workflows that the user can access:
         - Workflows created by the user
diff --git a/backend/workflow_manager/workflow_v2/views.py b/backend/workflow_manager/workflow_v2/views.py
index dc428d948a..629b52bb22 100644
--- a/backend/workflow_manager/workflow_v2/views.py
+++ b/backend/workflow_manager/workflow_v2/views.py
@@ -6,7 +6,6 @@
 from django.db import transaction
 from django.db.models.query import QuerySet
 from django.shortcuts import get_object_or_404
-from django.utils import timezone
 from django.views.decorators.csrf import csrf_exempt
 from permissions.permission import IsOwner, IsOwnerOrSharedUserOrSharedToOrg
 from pipeline_v2.models import Pipeline
@@ -528,7 +527,6 @@ def status(self, request, id=None):
                 if validated_data.get("execution_time") is not None:
                     execution.execution_time = validated_data["execution_time"]
 
-                execution.modified_at = timezone.now()
                 execution.save()
 
                 logger.info(

From fd532bec8c1b5479fffc89494115308d09c09594 Mon Sep 17 00:00:00 2001
From: harini-venkataraman
 <115449948+harini-venkataraman@users.noreply.github.com>
Date: Tue, 28 Apr 2026 10:58:43 +0530
Subject: [PATCH 4/7] UN-3403 [FEAT] Agentic table extractor plugin with
 multi-agent LLM-powered table extraction (#1914)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Execution backend - revamp

* async flow

* Streaming progress to FE

* Removing multi hop in Prompt studio ide and structure tool

* UN-3234 [FIX] Add beta tag to agentic prompt studio navigation item

* Added executors for agentic prompt studio

* Added executors for agentic prompt studio

* Removed redundant envs

* Removed redundant envs

* Removed redundant envs

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Removed redundant envs

* Removed redundant envs

* Removed redundant envs

* Removed redundant envs

* Removed redundant envs

* Removed redundant envs

* Removed redundant envs

* Removed redundant envs

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Removed redundant envs

* adding worker for callbacks

* adding worker for callbacks

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* adding worker for callbacks

* adding worker for callbacks

* adding worker for callbacks

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Pluggable apps and plugins to fit the new async prompt execution architecture

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Pluggable apps and plugins to fit the new async prompt execution architecture

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Pluggable apps and plugins to fit the new async prompt execution architecture

* adding worker for callbacks

* adding worker for callbacks

* adding worker for callbacks

* adding worker for callbacks

* adding worker for callbacks

* adding worker for callbacks

* adding worker for callbacks

* adding worker for callbacks

* fix: write output files in agentic extraction pipeline

Agentic extraction returned early without writing INFILE (JSON) or
METADATA.json, causing destination connectors to read the original PDF
and fail with "Expected tool output type: TXT, got: application/pdf".

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* UN-3266 fix: replace hardcoded /tmp paths with secure temp dirs in tests (#1850)

* UN-3266 fix: replace hardcoded /tmp paths with secure temp dirs in tests

Replace hardcoded /tmp/ paths (SonarCloud S5443 security hotspots) with
pytest's tmp_path fixture or module-level tempfile.mkdtemp() constants
in all affected test files to avoid world-writable directory vulnerabilities.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Update docs

* UN-3266 fix: remove dead code with undefined names in fetch_response

Remove unreachable code block after the async callback return in
fetch_response that still referenced output_count_before and response
from the old synchronous implementation, causing ruff F821 errors.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* Un 3266 fix security hotspot tmp paths (#1851)

* UN-3266 fix: replace hardcoded /tmp paths with secure temp dirs in tests

Replace hardcoded /tmp/ paths (SonarCloud S5443 security hotspots) with
pytest's tmp_path fixture or module-level tempfile.mkdtemp() constants
in all affected test files to avoid world-writable directory vulnerabilities.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* UN-3266 fix: resolve ruff linting failures across multiple files

- B026: pass url positionally in worker_celery.py to avoid star-arg after keyword
- N803: rename MockAsyncResult to mock_async_result in test_tasks.py
- E501/I001: fix long line and import sort in llm_whisperer helper
- ANN401: replace Any with object|None in dispatcher.py; add noqa in test helpers
- F841: remove unused workflow_id and result assignments

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

* UN-3266 fix: resolve SonarCloud bugs S2259 and S1244 in PR #1849

- S2259: guard against None after _discover_plugins() in loader.py
  to satisfy static analysis on the dict[str,type]|None field type
- S1244: replace float equality checks with pytest.approx() in
  test_answer_prompt.py and test_phase2h.py

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* UN-3266 fix: resolve SonarCloud code smells in PR #1849

- S5799: Merge all implicit string concatenations in log messages
  (legacy_executor.py, tasks.py, dispatcher.py, orchestrator.py,
   registry.py, variable_replacement.py, structure_tool_task.py)
- S1192: Extract duplicate literal to _NO_CELERY_APP_MSG constant in
  dispatcher.py
- S1871: Merge identical elif/else branches in tasks.py and
  test_sanity_phase6j.py
- S1186: Add comment to empty stub method in test_sanity_phase6a.py
- S1481: Remove unused local variables in test_sanity_phase6d/e/f/g/h/j
  and test_phase5d.py
- S117: Rename PascalCase local variables to snake_case in
  test_sanity_phase3/5/6i.py
- S5655: Broaden tool type annotation to StreamMixin in
  IndexingUtils.generate_index_key and PlatformHelper.get_adapter_config
- docker:S7031: Merge consecutive RUN instructions in
  worker-unified.Dockerfile
- javascript:S1128: Remove unused pollForCompletion import in
  usePromptRun.js

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* UN-3266 fix: wrap long log message in dispatcher.py to fix E501

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* UN-3266 fix: resolve remaining SonarCloud S117 naming violations

Rename PascalCase local variables to snake_case to comply with S117:

- legacy_executor.py: rename tuple-unpacked _get_prompt_deps() results
  (AnswerPromptService→answer_prompt_svc, RetrievalService→retrieval_svc,
  VariableReplacementService→variable_replacement_svc, LLM→llm_cls,
  EmbeddingCompat→embedding_compat_cls, VectorDB→vector_db_cls) and
  update all downstream usages including _apply_type_conversion and
  _handle_summarize
- test_phase1_log_streaming.py: rename Mock* local variables to
  mock_* snake_case equivalents
- test_sanity_phase3.py: rename MockDispatcher→mock_dispatcher_cls
  and MockShim→mock_shim_cls across all 10 test methods
- test_sanity_phase5.py: rename MockShim→mock_shim, MockX2Text→mock_x2text
  in 6 test methods; MockDispatcher→mock_dispatcher_cls in dispatch test;
  fix LLM_cls→llm_cls, EmbeddingCompat→embedding_compat_cls,
  VectorDB→vector_db_cls in _mock_prompt_deps helper

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* UN-3266 fix: resolve remaining SonarCloud code smells in PR #1849

- test_sanity_phase2/4.py, test_answer_prompt.py: rename PascalCase
  local variables in _mock_prompt_deps/_mock_deps to snake_case
  (RetrievalService→retrieval_svc, VariableReplacementService→
  variable_replacement_svc, Index→index_cls, LLM_cls→llm_cls,
  EmbeddingCompat→embedding_compat_cls, VectorDB→vector_db_cls,
  AnswerPromptService→answer_prompt_svc_cls) — fixes S117
- test_sanity_phase3.py: remove unused local variable "result" — fixes S1481
- structure_tool_task.py: remove redundant json.JSONDecodeError from
  except clause (subclass of ValueError) — fixes S5713
- shared/workflow/execution/service.py: replace generic Exception with
  RuntimeError for structure tool failure — fixes S112
- run-worker-docker.sh: define EXECUTOR_WORKER_TYPE constant and
  replace 10 literal "executor" occurrences — fixes S1192

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* UN-3266 fix: resolve SonarCloud cognitive complexity and code smell violations

- Reduce cognitive complexity in answer_prompt.py:
  - Extract _build_grammar_notes, _run_webhook_postprocess helpers
  - _is_safe_public_url: extracted _resolve_host_addresses helper
  - handle_json: early-return pattern eliminates nesting
  - construct_prompt: delegates grammar loop to _build_grammar_notes
- Reduce cognitive complexity in legacy_executor.py:
  - Extract _execute_single_prompt, _run_table_extraction helpers
  - Extract _run_challenge_if_enabled, _run_evaluation_if_enabled
  - Extract _inject_table_settings, _finalize_pipeline_result
  - Extract _convert_number_answer, _convert_scalar_answer
  - Extract _sanitize_dict_values helper
  - _handle_answer_prompt CC reduced from 50 to ~7
- Reduce CC in structure_tool_task.py: guard-clause refactor
- Reduce CC in backend: dto.py, deployment_helper.py,
  api_deployment_views.py, prompt_studio_helper.py
- Fix S117: rename PascalCase local vars in test_answer_prompt.py
- Fix S1192: extract EXECUTOR_WORKER_TYPE constant in run-worker.sh
- Fix S1172: remove unused params from structure_tool_task.py
- Fix S5713: remove redundant JSONDecodeError in json_repair_helper.py
- Fix S112/S5727 in test_execution.py

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* UN-3266 fix: remove unused RetrievalStrategy import from _handle_answer_prompt

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* UN-3266 fix: rename UsageHelper params to lowercase (N803)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* UN-3266 fix: resolve remaining SonarCloud issues from check run 66691002192

- Add @staticmethod to _sanitize_null_values (fixes S2325 missing self)
- Reduce _execute_single_prompt params from 25 to 11 (S107)
  by grouping services as deps tuple and extracting exec params
  from context.executor_params
- Add NOSONAR suppression for raise exc in test helper (S112)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* UN-3266 fix: remove unused locals in _handle_answer_prompt (F841)

execution_id, file_hash, log_events_id, custom_data are now extracted
inside _execute_single_prompt from context.executor_params.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* fix: resolve Biome linting errors in frontend source files

Auto-fixed 48 lint errors across 56 files: import ordering, block
statements, unused variable prefixing, and formatting issues.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* fix: replace dynamic import of SharePermission with static import in Workflows

Resolves vite build warning about SharePermission.jsx being both
dynamically and statically imported across the codebase.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* fix: resolve SonarCloud warnings in frontend components

- Remove unnecessary try-catch around PostHog event calls
- Flip negated condition in PromptOutput.handleTable for clarity

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Address PR #1849 review comments: fix null guards, dead code, and test drift

- Remove redundant inline `import uuid as _uuid` in views.py (use module-level uuid)
- URL-encode DB_USER in worker_celery.py result backend connection string
- Remove misleading task_queues=[Queue("executor")] from dispatch-only Celery app
- Remove dead `if not tool:` guards after objects.get() (already raises DoesNotExist)
- Move profile_manager/default_profile null checks before first dereference
- Reorder ProfileManager.objects.get before mark_document_indexed in tasks.py
- Handle ProfileManager.DoesNotExist as warning, not hard failure
- Wrap PostHog analytics in try/catch so failures don't block prompt execution
- Handle pending-indexing 200 response in usePromptRun.js (clear RUNNING status)
- Reset formData when metadata is missing in ConfigureDs.jsx
- Fix test_should_skip_extraction tests: function now takes 1 arg (outputs only)
- Fix agentic routing tests: mock X2Text.process, remove stale platform_helper kwarg

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Fix missing llm_usage_reason for summarize LLM usage tracking

Add PSKeys.LLM_USAGE_REASON to usage_kwargs in _handle_summarize() so
summarization costs appear under summarize_llm in API response metadata.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* UN-3266 [FIX] Fix single-pass extraction routing in LegacyExecutor

- Route _handle_structure_pipeline to _handle_single_pass_extraction when
  is_single_pass=True (was always calling _handle_answer_prompt)
- Delegate _handle_single_pass_extraction to cloud plugin via ExecutorRegistry,
  falling back to _handle_answer_prompt if plugin not installed

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* Fixing API depployment response mismatches

* Add complete_vision() method to SDK1 LLM for multimodal completions

Adds a new complete_vision() method alongside existing complete() that
accepts pre-built multimodal messages (text + image_url) in OpenAI-style
format. LiteLLM auto-translates for Anthropic/Bedrock/Vertex providers.
This enables the agentic table extractor plugin to send page images
alongside text prompts for VLM-based table detection and extraction.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* UN-3266 [FIX] Gate Run button by agentic table readiness checklist

- PromptCardItems loads AgenticTableChecklist plugin and owns the
  isAgenticTableReady state, rendering the checklist above the prompt
  text area and delegating the settings gear visibility to the plugin.
- Header and PromptOutput disable their Run buttons when
  isAgenticTableReady is false (default true for non-agentic types).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* [FIX] Use correct primary key field in prompt count subquery (#1905)

ToolStudioPrompt uses prompt_id as its primary key, not id.
Count("id") causes FieldError on the list endpoint (500).

Co-authored-by: Chandrasekharan M <chandrasekharan@zipstack.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* [FIX] Add agentic_table as valid enforce_type choice

The cloud build adds "agentic_table" to the prompt enforce_type
dropdown, but the OSS ToolStudioPrompt model rejected it as an
invalid choice. Add AGENTIC_TABLE to EnforceType and ship a
matching migration so the value can be persisted.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* UN-3266 [FIX] Wire agentic_table enforce_type to executor dispatch

The single-prompt run flow had no branch for prompts with
enforce_type=agentic_table, so clicking Run silently fell through to
the legacy prompt-service path and never invoked the agentic_table
executor. Adds an AGENTIC_TABLE constant to TSPKeys, includes it in
the OperationNotSupported guard, and dispatches to
PayloadModifier.execute_agentic_table when the plugin is available
so the result still flows through _handle_response.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* UN-3266 [FIX] Add agentic_table queue to executor worker defaults

The ExecutionDispatcher derives the queue name from the executor name
(celery_executor_{name}), so dispatches to the agentic_table executor
land on celery_executor_agentic_table. The local docker-compose default
only listed celery_executor_legacy and celery_executor_agentic, so no
worker consumed the new queue and dispatch hung for the full 1-hour
result timeout. Adds the missing queue to the docker-compose default.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* UN-3266 [FIX] Dispatch agentic_table prompts to executor on IDE Run

The IDE Run button was building a legacy answer_prompt payload for
agentic_table prompts, so the agentic table executor was never
invoked. Branch fetch_response on enforce_type so agentic_table
prompts are built via the cloud payload_modifier plugin and
dispatched directly to celery_executor_agentic_table. Add the
enforce_type to the OSS dropdown choices and the JSON-dump set in
OutputManagerHelper so the persisted output is parseable by the FE
table renderer.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* UN-3266 [FIX] Reshape agentic_table executor output in IDE callback

The agentic_table executor returns {"output": {"tables": [...],
"page_count": ..., "headers": [...], ...}}, but
OutputManagerHelper.handle_prompt_output_update reads
outputs[prompt.prompt_key] when persisting prompt output. Without a
reshape the table list never lands under the prompt key and the FE
sees an empty result.

When cb_kwargs carries is_agentic_table=True and prompt_key (set by
the cloud build_agentic_table_payload), reshape outputs to
{prompt_key: tables} before calling update_prompt_output. The
executor itself also shapes its envelope, so this is a defensive
double-keying that keeps the legacy answer_prompt path untouched.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* Fixing timeout issues

* API deployment fixes for Agentic table extractor

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Fixing syntax issues

* Fix agentic_table executor reading INFILE after JSON overwrite

Read from SOURCE instead of INFILE when dispatching to the
agentic_table executor. INFILE gets overwritten with JSON output
by the regular pipeline, causing PDFium parse errors when the
agentic_table executor tries to process it as a PDF.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

---------

Signed-off-by: harini-venkataraman <115449948+harini-venkataraman@users.noreply.github.com>
Co-authored-by: Ghost Jake <89829542+Deepak-Kesavan@users.noreply.github.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
Co-authored-by: Ritwik G <100672805+ritwik-g@users.noreply.github.com>
Co-authored-by: Chandrasekharan M <chandrasekharan@zipstack.com>
---
 .../prompt_studio_core_v2/constants.py        |   1 +
 .../prompt_studio_helper.py                   |  40 +++--
 .../static/select_choices.json                |   1 +
 .../prompt_studio_core_v2/views.py            |  59 ++++++-
 .../output_manager_helper.py                  |   8 +-
 .../prompt_studio_registry_v2/constants.py    |   1 +
 .../prompt_studio_registry_helper.py          |   9 +
 ...014_alter_toolstudioprompt_enforce_type.py |  41 +++++
 .../prompt_studio/prompt_studio_v2/models.py  |   1 +
 docker/docker-compose.yaml                    |   2 +-
 .../custom-tools/prompt-card/Header.jsx       |   4 +
 .../prompt-card/PromptCardItems.jsx           |  24 ++-
 .../custom-tools/prompt-card/PromptOutput.jsx |  14 +-
 frontend/src/hooks/usePromptRun.js            |   6 +-
 unstract/sdk1/src/unstract/sdk1/llm.py        |  90 ++++++++++
 workers/executor/executors/legacy_executor.py |  37 ++--
 .../executor/executors/retrievers/fusion.py   |   2 +-
 .../executors/retrievers/keyword_table.py     |   2 +-
 .../file_processing/structure_tool_task.py    | 161 ++++++++++++++----
 workers/ide_callback/tasks.py                 |  10 ++
 workers/tests/test_answer_prompt.py           |  16 +-
 21 files changed, 451 insertions(+), 78 deletions(-)
 create mode 100644 backend/prompt_studio/prompt_studio_v2/migrations/0014_alter_toolstudioprompt_enforce_type.py

diff --git a/backend/prompt_studio/prompt_studio_core_v2/constants.py b/backend/prompt_studio/prompt_studio_core_v2/constants.py
index c1ab14d380..03bd68c1d8 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/constants.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/constants.py
@@ -92,6 +92,7 @@ class ToolStudioPromptKeys:
     INCLUDE_METADATA = "include_metadata"
     TXT_EXTENTION = ".txt"
     TABLE = "table"
+    AGENTIC_TABLE = "agentic_table"
     PLATFORM_POSTAMBLE = "platform_postamble"
     WORD_CONFIDENCE_POSTAMBLE = "word_confidence_postamble"
     SUMMARIZE_AS_SOURCE = "summarize_as_source"
diff --git a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py
index d0ffef3114..ff2b345505 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py
@@ -1540,6 +1540,7 @@ def _execute_single_prompt(
         if (
             prompt_instance.enforce_type == TSPKeys.TABLE
             or prompt_instance.enforce_type == TSPKeys.RECORD
+            or prompt_instance.enforce_type == TSPKeys.AGENTIC_TABLE
         ) and not payload_modifier_plugin:
             raise OperationNotSupported()
 
@@ -1570,17 +1571,34 @@ def _execute_single_prompt(
             "Invoking prompt service",
         )
         try:
-            response = PromptStudioHelper._fetch_response(
-                doc_path=doc_path,
-                doc_name=doc_name,
-                tool=tool,
-                prompt=prompt_instance,
-                org_id=org_id,
-                document_id=document_id,
-                run_id=run_id,
-                profile_manager_id=profile_manager_id,
-                user_id=user_id,
-            )
+            if (
+                prompt_instance.enforce_type == TSPKeys.AGENTIC_TABLE
+                and payload_modifier_plugin
+            ):
+                modifier_service = payload_modifier_plugin["service_class"]()
+                response = modifier_service.execute_agentic_table(
+                    tool_id=tool_id,
+                    prompt_id=str(prompt_instance.prompt_id),
+                    prompt_key=prompt_name,
+                    prompt=prompt_instance.prompt,
+                    doc_path=doc_path,
+                    doc_name=doc_name,
+                    org_id=org_id,
+                    user_id=user_id,
+                    run_id=run_id,
+                )
+            else:
+                response = PromptStudioHelper._fetch_response(
+                    doc_path=doc_path,
+                    doc_name=doc_name,
+                    tool=tool,
+                    prompt=prompt_instance,
+                    org_id=org_id,
+                    document_id=document_id,
+                    run_id=run_id,
+                    profile_manager_id=profile_manager_id,
+                    user_id=user_id,
+                )
             return PromptStudioHelper._handle_response(
                 response=response,
                 run_id=run_id,
diff --git a/backend/prompt_studio/prompt_studio_core_v2/static/select_choices.json b/backend/prompt_studio/prompt_studio_core_v2/static/select_choices.json
index 3d59b2bf08..ee4c218ca2 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/static/select_choices.json
+++ b/backend/prompt_studio/prompt_studio_core_v2/static/select_choices.json
@@ -15,6 +15,7 @@
         "boolean":"boolean",
         "json":"json",
         "table":"table",
+        "agentic_table":"agentic_table",
         "line-item":"line-item"
     },
     "output_processing":{
diff --git a/backend/prompt_studio/prompt_studio_core_v2/views.py b/backend/prompt_studio/prompt_studio_core_v2/views.py
index 22c1a378bb..99edefd0b4 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/views.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/views.py
@@ -50,6 +50,7 @@
 from prompt_studio.prompt_studio_core_v2.exceptions import (
     DeploymentUsageCheckError,
     MaxProfilesReachedError,
+    OperationNotSupported,
     ToolDeleteError,
 )
 from prompt_studio.prompt_studio_core_v2.migration_utils import SummarizeMigrationUtils
@@ -509,6 +510,61 @@ def fetch_response(self, request: HttpRequest, pk: Any = None) -> Response:
         document: DocumentManager = DocumentManager.objects.get(pk=document_id)
         doc_path = str(Path(doc_path) / document.document_name)
 
+        # Agentic table prompts have a separate executor worker. Build the
+        # payload via the cloud payload_modifier plugin and dispatch directly
+        # so the legacy answer_prompt path is bypassed.
+        if prompt.enforce_type == ToolStudioPromptKeys.AGENTIC_TABLE:
+            payload_modifier_plugin = get_plugin("payload_modifier")
+            if not payload_modifier_plugin:
+                raise OperationNotSupported()
+            modifier = payload_modifier_plugin["service_class"]()
+            context, cb_kwargs = modifier.build_agentic_table_payload(
+                tool=custom_tool,
+                prompt=prompt,
+                doc_path=doc_path,
+                doc_name=document.document_name,
+                org_id=org_id,
+                user_id=user_id,
+                document_id=document_id,
+                run_id=run_id,
+                profile_manager_id=profile_manager_id,
+            )
+
+            from prompt_studio.prompt_studio_output_manager_v2.models import (
+                PromptStudioOutputManager,
+            )
+
+            cb_kwargs["hubspot_user_id"] = request.user.pk
+            cb_kwargs[
+                "is_first_prompt_run"
+            ] = not PromptStudioOutputManager.objects.filter(
+                tool_id__in=CustomTool.objects.values_list("tool_id", flat=True)
+            ).exists()
+
+            dispatcher = PromptStudioHelper._get_dispatcher()
+            executor_task_id = str(uuid.uuid4())
+            cb_kwargs["executor_task_id"] = executor_task_id
+            cb_kwargs["dispatch_time"] = time.time()
+
+            task = dispatcher.dispatch_with_callback(
+                context,
+                on_success=signature(
+                    "ide_prompt_complete",
+                    kwargs={"callback_kwargs": cb_kwargs},
+                    queue="ide_callback",
+                ),
+                on_error=signature(
+                    "ide_prompt_error",
+                    kwargs={"callback_kwargs": cb_kwargs},
+                    queue="ide_callback",
+                ),
+                task_id=executor_task_id,
+            )
+            return Response(
+                {"task_id": task.id, "run_id": run_id, "status": "accepted"},
+                status=status.HTTP_202_ACCEPTED,
+            )
+
         context, cb_kwargs = PromptStudioHelper.build_fetch_response_payload(
             tool=custom_tool,
             doc_path=doc_path,
@@ -705,7 +761,7 @@ def single_pass_extraction(self, request: HttpRequest, pk: uuid) -> Response:
 
         # Fetch prompts eligible for single-pass extraction.
         # Mirrors the filtering in _execute_prompts_in_single_pass:
-        # only active, non-NOTES, non-TABLE/RECORD prompts.
+        # only active, non-NOTES, non-TABLE/RECORD/AGENTIC_TABLE prompts.
         prompts = list(
             ToolStudioPrompt.objects.filter(tool_id=custom_tool.tool_id).order_by(
                 "sequence_number"
@@ -718,6 +774,7 @@ def single_pass_extraction(self, request: HttpRequest, pk: uuid) -> Response:
             and p.active
             and p.enforce_type != ToolStudioPromptKeys.TABLE
             and p.enforce_type != ToolStudioPromptKeys.RECORD
+            and p.enforce_type != ToolStudioPromptKeys.AGENTIC_TABLE
         ]
         if not prompts:
             return Response(
diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py b/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py
index 405b91e00f..64932eb3f9 100644
--- a/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py
+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py
@@ -170,7 +170,13 @@ def update_or_create_prompt_output(
 
             # TODO: use enums here
             output = outputs.get(prompt.prompt_key)
-            if prompt.enforce_type in {"json", "table", "record", "line-item"}:
+            if prompt.enforce_type in {
+                "json",
+                "table",
+                "record",
+                "line-item",
+                "agentic_table",
+            }:
                 output = json.dumps(output)
             eval_metrics = outputs.get(f"{prompt.prompt_key}__evaluation", [])
             profile_manager = default_profile
diff --git a/backend/prompt_studio/prompt_studio_registry_v2/constants.py b/backend/prompt_studio/prompt_studio_registry_v2/constants.py
index 785e77484c..35d6654851 100644
--- a/backend/prompt_studio/prompt_studio_registry_v2/constants.py
+++ b/backend/prompt_studio/prompt_studio_registry_v2/constants.py
@@ -12,6 +12,7 @@ class PromptStudioRegistryKeys:
     TABLE = "table"
     RECORD = "record"
     LINE_ITEM = "line-item"
+    AGENTIC_TABLE = "agentic_table"
 
 
 class PromptStudioRegistryErrors:
diff --git a/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py b/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py
index 6ce1f72095..08d2c27baa 100644
--- a/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py
+++ b/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py
@@ -371,6 +371,15 @@ def frame_export_json(
                         prompt_id=prompt.prompt_id,
                         prompt=prompt.prompt,
                     )
+            elif prompt.enforce_type == PromptStudioRegistryKeys.AGENTIC_TABLE:
+                payload_modifier_plugin = get_plugin("payload_modifier")
+                if payload_modifier_plugin:
+                    modifier_service = payload_modifier_plugin["service_class"]()
+                    output = modifier_service.export_agentic_table_settings(
+                        output=output,
+                        tool_id=tool.tool_id,
+                        prompt_id=prompt.prompt_id,
+                    )
 
             outputs.append(output)
             output = {}
diff --git a/backend/prompt_studio/prompt_studio_v2/migrations/0014_alter_toolstudioprompt_enforce_type.py b/backend/prompt_studio/prompt_studio_v2/migrations/0014_alter_toolstudioprompt_enforce_type.py
new file mode 100644
index 0000000000..beae260ce2
--- /dev/null
+++ b/backend/prompt_studio/prompt_studio_v2/migrations/0014_alter_toolstudioprompt_enforce_type.py
@@ -0,0 +1,41 @@
+# Generated for agentic table enforce type
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        (
+            "prompt_studio_v2",
+            "0013_toolstudioprompt_enable_postprocessing_webhook_and_more",
+        ),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="toolstudioprompt",
+            name="enforce_type",
+            field=models.TextField(
+                blank=True,
+                choices=[
+                    ("text", "Response sent as Text"),
+                    ("number", "Response sent as number"),
+                    ("email", "Response sent as email"),
+                    ("date", "Response sent as date"),
+                    ("boolean", "Response sent as boolean"),
+                    ("json", "Response sent as json"),
+                    (
+                        "line-item",
+                        "Response sent as line-item which is large a JSON output. If extraction stopped due to token limitation, we try to continue extraction from where it stopped",
+                    ),
+                    ("table", "Response sent as json"),
+                    (
+                        "agentic_table",
+                        "Response sent as agentic table extraction",
+                    ),
+                ],
+                db_comment="Field to store the type in             which the response to be returned.",
+                default="text",
+            ),
+        ),
+    ]
diff --git a/backend/prompt_studio/prompt_studio_v2/models.py b/backend/prompt_studio/prompt_studio_v2/models.py
index 74739bfe5a..b4c64f0912 100644
--- a/backend/prompt_studio/prompt_studio_v2/models.py
+++ b/backend/prompt_studio/prompt_studio_v2/models.py
@@ -31,6 +31,7 @@ class EnforceType(models.TextChoices):
             ),
         )
         TABLE = "table", "Response sent as json"
+        AGENTIC_TABLE = "agentic_table", "Response sent as agentic table extraction"
 
     class PromptType(models.TextChoices):
         PROMPT = "PROMPT", "Response sent as Text"
diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml
index 4dd8c38667..bb40b324ef 100644
--- a/docker/docker-compose.yaml
+++ b/docker/docker-compose.yaml
@@ -529,7 +529,7 @@ services:
       - EXECUTOR_METRICS_PORT=8088
       - HEALTH_PORT=8088
       # Configurable Celery options
-      - CELERY_QUEUES_EXECUTOR=${CELERY_QUEUES_EXECUTOR:-celery_executor_legacy,celery_executor_agentic}
+      - CELERY_QUEUES_EXECUTOR=${CELERY_QUEUES_EXECUTOR:-celery_executor_legacy,celery_executor_agentic,celery_executor_agentic_table}
       - CELERY_POOL=${WORKER_EXECUTOR_POOL:-prefork}
       - CELERY_PREFETCH_MULTIPLIER=${WORKER_EXECUTOR_PREFETCH_MULTIPLIER:-1}
       - CELERY_CONCURRENCY=${WORKER_EXECUTOR_CONCURRENCY:-2}
diff --git a/frontend/src/components/custom-tools/prompt-card/Header.jsx b/frontend/src/components/custom-tools/prompt-card/Header.jsx
index 2b6d6b99c1..cea5e2d6d3 100644
--- a/frontend/src/components/custom-tools/prompt-card/Header.jsx
+++ b/frontend/src/components/custom-tools/prompt-card/Header.jsx
@@ -59,6 +59,7 @@ function Header({
   spsLoading,
   handleSpsLoading,
   enforceType,
+  isAgenticTableReady = true,
 }) {
   const {
     selectedDoc,
@@ -357,6 +358,7 @@ function Header({
                   )
                 }
                 disabled={
+                  !isAgenticTableReady ||
                   (updateStatus?.promptId === promptDetails?.prompt_id &&
                     updateStatus?.status ===
                       promptStudioUpdateStatus?.isUpdating) ||
@@ -380,6 +382,7 @@ function Header({
                   )
                 }
                 disabled={
+                  !isAgenticTableReady ||
                   (updateStatus?.promptId === promptDetails?.prompt_id &&
                     updateStatus?.status ===
                       promptStudioUpdateStatus?.isUpdating) ||
@@ -436,6 +439,7 @@ Header.propTypes = {
   spsLoading: PropTypes.object,
   handleSpsLoading: PropTypes.func.isRequired,
   enforceType: PropTypes.string,
+  isAgenticTableReady: PropTypes.bool,
 };
 
 export { Header };
diff --git a/frontend/src/components/custom-tools/prompt-card/PromptCardItems.jsx b/frontend/src/components/custom-tools/prompt-card/PromptCardItems.jsx
index 90f4936a3a..6fd8b1bc6f 100644
--- a/frontend/src/components/custom-tools/prompt-card/PromptCardItems.jsx
+++ b/frontend/src/components/custom-tools/prompt-card/PromptCardItems.jsx
@@ -15,7 +15,6 @@ import { useEffect, useRef, useState } from "react";
 import { useCustomToolStore } from "../../../store/custom-tool-store";
 import { SpinnerLoader } from "../../widgets/spinner-loader/SpinnerLoader";
 import { EditableText } from "../editable-text/EditableText";
-import { TABLE } from "./constants";
 import { Header } from "./Header";
 import { OutputForIndex } from "./OutputForIndex";
 import { PromptOutput } from "./PromptOutput";
@@ -30,6 +29,16 @@ try {
   // The component will remain null of it is not available
 }
 
+let AgenticTableChecklist;
+try {
+  const mod = await import(
+    "../../../plugins/prompt-card/AgenticTableChecklist"
+  );
+  AgenticTableChecklist = mod.AgenticTableChecklist;
+} catch {
+  // The component will remain null of it is not available
+}
+
 function PromptCardItems({
   promptDetails,
   enforceTypeList,
@@ -82,6 +91,7 @@ function PromptCardItems({
   const divRef = useRef(null);
   const [enforceType, setEnforceType] = useState("");
   const [tableSettings, setTableSettings] = useState({});
+  const [isAgenticTableReady, setIsAgenticTableReady] = useState(true);
   const promptId = promptDetails?.prompt_id;
 
   useEffect(() => {
@@ -212,6 +222,7 @@ function PromptCardItems({
             spsLoading={spsLoading}
             handleSpsLoading={handleSpsLoading}
             enforceType={enforceType}
+            isAgenticTableReady={isAgenticTableReady}
           />
         </Space>
       </div>
@@ -222,6 +233,14 @@ function PromptCardItems({
       >
         <Collapse.Panel key={"1"} showArrow={false}>
           <div className="prompt-card-div-body">
+            {AgenticTableChecklist && (
+              <AgenticTableChecklist
+                promptId={promptDetails?.prompt_id}
+                promptText={promptText}
+                enforceType={enforceType}
+                onReadinessChange={setIsAgenticTableReady}
+              />
+            )}
             <EditableText
               isEditing={isEditingPrompt}
               setIsEditing={setIsEditingPrompt}
@@ -278,7 +297,7 @@ function PromptCardItems({
                         )}
                     </Space>
                     <Space>
-                      {enforceType === TABLE && TableExtractionSettingsBtn && (
+                      {TableExtractionSettingsBtn && (
                         <TableExtractionSettingsBtn
                           promptId={promptDetails?.prompt_id}
                           enforceType={enforceType}
@@ -324,6 +343,7 @@ function PromptCardItems({
               isChallenge={isChallenge}
               handleSelectHighlight={handleSelectHighlight}
               progressMsg={progressMsg}
+              isAgenticTableReady={isAgenticTableReady}
             />
           </Row>
         </Collapse.Panel>
diff --git a/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx b/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx
index 88df329744..38f7d924a6 100644
--- a/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx
+++ b/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx
@@ -67,6 +67,7 @@ function PromptOutput({
   isChallenge,
   handleSelectHighlight,
   progressMsg,
+  isAgenticTableReady = true,
 }) {
   const [openExpandModal, setOpenExpandModal] = useState(false);
   const { width: windowWidth } = useWindowDimensions();
@@ -386,7 +387,11 @@ function PromptOutput({
                             selectedDoc?.document_id,
                           )
                         }
-                        disabled={isPromptLoading || isPublicSource}
+                        disabled={
+                          !isAgenticTableReady ||
+                          isPromptLoading ||
+                          isPublicSource
+                        }
                       >
                         <PlayCircleOutlined className="prompt-card-actions-head" />
                       </Button>
@@ -404,7 +409,11 @@ function PromptOutput({
                             null,
                           )
                         }
-                        disabled={isPromptLoading || isPublicSource}
+                        disabled={
+                          !isAgenticTableReady ||
+                          isPromptLoading ||
+                          isPublicSource
+                        }
                       >
                         <PlayCircleFilled className="prompt-card-actions-head" />
                       </Button>
@@ -484,6 +493,7 @@ PromptOutput.propTypes = {
   isChallenge: PropTypes.bool,
   handleSelectHighlight: PropTypes.func.isRequired,
   progressMsg: PropTypes.object,
+  isAgenticTableReady: PropTypes.bool,
 };
 
 export { PromptOutput };
diff --git a/frontend/src/hooks/usePromptRun.js b/frontend/src/hooks/usePromptRun.js
index 44df3811c5..34fb42e7d7 100644
--- a/frontend/src/hooks/usePromptRun.js
+++ b/frontend/src/hooks/usePromptRun.js
@@ -16,7 +16,11 @@ import usePromptOutput from "./usePromptOutput";
 // Tracks the latest run nonce per (promptId, statusKey) so stale timeouts
 // from a previous run don't falsely cancel a newer run of the same combo.
 const runNonceMap = new Map();
-const SOCKET_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes
+// Safety-net timeout for the fire-and-forget prompt run flow. Must trail the
+// server-side LLM adapter timeout (default 900s in
+// unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/openai.json) so the UI
+// never gives up before the backend does. 16 minutes leaves a 1-minute buffer.
+const SOCKET_TIMEOUT_MS = 16 * 60 * 1000; // 16 minutes
 
 const usePromptRun = () => {
   const { pushPromptRunApi, freeActiveApi } = usePromptRunQueueStore();
diff --git a/unstract/sdk1/src/unstract/sdk1/llm.py b/unstract/sdk1/src/unstract/sdk1/llm.py
index c1730e6613..4588e6946c 100644
--- a/unstract/sdk1/src/unstract/sdk1/llm.py
+++ b/unstract/sdk1/src/unstract/sdk1/llm.py
@@ -363,6 +363,96 @@ def complete(self, prompt: str, **kwargs: object) -> dict[str, object]:
                 message=error_msg, status_code=status_code, actual_err=e
             ) from e
 
+    @capture_metrics
+    def complete_vision(
+        self,
+        messages: list[dict[str, Any]],
+        **kwargs: object,
+    ) -> dict[str, object]:
+        """Chat completion with multimodal (text + image) messages.
+
+        Accepts pre-built messages with image_url content blocks::
+
+            [
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "text", "text": "..."},
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": "data:image/png;base64,..."},
+                        },
+                    ],
+                }
+            ]
+
+        LiteLLM auto-translates the OpenAI-style image format for
+        Anthropic, Bedrock, Vertex, and other providers.
+
+        Same error handling, usage tracking, and metrics as complete().
+
+        Args:
+            messages: List of message dicts with multimodal content.
+            **kwargs: Additional arguments passed to litellm.completion().
+
+        Returns:
+            dict with "response" key containing LLMResponseCompat.
+        """
+        try:
+            litellm.drop_params = True
+
+            logger.debug(
+                f"[sdk1][LLM]Invoking {self.adapter.get_provider()} "
+                f"vision completion API"
+            )
+
+            completion_kwargs = self.adapter.validate({**self.kwargs, **kwargs})
+            completion_kwargs.pop("cost_model", None)
+
+            response: dict[str, object] = litellm.completion(
+                messages=messages,
+                **completion_kwargs,
+            )
+
+            response_text = response["choices"][0]["message"]["content"]
+            finish_reason = response["choices"][0].get("finish_reason")
+
+            self._record_usage(
+                self._cost_model or self.kwargs["model"],
+                messages,
+                response.get("usage"),
+                "complete_vision",
+            )
+
+            if response_text is None:
+                self._raise_for_empty_response(finish_reason)
+
+            response_object = LLMResponseCompat(response_text)
+            response_object.raw = response
+            return {"response": response_object}
+
+        except LLMError:
+            raise
+        except SdkError:
+            raise
+        except Exception as e:
+            logger.error(f"[sdk1][LLM] Error during vision completion: {e}")
+
+            status_code = None
+            if hasattr(e, "status_code"):
+                status_code = e.status_code
+            elif hasattr(e, "http_status"):
+                status_code = e.http_status
+
+            error_msg = (
+                f"Error from LLM adapter '{self._get_adapter_info()}': "
+                f"{strip_litellm_prefix(str(e))}"
+            )
+
+            raise LLMError(
+                message=error_msg, status_code=status_code, actual_err=e
+            ) from e
+
     def stream_complete(
         self,
         prompt: str,
diff --git a/workers/executor/executors/legacy_executor.py b/workers/executor/executors/legacy_executor.py
index 2a7dfc160d..3760aea067 100644
--- a/workers/executor/executors/legacy_executor.py
+++ b/workers/executor/executors/legacy_executor.py
@@ -1909,25 +1909,36 @@ def _apply_type_conversion(
         prompt_name = output[PSKeys.NAME]
         output_type = output[PSKeys.TYPE]
 
+        # Defensive guard: agentic_table prompts must be dispatched to
+        # the dedicated agentic_table executor by the worker (Layer 2 in
+        # workers/file_processing/structure_tool_task.py). If one ever
+        # reaches this method, the legacy fallthrough below would store
+        # the raw LLM completion as a string. Skip silently with a
+        # warning so the caller's existing entry (if any) survives.
+        if output_type == "agentic_table":
+            logger.warning(
+                "Skipping agentic_table prompt %s in legacy executor — "
+                "should have been dispatched to agentic_table executor",
+                prompt_name,
+            )
+            return
+
         if output_type == PSKeys.NUMBER:
             structured_output[prompt_name] = LegacyExecutor._convert_number_answer(
                 answer, llm, answer_prompt_svc
             )
 
         elif output_type == PSKeys.EMAIL:
-            if answer.lower() == "na":
-                structured_output[prompt_name] = answer
-            else:
-                email_prompt = (
-                    f"Extract the email from the following text:\n{answer}"
-                    f"\n\nOutput just the email. "
-                    f"The email should be directly assignable to a string "
-                    f"variable. No explanation is required. If you cannot "
-                    f'extract the email, output "NA".'
-                )
-                structured_output[prompt_name] = answer_prompt_svc.run_completion(
-                    llm=llm, prompt=email_prompt
-                )
+            email_prompt = (
+                f"Extract the email from the following text:\n{answer}"
+                f"\n\nOutput just the email. "
+                f"The email should be directly assignable to a string "
+                f"variable. No explanation is required. If you cannot "
+                f'extract the email, output "NA".'
+            )
+            structured_output[prompt_name] = LegacyExecutor._convert_scalar_answer(
+                answer, llm, answer_prompt_svc, email_prompt
+            )
 
         elif output_type == PSKeys.DATE:
             date_prompt = (
diff --git a/workers/executor/executors/retrievers/fusion.py b/workers/executor/executors/retrievers/fusion.py
index 7449ddb8e6..be6810ad28 100644
--- a/workers/executor/executors/retrievers/fusion.py
+++ b/workers/executor/executors/retrievers/fusion.py
@@ -64,7 +64,7 @@ def retrieve(self) -> set[str]:
                 mode="simple",  # Use simple fusion mode (reciprocal rank fusion)
                 use_async=False,
                 verbose=True,
-                llm=llm,
+                llm=llm,  # LLM generates query variations
             )
 
             # Retrieve nodes using fusion technique
diff --git a/workers/executor/executors/retrievers/keyword_table.py b/workers/executor/executors/retrievers/keyword_table.py
index 912b2fb7f8..5c7db9ab4e 100644
--- a/workers/executor/executors/retrievers/keyword_table.py
+++ b/workers/executor/executors/retrievers/keyword_table.py
@@ -48,7 +48,7 @@ def retrieve(self) -> set[str]:
             keyword_index = KeywordTableIndex(
                 nodes=[node.node for node in all_nodes],
                 show_progress=True,
-                llm=llm,
+                llm=llm,  # Use the provided LLM instead of defaulting to OpenAI
             )
 
             # Create retriever from keyword index
diff --git a/workers/file_processing/structure_tool_task.py b/workers/file_processing/structure_tool_task.py
index 87c9426376..c0fdee1c00 100644
--- a/workers/file_processing/structure_tool_task.py
+++ b/workers/file_processing/structure_tool_task.py
@@ -301,7 +301,35 @@ def _execute_structure_tool_impl(params: dict) -> dict:
 
     tool_id = tool_metadata[_SK.TOOL_ID]
     tool_settings = tool_metadata[_SK.TOOL_SETTINGS]
-    outputs = tool_metadata[_SK.OUTPUTS]
+    all_outputs = tool_metadata[_SK.OUTPUTS]
+
+    # ---- Partition prompts by enforce_type ----
+    # Agentic table prompts run via a dedicated executor (page-by-page
+    # extraction + Agent-5 schema cleanup). Regular prompts continue
+    # through the legacy structure_pipeline. Use local variables so
+    # tool_metadata[_SK.OUTPUTS] is preserved for METADATA.json
+    # serialization downstream in _write_tool_result.
+    agentic_table_outputs = [o for o in all_outputs if o.get("type") == "agentic_table"]
+    regular_outputs = [o for o in all_outputs if o.get("type") != "agentic_table"]
+
+    # Validate readiness for each agentic_table prompt: if the export
+    # step did not populate agentic_table_settings, fail loudly so the
+    # user knows to re-export the tool instead of producing the
+    # legacy stringified-truncated output.
+    for at_output in agentic_table_outputs:
+        at_settings = at_output.get("agentic_table_settings") or {}
+        if not at_settings.get("target_table") or not at_settings.get("json_structure"):
+            return ExecutionResult.failure(
+                error=(
+                    f"Agentic table prompt '{at_output[_SK.NAME]}' is missing "
+                    f"agentic_table_settings in the exported tool metadata. "
+                    f"Re-export the tool from Prompt Studio after the fix is "
+                    f"deployed to populate target_table / json_structure / "
+                    f"instructions."
+                )
+            ).to_dict()
+
+    outputs = regular_outputs
 
     # Inject workflow-level settings into tool_settings
     tool_settings[_SK.CHALLENGE_LLM] = challenge_llm
@@ -388,42 +416,103 @@ def _execute_structure_tool_impl(params: dict) -> dict:
             "prompt_keys": prompt_keys,
         }
 
-    # ---- Step 6: Single dispatch to executor ----
-    logger.info(
-        "Dispatching structure_pipeline: tool_id=%s "
-        "skip_extract=%s summarize=%s single_pass=%s",
-        tool_id,
-        skip_extraction_and_indexing,
-        is_summarization_enabled,
-        is_single_pass_enabled,
-    )
-
-    pipeline_ctx = ExecutionContext(
-        executor_name="legacy",
-        operation="structure_pipeline",
-        run_id=file_execution_id,
-        execution_source="tool",
-        organization_id=organization_id,
-        request_id=file_execution_id,
-        log_events_id=StateStore.get("LOG_EVENTS_ID") or "",
-        execution_id=execution_id,
-        file_execution_id=file_execution_id,
-        executor_params={
-            "extract_params": extract_params,
-            "index_template": index_template,
-            "answer_params": answer_params,
-            "pipeline_options": pipeline_options,
-            "summarize_params": summarize_params,
-        },
-    )
-    pipeline_start = time.monotonic()
-    pipeline_result = dispatcher.dispatch(pipeline_ctx, timeout=EXECUTOR_TIMEOUT)
-    pipeline_elapsed = time.monotonic() - pipeline_start
-
-    if not pipeline_result.success:
-        return pipeline_result.to_dict()
+    # ---- Step 6a: Dispatch agentic_table prompts ----
+    # Each agentic_table prompt runs in its own executor invocation.
+    # The executor handles X2Text extraction internally; we just
+    # forward the document path and the per-prompt settings unpacked
+    # from agentic_table_settings (populated by Layer 1 export).
+    #
+    # Important: read from SOURCE, not INFILE. INFILE gets overwritten
+    # with JSON output at the end of this function (line ~508), so any
+    # subsequent reuse of the same file_execution_dir would surface JSON
+    # bytes to the agentic_table executor and fail PDF parsing
+    # ("PDFium: Data format error"). SOURCE is the immutable original
+    # PDF written alongside INFILE by the source connector.
+    agentic_source_path = str(execution_run_data_folder / "SOURCE")
+    agentic_results: dict[str, Any] = {}
+    for at_output in agentic_table_outputs:
+        at_settings = at_output.get("agentic_table_settings") or {}
+        json_structure = at_settings.get("json_structure")
+        if isinstance(json_structure, dict):
+            json_structure = json.dumps(json_structure)
+        agentic_params = {
+            "llm_adapter_instance_id": at_output["llm"],
+            "lite_llm_adapter_instance_id": at_settings.get(
+                "lite_llm_adapter_instance_id", ""
+            ),
+            "x2text_adapter_instance_id": tool_settings[_SK.X2TEXT_ADAPTER],
+            "input_file": agentic_source_path,
+            "source_file_name": source_file_name,
+            "target_table": at_settings.get("target_table", ""),
+            "json_structure": json_structure,
+            "instructions": at_settings.get("instructions", ""),
+            "starting_page": at_settings.get("start_page", 1),
+            "ending_page": at_settings.get("end_page") or None,
+            "parallel_pages": at_settings.get("parallel_pages", 4),
+            "execution_id": execution_id,
+            "PLATFORM_SERVICE_API_KEY": platform_service_api_key,
+        }
+        at_ctx = ExecutionContext(
+            executor_name="agentic_table",
+            operation="table_extract",
+            run_id=file_execution_id,
+            execution_source="tool",
+            organization_id=organization_id,
+            request_id=file_execution_id,
+            executor_params=agentic_params,
+        )
+        at_result = dispatcher.dispatch(at_ctx, timeout=EXECUTOR_TIMEOUT)
+        if not at_result.success:
+            return at_result.to_dict()
+        at_output_data = at_result.data.get("output", {}) or {}
+        agentic_results[at_output[_SK.NAME]] = at_output_data.get("tables", [])
+
+    # ---- Step 6b: Dispatch legacy structure_pipeline ----
+    # Skipped entirely when every prompt is agentic_table — the legacy
+    # pipeline has no work to do and the agentic_table executor does
+    # its own X2Text inside the runner.
+    if regular_outputs:
+        logger.info(
+            "Dispatching structure_pipeline: tool_id=%s "
+            "skip_extract=%s summarize=%s single_pass=%s",
+            tool_id,
+            skip_extraction_and_indexing,
+            is_summarization_enabled,
+            is_single_pass_enabled,
+        )
 
-    structured_output = pipeline_result.data
+        pipeline_ctx = ExecutionContext(
+            executor_name="legacy",
+            operation="structure_pipeline",
+            run_id=file_execution_id,
+            execution_source="tool",
+            organization_id=organization_id,
+            request_id=file_execution_id,
+            executor_params={
+                "extract_params": extract_params,
+                "index_template": index_template,
+                "answer_params": answer_params,
+                "pipeline_options": pipeline_options,
+                "summarize_params": summarize_params,
+            },
+        )
+        pipeline_start = time.monotonic()
+        pipeline_result = dispatcher.dispatch(pipeline_ctx, timeout=EXECUTOR_TIMEOUT)
+        pipeline_elapsed = time.monotonic() - pipeline_start
+
+        if not pipeline_result.success:
+            return pipeline_result.to_dict()
+
+        structured_output = pipeline_result.data
+        if agentic_results:
+            structured_output.setdefault("output", {}).update(agentic_results)
+    else:
+        # All-agentic case: skip the legacy pipeline entirely.
+        structured_output = {
+            "output": agentic_results,
+            "metadata": {"agentic_only": True},
+        }
+        pipeline_elapsed = 0.0
 
     # ---- Step 7: Write output files ----
     # (metadata/metrics merging already done by executor pipeline)
diff --git a/workers/ide_callback/tasks.py b/workers/ide_callback/tasks.py
index cb610678b1..d0dbce2563 100644
--- a/workers/ide_callback/tasks.py
+++ b/workers/ide_callback/tasks.py
@@ -392,6 +392,16 @@ def ide_prompt_complete(
         outputs = _json_safe(data.get("output", {}))
         metadata = _json_safe(data.get("metadata", {}))
 
+        # Agentic table executor returns {"tables": [...], "page_count": ...,
+        # "headers": [...], ...}, but OutputManagerHelper expects
+        # outputs[prompt.prompt_key] to be the value for that prompt. Reshape
+        # so the table list lands under the prompt key.
+        if cb.get("is_agentic_table"):
+            prompt_key = cb.get("prompt_key", "")
+            if prompt_key:
+                tables = outputs.get("tables", []) if isinstance(outputs, dict) else []
+                outputs = {prompt_key: tables}
+
         logger.info(
             "ide_prompt_complete: operation=%s output_keys=%s prompt_ids=%s "
             "doc=%s profile=%s",
diff --git a/workers/tests/test_answer_prompt.py b/workers/tests/test_answer_prompt.py
index 6c9fb9fce9..4b58c4c1ea 100644
--- a/workers/tests/test_answer_prompt.py
+++ b/workers/tests/test_answer_prompt.py
@@ -622,8 +622,8 @@ def test_invalid_strategy_skips_retrieval(
         )
         result = executor._handle_answer_prompt(ctx)
 
-        # Answer stays "NA" (top-level NA is preserved, not sanitized)
-        assert result.data[PSKeys.OUTPUT]["field_a"] == "NA"
+        # Answer stays "NA" which gets sanitized to None
+        assert result.data[PSKeys.OUTPUT]["field_a"] is None
 
 
 class TestHandleAnswerPromptMultiPrompt:
@@ -747,21 +747,21 @@ def test_vectordb_closed(self, mock_shim_cls, mock_deps):
 class TestNullSanitization:
     """Tests for _sanitize_null_values."""
 
-    def test_na_string_preserved(self):
-        """Top-level 'NA' string is preserved (not sanitized to None)."""
+    def test_na_string_becomes_none(self):
+        """Top-level 'NA' string → None."""
         from executor.executors.legacy_executor import LegacyExecutor
 
         output = {"field": "NA"}
         result = LegacyExecutor._sanitize_null_values(output)
-        assert result["field"] == "NA"
+        assert result["field"] is None
 
-    def test_na_case_insensitive_preserved(self):
-        """Top-level 'na' (lowercase) is preserved (not sanitized to None)."""
+    def test_na_case_insensitive(self):
+        """'na' (lowercase) → None."""
         from executor.executors.legacy_executor import LegacyExecutor
 
         output = {"field": "na"}
         result = LegacyExecutor._sanitize_null_values(output)
-        assert result["field"] == "na"
+        assert result["field"] is None
 
     def test_nested_list_na(self):
         """NA in nested list items → None."""

From 5d591dc4e35447bd67081f770a488f9fd88cc878 Mon Sep 17 00:00:00 2001
From: Kirtiman Mishra
 <110175055+kirtimanmishrazipstack@users.noreply.github.com>
Date: Tue, 28 Apr 2026 17:08:43 +0530
Subject: [PATCH 5/7] UN-3358 [FIX] Drop cross-region S3 buckets from connector
 listing (#1931)

* list bucket

* greptile review
---
 .../filesystems/minio/exceptions.py           | 12 ++++++----
 .../tests/filesystems/test_miniofs.py         | 22 ++++++++++++++++---
 2 files changed, 27 insertions(+), 7 deletions(-)

diff --git a/unstract/connectors/src/unstract/connectors/filesystems/minio/exceptions.py b/unstract/connectors/src/unstract/connectors/filesystems/minio/exceptions.py
index 2a4149ecaa..dd46e61b16 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/minio/exceptions.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/minio/exceptions.py
@@ -8,20 +8,24 @@
 class BucketProbeDisposition(Enum):
     """Action for a `list_objects_v2` probe failure, per S3 error code."""
 
-    DROP = "drop"  # Hide the bucket (no access, or bucket gone).
-    FAIL_OPEN = "fail_open"  # Region mismatch — keep bucket visible.
+    DROP = "drop"  # Hide the bucket (no access, bucket gone, or unreachable).
+    FAIL_OPEN = "fail_open"  # Keep the bucket visible despite the probe error.
     RETRY_FAIL_OPEN = "retry_fail_open"  # Throttled — retry once, then keep.
 
 
 # S3 `Error.Code` → probe disposition. Unlisted codes propagate.
 # `RequestTimeTooSkewed` is omitted deliberately: it's a system-wide clock
 # issue, not a bucket outcome — let it surface via `handle_s3fs_exception`.
+# `PermanentRedirect` / `IllegalLocationConstraintException` are dropped: the
+# connector is pinned to one `endpoint_url`, so cross-region buckets can't
+# be browsed through it regardless of IAM — listing them only surfaces a
+# confusing `[Errno 78]` on click.
 BUCKET_PROBE_DISPOSITION: dict[str, BucketProbeDisposition] = {
     "AccessDenied": BucketProbeDisposition.DROP,
     "AllAccessDisabled": BucketProbeDisposition.DROP,
     "NoSuchBucket": BucketProbeDisposition.DROP,
-    "PermanentRedirect": BucketProbeDisposition.FAIL_OPEN,
-    "IllegalLocationConstraintException": BucketProbeDisposition.FAIL_OPEN,
+    "PermanentRedirect": BucketProbeDisposition.DROP,
+    "IllegalLocationConstraintException": BucketProbeDisposition.DROP,
     "SlowDown": BucketProbeDisposition.RETRY_FAIL_OPEN,
     "Throttling": BucketProbeDisposition.RETRY_FAIL_OPEN,
     "ThrottlingException": BucketProbeDisposition.RETRY_FAIL_OPEN,
diff --git a/unstract/connectors/tests/filesystems/test_miniofs.py b/unstract/connectors/tests/filesystems/test_miniofs.py
index 3cf9ade1f2..9da9a1dcf0 100644
--- a/unstract/connectors/tests/filesystems/test_miniofs.py
+++ b/unstract/connectors/tests/filesystems/test_miniofs.py
@@ -104,15 +104,31 @@ def test_no_such_bucket_is_dropped(self) -> None:
         ):
             self.assertFalse(asyncio.run(fs._is_bucket_accessible("deleted")))
 
-    def test_permanent_redirect_bucket_is_kept(self) -> None:
-        # Fail-open: region mismatch keeps the bucket listed.
+    def test_permanent_redirect_bucket_is_dropped(self) -> None:
+        # Connector pins one endpoint_url; cross-region buckets are unreachable
+        # via this connector regardless of IAM, so drop them rather than fail
+        # later with [Errno 78] on click.
         fs = self._make_fs()
         with patch.object(
             fs,
             "_call_s3",
             new=AsyncMock(side_effect=_translated_error("PermanentRedirect")),
         ):
-            self.assertTrue(asyncio.run(fs._is_bucket_accessible("other-region")))
+            self.assertFalse(asyncio.run(fs._is_bucket_accessible("other-region")))
+
+    def test_illegal_location_constraint_bucket_is_dropped(self) -> None:
+        # Companion to PermanentRedirect: same cross-region condition surfaced
+        # as a 400 instead of a 301. Guards against a future edit reverting
+        # only one of the two cross-region entries in BUCKET_PROBE_DISPOSITION.
+        fs = self._make_fs()
+        with patch.object(
+            fs,
+            "_call_s3",
+            new=AsyncMock(
+                side_effect=_translated_error("IllegalLocationConstraintException")
+            ),
+        ):
+            self.assertFalse(asyncio.run(fs._is_bucket_accessible("wrong-region")))
 
     def test_throttling_retries_then_fails_open(self) -> None:
         fs = self._make_fs()

From 019f33ccf4cab9da8ce9c258bb3c1b6fcca92660 Mon Sep 17 00:00:00 2001
From: kirtimanmishrazipstack <kirtiman.mishra@zipstack.com>
Date: Tue, 5 May 2026 20:49:03 +0530
Subject: [PATCH 6/7] payload metadata in api deployment

---
 backend/api_v2/notification.py                |  46 +++--
 backend/notification_v2/enums.py              |  12 --
 backend/notification_v2/internal_api_views.py | 195 ++++++++++--------
 .../migrations/0002_notification_notify_on.py |  30 ---
 .../0002_notification_notify_on_failures.py   |  23 +++
 backend/notification_v2/models.py             |  15 +-
 backend/notification_v2/serializers.py        |   8 +-
 backend/pipeline_v2/dto.py                    |  11 +-
 backend/pipeline_v2/notification.py           |  73 +++++--
 .../workflow_manager/internal_serializers.py  |   2 +
 backend/workflow_manager/internal_views.py    |  13 +-
 .../0020_workflowexecution_file_counts.py     |  36 ++++
 .../workflow_v2/models/execution.py           |  31 ++-
 .../notification-modal/CreateNotification.jsx |  23 +--
 .../core/src/unstract/core/data_models.py     |  12 ++
 workers/callback/tasks.py                     |   4 +
 workers/scheduler/tasks.py                    |   1 +
 workers/shared/api/internal_client.py         |   4 +
 workers/shared/clients/execution_client.py    |   8 +
 .../shared/patterns/notification/helper.py    |  33 ++-
 20 files changed, 361 insertions(+), 219 deletions(-)
 delete mode 100644 backend/notification_v2/migrations/0002_notification_notify_on.py
 create mode 100644 backend/notification_v2/migrations/0002_notification_notify_on_failures.py
 create mode 100644 backend/workflow_manager/workflow_v2/migrations/0020_workflowexecution_file_counts.py

diff --git a/backend/api_v2/notification.py b/backend/api_v2/notification.py
index a41ebc3000..57e810c0e9 100644
--- a/backend/api_v2/notification.py
+++ b/backend/api_v2/notification.py
@@ -1,6 +1,5 @@
 import logging
 
-from notification_v2.enums import NotificationTrigger
 from notification_v2.helper import NotificationHelper
 from notification_v2.models import Notification
 from pipeline_v2.dto import PipelineStatusPayload
@@ -12,6 +11,9 @@
 logger = logging.getLogger(__name__)
 
 
+_FAILURE_STATUSES = {ExecutionStatus.ERROR.value, ExecutionStatus.STOPPED.value}
+
+
 class APINotification:
     def __init__(self, api: APIDeployment, workflow_execution: WorkflowExecution) -> None:
         self.notifications = Notification.objects.filter(api=api, is_active=True)
@@ -19,31 +21,32 @@ def __init__(self, api: APIDeployment, workflow_execution: WorkflowExecution) ->
         self.workflow_execution = workflow_execution
 
     def send(self) -> None:
-        # Partition notifications by the run outcome so each row's notify_on
-        # preference is honored. STOPPED and any other non-terminal status
-        # fire only for ALL — explicit opt-ins to FAILURES/SUCCESS shouldn't.
-        status = self.workflow_execution.status
-        if status == ExecutionStatus.ERROR.value:
-            self.notifications = self.notifications.exclude(
-                notify_on=NotificationTrigger.SUCCESS_ONLY.value
-            )
-        elif status == ExecutionStatus.COMPLETED.value:
-            self.notifications = self.notifications.exclude(
-                notify_on=NotificationTrigger.FAILURES_ONLY.value
-            )
-        else:
-            self.notifications = self.notifications.filter(
-                notify_on=NotificationTrigger.ALL.value
-            )
+        # Failure if the run hit a non-success terminal state OR any file errored.
+        # Partial-success runs land as status=COMPLETED with failed_files>0, so the
+        # status check alone misses them — see callback aggregation rules.
+        failed_files = self.workflow_execution.failed_files or 0
+        is_failure = (
+            self.workflow_execution.status in _FAILURE_STATUSES or failed_files > 0
+        )
+        if not is_failure:
+            # Success path: skip rows that opted into failure-only alerts.
+            self.notifications = self.notifications.filter(notify_on_failures=False)
 
         if not self.notifications.exists():
             logger.info(
-                "No notifications to dispatch for api %s (status=%s)",
+                "No notifications to dispatch for api %s (status=%s, failed_files=%s)",
                 self.api,
-                status,
+                self.workflow_execution.status,
+                failed_files,
             )
             return
-        logger.info("Sending api status notification for api %s", self.api)
+        logger.info(
+            "Sending api status notification for api %s (status=%s, successful=%s, failed=%s)",
+            self.api,
+            self.workflow_execution.status,
+            self.workflow_execution.successful_files or 0,
+            failed_files,
+        )
 
         payload_dto = PipelineStatusPayload(
             type="API",
@@ -52,6 +55,9 @@ def send(self) -> None:
             status=self.workflow_execution.status,
             execution_id=self.workflow_execution.id,
             error_message=self.workflow_execution.error_message,
+            total_files=self.workflow_execution.total_files,
+            successful_files=self.workflow_execution.successful_files,
+            failed_files=failed_files,
         )
 
         NotificationHelper.send_notification(
diff --git a/backend/notification_v2/enums.py b/backend/notification_v2/enums.py
index 516b34074b..991b08cac9 100644
--- a/backend/notification_v2/enums.py
+++ b/backend/notification_v2/enums.py
@@ -36,15 +36,3 @@ class PlatformType(Enum):
     @classmethod
     def choices(cls):
         return [(e.value, e.name.replace("_", " ").capitalize()) for e in cls]
-
-
-class NotificationTrigger(Enum):
-    """Controls which run outcomes fire a notification."""
-
-    ALL = "ALL"
-    FAILURES_ONLY = "FAILURES_ONLY"
-    SUCCESS_ONLY = "SUCCESS_ONLY"
-
-    @classmethod
-    def choices(cls):
-        return [(e.value, e.name.replace("_", " ").capitalize()) for e in cls]
diff --git a/backend/notification_v2/internal_api_views.py b/backend/notification_v2/internal_api_views.py
index 6843d5a50c..3e3f386b17 100644
--- a/backend/notification_v2/internal_api_views.py
+++ b/backend/notification_v2/internal_api_views.py
@@ -10,14 +10,18 @@
 """
 
 import logging
+from typing import Any, cast
 
 from api_v2.models import APIDeployment
-from django.http import JsonResponse
+from django.db.models import QuerySet
+from django.http import HttpRequest, JsonResponse
 from django.shortcuts import get_object_or_404
 from django.views.decorators.csrf import csrf_exempt
 from django.views.decorators.http import require_http_methods
 from pipeline_v2.models import Pipeline
 from utils.organization_utils import filter_queryset_by_organization
+from workflow_manager.workflow_v2.enums import ExecutionStatus
+from workflow_manager.workflow_v2.models.execution import WorkflowExecution
 
 from notification_v2.models import Notification
 
@@ -26,10 +30,71 @@
 # Constants for error messages
 INTERNAL_SERVER_ERROR_MSG = "Internal server error"
 
+_FAILURE_STATUSES = {ExecutionStatus.ERROR.value, ExecutionStatus.STOPPED.value}
+
+
+def _load_execution(execution_id: str | None) -> WorkflowExecution | None:
+    """Best-effort lookup; returns None on missing id or unknown row."""
+    if not execution_id:
+        return None
+    try:
+        return cast(WorkflowExecution, WorkflowExecution.objects.get(id=execution_id))
+    except WorkflowExecution.DoesNotExist:
+        logger.warning("WorkflowExecution %s not found", execution_id)
+        return None
+
+
+def _apply_failure_filter(
+    notifications_qs: QuerySet[Notification],
+    execution: WorkflowExecution | None,
+) -> QuerySet[Notification]:
+    """Drop notify_on_failures=True rows on success runs.
+
+    Mirrors the dispatch-side rule in backend/api_v2/notification.py and
+    backend/pipeline_v2/notification.py so both code paths agree on what
+    counts as a failure (status ∈ {ERROR, STOPPED} OR any file errored).
+
+    No execution → no filter, preserving legacy "return every active row"
+    behavior for callers that don't pass execution_id.
+    """
+    if execution is None:
+        return notifications_qs
+    failed_files = execution.failed_files or 0
+    is_failure = execution.status in _FAILURE_STATUSES or failed_files > 0
+    if not is_failure:
+        notifications_qs = notifications_qs.filter(notify_on_failures=False)
+    return notifications_qs
+
+
+def _execution_counts(execution: WorkflowExecution | None) -> dict[str, int]:
+    """File counts surfaced into webhook payloads. Empty dict on no execution."""
+    if execution is None:
+        return {}
+    return {
+        "total_files": execution.total_files or 0,
+        "successful_files": execution.successful_files or 0,
+        "failed_files": execution.failed_files or 0,
+    }
+
+
+def _serialize_notification(n: Notification) -> dict[str, Any]:
+    return {
+        "id": str(n.id),
+        "notification_type": n.notification_type,
+        "platform": n.platform,
+        "url": n.url,
+        "authorization_type": n.authorization_type,
+        "authorization_key": n.authorization_key,
+        "authorization_header": n.authorization_header,
+        "max_retries": n.max_retries,
+        "is_active": n.is_active,
+        "notify_on_failures": n.notify_on_failures,
+    }
+
 
 @csrf_exempt  # Safe: Internal API with Bearer token auth, service-to-service only
 @require_http_methods(["GET"])
-def get_pipeline_notifications(request, pipeline_id):
+def get_pipeline_notifications(request: HttpRequest, pipeline_id: str) -> JsonResponse:
     """Get active notifications for a pipeline or API deployment.
 
     Used by callback worker to fetch notification configuration.
@@ -41,83 +106,53 @@ def get_pipeline_notifications(request, pipeline_id):
             pipeline_queryset, request, "organization"
         )
 
+        execution = _load_execution(request.GET.get("execution_id"))
+        counts = _execution_counts(execution)
+
         if pipeline_queryset.exists():
             pipeline = pipeline_queryset.first()
-
-            # Get active notifications for this pipeline
             notifications = Notification.objects.filter(pipeline=pipeline, is_active=True)
-
-            notifications_data = []
-            for notification in notifications:
-                notifications_data.append(
-                    {
-                        "id": str(notification.id),
-                        "notification_type": notification.notification_type,
-                        "platform": notification.platform,
-                        "url": notification.url,
-                        "authorization_type": notification.authorization_type,
-                        "authorization_key": notification.authorization_key,
-                        "authorization_header": notification.authorization_header,
-                        "max_retries": notification.max_retries,
-                        "is_active": notification.is_active,
-                    }
-                )
-
+            notifications = _apply_failure_filter(notifications, execution)
+            serialized = [_serialize_notification(n) for n in notifications]
             return JsonResponse(
                 {
                     "status": "success",
                     "pipeline_id": str(pipeline.id),
                     "pipeline_name": pipeline.pipeline_name,
                     "pipeline_type": pipeline.pipeline_type,
-                    "notifications": notifications_data,
+                    "notifications": serialized,
+                    "execution_counts": counts,
                 }
             )
-        else:
-            # If not found in Pipeline, try APIDeployment model
-            api_queryset = APIDeployment.objects.filter(id=pipeline_id)
-            api_queryset = filter_queryset_by_organization(
-                api_queryset, request, "organization"
+
+        # If not found in Pipeline, try APIDeployment model
+        api_queryset = APIDeployment.objects.filter(id=pipeline_id)
+        api_queryset = filter_queryset_by_organization(
+            api_queryset, request, "organization"
+        )
+        if api_queryset.exists():
+            api = api_queryset.first()
+            notifications = Notification.objects.filter(api=api, is_active=True)
+            notifications = _apply_failure_filter(notifications, execution)
+            serialized = [_serialize_notification(n) for n in notifications]
+            return JsonResponse(
+                {
+                    "status": "success",
+                    "pipeline_id": str(api.id),
+                    "pipeline_name": api.api_name,
+                    "pipeline_type": "API",
+                    "notifications": serialized,
+                    "execution_counts": counts,
+                }
             )
 
-            if api_queryset.exists():
-                api = api_queryset.first()
-
-                # Get active notifications for this API deployment
-                notifications = Notification.objects.filter(api=api, is_active=True)
-
-                notifications_data = []
-                for notification in notifications:
-                    notifications_data.append(
-                        {
-                            "id": str(notification.id),
-                            "notification_type": notification.notification_type,
-                            "platform": notification.platform,
-                            "url": notification.url,
-                            "authorization_type": notification.authorization_type,
-                            "authorization_key": notification.authorization_key,
-                            "authorization_header": notification.authorization_header,
-                            "max_retries": notification.max_retries,
-                            "is_active": notification.is_active,
-                        }
-                    )
-
-                return JsonResponse(
-                    {
-                        "status": "success",
-                        "pipeline_id": str(api.id),
-                        "pipeline_name": api.api_name,
-                        "pipeline_type": "API",
-                        "notifications": notifications_data,
-                    }
-                )
-            else:
-                return JsonResponse(
-                    {
-                        "status": "error",
-                        "message": "Pipeline or API deployment not found",
-                    },
-                    status=404,
-                )
+        return JsonResponse(
+            {
+                "status": "error",
+                "message": "Pipeline or API deployment not found",
+            },
+            status=404,
+        )
     except Exception as e:
         logger.error(f"Error getting pipeline notifications for {pipeline_id}: {e}")
         return JsonResponse(
@@ -127,7 +162,7 @@ def get_pipeline_notifications(request, pipeline_id):
 
 @csrf_exempt  # Safe: Internal API with Bearer token auth, service-to-service only
 @require_http_methods(["GET"])
-def get_api_notifications(request, api_id):
+def get_api_notifications(request: HttpRequest, api_id: str) -> JsonResponse:
     """Get active notifications for an API deployment.
 
     Used by callback worker to fetch notification configuration.
@@ -140,24 +175,9 @@ def get_api_notifications(request, api_id):
         )
         api = get_object_or_404(api_queryset)
 
-        # Get active notifications for this API
+        execution = _load_execution(request.GET.get("execution_id"))
         notifications = Notification.objects.filter(api=api, is_active=True)
-
-        notifications_data = []
-        for notification in notifications:
-            notifications_data.append(
-                {
-                    "id": str(notification.id),
-                    "notification_type": notification.notification_type,
-                    "platform": notification.platform,
-                    "url": notification.url,
-                    "authorization_type": notification.authorization_type,
-                    "authorization_key": notification.authorization_key,
-                    "authorization_header": notification.authorization_header,
-                    "max_retries": notification.max_retries,
-                    "is_active": notification.is_active,
-                }
-            )
+        notifications = _apply_failure_filter(notifications, execution)
 
         return JsonResponse(
             {
@@ -165,7 +185,8 @@ def get_api_notifications(request, api_id):
                 "api_id": str(api.id),
                 "api_name": api.api_name,
                 "display_name": api.display_name,
-                "notifications": notifications_data,
+                "notifications": [_serialize_notification(n) for n in notifications],
+                "execution_counts": _execution_counts(execution),
             }
         )
 
@@ -182,7 +203,7 @@ def get_api_notifications(request, api_id):
 
 @csrf_exempt  # Safe: Internal API with Bearer token auth, service-to-service only
 @require_http_methods(["GET"])
-def get_pipeline_data(request, pipeline_id):
+def get_pipeline_data(request: HttpRequest, pipeline_id: str) -> JsonResponse:
     """Get basic pipeline data for notification purposes.
 
     Used by callback worker to determine pipeline type and name.
@@ -218,7 +239,7 @@ def get_pipeline_data(request, pipeline_id):
 
 @csrf_exempt  # Safe: Internal API with Bearer token auth, service-to-service only
 @require_http_methods(["GET"])
-def get_api_data(request, api_id):
+def get_api_data(request: HttpRequest, api_id: str) -> JsonResponse:
     """Get basic API deployment data for notification purposes.
 
     Used by callback worker to determine API name and details.
diff --git a/backend/notification_v2/migrations/0002_notification_notify_on.py b/backend/notification_v2/migrations/0002_notification_notify_on.py
deleted file mode 100644
index 53c1180126..0000000000
--- a/backend/notification_v2/migrations/0002_notification_notify_on.py
+++ /dev/null
@@ -1,30 +0,0 @@
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("notification_v2", "0001_initial"),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name="notification",
-            name="notify_on",
-            field=models.CharField(
-                max_length=50,
-                choices=[
-                    ("ALL", "All"),
-                    ("FAILURES_ONLY", "Failures only"),
-                    ("SUCCESS_ONLY", "Success only"),
-                ],
-                default="ALL",
-                db_comment=(
-                    "Controls which run outcomes trigger this notification. ALL "
-                    "(default) preserves the historical 'notify on every "
-                    "completion' behavior; FAILURES_ONLY fires only on failed "
-                    "runs (ERROR for API deployments, FAILURE for ETL "
-                    "pipelines); SUCCESS_ONLY fires only on successful runs."
-                ),
-            ),
-        ),
-    ]
diff --git a/backend/notification_v2/migrations/0002_notification_notify_on_failures.py b/backend/notification_v2/migrations/0002_notification_notify_on_failures.py
new file mode 100644
index 0000000000..ce0c3535b5
--- /dev/null
+++ b/backend/notification_v2/migrations/0002_notification_notify_on_failures.py
@@ -0,0 +1,23 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("notification_v2", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="notification",
+            name="notify_on_failures",
+            field=models.BooleanField(
+                default=False,
+                db_comment=(
+                    "When True, fire only on failed runs — terminal status "
+                    "ERROR/STOPPED or any file in the run errored (partial "
+                    "failure). When False (default), fire on every terminal "
+                    "completion."
+                ),
+            ),
+        ),
+    ]
diff --git a/backend/notification_v2/models.py b/backend/notification_v2/models.py
index 4fe21cd6c4..e5238ec176 100644
--- a/backend/notification_v2/models.py
+++ b/backend/notification_v2/models.py
@@ -5,7 +5,7 @@
 from pipeline_v2.models import Pipeline
 from utils.models.base_model import BaseModel
 
-from .enums import AuthorizationType, NotificationTrigger, NotificationType, PlatformType
+from .enums import AuthorizationType, NotificationType, PlatformType
 
 NOTIFICATION_NAME_MAX_LENGTH = 255
 
@@ -47,15 +47,12 @@ class Notification(BaseModel):
         default=True,
         db_comment="Flag indicating whether the notification is active or not.",
     )
-    notify_on = models.CharField(
-        max_length=50,
-        choices=NotificationTrigger.choices(),
-        default=NotificationTrigger.ALL.value,
+    notify_on_failures = models.BooleanField(
+        default=False,
         db_comment=(
-            "Controls which run outcomes trigger this notification. ALL (default) "
-            "preserves the historical 'notify on every completion' behavior; "
-            "FAILURES_ONLY fires only on failed runs (ERROR for API deployments, "
-            "FAILURE for ETL pipelines); SUCCESS_ONLY fires only on successful runs."
+            "When True, fire only on failed runs — terminal status ERROR/STOPPED "
+            "or any file in the run errored (partial failure). When False "
+            "(default), fire on every terminal completion."
         ),
     )
     # Foreign keys to specific models
diff --git a/backend/notification_v2/serializers.py b/backend/notification_v2/serializers.py
index 784ec75413..4cb4f3c4cb 100644
--- a/backend/notification_v2/serializers.py
+++ b/backend/notification_v2/serializers.py
@@ -1,7 +1,7 @@
 from rest_framework import serializers
 from utils.input_sanitizer import validate_name_field
 
-from .enums import AuthorizationType, NotificationTrigger, NotificationType, PlatformType
+from .enums import AuthorizationType, NotificationType, PlatformType
 from .models import Notification
 
 
@@ -12,11 +12,7 @@ class NotificationSerializer(serializers.ModelSerializer):
     max_retries = serializers.IntegerField(
         max_value=4, min_value=0, default=0, required=False
     )
-    notify_on = serializers.ChoiceField(
-        choices=NotificationTrigger.choices(),
-        default=NotificationTrigger.ALL.value,
-        required=False,
-    )
+    notify_on_failures = serializers.BooleanField(default=False, required=False)
 
     class Meta:
         model = Notification
diff --git a/backend/pipeline_v2/dto.py b/backend/pipeline_v2/dto.py
index 5a87ba0825..b8ad7c8707 100644
--- a/backend/pipeline_v2/dto.py
+++ b/backend/pipeline_v2/dto.py
@@ -10,6 +10,9 @@ def __init__(
         status: str,
         execution_id: str | None = None,
         error_message: str | None = None,
+        total_files: int | None = None,
+        successful_files: int | None = None,
+        failed_files: int | None = None,
     ):
         self.type = type
         self.pipeline_id = pipeline_id
@@ -17,14 +20,20 @@ def __init__(
         self.status = status
         self.execution_id = execution_id
         self.error_message = error_message
+        self.total_files = total_files
+        self.successful_files = successful_files
+        self.failed_files = failed_files
 
     def to_dict(self) -> dict[str, Any]:
         """Convert the payload DTO to a dictionary."""
-        payload = {
+        payload: dict[str, Any] = {
             "type": self.type,
             "pipeline_id": str(self.pipeline_id),
             "pipeline_name": self.pipeline_name,
             "status": self.status,
+            "total_files": self.total_files or 0,
+            "successful_files": self.successful_files or 0,
+            "failed_files": self.failed_files or 0,
         }
         if self.execution_id:
             payload["execution_id"] = str(self.execution_id)
diff --git a/backend/pipeline_v2/notification.py b/backend/pipeline_v2/notification.py
index 64df6f1126..ec82145054 100644
--- a/backend/pipeline_v2/notification.py
+++ b/backend/pipeline_v2/notification.py
@@ -1,8 +1,9 @@
 import logging
 
-from notification_v2.enums import NotificationTrigger
 from notification_v2.helper import NotificationHelper
 from notification_v2.models import Notification
+from workflow_manager.workflow_v2.enums import ExecutionStatus
+from workflow_manager.workflow_v2.models.execution import WorkflowExecution
 
 from pipeline_v2.dto import PipelineStatusPayload
 from pipeline_v2.models import Pipeline
@@ -10,6 +11,9 @@
 logger = logging.getLogger(__name__)
 
 
+_FAILURE_STATUSES = {ExecutionStatus.ERROR.value, ExecutionStatus.STOPPED.value}
+
+
 class PipelineNotification:
     def __init__(
         self,
@@ -24,33 +28,57 @@ def __init__(
         self.error_message = error_message
         self.execution_id = execution_id
 
-    def send(self) -> None:
-        # Partition notifications by the run outcome so each row's notify_on
-        # preference is honored. PipelineUtils.update_pipeline_status collapses
-        # both ERROR and STOPPED execution statuses into PipelineStatus.FAILURE,
-        # so FAILURES_ONLY subscribers get alerts for both on the pipeline side.
-        status = self.pipeline.last_run_status
-        if status == Pipeline.PipelineStatus.FAILURE:
-            self.notifications = self.notifications.exclude(
-                notify_on=NotificationTrigger.SUCCESS_ONLY.value
-            )
-        elif status == Pipeline.PipelineStatus.SUCCESS:
-            self.notifications = self.notifications.exclude(
-                notify_on=NotificationTrigger.FAILURES_ONLY.value
-            )
-        else:
-            self.notifications = self.notifications.filter(
-                notify_on=NotificationTrigger.ALL.value
+    def _load_execution(self) -> WorkflowExecution | None:
+        """Load the WorkflowExecution row for this dispatch, if available.
+
+        Falls back to None when no execution_id was supplied (e.g. legacy
+        callers); callers must handle the None case.
+        """
+        if not self.execution_id:
+            return None
+        try:
+            return WorkflowExecution.objects.get(id=self.execution_id)
+        except WorkflowExecution.DoesNotExist:
+            logger.warning(
+                "WorkflowExecution %s not found for pipeline notification",
+                self.execution_id,
             )
+            return None
+
+    def send(self) -> None:
+        execution = self._load_execution()
+        # Source of truth for partial-failure detection is the per-run aggregate
+        # written by the worker callback. Pipeline.last_run_status is a coarse
+        # collapse (ERROR/STOPPED → FAILURE) that hides per-file errors when
+        # at least one file succeeded.
+        failed_files = (execution.failed_files or 0) if execution else 0
+        execution_status = execution.status if execution else None
+        is_failure = (
+            execution_status in _FAILURE_STATUSES
+            or failed_files > 0
+            or self.pipeline.last_run_status == Pipeline.PipelineStatus.FAILURE
+        )
+        if not is_failure:
+            self.notifications = self.notifications.filter(notify_on_failures=False)
 
         if not self.notifications.exists():
             logger.info(
-                "No notifications to dispatch for pipeline %s (status=%s)",
+                "No notifications to dispatch for pipeline %s (status=%s, failed_files=%s)",
                 self.pipeline,
-                status,
+                self.pipeline.last_run_status,
+                failed_files,
             )
             return
-        logger.info("Sending pipeline status notification for pipeline %s", self.pipeline)
+        successful_files = (execution.successful_files or 0) if execution else 0
+        total_files = execution.total_files if execution else None
+        logger.info(
+            "Sending pipeline status notification for pipeline %s "
+            "(status=%s, successful=%s, failed=%s)",
+            self.pipeline,
+            self.pipeline.last_run_status,
+            successful_files,
+            failed_files,
+        )
         payload_dto = PipelineStatusPayload(
             type=self.pipeline.pipeline_type,
             pipeline_id=str(self.pipeline.id),
@@ -58,6 +86,9 @@ def send(self) -> None:
             status=self.pipeline.last_run_status,
             execution_id=self.execution_id,
             error_message=self.error_message,
+            total_files=total_files,
+            successful_files=successful_files,
+            failed_files=failed_files,
         )
 
         NotificationHelper.send_notification(
diff --git a/backend/workflow_manager/internal_serializers.py b/backend/workflow_manager/internal_serializers.py
index bed98c6853..cd221470cb 100644
--- a/backend/workflow_manager/internal_serializers.py
+++ b/backend/workflow_manager/internal_serializers.py
@@ -178,6 +178,8 @@ class WorkflowExecutionStatusUpdateSerializer(serializers.Serializer):
     total_files = serializers.IntegerField(
         required=False, min_value=0
     )  # Allow 0 but backend will only update if > 0
+    successful_files = serializers.IntegerField(required=False, min_value=0)
+    failed_files = serializers.IntegerField(required=False, min_value=0)
     attempts = serializers.IntegerField(required=False, min_value=0)
     execution_time = serializers.FloatField(required=False, min_value=0)
 
diff --git a/backend/workflow_manager/internal_views.py b/backend/workflow_manager/internal_views.py
index c822e5e7b5..52f100196a 100644
--- a/backend/workflow_manager/internal_views.py
+++ b/backend/workflow_manager/internal_views.py
@@ -513,10 +513,19 @@ def update_status(self, request, id=None):
                     increment_attempt=increment_attempt,
                 )
 
-                # Update total_files separately (not handled by update_execution)
+                # Update total_files / per-file aggregates separately (not handled by update_execution)
+                update_fields: list[str] = []
                 if validated_data.get("total_files") is not None:
                     execution.total_files = validated_data["total_files"]
-                    execution.save()
+                    update_fields.append("total_files")
+                if validated_data.get("successful_files") is not None:
+                    execution.successful_files = validated_data["successful_files"]
+                    update_fields.append("successful_files")
+                if validated_data.get("failed_files") is not None:
+                    execution.failed_files = validated_data["failed_files"]
+                    update_fields.append("failed_files")
+                if update_fields:
+                    execution.save(update_fields=update_fields)
 
                 logger.info(
                     f"Updated workflow execution {id} status to {validated_data['status']}"
diff --git a/backend/workflow_manager/workflow_v2/migrations/0020_workflowexecution_file_counts.py b/backend/workflow_manager/workflow_v2/migrations/0020_workflowexecution_file_counts.py
new file mode 100644
index 0000000000..6dba25f5be
--- /dev/null
+++ b/backend/workflow_manager/workflow_v2/migrations/0020_workflowexecution_file_counts.py
@@ -0,0 +1,36 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("workflow_v2", "0019_remove_filehistory_trigram_index"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="workflowexecution",
+            name="successful_files",
+            field=models.PositiveIntegerField(
+                blank=True,
+                null=True,
+                db_comment=(
+                    "Per-run aggregate of files that completed successfully. "
+                    "Written by the worker callback at terminal state. Null on "
+                    "rows created before this column was added."
+                ),
+            ),
+        ),
+        migrations.AddField(
+            model_name="workflowexecution",
+            name="failed_files",
+            field=models.PositiveIntegerField(
+                blank=True,
+                null=True,
+                db_comment=(
+                    "Per-run aggregate of files that errored. Written by the "
+                    "worker callback at terminal state. Null on rows created "
+                    "before this column was added."
+                ),
+            ),
+        ),
+    ]
diff --git a/backend/workflow_manager/workflow_v2/models/execution.py b/backend/workflow_manager/workflow_v2/models/execution.py
index 45886bd64e..4e3656b68d 100644
--- a/backend/workflow_manager/workflow_v2/models/execution.py
+++ b/backend/workflow_manager/workflow_v2/models/execution.py
@@ -16,7 +16,6 @@
 
 from workflow_manager.execution.dto import ExecutionCache
 from workflow_manager.execution.execution_cache_utils import ExecutionCacheUtils
-from workflow_manager.file_execution.models import WorkflowFileExecution
 from workflow_manager.workflow_v2.enums import ExecutionStatus
 from workflow_manager.workflow_v2.models import Workflow
 
@@ -174,6 +173,24 @@ class Type(models.TextChoices):
     total_files = models.PositiveIntegerField(
         default=0, verbose_name="Total files", db_comment="Number of files to process"
     )
+    successful_files = models.PositiveIntegerField(
+        null=True,
+        blank=True,
+        db_comment=(
+            "Per-run aggregate of files that completed successfully. Written by "
+            "the worker callback at terminal state. Null on rows created before "
+            "this column was added."
+        ),
+    )
+    failed_files = models.PositiveIntegerField(
+        null=True,
+        blank=True,
+        db_comment=(
+            "Per-run aggregate of files that errored. Written by the worker "
+            "callback at terminal state. Null on rows created before this "
+            "column was added."
+        ),
+    )
     error_message = models.CharField(
         max_length=EXECUTION_ERROR_LENGTH,
         blank=True,
@@ -420,16 +437,8 @@ def get_last_run_statuses(cls, pipeline_id: uuid.UUID, limit: int = 5) -> list[d
 
         result = []
         for e in executions:
-            # TODO: Optimize by storing successful/failed counts directly in
-            # WorkflowExecution model. Current approach causes N+1 queries
-            # (2 queries per execution). Denormalized counts would eliminate
-            # these queries entirely.
-            successful = WorkflowFileExecution.objects.filter(
-                workflow_execution_id=e.id, status="COMPLETED"
-            ).count()
-            failed = WorkflowFileExecution.objects.filter(
-                workflow_execution_id=e.id, status="ERROR"
-            ).count()
+            successful = e.successful_files or 0
+            failed = e.failed_files or 0
 
             # Compute display_status: PARTIAL_SUCCESS if completed with mixed results
             display_status = e.status
diff --git a/frontend/src/components/pipelines-or-deployments/notification-modal/CreateNotification.jsx b/frontend/src/components/pipelines-or-deployments/notification-modal/CreateNotification.jsx
index 9302f9f1dc..65460376d4 100644
--- a/frontend/src/components/pipelines-or-deployments/notification-modal/CreateNotification.jsx
+++ b/frontend/src/components/pipelines-or-deployments/notification-modal/CreateNotification.jsx
@@ -1,4 +1,4 @@
-import { Button, Form, Input, Select, Space } from "antd";
+import { Button, Checkbox, Form, Input, Select, Space } from "antd";
 import PropTypes from "prop-types";
 import { useEffect, useState } from "react";
 import { getBackendErrorDetail } from "../../../helpers/GetStaticData";
@@ -12,7 +12,7 @@ const DEFAULT_FORM_DETAILS = {
   authorization_key: "",
   is_active: false,
   max_retries: 0,
-  notify_on: "ALL",
+  notify_on_failures: false,
   pipeline: "",
   api: "",
   url: "",
@@ -55,12 +55,6 @@ const AUTHORIZATION_TYPES = [
   },
 ];
 
-const NOTIFY_ON_OPTIONS = [
-  { value: "ALL", label: "On every completion" },
-  { value: "FAILURES_ONLY", label: "On failures only" },
-  { value: "SUCCESS_ONLY", label: "On success only" },
-];
-
 function CreateNotification({
   setIsForm,
   type,
@@ -199,12 +193,6 @@ function CreateNotification({
       tooltip:
         "Specify the maximum number of times the notification should be retried if it fails.",
     },
-    {
-      label: "Notify on",
-      name: "notify_on",
-      component: <Select options={NOTIFY_ON_OPTIONS} />,
-      tooltip: "Choose which run outcomes should trigger this webhook.",
-    },
   ];
 
   return (
@@ -233,6 +221,13 @@ function CreateNotification({
             </Form.Item>
           ),
       )}
+      <Form.Item
+        name="notify_on_failures"
+        valuePropName="checked"
+        tooltip="When enabled, only runs with at least one failed file or a run-level error/stop trigger this notification. Otherwise notifications fire on every completion."
+      >
+        <Checkbox>Notify on failures only</Checkbox>
+      </Form.Item>
       <Form.Item className="display-flex-right">
         <Space>
           <Button onClick={() => setIsForm(false)}>Cancel</Button>
diff --git a/unstract/core/src/unstract/core/data_models.py b/unstract/core/src/unstract/core/data_models.py
index 7e8e984a04..82da16a72e 100644
--- a/unstract/core/src/unstract/core/data_models.py
+++ b/unstract/core/src/unstract/core/data_models.py
@@ -510,6 +510,12 @@ class NotificationPayload:
     error_message: str | None = None
     organization_id: str | None = None
 
+    # Per-run file aggregates surfaced into webhook payloads.
+    # Default 0 lets receivers switch on a numeric value without None-checks.
+    total_files: int = 0
+    successful_files: int = 0
+    failed_files: int = 0
+
     # Metadata
     timestamp: datetime = field(default_factory=lambda: datetime.now(UTC))
     additional_data: dict[str, Any] = field(default_factory=dict)
@@ -565,6 +571,9 @@ def from_execution_status(
         error_message: str | None = None,
         organization_id: str | None = None,
         additional_data: dict[str, Any] | None = None,
+        total_files: int = 0,
+        successful_files: int = 0,
+        failed_files: int = 0,
     ) -> "NotificationPayload":
         """Create notification payload from execution status.
 
@@ -607,6 +616,9 @@ def from_execution_status(
             execution_id=execution_id,
             error_message=error_message,
             organization_id=organization_id,
+            total_files=total_files,
+            successful_files=successful_files,
+            failed_files=failed_files,
             additional_data=additional_data or {},
             _source=source,
         )
diff --git a/workers/callback/tasks.py b/workers/callback/tasks.py
index 42599f0659..eee5b97da3 100644
--- a/workers/callback/tasks.py
+++ b/workers/callback/tasks.py
@@ -381,12 +381,16 @@ def _update_execution_status_unified(
     try:
         # Consistent workflow execution status update across all callback types
         total_files = aggregated_results.get("total_files", 0)
+        successful_files = aggregated_results.get("successful_files", 0)
+        failed_files = aggregated_results.get("failed_files", 0)
 
         # Make the unified API call
         api_client.update_workflow_execution_status(
             execution_id=execution_id,
             status=final_status,
             total_files=total_files,
+            successful_files=successful_files,
+            failed_files=failed_files,
             organization_id=organization_id,
             error_message=error_message,
         )
diff --git a/workers/scheduler/tasks.py b/workers/scheduler/tasks.py
index 2e22946a58..65e5bd7af1 100644
--- a/workers/scheduler/tasks.py
+++ b/workers/scheduler/tasks.py
@@ -81,6 +81,7 @@ def _send_pipeline_status_notification(
             pipeline_id=pipeline_id,
             pipeline_name=pipeline_name,
             notification_payload=notification,
+            execution_id=execution_id,
         )
         logger.info(f"Notification sent successfully for {pipeline_type} {pipeline_id}")
     except Exception as notification_error:
diff --git a/workers/shared/api/internal_client.py b/workers/shared/api/internal_client.py
index def90bd86a..4964a2b377 100644
--- a/workers/shared/api/internal_client.py
+++ b/workers/shared/api/internal_client.py
@@ -603,6 +603,8 @@ def update_workflow_execution_status(
         status: str,
         error_message: str | None = None,
         total_files: int | None = None,
+        successful_files: int | None = None,
+        failed_files: int | None = None,
         attempts: int | None = None,
         execution_time: float | None = None,
         organization_id: str | None = None,
@@ -613,6 +615,8 @@ def update_workflow_execution_status(
             status,
             error_message,
             total_files,
+            successful_files,
+            failed_files,
             attempts,
             execution_time,
             organization_id,
diff --git a/workers/shared/clients/execution_client.py b/workers/shared/clients/execution_client.py
index 80e9ca6568..e1373eb9f5 100644
--- a/workers/shared/clients/execution_client.py
+++ b/workers/shared/clients/execution_client.py
@@ -265,6 +265,8 @@ def update_workflow_execution_status(
         status: str | TaskStatus,
         error_message: str | None = None,
         total_files: int | None = None,
+        successful_files: int | None = None,
+        failed_files: int | None = None,
         attempts: int | None = None,
         execution_time: float | None = None,
         organization_id: str | None = None,
@@ -276,6 +278,8 @@ def update_workflow_execution_status(
             status: New status (TaskStatus enum or string)
             error_message: Optional error message
             total_files: Optional total files count
+            successful_files: Optional count of files that completed successfully
+            failed_files: Optional count of files that errored
             attempts: Optional attempts count
             execution_time: Optional execution time
             organization_id: Optional organization ID override
@@ -292,6 +296,10 @@ def update_workflow_execution_status(
             data["error_message"] = error_message
         if total_files is not None:
             data["total_files"] = total_files
+        if successful_files is not None:
+            data["successful_files"] = successful_files
+        if failed_files is not None:
+            data["failed_files"] = failed_files
         if attempts is not None:
             data["attempts"] = attempts
         if execution_time is not None:
diff --git a/workers/shared/patterns/notification/helper.py b/workers/shared/patterns/notification/helper.py
index 977f5a875f..c4f4c5d518 100644
--- a/workers/shared/patterns/notification/helper.py
+++ b/workers/shared/patterns/notification/helper.py
@@ -104,6 +104,7 @@ def trigger_notification(
     pipeline_id: str,
     pipeline_name: str,
     notification_payload: NotificationPayload,
+    execution_id: str | None = None,
 ) -> None:
     """Trigger notifications for pipeline status updates.
 
@@ -111,10 +112,13 @@ def trigger_notification(
     Uses API client to fetch notification configuration.
     """
     try:
-        # Fetch pipeline notifications via API
+        # Pass execution_id so the backend filter respects notify_on_failures
+        # (see trigger_pipeline_notifications for the rationale).
+        params = {"execution_id": execution_id} if execution_id else None
         response_data = api_client._make_request(
             method="GET",
             endpoint=f"v1/webhook/pipeline/{pipeline_id}/notifications/",
+            params=params,
             timeout=10,
         )
 
@@ -176,10 +180,14 @@ def trigger_pipeline_notifications(
         return
 
     try:
-        # Fetch pipeline notifications via API
+        # Pass execution_id so the backend can drop notify_on_failures=True rows
+        # on success runs. Without it the endpoint is a no-op and we'd fire on
+        # every active row regardless of trigger preference.
+        params = {"execution_id": execution_id} if execution_id else None
         response_data = api_client._make_request(
             method="GET",
             endpoint=f"v1/webhook/pipeline/{pipeline_id}/notifications/",
+            params=params,
             timeout=10,
         )
 
@@ -204,7 +212,9 @@ def trigger_pipeline_notifications(
         else:
             workflow_type = WorkflowType.ETL  # Default fallback
 
-        # Create notification payload using dataclass
+        # File counts come from WorkflowExecution via the same endpoint so
+        # webhook receivers (Slack, raw API) see partial-success breakdowns.
+        counts = response_data.get("execution_counts") or {}
         payload = NotificationPayload.from_execution_status(
             pipeline_id=pipeline_id,
             pipeline_name=pipeline_name,
@@ -213,6 +223,9 @@ def trigger_pipeline_notifications(
             source=NotificationSource.CALLBACK_WORKER,
             execution_id=execution_id,
             error_message=error_message,
+            total_files=counts.get("total_files", 0),
+            successful_files=counts.get("successful_files", 0),
+            failed_files=counts.get("failed_files", 0),
         )
 
         logger.info(
@@ -261,9 +274,14 @@ def trigger_api_notifications(
         return
 
     try:
-        # Fetch API notifications via API
+        # See trigger_pipeline_notifications: execution_id powers the backend
+        # filter that respects notify_on_failures.
+        params = {"execution_id": execution_id} if execution_id else None
         response_data = api_client._make_request(
-            method="GET", endpoint=f"v1/webhook/api/{api_id}/notifications/", timeout=10
+            method="GET",
+            endpoint=f"v1/webhook/api/{api_id}/notifications/",
+            params=params,
+            timeout=10,
         )
 
         # _make_request already handles status codes and returns parsed data
@@ -277,7 +295,7 @@ def trigger_api_notifications(
             logger.info(f"No active notifications found for API {api_id}")
             return
 
-        # Create notification payload using dataclass
+        counts = response_data.get("execution_counts") or {}
         payload = NotificationPayload.from_execution_status(
             pipeline_id=api_id,
             pipeline_name=api_name,
@@ -286,6 +304,9 @@ def trigger_api_notifications(
             source=NotificationSource.CALLBACK_WORKER,
             execution_id=execution_id,
             error_message=error_message,
+            total_files=counts.get("total_files", 0),
+            successful_files=counts.get("successful_files", 0),
+            failed_files=counts.get("failed_files", 0),
         )
 
         logger.info(

From 9cd8eb1984891231e9cb1e673c0225be24534a55 Mon Sep 17 00:00:00 2001
From: kirtimanmishrazipstack <kirtiman.mishra@zipstack.com>
Date: Tue, 5 May 2026 22:39:48 +0530
Subject: [PATCH 7/7] slack webhook payload

---
 .../tests/test_notification_filter.py         | 321 ------------------
 backend/pipeline_v2/dto.py                    |  14 +-
 .../core/src/unstract/core/data_models.py     |  24 +-
 .../notification/providers/slack_webhook.py   |   9 +-
 .../shared/patterns/notification/helper.py    |   3 +-
 5 files changed, 30 insertions(+), 341 deletions(-)
 delete mode 100644 backend/notification_v2/tests/test_notification_filter.py

diff --git a/backend/notification_v2/tests/test_notification_filter.py b/backend/notification_v2/tests/test_notification_filter.py
deleted file mode 100644
index 7f887be500..0000000000
--- a/backend/notification_v2/tests/test_notification_filter.py
+++ /dev/null
@@ -1,321 +0,0 @@
-"""Unit tests for the ``notify_on`` dispatch partition on notifications.
-
-Covers both dispatch paths that fan out to ``NotificationHelper.send_notification``:
-
-* ``APINotification.send`` — keyed on ``ExecutionStatus`` (ERROR, COMPLETED, STOPPED)
-* ``PipelineNotification.send`` — keyed on ``Pipeline.PipelineStatus``
-  (FAILURE, SUCCESS, INPROGRESS)
-
-Follows the repo convention (see ``usage_v2/tests/test_helper.py``) of stubbing
-Django-heavy modules at import time so the tests run without a live DB.
-"""
-
-from __future__ import annotations
-
-import sys
-import types
-from unittest.mock import MagicMock, patch
-
-
-# ---------------------------------------------------------------------------
-# Module-level stubs — must be installed BEFORE importing the modules under
-# test so Django's ORM imports resolve to our MagicMock-backed fakes.
-# ---------------------------------------------------------------------------
-
-
-def _ensure_mod(name: str) -> types.ModuleType:
-    """Force-install a fresh stub module in ``sys.modules``."""
-    mod = types.ModuleType(name)
-    sys.modules[name] = mod
-    return mod
-
-
-def _install_stubs() -> None:
-    # Only stub leaf modules that pull in Django ORM. Parent packages
-    # (api_v2, pipeline_v2, notification_v2, workflow_manager*) load normally.
-
-    exec_enums = _ensure_mod("workflow_manager.workflow_v2.enums")
-
-    class _ExecStatusNS:
-        class ERROR:
-            value = "ERROR"
-
-        class COMPLETED:
-            value = "COMPLETED"
-
-        class STOPPED:
-            value = "STOPPED"
-
-    exec_enums.ExecutionStatus = _ExecStatusNS  # type: ignore[attr-defined]
-
-    exec_models = _ensure_mod("workflow_manager.workflow_v2.models.execution")
-    exec_models.WorkflowExecution = MagicMock(name="WorkflowExecution")  # type: ignore[attr-defined]
-
-    api_models = _ensure_mod("api_v2.models")
-    api_models.APIDeployment = MagicMock(name="APIDeployment")  # type: ignore[attr-defined]
-
-    # notification_v2.models.Notification with a patchable ``objects``.
-    notif_models = _ensure_mod("notification_v2.models")
-
-    class _FakeNotification:
-        objects = MagicMock(name="Notification.objects")
-
-    notif_models.Notification = _FakeNotification  # type: ignore[attr-defined]
-
-    # notification_v2.helper.NotificationHelper
-    notif_helper = _ensure_mod("notification_v2.helper")
-
-    class _FakeHelper:
-        send_notification = MagicMock(name="NotificationHelper.send_notification")
-
-    notif_helper.NotificationHelper = _FakeHelper  # type: ignore[attr-defined]
-
-    # pipeline_v2.dto.PipelineStatusPayload
-    pipeline_dto = _ensure_mod("pipeline_v2.dto")
-    pipeline_dto.PipelineStatusPayload = MagicMock(name="PipelineStatusPayload")  # type: ignore[attr-defined]
-
-    # pipeline_v2.models.Pipeline with a PipelineStatus text-choices surface.
-    pipeline_models = _ensure_mod("pipeline_v2.models")
-
-    class _PipelineStatus:
-        SUCCESS = "SUCCESS"
-        FAILURE = "FAILURE"
-        INPROGRESS = "INPROGRESS"
-
-    class _FakePipeline:
-        PipelineStatus = _PipelineStatus
-
-    pipeline_models.Pipeline = _FakePipeline  # type: ignore[attr-defined]
-
-
-_install_stubs()
-
-
-# Now safe to import the modules under test.
-from api_v2 import notification as api_notification_mod  # noqa: E402
-from notification_v2.enums import NotificationTrigger  # noqa: E402
-from notification_v2.helper import NotificationHelper  # noqa: E402
-from notification_v2.models import Notification  # noqa: E402
-from pipeline_v2 import notification as pipeline_notification_mod  # noqa: E402
-from pipeline_v2.models import Pipeline  # noqa: E402
-
-
-# ---------------------------------------------------------------------------
-# Test helpers
-# ---------------------------------------------------------------------------
-
-
-def _make_queryset(notifications: list[MagicMock]) -> MagicMock:
-    """Return a MagicMock that mimics the chained QuerySet surface we use.
-
-    Supports:
-        qs.filter(notify_on=<value>)   -> qs with matching rows
-        qs.exclude(notify_on=<value>)  -> qs with non-matching rows
-        qs.exists()                    -> bool based on contents
-        iter(qs)                       -> notifications
-    """
-    qs = MagicMock(name="qs")
-    qs.__iter__ = lambda self: iter(notifications)
-
-    def _filter(**kwargs):
-        if "notify_on" in kwargs:
-            target = kwargs["notify_on"]
-            kept = [n for n in notifications if n.notify_on == target]
-            return _make_queryset(kept)
-        return _make_queryset(notifications)
-
-    def _exclude(**kwargs):
-        if "notify_on" in kwargs:
-            target = kwargs["notify_on"]
-            kept = [n for n in notifications if n.notify_on != target]
-            return _make_queryset(kept)
-        return _make_queryset(notifications)
-
-    qs.filter.side_effect = _filter
-    qs.exclude.side_effect = _exclude
-    qs.exists.return_value = bool(notifications)
-    qs.count.return_value = len(notifications)
-    return qs
-
-
-def _make_notification(*, notify_on: str) -> MagicMock:
-    n = MagicMock(name="Notification")
-    n.notify_on = notify_on
-    return n
-
-
-# ---------------------------------------------------------------------------
-# APINotification — 3 modes × 3 statuses
-# ---------------------------------------------------------------------------
-
-
-class TestAPINotificationFilter:
-    def _setup(self, *, status: str, notifications: list[MagicMock]):
-        Notification.objects.filter.reset_mock()
-        Notification.objects.filter.side_effect = None
-        Notification.objects.filter.return_value = _make_queryset(notifications)
-        NotificationHelper.send_notification.reset_mock()
-
-        api = MagicMock(name="APIDeployment")
-        api.api_name = "test-api"
-        api.id = "api-uuid"
-
-        execution = MagicMock(name="WorkflowExecution")
-        execution.status = status
-        execution.id = "exec-uuid"
-        execution.error_message = "boom" if status == "ERROR" else None
-
-        return api_notification_mod.APINotification(api=api, workflow_execution=execution)
-
-    # --- ALL: fires on every status ---
-    def test_all_fires_on_completed(self):
-        n = _make_notification(notify_on=NotificationTrigger.ALL.value)
-        self._setup(status="COMPLETED", notifications=[n]).send()
-        assert NotificationHelper.send_notification.call_count == 1
-
-    def test_all_fires_on_error(self):
-        n = _make_notification(notify_on=NotificationTrigger.ALL.value)
-        self._setup(status="ERROR", notifications=[n]).send()
-        assert NotificationHelper.send_notification.call_count == 1
-
-    def test_all_fires_on_stopped(self):
-        n = _make_notification(notify_on=NotificationTrigger.ALL.value)
-        self._setup(status="STOPPED", notifications=[n]).send()
-        assert NotificationHelper.send_notification.call_count == 1
-
-    # --- FAILURES_ONLY: fires on ERROR only ---
-    def test_failures_only_suppressed_on_completed(self):
-        n = _make_notification(notify_on=NotificationTrigger.FAILURES_ONLY.value)
-        self._setup(status="COMPLETED", notifications=[n]).send()
-        NotificationHelper.send_notification.assert_not_called()
-
-    def test_failures_only_fires_on_error(self):
-        n = _make_notification(notify_on=NotificationTrigger.FAILURES_ONLY.value)
-        self._setup(status="ERROR", notifications=[n]).send()
-        assert NotificationHelper.send_notification.call_count == 1
-
-    def test_failures_only_suppressed_on_stopped(self):
-        n = _make_notification(notify_on=NotificationTrigger.FAILURES_ONLY.value)
-        self._setup(status="STOPPED", notifications=[n]).send()
-        NotificationHelper.send_notification.assert_not_called()
-
-    # --- SUCCESS_ONLY: fires on COMPLETED only ---
-    def test_success_only_fires_on_completed(self):
-        n = _make_notification(notify_on=NotificationTrigger.SUCCESS_ONLY.value)
-        self._setup(status="COMPLETED", notifications=[n]).send()
-        assert NotificationHelper.send_notification.call_count == 1
-
-    def test_success_only_suppressed_on_error(self):
-        n = _make_notification(notify_on=NotificationTrigger.SUCCESS_ONLY.value)
-        self._setup(status="ERROR", notifications=[n]).send()
-        NotificationHelper.send_notification.assert_not_called()
-
-    def test_success_only_suppressed_on_stopped(self):
-        n = _make_notification(notify_on=NotificationTrigger.SUCCESS_ONLY.value)
-        self._setup(status="STOPPED", notifications=[n]).send()
-        NotificationHelper.send_notification.assert_not_called()
-
-    # --- Mixed partition on a COMPLETED run: ALL + SUCCESS_ONLY fire, FAILURES_ONLY doesn't ---
-    def test_mixed_partition_on_completed(self):
-        all_mode = _make_notification(notify_on=NotificationTrigger.ALL.value)
-        failures_only = _make_notification(notify_on=NotificationTrigger.FAILURES_ONLY.value)
-        success_only = _make_notification(notify_on=NotificationTrigger.SUCCESS_ONLY.value)
-        notifier = self._setup(
-            status="COMPLETED", notifications=[all_mode, failures_only, success_only]
-        )
-        with patch.object(api_notification_mod, "PipelineStatusPayload") as payload_cls:
-            payload_cls.return_value.to_dict.return_value = {}
-            notifier.send()
-
-        assert NotificationHelper.send_notification.call_count == 1
-        kwargs = NotificationHelper.send_notification.call_args.kwargs
-        dispatched = sorted(n.notify_on for n in kwargs["notifications"])
-        assert dispatched == ["ALL", "SUCCESS_ONLY"]
-
-
-# ---------------------------------------------------------------------------
-# PipelineNotification — 3 modes × 3 statuses
-# ---------------------------------------------------------------------------
-
-
-class TestPipelineNotificationFilter:
-    def _setup(self, *, last_run_status: str, notifications: list[MagicMock]):
-        Notification.objects.filter.reset_mock()
-        Notification.objects.filter.side_effect = None
-        Notification.objects.filter.return_value = _make_queryset(notifications)
-        NotificationHelper.send_notification.reset_mock()
-
-        pipeline = MagicMock(name="Pipeline")
-        pipeline.id = "pipeline-uuid"
-        pipeline.pipeline_name = "test-pipeline"
-        pipeline.pipeline_type = "ETL"
-        pipeline.last_run_status = last_run_status
-
-        return pipeline_notification_mod.PipelineNotification(
-            pipeline=pipeline, execution_id="exec-uuid", error_message=None
-        )
-
-    # --- ALL ---
-    def test_all_fires_on_success(self):
-        n = _make_notification(notify_on=NotificationTrigger.ALL.value)
-        self._setup(
-            last_run_status=Pipeline.PipelineStatus.SUCCESS, notifications=[n]
-        ).send()
-        assert NotificationHelper.send_notification.call_count == 1
-
-    def test_all_fires_on_failure(self):
-        n = _make_notification(notify_on=NotificationTrigger.ALL.value)
-        self._setup(
-            last_run_status=Pipeline.PipelineStatus.FAILURE, notifications=[n]
-        ).send()
-        assert NotificationHelper.send_notification.call_count == 1
-
-    # --- FAILURES_ONLY ---
-    def test_failures_only_suppressed_on_success(self):
-        n = _make_notification(notify_on=NotificationTrigger.FAILURES_ONLY.value)
-        self._setup(
-            last_run_status=Pipeline.PipelineStatus.SUCCESS, notifications=[n]
-        ).send()
-        NotificationHelper.send_notification.assert_not_called()
-
-    def test_failures_only_fires_on_failure(self):
-        n = _make_notification(notify_on=NotificationTrigger.FAILURES_ONLY.value)
-        self._setup(
-            last_run_status=Pipeline.PipelineStatus.FAILURE, notifications=[n]
-        ).send()
-        assert NotificationHelper.send_notification.call_count == 1
-
-    # --- SUCCESS_ONLY ---
-    def test_success_only_fires_on_success(self):
-        n = _make_notification(notify_on=NotificationTrigger.SUCCESS_ONLY.value)
-        self._setup(
-            last_run_status=Pipeline.PipelineStatus.SUCCESS, notifications=[n]
-        ).send()
-        assert NotificationHelper.send_notification.call_count == 1
-
-    def test_success_only_suppressed_on_failure(self):
-        n = _make_notification(notify_on=NotificationTrigger.SUCCESS_ONLY.value)
-        self._setup(
-            last_run_status=Pipeline.PipelineStatus.FAILURE, notifications=[n]
-        ).send()
-        NotificationHelper.send_notification.assert_not_called()
-
-    # --- Mixed partition on a SUCCESS run ---
-    def test_mixed_partition_on_success(self):
-        all_mode = _make_notification(notify_on=NotificationTrigger.ALL.value)
-        failures_only = _make_notification(notify_on=NotificationTrigger.FAILURES_ONLY.value)
-        success_only = _make_notification(notify_on=NotificationTrigger.SUCCESS_ONLY.value)
-        notifier = self._setup(
-            last_run_status=Pipeline.PipelineStatus.SUCCESS,
-            notifications=[all_mode, failures_only, success_only],
-        )
-        with patch.object(
-            pipeline_notification_mod, "PipelineStatusPayload"
-        ) as payload_cls:
-            payload_cls.return_value.to_dict.return_value = {}
-            notifier.send()
-
-        assert NotificationHelper.send_notification.call_count == 1
-        kwargs = NotificationHelper.send_notification.call_args.kwargs
-        dispatched = sorted(n.notify_on for n in kwargs["notifications"])
-        assert dispatched == ["ALL", "SUCCESS_ONLY"]
diff --git a/backend/pipeline_v2/dto.py b/backend/pipeline_v2/dto.py
index b8ad7c8707..d0f27e7943 100644
--- a/backend/pipeline_v2/dto.py
+++ b/backend/pipeline_v2/dto.py
@@ -25,15 +25,21 @@ def __init__(
         self.failed_files = failed_files
 
     def to_dict(self) -> dict[str, Any]:
-        """Convert the payload DTO to a dictionary."""
+        """Convert the payload DTO to a dictionary.
+
+        File counts are nested in `additional_data` to match the worker-path
+        payload shape (NotificationPayload.from_execution_status).
+        """
         payload: dict[str, Any] = {
             "type": self.type,
             "pipeline_id": str(self.pipeline_id),
             "pipeline_name": self.pipeline_name,
             "status": self.status,
-            "total_files": self.total_files or 0,
-            "successful_files": self.successful_files or 0,
-            "failed_files": self.failed_files or 0,
+            "additional_data": {
+                "total_files": self.total_files or 0,
+                "successful_files": self.successful_files or 0,
+                "failed_files": self.failed_files or 0,
+            },
         }
         if self.execution_id:
             payload["execution_id"] = str(self.execution_id)
diff --git a/unstract/core/src/unstract/core/data_models.py b/unstract/core/src/unstract/core/data_models.py
index 82da16a72e..7642aa9ce1 100644
--- a/unstract/core/src/unstract/core/data_models.py
+++ b/unstract/core/src/unstract/core/data_models.py
@@ -510,14 +510,10 @@ class NotificationPayload:
     error_message: str | None = None
     organization_id: str | None = None
 
-    # Per-run file aggregates surfaced into webhook payloads.
-    # Default 0 lets receivers switch on a numeric value without None-checks.
-    total_files: int = 0
-    successful_files: int = 0
-    failed_files: int = 0
-
     # Metadata
     timestamp: datetime = field(default_factory=lambda: datetime.now(UTC))
+    # Per-run file aggregates (total/successful/failed) are nested here
+    # so receivers see them grouped rather than as top-level keys.
     additional_data: dict[str, Any] = field(default_factory=dict)
 
     # Internal tracking (not sent to external webhooks)
@@ -608,6 +604,17 @@ def from_execution_status(
                 f"Cannot create notification for non-final status: {execution_status}"
             )
 
+        # File counts are bundled inside additional_data so receivers see
+        # them grouped (e.g. Slack renders one "Additional Data" section).
+        # Caller-supplied additional_data takes precedence on key conflict.
+        merged_additional = {
+            "total_files": total_files,
+            "successful_files": successful_files,
+            "failed_files": failed_files,
+        }
+        if additional_data:
+            merged_additional.update(additional_data)
+
         return cls(
             type=workflow_type,
             pipeline_id=pipeline_id,
@@ -616,10 +623,7 @@ def from_execution_status(
             execution_id=execution_id,
             error_message=error_message,
             organization_id=organization_id,
-            total_files=total_files,
-            successful_files=successful_files,
-            failed_files=failed_files,
-            additional_data=additional_data or {},
+            additional_data=merged_additional,
             _source=source,
         )
 
diff --git a/workers/notification/providers/slack_webhook.py b/workers/notification/providers/slack_webhook.py
index 89206646a5..04e3532fa3 100644
--- a/workers/notification/providers/slack_webhook.py
+++ b/workers/notification/providers/slack_webhook.py
@@ -199,11 +199,10 @@ def _format_value(self, value: Any) -> str:
         elif isinstance(value, (list, tuple)):
             return "\n• " + "\n• ".join(str(item) for item in value)
         elif isinstance(value, dict):
-            # Format nested dictionary
-            items = []
-            for k, v in value.items():
-                items.append(f"  • {self._format_key(k)}: {v}")
-            return "\n" + "\n".join(items)
+            # Inline {Key: Value, Key: Value} so the receiver sees the
+            # whole dict on one line instead of a bulleted block.
+            items = [f"{self._format_key(k)}: {v}" for k, v in value.items()]
+            return "{" + ", ".join(items) + "}"
         elif value is None:
             return "_Not specified_"
         else:
diff --git a/workers/shared/patterns/notification/helper.py b/workers/shared/patterns/notification/helper.py
index c4f4c5d518..19f78fbb99 100644
--- a/workers/shared/patterns/notification/helper.py
+++ b/workers/shared/patterns/notification/helper.py
@@ -5,6 +5,7 @@
 """
 
 import logging
+from typing import Any
 
 from celery import current_app
 
@@ -335,7 +336,7 @@ def trigger_api_notifications(
 
 
 def handle_status_notifications(
-    api_client,
+    api_client: Any,
     pipeline_id: str,
     status: str,
     execution_id: str | None = None,