From a7786e34db314b6af5235a5546cda88dc2ea5993 Mon Sep 17 00:00:00 2001
From: Gabrielle Gauthier-Melancon <gabrielle.gm@servicenow.com>
Date: Tue, 21 Jul 2026 16:41:58 -0400
Subject: [PATCH 01/10] Add sub-metrics for conversation correctly finished

---
 .../diagnostic/conv_finish_classifier.py      | 543 ++++++++++++++++++
 .../conversation_correctly_finished.py        |  24 +-
 tests/fixtures/metric_signatures.json         |   4 +-
 .../metrics/test_conv_finish_classifier.py    | 234 ++++++++
 .../metrics/test_conv_finish_extraction.py    | 171 ++++++
 .../metrics/test_conv_finish_submetrics.py    | 119 ++++
 6 files changed, 1091 insertions(+), 4 deletions(-)
 create mode 100644 src/eva/metrics/diagnostic/conv_finish_classifier.py
 create mode 100644 tests/unit/metrics/test_conv_finish_classifier.py
 create mode 100644 tests/unit/metrics/test_conv_finish_extraction.py
 create mode 100644 tests/unit/metrics/test_conv_finish_submetrics.py

diff --git a/src/eva/metrics/diagnostic/conv_finish_classifier.py b/src/eva/metrics/diagnostic/conv_finish_classifier.py
new file mode 100644
index 00000000..8e5f6d2c
--- /dev/null
+++ b/src/eva/metrics/diagnostic/conv_finish_classifier.py
@@ -0,0 +1,543 @@
+"""Shared classifier for `conversation_correctly_finished` failures.
+
+Splits the single "agent went silent after the user spoke" failure bucket into one primary
+cause. Two layers:
+
+- ``ConvFinishSignals`` + ``classify_conv_finish_failure`` — **pure logic**: given the
+  extracted signals, apply a fixed priority order and return exactly one category (or ``None``
+  when the record isn't a parent failure at all). Unit-tested without any file I/O.
+- ``extract_conv_finish_signals`` — the I/O layer that reads the record's raw files and builds
+  a ``ConvFinishSignals``. (Kept separate so the logic stays trivially testable.)
+
+Design rationale, per-category detection, and validation evidence live in
+``docs/metrics/conv_finish_submetrics.md``.
+"""
+
+import csv
+import json
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from eva.metrics.processor import is_agent_timeout_on_user_turn
+from eva.models.results import MetricScore
+
+# Priority order: infra failures first (they invalidate the run and supersede apparent
+# behaviour), then model/pipeline-specific causes, then the residual catch-all.
+CATEGORY_PRIORITY = [
+    "tts_api_error",
+    "stt_api_error",
+    "llm_api_error",
+    "answer_lost_in_reasoning",
+    "stt_empty_transcription",
+    "stt_missing_transcription",
+    "ended_with_user_interruption",
+    "vad_no_turn_detected",
+    "unknown_reason",
+]
+
+# Categories that indicate an environmental/infra failure — the agent was never actually
+# exercised, so the record should be treated as an invalid run, not an agent regression.
+INFRA_CATEGORIES = frozenset({"llm_api_error", "tts_api_error", "stt_api_error"})
+
+
+@dataclass
+class ConvFinishSignals:
+    """Everything the classifier needs, extracted once from a record's raw files.
+
+    Defaults describe a *clean* parent-failure baseline (nothing detected → ``unknown_reason``).
+    """
+
+    # --- parent gate -------------------------------------------------------
+    is_parent_failure: bool = True  # inactivity_timeout AND user was last audio speaker
+    is_cascade: bool = True  # CASCADE stack (vs S2S / AUDIO_LLM)
+    last_conv_message_role: str | None = "assistant"
+
+    # --- #1 answer_lost_in_reasoning (agent_perf_stats last row) -----------
+    last_perf_response_empty: bool = False
+    last_perf_has_tool_call: bool = False
+    last_perf_reasoning: str = ""
+    num_llm_calls: int = 0
+
+    # --- #5 llm_api_error (logs.log litellm patterns) ----------------------
+    llm_api_error_terminal: bool = False
+    llm_error_type: str = ""
+    llm_error_excerpt: str = ""
+    num_llm_api_error_lines: int = 0
+
+    # --- #8 / #9 service errors (pipecat error frames) ---------------------
+    tts_service_error: bool = False
+    stt_service_error: bool = False
+    service_error_name: str = ""
+    service_error_excerpt: str = ""
+    num_service_error_frames: int = 0
+    assistant_audio_events: int = 1
+
+    # --- #2 / #3 stt no-transcription (logs.log) ---------------------------
+    vad_no_tx_warning_after_last_response: bool = False
+    empty_transcript_after_last_response: bool = False
+    num_vad_no_tx_warnings: int = 0
+
+    # --- #4 ended_with_user_interruption (logs.log) ------------------------
+    num_interruptions: int = 0
+    accepted_turn_before_last_interruption: str | None = None
+    response_after_last_interruption: bool = True
+    interruption_delay_after_turn_ms: int | None = None
+
+    # --- #7 vad_no_turn_detected (sim events vs pipecat VAD) ----------------
+    user_final_utterance_after_agent: bool = False
+    vad_onset_in_final_window: bool = True
+    user_final_words: str | None = None
+    assistant_still_speaking_before: bool = False
+
+    # extraction problems (missing files); still classifiable, noted in details
+    notes: list[str] = field(default_factory=list)
+
+
+@dataclass
+class Classification:
+    category: str | None  # None when the record is not a parent failure
+    details: dict[str, Any]
+
+
+def classify_conv_finish_failure(s: ConvFinishSignals) -> Classification:
+    """Return the single primary cause for a conv_finish failure (or ``None`` if not one)."""
+    if not s.is_parent_failure:
+        return Classification(None, {})
+
+    base: dict[str, Any] = {}
+    if s.notes:
+        base["notes"] = list(s.notes)
+
+    # 1-3. Infra / service errors (highest priority; mark invalid run).
+    if s.tts_service_error:
+        return Classification("tts_api_error", {**base, **_infra_details("tts", s)})
+    if s.stt_service_error:
+        return Classification("stt_api_error", {**base, **_infra_details("stt", s)})
+    if s.llm_api_error_terminal:
+        return Classification(
+            "llm_api_error",
+            {
+                **base,
+                "invalid_run": True,
+                "component": "llm",
+                "error_type": s.llm_error_type,
+                "error_excerpt": s.llm_error_excerpt,
+                "num_api_error_lines": s.num_llm_api_error_lines,
+                "responses_after_last_error": 0,
+            },
+        )
+
+    # 4. Answer trapped in the reasoning field (reasoning stacks).
+    if s.last_perf_response_empty and not s.last_perf_has_tool_call and s.last_perf_reasoning.strip():
+        return Classification(
+            "answer_lost_in_reasoning",
+            {
+                **base,
+                "final_response_empty": True,
+                "final_row_had_tool_call": False,
+                "recovered_answer": s.last_perf_reasoning[:300],
+                "reasoning_chars": len(s.last_perf_reasoning),
+                "num_llm_calls": s.num_llm_calls,
+            },
+        )
+
+    # 5-6. STT gave no usable transcription for a turn VAD detected.
+    if s.is_cascade and s.last_conv_message_role == "assistant" and s.vad_no_tx_warning_after_last_response:
+        category = "stt_empty_transcription" if s.empty_transcript_after_last_response else "stt_missing_transcription"
+        return Classification(
+            category,
+            {
+                **base,
+                "vad_fired": True,
+                "final_user_turn_in_llm_context": False,
+                "user_turn_stopped_emitted": s.empty_transcript_after_last_response,
+                "user_said_ground_truth": s.user_final_words,
+                "num_vad_no_transcription_warnings": s.num_vad_no_tx_warnings,
+            },
+        )
+
+    # 7. Response cancelled by an interruption and never resumed.
+    if s.num_interruptions > 0 and s.accepted_turn_before_last_interruption and not s.response_after_last_interruption:
+        return Classification(
+            "ended_with_user_interruption",
+            {
+                **base,
+                "accepted_user_turn": s.accepted_turn_before_last_interruption,
+                "interruption_delay_after_turn_ms": s.interruption_delay_after_turn_ms,
+                "num_interruptions": s.num_interruptions,
+                "responses_after_last_interruption": 0,
+            },
+        )
+
+    # 8. VAD never fired for the user's final (emitted) utterance.
+    if (
+        s.user_final_utterance_after_agent
+        and not s.vad_onset_in_final_window
+        and s.last_conv_message_role == "assistant"
+    ):
+        return Classification(
+            "vad_no_turn_detected",
+            {
+                **base,
+                "user_said_ground_truth": s.user_final_words,
+                "assistant_still_speaking_before": s.assistant_still_speaking_before,
+                "vad_onsets_in_window": 0,
+            },
+        )
+
+    # 9. Residual — unexplained failure; candidate for a new sub-metric.
+    return Classification(
+        "unknown_reason",
+        {
+            **base,
+            "last_conversation_message_role": s.last_conv_message_role,
+            "num_interruptions": s.num_interruptions,
+            "final_user_turn_transcript": s.user_final_words,
+            "note": "no known sub-metric matched — candidate for a new sub-metric",
+        },
+    )
+
+
+# --- extraction (I/O layer) -------------------------------------------------
+
+csv.field_size_limit(10**7)  # agent_perf_stats rows embed full prompts
+
+_RE_RESP = re.compile(r"complete response: '(.*)'$")
+_RE_VADNOTX = re.compile(r"VAD fired but no previous transcription timestamp found")
+_RE_TXSTOP_EMPTY = re.compile(r"User turn stopped - complete transcript: ''")
+_RE_PROC = re.compile(r"Processing complete user turn: (.+)$")
+_RE_INTR = re.compile(r"Interruption received - cancelling ongoing")
+_RE_STILL = re.compile(r"Assistant still speaking - resetting inactivity")
+# Fatal LLM API-error signatures — allowlist, not a broad `litellm.*Error` match.
+_RE_LLM_ERR = re.compile(
+    r"MidStreamFallbackError|APIConnectionError|Retryable streaming error|RateLimitError|InternalServerError"
+)
+_RE_SERVICE = re.compile(r"([A-Za-z0-9]+(?:TTS|STT)Service)")
+_VAD_WINDOW_S = 1.5
+
+
+def _load_jsonl(path: Path) -> list[dict]:
+    out: list[dict] = []
+    if not path.exists():
+        return out
+    for line in path.read_text().splitlines():
+        line = line.strip()
+        if line:
+            try:
+                out.append(json.loads(line))
+            except json.JSONDecodeError:
+                continue
+    return out
+
+
+def extract_conv_finish_signals(context) -> ConvFinishSignals:  # noqa: ANN001 (MetricContext)
+    """Read a record's raw files and build the signals the classifier needs."""
+    s = ConvFinishSignals()
+    s.is_parent_failure = is_agent_timeout_on_user_turn(
+        context.conversation_ended_reason,
+        context.audio_timestamps_user_turns,
+        context.audio_timestamps_assistant_turns,
+    )
+    s.is_cascade = not context.is_audio_native
+    out = Path(context.output_dir) if context.output_dir else None
+    if out is None:
+        s.notes.append("no output_dir")
+        return s
+
+    # audit_log → last conversation message role
+    audit = out / "audit_log.json"
+    if audit.exists():
+        try:
+            cm = json.loads(audit.read_text()).get("conversation_messages", [])
+            s.last_conv_message_role = cm[-1]["role"] if cm else None
+        except (json.JSONDecodeError, KeyError):
+            s.notes.append("audit_log unreadable")
+
+    # agent_perf_stats.csv → last row (answer_lost signals)
+    perf = out / "agent_perf_stats.csv"
+    if perf.exists():
+        try:
+            rows = list(csv.DictReader(perf.open(newline="")))
+            s.num_llm_calls = len(rows)
+            if rows:
+                last = rows[-1]
+                s.last_perf_response_empty = not (last.get("response") or "").strip()
+                s.last_perf_has_tool_call = bool((last.get("tool_calls") or "").strip())
+                s.last_perf_reasoning = (last.get("reasoning") or "").strip()
+        except (OSError, csv.Error):
+            s.notes.append("agent_perf_stats unreadable")
+
+    # user_simulator_events → ground-truth audio timeline + final words + assistant audio count
+    ev = _load_jsonl(out / "user_simulator_events.jsonl")
+    user_speech = [e for e in ev if e.get("type") == "user_speech"]
+    s.user_final_words = user_speech[-1]["data"]["text"] if user_speech else None
+    asst_ends = [
+        e["audio_timestamp"] for e in ev if e.get("event_type") == "audio_end" and e.get("user") == "assistant"
+    ]
+    s.assistant_audio_events = sum(
+        1 for e in ev if e.get("event_type") == "audio_start" and e.get("user") == "assistant"
+    )
+    u_starts = [
+        e["audio_timestamp"] for e in ev if e.get("event_type") == "audio_start" and e.get("user") == "simulated_user"
+    ]
+    u_ends = [
+        e["audio_timestamp"] for e in ev if e.get("event_type") == "audio_end" and e.get("user") == "simulated_user"
+    ]
+
+    # pipecat_logs → service-error frames + VAD onsets
+    pl = _load_jsonl(out / "pipecat_logs.jsonl")
+    for e in pl:
+        if e.get("type") == "error":
+            frame = str(e.get("data", {}).get("frame", ""))
+            m = _RE_SERVICE.search(frame)
+            if m:
+                s.num_service_error_frames += 1
+                svc = m.group(1)
+                if "TTS" in svc:
+                    s.tts_service_error = True
+                    s.service_error_name = svc
+                elif "STT" in svc:
+                    s.stt_service_error = True
+                    s.service_error_name = svc
+                s.service_error_excerpt = frame[:160]
+    vad_onsets = [e["timestamp"] / 1000.0 for e in pl if e.get("type") == "user_started_speaking"]
+
+    # #7: user's final emitted utterance after the agent, with no VAD onset in its window
+    if u_starts and asst_ends:
+        fu_start = max(u_starts)
+        fu_end = max([e for e in u_ends if e >= fu_start], default=fu_start + 1.0)
+        s.user_final_utterance_after_agent = fu_start >= max(asst_ends) - 0.5
+        s.vad_onset_in_final_window = any(fu_start - _VAD_WINDOW_S <= t <= fu_end + _VAD_WINDOW_S for t in vad_onsets)
+
+    # logs.log → response/interruption/STT/LLM-error line analysis
+    log_path = out / "logs.log"
+    if log_path.exists():
+        _extract_log_signals(log_path.read_text().splitlines(), s)
+    elif not (s.tts_service_error or s.stt_service_error or perf.exists()):
+        s.notes.append("no logs.log")
+
+    return s
+
+
+def _msg(line: str) -> str:
+    """The message portion of a log line ('… | msg'), or the whole line."""
+    return line.rsplit(" | ", 1)[-1]
+
+
+def _extract_log_signals(lines: list[str], s: ConvFinishSignals) -> None:
+    resp_idx = [i for i, ln in enumerate(lines) if (m := _RE_RESP.search(_msg(ln))) and m.group(1).strip()]
+    last_resp = resp_idx[-1] if resp_idx else -1
+    intr_idx = [i for i, ln in enumerate(lines) if _RE_INTR.search(ln)]
+    proc = [(i, m.group(1).strip()) for i, ln in enumerate(lines) if (m := _RE_PROC.search(_msg(ln)))]
+    err_idx = [i for i, ln in enumerate(lines) if _RE_LLM_ERR.search(ln)]
+    vadnotx_idx = [i for i, ln in enumerate(lines) if _RE_VADNOTX.search(ln)]
+    empty_tx_idx = [i for i, ln in enumerate(lines) if _RE_TXSTOP_EMPTY.search(ln)]
+    s.assistant_still_speaking_before = any(_RE_STILL.search(ln) for ln in lines)
+
+    # STT no-transcription: warning/empty-transcript after the last real assistant response
+    s.num_vad_no_tx_warnings = len(vadnotx_idx)
+    s.vad_no_tx_warning_after_last_response = any(i > last_resp for i in vadnotx_idx)
+    s.empty_transcript_after_last_response = any(i > last_resp for i in empty_tx_idx)
+
+    # LLM API error: fatal error with no non-empty response after the LAST error line
+    if err_idx:
+        last_err = err_idx[-1]
+        s.num_llm_api_error_lines = len(err_idx)
+        s.llm_api_error_terminal = not any(i > last_err for i in resp_idx)
+        if s.llm_api_error_terminal:
+            excerpt = _RE_LLM_ERR.search(lines[last_err])
+            s.llm_error_type = excerpt.group(0) if excerpt else ""
+            s.llm_error_excerpt = _msg(lines[last_err])[:160]
+
+    # Interruption: cancelled a real accepted turn, no response after the last interruption
+    if intr_idx:
+        last_intr = intr_idx[-1]
+        s.num_interruptions = len(intr_idx)
+        proc_before = [t for (i, t) in proc if i < last_intr]
+        s.accepted_turn_before_last_interruption = proc_before[-1] if proc_before else None
+        s.response_after_last_interruption = any(i > last_intr for i in resp_idx)
+
+
+# Final-turn input-characteristic heuristics — kept in sync with the analysis app
+# (EVABench `apps/analysis.py`: `_strip_annotations`, `_CONFIRM_START_RE`, `_spelled_out_signals`).
+# Prosody/annotation tags ([slow], [neutral], [user interrupts]) inflate word counts and hide the
+# acknowledgement lead, so strip them first.
+_ANNOTATION_RE = re.compile(r"\[[^\]]*\]")
+# Acknowledgement lead — mirrors EVABench `_CONFIRM_START_RE` (includes negations no/nope, tolerant
+# of leading [tags]); scope is "starts with a confirmation/acknowledgement".
+_ACKNOWLEDGEMENT_LEAD = re.compile(
+    r"^\s*(?:\[[^\]]*\]\s*)*\b(?:yes|no|nope|yep|yeah|sure|okay|ok|alright|correct|right)\b",
+    re.IGNORECASE,
+)
+# Spelled-entity signals (ported from EVABench `_spelled_out_signals`).
+# Single-letter / single-digit runs require ≥3 chars — 2 chars ("I T" = the word IT, "G A") isn't
+# spelling.
+_SEPS = r"[\s.,;:\-…]+"
+_SINGLE_LETTER_RUN_RE = re.compile(rf"(?:\b[a-zA-Z]\b{_SEPS}){{2,}}\b[a-zA-Z]\b")
+_SINGLE_DIGIT_RUN_RE = re.compile(rf"(?:\b\d\b{_SEPS}){{2,}}\b\d\b")
+_AS_IN_RE = re.compile(r"\b[a-zA-Z]\s+as\s+in\s+\w+", re.IGNORECASE)
+_CAPS_ALNUM_CODE_RE = re.compile(r"\b(?=[A-Z0-9]{3,})(?=.*[A-Z])(?=.*\d)[A-Z0-9]+\b")
+_CAPS_ACRONYM_RE = re.compile(r"\b[A-Z]{3,}\b")
+_DIGIT_WORD = r"(?:zero|one|two|three|four|five|six|seven|eight|nine|ten)"
+_SPOKEN_DIGIT_RUN_RE = re.compile(rf"(?:\b{_DIGIT_WORD}\b[^a-zA-Z]+){{2,}}\b{_DIGIT_WORD}\b", re.IGNORECASE)
+_NATO_WORDS = (
+    "alpha",
+    "bravo",
+    "charlie",
+    "delta",
+    "echo",
+    "foxtrot",
+    "golf",
+    "hotel",
+    "india",
+    "juliet",
+    "juliett",
+    "kilo",
+    "lima",
+    "mike",
+    "november",
+    "oscar",
+    "papa",
+    "quebec",
+    "romeo",
+    "sierra",
+    "tango",
+    "uniform",
+    "victor",
+    "whiskey",
+    "x-ray",
+    "xray",
+    "yankee",
+    "zulu",
+)
+_NATO_WORD_RE = re.compile(r"\b(?:" + "|".join(re.escape(w) for w in _NATO_WORDS) + r")\b", re.IGNORECASE)
+_NATO_SET = set(_NATO_WORDS)
+_DIGIT_WORD_SET = {"zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten"}
+# Length caps (annotation-stripped word counts). A genuine acknowledgement is short; a long sentence
+# that merely starts "yeah …" isn't one. A spelled entity must *dominate* the turn — so we cap the
+# surrounding non-spelled (prose) words rather than total words (real codes are long).
+_ACKNOWLEDGEMENT_MAX_WORDS = 6
+_SPELL_MAX_EXTRA_WORDS = 10
+
+
+def _strip_annotations(text: str) -> str:
+    return _ANNOTATION_RE.sub(" ", text or "")
+
+
+def _non_spell_word_count(text: str) -> int:
+    """Words that are NOT part of a spell-out (single letters, digit-words, 'dash', NATO, caps codes)."""
+    n = 0
+    for raw in text.split():
+        tok = raw.strip(".,!?;:'\"()[]…-")
+        if not tok:
+            continue
+        low = tok.lower()
+        is_spell_tok = (
+            (len(tok) == 1 and tok.isalpha())
+            or low in _DIGIT_WORD_SET
+            or low == "dash"
+            or low in _NATO_SET
+            or (tok.isupper() and len(tok) >= 3)  # caps acronym (VPN, HIPAA)
+            or (len(tok) >= 3 and tok.isalnum() and any(c.isupper() for c in tok) and any(c.isdigit() for c in tok))
+        )
+        if not is_spell_tok:
+            n += 1
+    return n
+
+
+def _looks_spelled(text: str) -> bool:
+    """Any spelled-out signal — single-letter/digit runs, NATO, 'X as in Y', caps codes/acronyms."""
+    return bool(
+        _SINGLE_LETTER_RUN_RE.search(text)
+        or _SINGLE_DIGIT_RUN_RE.search(text)
+        or _AS_IN_RE.search(text)
+        or _NATO_WORD_RE.search(text)
+        or _CAPS_ALNUM_CODE_RE.search(text)
+        or _CAPS_ACRONYM_RE.search(text)
+        or _SPOKEN_DIGIT_RUN_RE.search(text)
+    )
+
+
+def final_turn_input_flags(text: str | None) -> dict[str, bool]:
+    """Orthogonal input-characteristic flags for the user's final intended utterance.
+
+    These are NOT pipeline causes — they describe the *input* and cut across the cause partition
+    (a failure can be both `vad_no_turn_detected` and have an acknowledgement final turn). Used to
+    test whether short / acknowledgement / spelled-out final turns correlate with failures.
+    Heuristics mirror the analysis app (EVABench `apps/analysis.py`) so the two tools agree.
+
+    - ``short``: 1-2 words (< 3), counted **after stripping `[annotation]` tags**.
+    - ``acknowledgement``: leads with a confirmation/ack ("Yes, that is correct.", "Ok thanks",
+      "[neutral] Okay.", "No.") AND is at most ``_ACKNOWLEDGEMENT_MAX_WORDS`` words — a long sentence
+      that merely starts "yeah …" is not counted.
+    - ``spelled_entity``: letter/digit spell-out of an ID/code/name ("E M P eight nine …", NATO,
+      "V as in Victor", caps codes like EMP358) that **dominates** the turn (≤ ``_SPELL_MAX_EXTRA_WORDS``
+      non-spelled words) — a long sentence with a small spelled fragment is not counted. Single-char
+      runs need ≥3 chars, so 2-letter words like "IT" ("I T") don't count.
+    """
+    if not text or not text.strip():
+        return {"short": False, "acknowledgement": False, "spelled_entity": False}
+    stripped = _strip_annotations(text).strip()
+    word_count = len(stripped.split())
+    return {
+        "short": 0 < word_count < 3,
+        "acknowledgement": word_count <= _ACKNOWLEDGEMENT_MAX_WORDS and bool(_ACKNOWLEDGEMENT_LEAD.match(stripped)),
+        "spelled_entity": _looks_spelled(stripped) and _non_spell_word_count(stripped) <= _SPELL_MAX_EXTRA_WORDS,
+    }
+
+
+def build_final_turn_flag_sub_metrics(
+    user_final_text: str | None,
+    parent_name: str = "conversation_correctly_finished",
+) -> dict[str, "MetricScore"]:
+    """Binary flags (``1.0`` = present) for final-turn input characteristics — orthogonal to causes."""
+    flags = final_turn_input_flags(user_final_text)
+    subs: dict[str, MetricScore] = {}
+    for key in ("short", "acknowledgement", "spelled_entity"):
+        occurred = flags[key]
+        score = 1.0 if occurred else 0.0
+        sub_key = f"final_turn_{key}_rate"
+        subs[sub_key] = MetricScore(
+            name=f"{parent_name}.{sub_key}",
+            score=score,
+            normalized_score=score,
+            details={"final_user_text": user_final_text} if occurred else {},
+        )
+    return subs
+
+
+def build_conv_finish_sub_metrics(
+    classification: Classification,
+    parent_name: str = "conversation_correctly_finished",
+) -> dict[str, "MetricScore"]:
+    """Per-cause binary-flag sub-metrics (framework convention: ``1.0`` = this cause occurred).
+
+    One flag per category in ``CATEGORY_PRIORITY``, keyed ``<cause>_rate`` (the ``_rate`` suffix
+    signals lower-is-better to ``direction_for_sub_metric`` and makes the cross-record mean read
+    as "fraction of failures with this cause"). The fired cause carries the evidence details.
+    """
+    subs: dict[str, MetricScore] = {}
+    for cause in CATEGORY_PRIORITY:
+        occurred = cause == classification.category
+        score = 1.0 if occurred else 0.0
+        sub_key = f"{cause}_rate"
+        subs[sub_key] = MetricScore(
+            name=f"{parent_name}.{sub_key}",
+            score=score,
+            normalized_score=score,
+            details=classification.details if occurred else {},
+        )
+    return subs
+
+
+def _infra_details(component: str, s: ConvFinishSignals) -> dict[str, Any]:
+    return {
+        "invalid_run": True,
+        "component": component,
+        "service": s.service_error_name,
+        "error_excerpt": s.service_error_excerpt,
+        "num_error_frames": s.num_service_error_frames,
+        "assistant_audio_events": s.assistant_audio_events,
+    }
diff --git a/src/eva/metrics/diagnostic/conversation_correctly_finished.py b/src/eva/metrics/diagnostic/conversation_correctly_finished.py
index c38adb7d..08ea39b6 100644
--- a/src/eva/metrics/diagnostic/conversation_correctly_finished.py
+++ b/src/eva/metrics/diagnostic/conversation_correctly_finished.py
@@ -1,6 +1,12 @@
 """Conversation-correctly-finished diagnostic metric."""
 
 from eva.metrics.base import CodeMetric, MetricContext
+from eva.metrics.diagnostic.conv_finish_classifier import (
+    build_conv_finish_sub_metrics,
+    build_final_turn_flag_sub_metrics,
+    classify_conv_finish_failure,
+    extract_conv_finish_signals,
+)
 from eva.metrics.processor import is_agent_timeout_on_user_turn, last_audio_speaker
 from eva.metrics.registry import register_metric
 from eva.models.results import MetricScore
@@ -8,10 +14,16 @@
 
 @register_metric
 class ConversationCorrectlyFinishedMetric(CodeMetric):
-    """0.0 when the agent timed out on the user's final turn; 1.0 otherwise."""
+    """0.0 when the agent timed out on the user's final turn; 1.0 otherwise.
+
+    On failures (score 0.0) it also emits per-cause diagnostic sub-metrics
+    (``conversation_correctly_finished.<cause>_rate``) splitting *why* the agent went silent.
+    Causes are only classified when the conversation failed — see
+    ``docs/metrics/conv_finish_submetrics.md``.
+    """
 
     name = "conversation_correctly_finished"
-    version = "v0.1"
+    version = "v0.2"
     description = "Diagnostic metric: 0.0 when agent failed to respond to the user's final turn"
     category = "diagnostic"
     exclude_from_pass_at_k = True
@@ -30,8 +42,15 @@ async def compute(self, context: MetricContext) -> MetricScore:
             )
             score = 0.0 if missed_turn else 1.0
 
+            # Classify the cause only when the conversation actually failed.
+            sub_metrics = None
             if missed_turn:
                 human_reason = "conversation ended with inactivity_timeout and user was the last speaker"
+                signals = extract_conv_finish_signals(context)
+                classification = classify_conv_finish_failure(signals)
+                sub_metrics = build_conv_finish_sub_metrics(classification, self.name)
+                # Orthogonal input-characteristic flags (short / acknowledgement / spelled final turn).
+                sub_metrics.update(build_final_turn_flag_sub_metrics(signals.user_final_words, self.name))
             elif reason == "inactivity_timeout":
                 human_reason = f"inactivity_timeout but last speaker was {speaker!r}"
             else:
@@ -46,6 +65,7 @@ async def compute(self, context: MetricContext) -> MetricScore:
                     "last_audio_speaker": speaker,
                     "reason": human_reason,
                 },
+                sub_metrics=sub_metrics,
             )
 
         except Exception as e:
diff --git a/tests/fixtures/metric_signatures.json b/tests/fixtures/metric_signatures.json
index 4dc4639f..f1f3c256 100644
--- a/tests/fixtures/metric_signatures.json
+++ b/tests/fixtures/metric_signatures.json
@@ -14,8 +14,8 @@
   "ConversationCorrectlyFinishedMetric": {
     "name": "conversation_correctly_finished",
     "prompt_hash": null,
-    "source_hash": "3f495631946f",
-    "version": "v0.1"
+    "source_hash": "dd402832de14",
+    "version": "v0.2"
   },
   "ConversationProgressionJudgeMetric": {
     "name": "conversation_progression",
diff --git a/tests/unit/metrics/test_conv_finish_classifier.py b/tests/unit/metrics/test_conv_finish_classifier.py
new file mode 100644
index 00000000..e8679416
--- /dev/null
+++ b/tests/unit/metrics/test_conv_finish_classifier.py
@@ -0,0 +1,234 @@
+"""Tests for the shared conv_finish failure classifier (pure logic over signals)."""
+
+import pytest
+
+from eva.metrics.diagnostic.conv_finish_classifier import (
+    ConvFinishSignals,
+    classify_conv_finish_failure,
+)
+
+
+def sig(**overrides) -> ConvFinishSignals:
+    """A clean parent-failure baseline (fires nothing → unknown_reason), overridable."""
+    return ConvFinishSignals(**overrides)
+
+
+def cat(**overrides) -> str | None:
+    return classify_conv_finish_failure(sig(**overrides)).category
+
+
+# --- not applicable ---------------------------------------------------------
+def test_not_parent_failure_returns_none():
+    assert classify_conv_finish_failure(sig(is_parent_failure=False)).category is None
+
+
+def test_clean_parent_failure_is_unknown():
+    assert cat() == "unknown_reason"
+
+
+# --- #1 answer_lost_in_reasoning -------------------------------------------
+def test_answer_lost_in_reasoning():
+    assert cat(last_perf_response_empty=True, last_perf_reasoning="Great, I booked it.") == ("answer_lost_in_reasoning")
+
+
+def test_empty_response_but_no_reasoning_is_not_answer_lost():
+    # genuine empty generation, not answer-lost
+    assert cat(last_perf_response_empty=True, last_perf_reasoning="") == "unknown_reason"
+
+
+def test_empty_response_with_tool_call_is_not_answer_lost():
+    assert cat(last_perf_response_empty=True, last_perf_reasoning="x", last_perf_has_tool_call=True) == (
+        "unknown_reason"
+    )
+
+
+# --- #2 / #3 stt no-transcription -------------------------------------------
+def test_stt_empty_transcription():
+    assert (
+        cat(
+            is_cascade=True,
+            last_conv_message_role="assistant",
+            vad_no_tx_warning_after_last_response=True,
+            empty_transcript_after_last_response=True,
+        )
+        == "stt_empty_transcription"
+    )
+
+
+def test_stt_missing_transcription():
+    assert (
+        cat(
+            is_cascade=True,
+            last_conv_message_role="assistant",
+            vad_no_tx_warning_after_last_response=True,
+            empty_transcript_after_last_response=False,
+        )
+        == "stt_missing_transcription"
+    )
+
+
+def test_stt_needs_cascade():
+    assert (
+        cat(
+            is_cascade=False,
+            last_conv_message_role="assistant",
+            vad_no_tx_warning_after_last_response=True,
+            empty_transcript_after_last_response=True,
+        )
+        == "unknown_reason"
+    )
+
+
+def test_stt_needs_assistant_last_role():
+    # final user turn reached the LLM (role=user) → not an STT drop
+    assert (
+        cat(
+            is_cascade=True,
+            last_conv_message_role="user",
+            vad_no_tx_warning_after_last_response=True,
+            empty_transcript_after_last_response=True,
+        )
+        == "unknown_reason"
+    )
+
+
+# --- #4 ended_with_user_interruption ---------------------------------------
+def test_ended_with_user_interruption():
+    assert (
+        cat(
+            num_interruptions=1,
+            accepted_turn_before_last_interruption="the code is 610311",
+            response_after_last_interruption=False,
+        )
+        == "ended_with_user_interruption"
+    )
+
+
+def test_interruption_but_agent_recovered_is_not_it():
+    assert (
+        cat(
+            num_interruptions=1,
+            accepted_turn_before_last_interruption="x",
+            response_after_last_interruption=True,
+        )
+        == "unknown_reason"
+    )
+
+
+# --- #5 llm_api_error -------------------------------------------------------
+def test_llm_api_error():
+    assert cat(llm_api_error_terminal=True) == "llm_api_error"
+
+
+def test_llm_api_error_outranks_interruption():
+    # rec 42 shape: has interruption-like state AND a terminal API error → API error wins
+    assert (
+        cat(
+            llm_api_error_terminal=True,
+            num_interruptions=1,
+            accepted_turn_before_last_interruption="x",
+            response_after_last_interruption=False,
+        )
+        == "llm_api_error"
+    )
+
+
+# --- #7 vad_no_turn_detected ------------------------------------------------
+def test_vad_no_turn_detected():
+    assert (
+        cat(
+            user_final_utterance_after_agent=True,
+            vad_onset_in_final_window=False,
+            last_conv_message_role="assistant",
+        )
+        == "vad_no_turn_detected"
+    )
+
+
+def test_vad_fired_is_not_vad_no_turn():
+    assert (
+        cat(
+            user_final_utterance_after_agent=True,
+            vad_onset_in_final_window=True,
+            last_conv_message_role="assistant",
+        )
+        == "unknown_reason"
+    )
+
+
+# --- #8 / #9 service errors (infra) — highest priority ----------------------
+def test_tts_api_error():
+    assert cat(tts_service_error=True) == "tts_api_error"
+
+
+def test_stt_api_error():
+    assert cat(stt_service_error=True) == "stt_api_error"
+
+
+def test_tts_error_outranks_everything():
+    assert (
+        cat(
+            tts_service_error=True,
+            user_final_utterance_after_agent=True,
+            vad_onset_in_final_window=False,
+            num_interruptions=1,
+            accepted_turn_before_last_interruption="x",
+            response_after_last_interruption=False,
+        )
+        == "tts_api_error"
+    )
+
+
+# --- details carried ---------------------------------------------------------
+def test_details_include_evidence_for_answer_lost():
+    r = classify_conv_finish_failure(sig(last_perf_response_empty=True, last_perf_reasoning="Booked room 201."))
+    assert r.category == "answer_lost_in_reasoning"
+    assert r.details.get("recovered_answer", "").startswith("Booked room 201")
+
+
+def test_infra_details_flag_invalid_run():
+    r = classify_conv_finish_failure(sig(tts_service_error=True, service_error_name="DeepgramTTSService"))
+    assert r.details.get("invalid_run") is True
+    assert r.details.get("component") == "tts"
+
+
+# --- final-turn input flags (orthogonal to the cause classifier) ------------
+from eva.metrics.diagnostic.conv_finish_classifier import final_turn_input_flags  # noqa: E402
+
+
+@pytest.mark.parametrize(
+    "text,short,acknowledgement,spelled",
+    [
+        ("Yes.", True, True, False),
+        ("Yes, that is correct.", False, True, False),
+        ("Ok thanks", True, True, False),
+        ("Correct.", True, True, False),
+        ("Confirm.", True, False, False),  # EVABench scope excludes "confirm"
+        ("I need a replacement laptop", False, False, False),
+        ("No, that is wrong.", False, True, False),  # EVABench confirmation-start includes "no"
+        ("floor two", True, False, False),
+        ("", False, False, False),
+        (None, False, False, False),
+        # annotation tags are stripped before short/acknowledgement (EVABench parity)
+        ("[neutral] Okay.", True, True, False),
+        ("[slightly impatient] Okay.", True, True, False),
+        # spelled entities — single-letter / spoken-digit runs, NATO, "as in", caps codes
+        ("It is D I A G dash two X M nine five P L Q.", False, False, True),
+        ("My employee ID is E M P eight nine seven three zero five", False, False, True),
+        ("It is four nine one seven.", False, False, True),
+        ("Room two zero one on the second floor.", False, False, True),
+        ("The clearance code is C L R dash O C C dash nine five.", False, False, True),
+        ("The state abbreviation is G A.", False, False, False),  # only 2 single letters → not a run (≥3)
+        ("My confirmation is Alpha Bravo Charlie.", False, False, True),  # NATO
+        ("My ID is EMP358.", False, False, True),  # caps alnum code
+        ("V as in Victor.", False, False, True),  # phonetic "as in"
+        # length caps: long sentences that merely start with an acknowledgement / contain a spell fragment
+        ("Yes, that is correct, September fourteenth at three o clock in the afternoon.", False, False, False),
+        ("I was calling about my order and my confirmation number is four nine one seven please.", False, False, False),
+    ],
+)
+def test_final_turn_input_flags(text, short, acknowledgement, spelled):
+    flags = final_turn_input_flags(text)
+    assert flags["short"] is short
+    assert flags["acknowledgement"] is acknowledgement
+    assert flags["spelled_entity"] is spelled
diff --git a/tests/unit/metrics/test_conv_finish_extraction.py b/tests/unit/metrics/test_conv_finish_extraction.py
new file mode 100644
index 00000000..a4a345ff
--- /dev/null
+++ b/tests/unit/metrics/test_conv_finish_extraction.py
@@ -0,0 +1,171 @@
+"""Tests for extract_conv_finish_signals (reads a record dir → ConvFinishSignals)."""
+
+import json
+
+from eva.metrics.diagnostic.conv_finish_classifier import (
+    classify_conv_finish_failure,
+    extract_conv_finish_signals,
+)
+
+from .conftest import make_metric_context
+
+TS = "2026-07-20 10:00:00,000 | INFO     | {src} | {msg}"
+
+
+def _write(d, **files):
+    for name, content in files.items():
+        (d / name).write_text(content)
+
+
+def _ctx(tmp_path, reason="inactivity_timeout", user_last=True, **overrides):
+    # user_last → user audio ends after assistant → is_agent_timeout_on_user_turn True
+    user_ts = {0: [(0.0, 5.0)]} if user_last else {0: [(0.0, 2.0)]}
+    asst_ts = {0: [(1.0, 2.0)]} if user_last else {0: [(1.0, 5.0)]}
+    return make_metric_context(
+        output_dir=str(tmp_path),
+        conversation_ended_reason=reason,
+        audio_timestamps_user_turns=user_ts,
+        audio_timestamps_assistant_turns=asst_ts,
+        **overrides,
+    )
+
+
+def test_parent_gate_false_on_goodbye(tmp_path):
+    _write(tmp_path, **{"audit_log.json": json.dumps({"conversation_messages": []})})
+    s = extract_conv_finish_signals(_ctx(tmp_path, reason="goodbye"))
+    assert s.is_parent_failure is False
+    assert classify_conv_finish_failure(s).category is None
+
+
+def test_answer_lost_from_perf_csv(tmp_path):
+    csv = 'response,tool_calls,reasoning\n,,"Great. I booked room 201 for you."\n'
+    _write(
+        tmp_path,
+        **{
+            "agent_perf_stats.csv": csv,
+            "audit_log.json": json.dumps({"conversation_messages": [{"role": "assistant", "content": ""}]}),
+        },
+    )
+    s = extract_conv_finish_signals(_ctx(tmp_path))
+    assert s.last_perf_response_empty is True
+    assert s.last_perf_has_tool_call is False
+    assert "booked room 201" in s.last_perf_reasoning.lower()
+    assert classify_conv_finish_failure(s).category == "answer_lost_in_reasoning"
+
+
+def test_tts_service_error_from_pipecat_frames(tmp_path):
+    pl = "\n".join(
+        json.dumps(x)
+        for x in [
+            {
+                "type": "error",
+                "data": {
+                    "frame": "DeepgramTTSService#1 error: server rejected WebSocket connection: HTTP 402, fatal: False"
+                },
+            },
+            {
+                "type": "error",
+                "data": {
+                    "frame": "DeepgramTTSService#1 error: server rejected WebSocket connection: HTTP 402, fatal: False"
+                },
+            },
+        ]
+    )
+    _write(
+        tmp_path,
+        **{
+            "pipecat_logs.jsonl": pl,
+            "user_simulator_events.jsonl": "",  # no assistant audio events
+            "audit_log.json": json.dumps({"conversation_messages": [{"role": "assistant", "content": ""}]}),
+        },
+    )
+    s = extract_conv_finish_signals(_ctx(tmp_path))
+    assert s.tts_service_error is True
+    assert s.assistant_audio_events == 0
+    r = classify_conv_finish_failure(s)
+    assert r.category == "tts_api_error"
+    assert r.details["invalid_run"] is True
+
+
+def test_llm_api_error_terminal_from_logs(tmp_path):
+    log = "\n".join(
+        [
+            TS.format(
+                src="eva.assistant.pipecat_server:738", msg="Assistant turn stopped - complete response: 'What floor?'"
+            ),
+            TS.format(src="eva.assistant.pipeline.agent_processor:196", msg="Processing complete user turn: floor two"),
+            TS.format(
+                src="eva.assistant.services.llm:354",
+                msg="Retryable streaming error on attempt 1/6: litellm.MidStreamFallbackError: APIConnectionError: XaiException - Timeout",
+            ),
+        ]
+    )
+    _write(
+        tmp_path,
+        **{
+            "logs.log": log,
+            "audit_log.json": json.dumps({"conversation_messages": [{"role": "tool", "content": "{}"}]}),
+        },
+    )
+    s = extract_conv_finish_signals(_ctx(tmp_path))
+    assert s.llm_api_error_terminal is True
+    assert classify_conv_finish_failure(s).category == "llm_api_error"
+
+
+def test_stt_empty_transcription_from_logs(tmp_path):
+    log = "\n".join(
+        [
+            TS.format(
+                src="eva.assistant.pipecat_server:738",
+                msg="Assistant turn stopped - complete response: 'Is that correct?'",
+            ),
+            TS.format(
+                src="eva.assistant.pipeline.agent_processor:380",
+                msg="VAD fired but no previous transcription timestamp found",
+            ),
+            TS.format(src="eva.assistant.pipecat_server:701", msg="User turn stopped - complete transcript: ''"),
+        ]
+    )
+    _write(
+        tmp_path,
+        **{
+            "logs.log": log,
+            "audit_log.json": json.dumps(
+                {"conversation_messages": [{"role": "assistant", "content": "Is that correct?"}]}
+            ),
+        },
+    )
+    s = extract_conv_finish_signals(_ctx(tmp_path))
+    assert s.vad_no_tx_warning_after_last_response is True
+    assert s.empty_transcript_after_last_response is True
+    assert classify_conv_finish_failure(s).category == "stt_empty_transcription"
+
+
+def test_vad_no_turn_detected_from_events(tmp_path):
+    events = "\n".join(
+        json.dumps(x)
+        for x in [
+            {"event_type": "audio_start", "user": "assistant", "audio_timestamp": 100.0},
+            {"event_type": "audio_end", "user": "assistant", "audio_timestamp": 108.0},
+            {"type": "user_speech", "data": {"text": "It is for end of life."}},
+            {"event_type": "audio_start", "user": "simulated_user", "audio_timestamp": 112.0},
+            {"event_type": "audio_end", "user": "simulated_user", "audio_timestamp": 113.5},
+        ]
+    )
+    # pipecat: only an EARLIER user_started_speaking, none in the final window (112-113.5)
+    pl = json.dumps({"type": "user_started_speaking", "timestamp": 90000})
+    _write(
+        tmp_path,
+        **{
+            "user_simulator_events.jsonl": events,
+            "pipecat_logs.jsonl": pl,
+            "audit_log.json": json.dumps(
+                {"conversation_messages": [{"role": "assistant", "content": "Just to confirm..."}]}
+            ),
+        },
+    )
+    s = extract_conv_finish_signals(_ctx(tmp_path))
+    assert s.user_final_utterance_after_agent is True
+    assert s.vad_onset_in_final_window is False
+    assert s.user_final_words == "It is for end of life."
+    assert classify_conv_finish_failure(s).category == "vad_no_turn_detected"
diff --git a/tests/unit/metrics/test_conv_finish_submetrics.py b/tests/unit/metrics/test_conv_finish_submetrics.py
new file mode 100644
index 00000000..06a9e29c
--- /dev/null
+++ b/tests/unit/metrics/test_conv_finish_submetrics.py
@@ -0,0 +1,119 @@
+"""conversation_correctly_finished emits per-cause sub-metrics on failures only."""
+
+import json
+
+import pytest
+
+from eva.metrics.diagnostic.conversation_correctly_finished import ConversationCorrectlyFinishedMetric
+
+from .conftest import make_metric_context
+
+
+@pytest.fixture
+def metric():
+    return ConversationCorrectlyFinishedMetric()
+
+
+def _fail_ctx(tmp_path, **overrides):
+    # inactivity_timeout + user last speaker → parent failure (score 0.0)
+    return make_metric_context(
+        output_dir=str(tmp_path),
+        conversation_ended_reason="inactivity_timeout",
+        audio_timestamps_user_turns={0: [(0.0, 5.0)]},
+        audio_timestamps_assistant_turns={0: [(1.0, 2.0)]},
+        **overrides,
+    )
+
+
+@pytest.mark.asyncio
+async def test_success_has_no_cause_sub_metrics(metric):
+    ctx = make_metric_context(
+        conversation_ended_reason="goodbye",
+        audio_timestamps_user_turns={0: [(0.0, 1.0)]},
+        audio_timestamps_assistant_turns={0: [(1.5, 3.0)]},
+    )
+    result = await metric.compute(ctx)
+    assert result.score == 1.0
+    assert not result.sub_metrics  # None or empty — causes only classified on failures
+
+
+@pytest.mark.asyncio
+async def test_failure_answer_lost_sub_metric_fires(metric, tmp_path):
+    (tmp_path / "agent_perf_stats.csv").write_text('response,tool_calls,reasoning\n,,"Booked room 201."\n')
+    (tmp_path / "audit_log.json").write_text(
+        json.dumps({"conversation_messages": [{"role": "assistant", "content": ""}]})
+    )
+    result = await metric.compute(_fail_ctx(tmp_path))
+    assert result.score == 0.0
+    subs = result.sub_metrics
+    assert subs["answer_lost_in_reasoning_rate"].score == 1.0
+    assert subs["stt_empty_transcription_rate"].score == 0.0
+    assert subs["unknown_reason_rate"].score == 0.0
+    # exactly one *cause* flag is hot (input-characteristic flags are orthogonal, excluded here)
+    from eva.metrics.diagnostic.conv_finish_classifier import CATEGORY_PRIORITY
+
+    assert sum(subs[f"{c}_rate"].score for c in CATEGORY_PRIORITY) == 1.0
+    # fired sub-metric name is namespaced under the parent
+    assert subs["answer_lost_in_reasoning_rate"].name == "conversation_correctly_finished.answer_lost_in_reasoning_rate"
+
+
+@pytest.mark.asyncio
+async def test_failure_with_no_files_is_unknown(metric, tmp_path):
+    (tmp_path / "audit_log.json").write_text(
+        json.dumps({"conversation_messages": [{"role": "assistant", "content": "x"}]})
+    )
+    result = await metric.compute(_fail_ctx(tmp_path))
+    assert result.score == 0.0
+    assert result.sub_metrics["unknown_reason_rate"].score == 1.0
+
+
+def _fixture_with_final_user_speech(tmp_path, text):
+    (tmp_path / "audit_log.json").write_text(
+        json.dumps({"conversation_messages": [{"role": "assistant", "content": "x"}]})
+    )
+    (tmp_path / "user_simulator_events.jsonl").write_text(json.dumps({"type": "user_speech", "data": {"text": text}}))
+    return tmp_path
+
+
+@pytest.mark.asyncio
+async def test_acknowledgement_final_turn_flag(metric, tmp_path):
+    result = await metric.compute(_fail_ctx(_fixture_with_final_user_speech(tmp_path, "Yes, that is correct.")))
+    subs = result.sub_metrics
+    assert subs["final_turn_acknowledgement_rate"].score == 1.0
+    assert subs["final_turn_short_rate"].score == 0.0  # 4 words
+
+
+@pytest.mark.asyncio
+async def test_short_final_turn_flag(metric, tmp_path):
+    subs = (await metric.compute(_fail_ctx(_fixture_with_final_user_speech(tmp_path, "Yes.")))).sub_metrics
+    assert subs["final_turn_short_rate"].score == 1.0
+    assert subs["final_turn_acknowledgement_rate"].score == 1.0
+
+
+@pytest.mark.asyncio
+async def test_spelled_entity_final_turn_flag(metric, tmp_path):
+    fx = _fixture_with_final_user_speech(tmp_path, "It is D I A G dash two X M nine five P L Q.")
+    subs = (await metric.compute(_fail_ctx(fx))).sub_metrics
+    assert subs["final_turn_spelled_entity_rate"].score == 1.0
+    assert subs["final_turn_acknowledgement_rate"].score == 0.0
+
+
+@pytest.mark.asyncio
+async def test_unknown_reason_fires_even_with_input_flags_present(metric, tmp_path):
+    # An input flag being set must NOT suppress unknown_reason — the flags are orthogonal to causes.
+    subs = (
+        await metric.compute(_fail_ctx(_fixture_with_final_user_speech(tmp_path, "Yes, that is correct.")))
+    ).sub_metrics
+    assert subs["final_turn_acknowledgement_rate"].score == 1.0  # input flag present
+    assert subs["unknown_reason_rate"].score == 1.0  # …and the record is still tagged unknown
+
+
+@pytest.mark.asyncio
+async def test_input_flags_absent_on_success(metric):
+    ctx = make_metric_context(
+        conversation_ended_reason="goodbye",
+        audio_timestamps_user_turns={0: [(0.0, 1.0)]},
+        audio_timestamps_assistant_turns={0: [(1.5, 3.0)]},
+    )
+    result = await metric.compute(ctx)
+    assert not result.sub_metrics  # only computed on failures

From d4afa98ab0f650f2ece0b2e15e6bae9bb899321d Mon Sep 17 00:00:00 2001
From: Gabrielle Gauthier-Melancon <gabrielle.gm@servicenow.com>
Date: Tue, 21 Jul 2026 16:42:44 -0400
Subject: [PATCH 02/10] Add sub-metrics for response speed

---
 src/eva/metrics/diagnostic/response_speed.py | 72 +++++++++++++++-
 tests/fixtures/metric_signatures.json        |  4 +-
 tests/unit/metrics/test_response_speed.py    | 91 ++++++++++++++++++++
 3 files changed, 164 insertions(+), 3 deletions(-)

diff --git a/src/eva/metrics/diagnostic/response_speed.py b/src/eva/metrics/diagnostic/response_speed.py
index f4900790..920aad38 100644
--- a/src/eva/metrics/diagnostic/response_speed.py
+++ b/src/eva/metrics/diagnostic/response_speed.py
@@ -36,6 +36,58 @@ def _load_component_latencies(output_dir: str) -> dict[str, dict]:
     return latencies
 
 
+def _load_audit_stats(output_dir: str) -> dict | None:
+    """Read audit_log.json once and compute LLM-call / tool-call stats.
+
+    - ``num_llm_calls``: number of LLM completions (``llm_prompts`` entries)
+    - ``num_turns``: user turns (``transcript`` entries with ``message_type == "user"``)
+    - ``num_tool_calls``: total tool calls across all completions
+      (sum of each completion's ``response_message.tool_calls`` length)
+    - ``calls_with_tool_calls`` / ``parallel_tool_call_completions``: completions
+      emitting >=1 / >1 tool call (the latter reveals parallel/batched tool calls)
+
+    Uses the same LLM-call and turn definitions as the cross-run scan so numbers
+    are comparable. Returns a stats dict, or None if the audit log is
+    missing/unreadable, has no user turns, or has no ``llm_prompts`` — the last
+    case covers models (e.g. GPT realtime / speech-to-speech) that don't log
+    prompt-level calls, for which these stats are undefined and must be omitted
+    rather than reported as zero.
+    """
+    audit_path = Path(output_dir) / "audit_log.json"
+    if not audit_path.exists():
+        return None
+
+    try:
+        audit = json.loads(audit_path.read_text())
+    except Exception:
+        return None
+
+    prompts = audit.get("llm_prompts")
+    prompts = prompts if isinstance(prompts, list) else []
+    num_calls = len(prompts)
+    num_turns = sum(1 for e in audit.get("transcript", []) if isinstance(e, dict) and e.get("message_type") == "user")
+    if num_turns <= 0 or num_calls <= 0:
+        return None
+
+    per_call_tools = []
+    for c in prompts:
+        rm = c.get("response_message") if isinstance(c, dict) else None
+        tool_calls = rm.get("tool_calls") if isinstance(rm, dict) else None
+        per_call_tools.append(len(tool_calls) if isinstance(tool_calls, list) else 0)
+    num_tool_calls = sum(per_call_tools)
+
+    return {
+        "num_llm_calls": num_calls,
+        "num_turns": num_turns,
+        "llm_calls_per_turn": round(num_calls / num_turns, 3),
+        "num_tool_calls": num_tool_calls,
+        "tools_per_llm_call": round(num_tool_calls / num_calls, 3),
+        "calls_with_tool_calls": sum(1 for n in per_call_tools if n >= 1),
+        "parallel_tool_call_completions": sum(1 for n in per_call_tools if n > 1),
+        "max_tools_per_call": max(per_call_tools) if per_call_tools else 0,
+    }
+
+
 def _split_by_tool_calls(
     context: MetricContext,
 ) -> tuple[list[float], list[float]]:
@@ -87,7 +139,7 @@ class ResponseSpeedMetric(CodeMetric):
     description = "Diagnostic metric: latency between user utterance end and assistant response start"
     exclude_from_pass_at_k = True
     higher_is_better = False  # Score is latency in seconds — lower is better.
-    version = "v0.2"
+    version = "v0.3"
 
     async def compute(self, context: MetricContext) -> MetricScore:
         try:
@@ -129,6 +181,24 @@ async def compute(self, context: MetricContext) -> MetricScore:
                         details=stats,
                     )
 
+            # LLM-call / tool-call efficiency diagnostics from audit_log.json.
+            audit_stats = _load_audit_stats(context.output_dir)
+            if audit_stats is not None:
+                # LLM completions per user turn (tool-loop / model efficiency).
+                sub_metrics["llm_calls_per_turn"] = MetricScore(
+                    name=f"{self.name}.llm_calls_per_turn",
+                    score=audit_stats["llm_calls_per_turn"],
+                    normalized_score=None,
+                    details=audit_stats,
+                )
+                # Tool calls per LLM completion (reveals sequential vs batched tool calls).
+                sub_metrics["tools_per_llm_call"] = MetricScore(
+                    name=f"{self.name}.tools_per_llm_call",
+                    score=audit_stats["tools_per_llm_call"],
+                    normalized_score=None,
+                    details=audit_stats,
+                )
+
             # Add per-component latency sub_metrics from result.json.
             # result.json stores these in milliseconds; convert to seconds here
             # to match the top-level response_speed score (already in seconds).
diff --git a/tests/fixtures/metric_signatures.json b/tests/fixtures/metric_signatures.json
index f1f3c256..7b85fbe1 100644
--- a/tests/fixtures/metric_signatures.json
+++ b/tests/fixtures/metric_signatures.json
@@ -44,8 +44,8 @@
   "ResponseSpeedMetric": {
     "name": "response_speed",
     "prompt_hash": null,
-    "source_hash": "fdac735ba008",
-    "version": "v0.2"
+    "source_hash": "101a7600f080",
+    "version": "v0.3"
   },
   "STTWERMetric": {
     "name": "stt_wer",
diff --git a/tests/unit/metrics/test_response_speed.py b/tests/unit/metrics/test_response_speed.py
index df4d11ed..8a4c69a5 100644
--- a/tests/unit/metrics/test_response_speed.py
+++ b/tests/unit/metrics/test_response_speed.py
@@ -1,11 +1,32 @@
 """Tests for the ResponseSpeedMetric."""
 
+import json
+
 import pytest
 
 from eva.metrics.diagnostic.response_speed import ResponseSpeedMetric
 
 from .conftest import make_metric_context
 
+
+def _write_audit_log(output_dir, n_llm_calls: int, n_user_turns: int, tools_per_call=None) -> None:
+    """Write a minimal audit_log.json with the given call/turn/tool counts.
+
+    tools_per_call: optional list giving the number of tool calls emitted by each
+    completion (defaults to no tool calls).
+    """
+    tools_per_call = tools_per_call or [0] * n_llm_calls
+    prompts = [
+        {"latency_ms": 100.0, "response_message": {"tool_calls": [{} for _ in range(tools_per_call[i])]}}
+        for i in range(n_llm_calls)
+    ]
+    audit = {
+        "llm_prompts": prompts,
+        "transcript": [{"message_type": "user", "value": "hi", "turn_id": i + 1} for i in range(n_user_turns)],
+    }
+    (output_dir / "audit_log.json").write_text(json.dumps(audit))
+
+
 # ---------------------------------------------------------------------------
 # Helpers
 # ---------------------------------------------------------------------------
@@ -172,6 +193,76 @@ async def test_tool_call_breakdown_filters_invalid_latencies(self):
         with_tc = result.sub_metrics["with_tool_calls"]
         assert with_tc.details["num_turns"] == 2  # only 5.0 and 3.0 pass the filter
 
+    @pytest.mark.asyncio
+    async def test_llm_calls_per_turn_sub_metric(self, tmp_path):
+        """llm_calls_per_turn sub-metric = total llm calls / user turns from audit_log.json."""
+        _write_audit_log(tmp_path, n_llm_calls=6, n_user_turns=3)
+        metric = ResponseSpeedMetric()
+        ctx = make_metric_context(
+            latency_assistant_turns={1: 1.0, 2: 2.0, 3: 3.0},
+            output_dir=str(tmp_path),
+        )
+
+        result = await metric.compute(ctx)
+
+        assert result.error is None
+        assert result.sub_metrics is not None
+        cpt = result.sub_metrics["llm_calls_per_turn"]
+        assert cpt.score == pytest.approx(2.0)
+        assert cpt.details["num_llm_calls"] == 6
+        assert cpt.details["num_turns"] == 3
+
+    @pytest.mark.asyncio
+    async def test_tools_per_llm_call_sub_metric(self, tmp_path):
+        """tools_per_llm_call = total tool calls / llm completions, with batching detail."""
+        # 4 completions: 2 with 1 tool, 1 with 2 tools (parallel), 1 with none → 4 tools / 4 calls.
+        _write_audit_log(tmp_path, n_llm_calls=4, n_user_turns=2, tools_per_call=[1, 2, 1, 0])
+        metric = ResponseSpeedMetric()
+        ctx = make_metric_context(
+            latency_assistant_turns={1: 1.0, 2: 2.0},
+            output_dir=str(tmp_path),
+        )
+
+        result = await metric.compute(ctx)
+
+        assert result.error is None
+        tpc = result.sub_metrics["tools_per_llm_call"]
+        assert tpc.score == pytest.approx(1.0)  # 4 tools / 4 calls
+        assert tpc.details["num_tool_calls"] == 4
+        assert tpc.details["calls_with_tool_calls"] == 3
+        assert tpc.details["parallel_tool_call_completions"] == 1
+        assert tpc.details["max_tools_per_call"] == 2
+
+    @pytest.mark.asyncio
+    async def test_call_sub_metrics_absent_without_llm_prompts(self, tmp_path):
+        """Models without llm_prompts (e.g. GPT realtime / S2S) omit the sub-metrics."""
+        # audit_log with user turns but empty llm_prompts (no prompt-level instrumentation).
+        _write_audit_log(tmp_path, n_llm_calls=0, n_user_turns=3)
+        metric = ResponseSpeedMetric()
+        ctx = make_metric_context(
+            latency_assistant_turns={1: 1.0, 2: 2.0, 3: 3.0},
+            output_dir=str(tmp_path),
+        )
+
+        result = await metric.compute(ctx)
+
+        assert result.error is None
+        # response_speed still computes latency stats, but the llm_prompts-based
+        # sub-metrics must be absent (aggregating a fabricated 0.0 would be wrong).
+        assert "llm_calls_per_turn" not in (result.sub_metrics or {})
+        assert "tools_per_llm_call" not in (result.sub_metrics or {})
+
+    @pytest.mark.asyncio
+    async def test_llm_calls_per_turn_absent_without_audit_log(self):
+        """No audit_log.json → llm_calls_per_turn sub-metric is omitted (no error)."""
+        metric = ResponseSpeedMetric()
+        ctx = make_metric_context(latency_assistant_turns={1: 1.0, 2: 2.0})
+
+        result = await metric.compute(ctx)
+
+        assert result.error is None
+        assert "llm_calls_per_turn" not in (result.sub_metrics or {})
+
     @pytest.mark.asyncio
     async def test_with_and_no_tool_split_is_exhaustive(self):
         """with_tool + no_tool latencies together cover all per_turn_latency values."""

From 0bc89ae2cd9dc5acab0544048f3afb0cf776ace9 Mon Sep 17 00:00:00 2001
From: Gabrielle Gauthier-Melancon <gabrielle.gm@servicenow.com>
Date: Tue, 21 Jul 2026 16:43:02 -0400
Subject: [PATCH 03/10] Fix config for old value in config

---
 src/eva/models/config.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/eva/models/config.py b/src/eva/models/config.py
index 5a4cf055..b5094725 100644
--- a/src/eva/models/config.py
+++ b/src/eva/models/config.py
@@ -117,7 +117,11 @@ class ModelConfig(BaseModel):
         "stt_model": "stt",
         "tts_model": "tts",
     }
-    _LEGACY_DROP: ClassVar[set[str]] = {"realtime_model", "realtime_model_params"}
+    _LEGACY_DROP: ClassVar[set[str]] = {
+        "realtime_model",
+        "realtime_model_params",
+        "audio_llm_full_audio_context",
+    }
 
     # STT models that perform their own (server-side) semantic endpointing. They drive turn
     # boundaries themselves, so they must run with the 'external' turn strategies and no local VAD.

From 39d58564dfe8f35f7be188341a4ca20829e3304a Mon Sep 17 00:00:00 2001
From: Gabrielle Gauthier-Melancon <gabrielle.gm@servicenow.com>
Date: Thu, 23 Jul 2026 14:09:19 -0400
Subject: [PATCH 04/10] Fix empty double-quote for reasoning content

---
 src/eva/assistant/agentic/system.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/eva/assistant/agentic/system.py b/src/eva/assistant/agentic/system.py
index c6ea0d96..68135c1b 100644
--- a/src/eva/assistant/agentic/system.py
+++ b/src/eva/assistant/agentic/system.py
@@ -316,7 +316,7 @@ async def _run_tool_loop(
                     "tool_calls": json.dumps(response_tool_calls_for_stats, ensure_ascii=False)
                     if response_tool_calls_for_stats
                     else "",
-                    "reasoning": f'"{reasoning_content_for_csv}"',
+                    "reasoning": reasoning_content_for_csv or None,
                     "reasoning_tokens": reasoning_tokens,
                 }
                 self.agent_perf_stats.append(perf_stat)

From df6dff88d2a9c7a51b45d206ba15c4bcff26efd8 Mon Sep 17 00:00:00 2001
From: Gabrielle Gauthier-Melancon <gabrielle.gm@servicenow.com>
Date: Thu, 23 Jul 2026 14:13:46 -0400
Subject: [PATCH 05/10] Improve reasoning sub-metrics

---
 .../diagnostic/conv_finish_classifier.py      | 56 +++++++++++++------
 .../metrics/test_conv_finish_classifier.py    | 42 ++++++++++----
 .../metrics/test_conv_finish_extraction.py    | 53 +++++++++++++++++-
 3 files changed, 122 insertions(+), 29 deletions(-)

diff --git a/src/eva/metrics/diagnostic/conv_finish_classifier.py b/src/eva/metrics/diagnostic/conv_finish_classifier.py
index 8e5f6d2c..86e9c166 100644
--- a/src/eva/metrics/diagnostic/conv_finish_classifier.py
+++ b/src/eva/metrics/diagnostic/conv_finish_classifier.py
@@ -29,7 +29,8 @@
     "tts_api_error",
     "stt_api_error",
     "llm_api_error",
-    "answer_lost_in_reasoning",
+    "reasoning_too_long",
+    "reasoning_only",
     "stt_empty_transcription",
     "stt_missing_transcription",
     "ended_with_user_interruption",
@@ -54,10 +55,12 @@ class ConvFinishSignals:
     is_cascade: bool = True  # CASCADE stack (vs S2S / AUDIO_LLM)
     last_conv_message_role: str | None = "assistant"
 
-    # --- #1 answer_lost_in_reasoning (agent_perf_stats last row) -----------
+    # --- #1 reasoning_only / reasoning_too_long (agent_perf_stats last row) -----------
     last_perf_response_empty: bool = False
     last_perf_has_tool_call: bool = False
     last_perf_reasoning: str = ""
+    last_perf_reasoning_tokens: int = 0  # hidden-reasoning models report tokens but no text
+    last_perf_stop_reason: str = ""  # 'length' ⇒ hit the token cap (reasoning_too_long)
     num_llm_calls: int = 0
 
     # --- #5 llm_api_error (logs.log litellm patterns) ----------------------
@@ -129,19 +132,30 @@ def classify_conv_finish_failure(s: ConvFinishSignals) -> Classification:
             },
         )
 
-    # 4. Answer trapped in the reasoning field (reasoning stacks).
-    if s.last_perf_response_empty and not s.last_perf_has_tool_call and s.last_perf_reasoning.strip():
-        return Classification(
-            "answer_lost_in_reasoning",
-            {
-                **base,
-                "final_response_empty": True,
-                "final_row_had_tool_call": False,
-                "recovered_answer": s.last_perf_reasoning[:300],
-                "reasoning_chars": len(s.last_perf_reasoning),
-                "num_llm_calls": s.num_llm_calls,
-            },
-        )
+    # 4. The model reasoned but produced no spoken answer / tool call. Reasoning is evidenced by
+    #    visible text OR a hidden-reasoning token count (gemini/gpt-5 report reasoning_tokens
+    #    without returning the thinking text). A truly empty generation (no text AND 0 tokens) is
+    #    NOT this — it falls through. Split by whether it hit the token cap:
+    #      - stop_reason == 'length' → `reasoning_too_long` (ran out of tokens mid-reasoning)
+    #      - otherwise             → `reasoning_only` (finished, but only reasoning came out; we
+    #                                 can't assume the answer was actually in the reasoning)
+    if (
+        s.last_perf_response_empty
+        and not s.last_perf_has_tool_call
+        and (s.last_perf_reasoning.strip() or s.last_perf_reasoning_tokens > 0)
+    ):
+        details = {
+            **base,
+            "final_response_empty": True,
+            "final_row_had_tool_call": False,
+            "reasoning_preview": s.last_perf_reasoning[:300],  # empty when reasoning is hidden
+            "reasoning_chars": len(s.last_perf_reasoning),
+            "reasoning_tokens": s.last_perf_reasoning_tokens,
+            "stop_reason": s.last_perf_stop_reason,
+            "num_llm_calls": s.num_llm_calls,
+        }
+        category = "reasoning_too_long" if s.last_perf_stop_reason == "length" else "reasoning_only"
+        return Classification(category, details)
 
     # 5-6. STT gave no usable transcription for a turn VAD detected.
     if s.is_cascade and s.last_conv_message_role == "assistant" and s.vad_no_tx_warning_after_last_response:
@@ -255,7 +269,7 @@ def extract_conv_finish_signals(context) -> ConvFinishSignals:  # noqa: ANN001 (
         except (json.JSONDecodeError, KeyError):
             s.notes.append("audit_log unreadable")
 
-    # agent_perf_stats.csv → last row (answer_lost signals)
+    # agent_perf_stats.csv → last row (reasoning_only / reasoning_too_long signals)
     perf = out / "agent_perf_stats.csv"
     if perf.exists():
         try:
@@ -265,7 +279,15 @@ def extract_conv_finish_signals(context) -> ConvFinishSignals:  # noqa: ANN001 (
                 last = rows[-1]
                 s.last_perf_response_empty = not (last.get("response") or "").strip()
                 s.last_perf_has_tool_call = bool((last.get("tool_calls") or "").strip())
-                s.last_perf_reasoning = (last.get("reasoning") or "").strip()
+                # The perf-stats writer wraps non-empty reasoning as f'"{content}"' and stores empty
+                # reasoning as an empty cell. Strip any wrapping quotes; both cases read as expected.
+                # (Older runs may still contain the literal '""' — stripping handles those too.)
+                s.last_perf_reasoning = (last.get("reasoning") or "").strip().strip('"').strip()
+                try:
+                    s.last_perf_reasoning_tokens = int(float(last.get("reasoning_tokens") or 0))
+                except (TypeError, ValueError):
+                    s.last_perf_reasoning_tokens = 0
+                s.last_perf_stop_reason = (last.get("stop_reason") or "").strip()
         except (OSError, csv.Error):
             s.notes.append("agent_perf_stats unreadable")
 
diff --git a/tests/unit/metrics/test_conv_finish_classifier.py b/tests/unit/metrics/test_conv_finish_classifier.py
index e8679416..ee6edd1c 100644
--- a/tests/unit/metrics/test_conv_finish_classifier.py
+++ b/tests/unit/metrics/test_conv_finish_classifier.py
@@ -26,22 +26,44 @@ def test_clean_parent_failure_is_unknown():
     assert cat() == "unknown_reason"
 
 
-# --- #1 answer_lost_in_reasoning -------------------------------------------
-def test_answer_lost_in_reasoning():
-    assert cat(last_perf_response_empty=True, last_perf_reasoning="Great, I booked it.") == ("answer_lost_in_reasoning")
+# --- #1 reasoning_only -------------------------------------------
+def test_reasoning_only():
+    assert cat(last_perf_response_empty=True, last_perf_reasoning="Great, I booked it.") == ("reasoning_only")
 
 
-def test_empty_response_but_no_reasoning_is_not_answer_lost():
-    # genuine empty generation, not answer-lost
-    assert cat(last_perf_response_empty=True, last_perf_reasoning="") == "unknown_reason"
+def test_empty_response_but_no_reasoning_is_not_reasoning_only():
+    # genuine empty generation (no text AND 0 reasoning tokens) → not reasoning-only
+    assert cat(last_perf_response_empty=True, last_perf_reasoning="", last_perf_reasoning_tokens=0) == "unknown_reason"
 
 
-def test_empty_response_with_tool_call_is_not_answer_lost():
+def test_hidden_reasoning_tokens_is_reasoning_only():
+    # reasoning text hidden (gemini/gpt-5) but token count proves the model reasoned → reasoning-only
+    assert cat(last_perf_response_empty=True, last_perf_reasoning="", last_perf_reasoning_tokens=26) == (
+        "reasoning_only"
+    )
+
+
+def test_empty_response_with_tool_call_is_not_reasoning_only():
     assert cat(last_perf_response_empty=True, last_perf_reasoning="x", last_perf_has_tool_call=True) == (
         "unknown_reason"
     )
 
 
+# --- reasoning_too_long (reasoned + hit the token cap) ----------------------
+def test_reasoning_too_long_when_stop_reason_length():
+    assert (
+        cat(last_perf_response_empty=True, last_perf_reasoning="thinking a lot…", last_perf_stop_reason="length")
+        == "reasoning_too_long"
+    )
+
+
+def test_reasoning_only_when_stop_reason_not_length():
+    assert (
+        cat(last_perf_response_empty=True, last_perf_reasoning="thinking…", last_perf_stop_reason="stop")
+        == "reasoning_only"
+    )
+
+
 # --- #2 / #3 stt no-transcription -------------------------------------------
 def test_stt_empty_transcription():
     assert (
@@ -180,10 +202,10 @@ def test_tts_error_outranks_everything():
 
 
 # --- details carried ---------------------------------------------------------
-def test_details_include_evidence_for_answer_lost():
+def test_details_include_evidence_for_reasoning_only():
     r = classify_conv_finish_failure(sig(last_perf_response_empty=True, last_perf_reasoning="Booked room 201."))
-    assert r.category == "answer_lost_in_reasoning"
-    assert r.details.get("recovered_answer", "").startswith("Booked room 201")
+    assert r.category == "reasoning_only"
+    assert r.details.get("reasoning_preview", "").startswith("Booked room 201")
 
 
 def test_infra_details_flag_invalid_run():
diff --git a/tests/unit/metrics/test_conv_finish_extraction.py b/tests/unit/metrics/test_conv_finish_extraction.py
index a4a345ff..81cefdfe 100644
--- a/tests/unit/metrics/test_conv_finish_extraction.py
+++ b/tests/unit/metrics/test_conv_finish_extraction.py
@@ -37,7 +37,7 @@ def test_parent_gate_false_on_goodbye(tmp_path):
     assert classify_conv_finish_failure(s).category is None
 
 
-def test_answer_lost_from_perf_csv(tmp_path):
+def test_reasoning_only_from_perf_csv(tmp_path):
     csv = 'response,tool_calls,reasoning\n,,"Great. I booked room 201 for you."\n'
     _write(
         tmp_path,
@@ -50,7 +50,56 @@ def test_answer_lost_from_perf_csv(tmp_path):
     assert s.last_perf_response_empty is True
     assert s.last_perf_has_tool_call is False
     assert "booked room 201" in s.last_perf_reasoning.lower()
-    assert classify_conv_finish_failure(s).category == "answer_lost_in_reasoning"
+    assert classify_conv_finish_failure(s).category == "reasoning_only"
+
+
+def _write_perf_csv(tmp_path, reasoning_value, reasoning_tokens="0", stop_reason="stop"):
+    # Mirrors the writer (system.py) which stores reasoning as f'"{content}"' — so empty reasoning
+    # content lands in the CSV as the literal 2-char string '""'. Use csv.writer so the parsed cell
+    # equals reasoning_value exactly.
+    import csv as _csv
+
+    with (tmp_path / "agent_perf_stats.csv").open("w", newline="") as f:
+        w = _csv.writer(f)
+        w.writerow(["response", "tool_calls", "reasoning", "reasoning_tokens", "stop_reason"])
+        w.writerow(["", "", reasoning_value, reasoning_tokens, stop_reason])
+    (tmp_path / "audit_log.json").write_text(
+        json.dumps({"conversation_messages": [{"role": "assistant", "content": ""}]})
+    )
+
+
+def test_reasoning_too_long_when_stop_reason_length(tmp_path):
+    # gemma-4 rec 40: huge reasoning, hit the token cap (stop_reason=length), empty response.
+    _write_perf_csv(tmp_path, '"lots of thinking…"', reasoning_tokens="2047", stop_reason="length")
+    s = extract_conv_finish_signals(_ctx(tmp_path))
+    assert s.last_perf_stop_reason == "length"
+    assert classify_conv_finish_failure(s).category == "reasoning_too_long"
+
+
+def test_hidden_reasoning_tokens_still_reasoning_only(tmp_path):
+    # gemini-3.5-flash-lite / gpt-5: reasoning_tokens>0 but NO reasoning text (writer stores '""').
+    # The model DID reason, the answer just never reached TTS → still reasoning_only.
+    _write_perf_csv(tmp_path, '""', reasoning_tokens="26")
+    s = extract_conv_finish_signals(_ctx(tmp_path))
+    assert s.last_perf_reasoning == ""  # text unwrapped to empty
+    assert s.last_perf_reasoning_tokens == 26
+    assert classify_conv_finish_failure(s).category == "reasoning_only"
+
+
+def test_truly_empty_generation_is_not_reasoning_only(tmp_path):
+    # No reasoning text AND 0 reasoning tokens → the model did not reason; not reasoning_only.
+    _write_perf_csv(tmp_path, '""', reasoning_tokens="0")
+    s = extract_conv_finish_signals(_ctx(tmp_path))
+    assert classify_conv_finish_failure(s).category != "reasoning_only"
+
+
+def test_quote_wrapped_real_reasoning_still_fires(tmp_path):
+    _write_perf_csv(tmp_path, '"Booked room 201."')  # writer-wrapped real content, 0 tokens
+    s = extract_conv_finish_signals(_ctx(tmp_path))
+    assert s.last_perf_reasoning == "Booked room 201."  # outer wrapper stripped
+    r = classify_conv_finish_failure(s)
+    assert r.category == "reasoning_only"
+    assert r.details["reasoning_preview"] == "Booked room 201."
 
 
 def test_tts_service_error_from_pipecat_frames(tmp_path):

From 1fef37b950f4ddd0560dd0fa8019ea49e9567996 Mon Sep 17 00:00:00 2001
From: Gabrielle Gauthier-Melancon <gabrielle.gm@servicenow.com>
Date: Thu, 23 Jul 2026 14:18:47 -0400
Subject: [PATCH 06/10] Change denominator so rates include non-failures

---
 .../diagnostic/conv_finish_classifier.py      |  6 ++--
 .../conversation_correctly_finished.py        | 25 +++++++++++-----
 tests/fixtures/metric_signatures.json         |  2 +-
 .../metrics/test_conv_finish_submetrics.py    | 30 ++++++++++++++-----
 4 files changed, 45 insertions(+), 18 deletions(-)

diff --git a/src/eva/metrics/diagnostic/conv_finish_classifier.py b/src/eva/metrics/diagnostic/conv_finish_classifier.py
index 86e9c166..948e4de8 100644
--- a/src/eva/metrics/diagnostic/conv_finish_classifier.py
+++ b/src/eva/metrics/diagnostic/conv_finish_classifier.py
@@ -537,8 +537,10 @@ def build_conv_finish_sub_metrics(
     """Per-cause binary-flag sub-metrics (framework convention: ``1.0`` = this cause occurred).
 
     One flag per category in ``CATEGORY_PRIORITY``, keyed ``<cause>_rate`` (the ``_rate`` suffix
-    signals lower-is-better to ``direction_for_sub_metric`` and makes the cross-record mean read
-    as "fraction of failures with this cause"). The fired cause carries the evidence details.
+    signals lower-is-better to ``direction_for_sub_metric``). Emitted on every record (all 0.0
+    when ``classification.category`` is None, i.e. a clean finish), so the cross-record mean reads
+    as "fraction of all conversations with this cause" — an all-records denominator matching
+    faithfulness. The fired cause carries the evidence details.
     """
     subs: dict[str, MetricScore] = {}
     for cause in CATEGORY_PRIORITY:
diff --git a/src/eva/metrics/diagnostic/conversation_correctly_finished.py b/src/eva/metrics/diagnostic/conversation_correctly_finished.py
index 08ea39b6..f017d1c5 100644
--- a/src/eva/metrics/diagnostic/conversation_correctly_finished.py
+++ b/src/eva/metrics/diagnostic/conversation_correctly_finished.py
@@ -2,6 +2,7 @@
 
 from eva.metrics.base import CodeMetric, MetricContext
 from eva.metrics.diagnostic.conv_finish_classifier import (
+    Classification,
     build_conv_finish_sub_metrics,
     build_final_turn_flag_sub_metrics,
     classify_conv_finish_failure,
@@ -16,10 +17,13 @@
 class ConversationCorrectlyFinishedMetric(CodeMetric):
     """0.0 when the agent timed out on the user's final turn; 1.0 otherwise.
 
-    On failures (score 0.0) it also emits per-cause diagnostic sub-metrics
+    It also emits per-cause diagnostic sub-metrics
     (``conversation_correctly_finished.<cause>_rate``) splitting *why* the agent went silent.
-    Causes are only classified when the conversation failed — see
-    ``docs/metrics/conv_finish_submetrics.md``.
+    These fire (``1.0``) only on failures, but are emitted as ``0.0`` on every clean finish too,
+    so their cross-record mean reads as "fraction of all conversations with this cause" —
+    matching how faithfulness dimension rates aggregate (denominator = all records, not just
+    failures). The orthogonal input-characteristic flags (``final_turn_*``) stay failure-only,
+    since they describe the unanswered final turn. See ``docs/metrics/conv_finish_submetrics.md``.
     """
 
     name = "conversation_correctly_finished"
@@ -42,8 +46,9 @@ async def compute(self, context: MetricContext) -> MetricScore:
             )
             score = 0.0 if missed_turn else 1.0
 
-            # Classify the cause only when the conversation actually failed.
-            sub_metrics = None
+            # Classify the cause only when the conversation actually failed. Emit the per-cause
+            # rate sub-metrics on every record regardless (all 0.0 on a clean finish) so the
+            # cross-record mean has an all-records denominator, matching faithfulness.
             if missed_turn:
                 human_reason = "conversation ended with inactivity_timeout and user was the last speaker"
                 signals = extract_conv_finish_signals(context)
@@ -51,10 +56,14 @@ async def compute(self, context: MetricContext) -> MetricScore:
                 sub_metrics = build_conv_finish_sub_metrics(classification, self.name)
                 # Orthogonal input-characteristic flags (short / acknowledgement / spelled final turn).
                 sub_metrics.update(build_final_turn_flag_sub_metrics(signals.user_final_words, self.name))
-            elif reason == "inactivity_timeout":
-                human_reason = f"inactivity_timeout but last speaker was {speaker!r}"
             else:
-                human_reason = f"conversation ended with reason={reason!r}"
+                # Clean finish: no cause fired, but emit every cause flag at 0.0 so this record
+                # counts toward the rate denominator.
+                sub_metrics = build_conv_finish_sub_metrics(Classification(None, {}), self.name)
+                if reason == "inactivity_timeout":
+                    human_reason = f"inactivity_timeout but last speaker was {speaker!r}"
+                else:
+                    human_reason = f"conversation ended with reason={reason!r}"
 
             return MetricScore(
                 name=self.name,
diff --git a/tests/fixtures/metric_signatures.json b/tests/fixtures/metric_signatures.json
index 7b85fbe1..2eacb778 100644
--- a/tests/fixtures/metric_signatures.json
+++ b/tests/fixtures/metric_signatures.json
@@ -14,7 +14,7 @@
   "ConversationCorrectlyFinishedMetric": {
     "name": "conversation_correctly_finished",
     "prompt_hash": null,
-    "source_hash": "dd402832de14",
+    "source_hash": "cee94b5782ac",
     "version": "v0.2"
   },
   "ConversationProgressionJudgeMetric": {
diff --git a/tests/unit/metrics/test_conv_finish_submetrics.py b/tests/unit/metrics/test_conv_finish_submetrics.py
index 06a9e29c..1f658e19 100644
--- a/tests/unit/metrics/test_conv_finish_submetrics.py
+++ b/tests/unit/metrics/test_conv_finish_submetrics.py
@@ -1,4 +1,10 @@
-"""conversation_correctly_finished emits per-cause sub-metrics on failures only."""
+"""conversation_correctly_finished emits per-cause rate sub-metrics on every record.
+
+Cause rates are emitted on all records (0.0 when the conversation finished correctly) so
+their cross-record mean reads as "fraction of all conversations with this cause" — matching
+how faithfulness dimension rates aggregate. The orthogonal input-characteristic flags
+(``final_turn_*``) remain failure-only, since they characterize the unanswered final turn.
+"""
 
 import json
 
@@ -26,7 +32,9 @@ def _fail_ctx(tmp_path, **overrides):
 
 
 @pytest.mark.asyncio
-async def test_success_has_no_cause_sub_metrics(metric):
+async def test_success_emits_zero_cause_rates(metric):
+    from eva.metrics.diagnostic.conv_finish_classifier import CATEGORY_PRIORITY
+
     ctx = make_metric_context(
         conversation_ended_reason="goodbye",
         audio_timestamps_user_turns={0: [(0.0, 1.0)]},
@@ -34,11 +42,18 @@ async def test_success_has_no_cause_sub_metrics(metric):
     )
     result = await metric.compute(ctx)
     assert result.score == 1.0
-    assert not result.sub_metrics  # None or empty — causes only classified on failures
+    # Cause rates are present on every record now (denominator = all records, like faithfulness),
+    # all 0.0 on a clean finish.
+    subs = result.sub_metrics
+    assert subs is not None
+    assert all(subs[f"{c}_rate"].score == 0.0 for c in CATEGORY_PRIORITY)
+    assert sum(subs[f"{c}_rate"].score for c in CATEGORY_PRIORITY) == 0.0
+    # Input-characteristic flags remain failure-only.
+    assert not any(k.startswith("final_turn_") for k in subs)
 
 
 @pytest.mark.asyncio
-async def test_failure_answer_lost_sub_metric_fires(metric, tmp_path):
+async def test_failure_reasoning_only_sub_metric_fires(metric, tmp_path):
     (tmp_path / "agent_perf_stats.csv").write_text('response,tool_calls,reasoning\n,,"Booked room 201."\n')
     (tmp_path / "audit_log.json").write_text(
         json.dumps({"conversation_messages": [{"role": "assistant", "content": ""}]})
@@ -46,7 +61,7 @@ async def test_failure_answer_lost_sub_metric_fires(metric, tmp_path):
     result = await metric.compute(_fail_ctx(tmp_path))
     assert result.score == 0.0
     subs = result.sub_metrics
-    assert subs["answer_lost_in_reasoning_rate"].score == 1.0
+    assert subs["reasoning_only_rate"].score == 1.0
     assert subs["stt_empty_transcription_rate"].score == 0.0
     assert subs["unknown_reason_rate"].score == 0.0
     # exactly one *cause* flag is hot (input-characteristic flags are orthogonal, excluded here)
@@ -54,7 +69,7 @@ async def test_failure_answer_lost_sub_metric_fires(metric, tmp_path):
 
     assert sum(subs[f"{c}_rate"].score for c in CATEGORY_PRIORITY) == 1.0
     # fired sub-metric name is namespaced under the parent
-    assert subs["answer_lost_in_reasoning_rate"].name == "conversation_correctly_finished.answer_lost_in_reasoning_rate"
+    assert subs["reasoning_only_rate"].name == "conversation_correctly_finished.reasoning_only_rate"
 
 
 @pytest.mark.asyncio
@@ -116,4 +131,5 @@ async def test_input_flags_absent_on_success(metric):
         audio_timestamps_assistant_turns={0: [(1.5, 3.0)]},
     )
     result = await metric.compute(ctx)
-    assert not result.sub_metrics  # only computed on failures
+    # Input-characteristic flags are only emitted on failures; cause rates are still present (all 0.0).
+    assert not any(k.startswith("final_turn_") for k in result.sub_metrics)

From 7baf250bcca828205f880f01aab9ffae3e405e7a Mon Sep 17 00:00:00 2001
From: Gabrielle Gauthier-Melancon <gabrielle.gm@servicenow.com>
Date: Thu, 23 Jul 2026 16:18:13 -0400
Subject: [PATCH 07/10] Refactor code

---
 .../diagnostic/conv_finish_classifier.py      | 567 ------------------
 .../conversation_correctly_finished.py        |  10 +-
 .../__init__.py                               |  27 +
 .../causes/__init__.py                        |   1 +
 .../causes/api_errors.py                      |  89 +++
 .../causes/interruption.py                    |  40 ++
 .../causes/reasoning.py                       |  62 ++
 .../causes/transcript_vad.py                  |  87 +++
 .../classifier.py                             | 201 +++++++
 .../final_turn.py                             | 132 ++++
 .../signals.py                                |  64 ++
 src/eva/utils/json_utils.py                   |  20 +
 src/eva/utils/log_processing.py               |  18 +
 tests/fixtures/metric_signatures.json         |   2 +-
 .../metrics/test_conv_finish_extraction.py    |   2 +-
 .../metrics/test_conv_finish_submetrics.py    |   4 +-
 ...=> test_conversation_finish_classifier.py} |   4 +-
 17 files changed, 752 insertions(+), 578 deletions(-)
 delete mode 100644 src/eva/metrics/diagnostic/conv_finish_classifier.py
 create mode 100644 src/eva/utils/conversation_correctly_finished/__init__.py
 create mode 100644 src/eva/utils/conversation_correctly_finished/causes/__init__.py
 create mode 100644 src/eva/utils/conversation_correctly_finished/causes/api_errors.py
 create mode 100644 src/eva/utils/conversation_correctly_finished/causes/interruption.py
 create mode 100644 src/eva/utils/conversation_correctly_finished/causes/reasoning.py
 create mode 100644 src/eva/utils/conversation_correctly_finished/causes/transcript_vad.py
 create mode 100644 src/eva/utils/conversation_correctly_finished/classifier.py
 create mode 100644 src/eva/utils/conversation_correctly_finished/final_turn.py
 create mode 100644 src/eva/utils/conversation_correctly_finished/signals.py
 rename tests/unit/metrics/{test_conv_finish_classifier.py => test_conversation_finish_classifier.py} (98%)

diff --git a/src/eva/metrics/diagnostic/conv_finish_classifier.py b/src/eva/metrics/diagnostic/conv_finish_classifier.py
deleted file mode 100644
index 948e4de8..00000000
--- a/src/eva/metrics/diagnostic/conv_finish_classifier.py
+++ /dev/null
@@ -1,567 +0,0 @@
-"""Shared classifier for `conversation_correctly_finished` failures.
-
-Splits the single "agent went silent after the user spoke" failure bucket into one primary
-cause. Two layers:
-
-- ``ConvFinishSignals`` + ``classify_conv_finish_failure`` — **pure logic**: given the
-  extracted signals, apply a fixed priority order and return exactly one category (or ``None``
-  when the record isn't a parent failure at all). Unit-tested without any file I/O.
-- ``extract_conv_finish_signals`` — the I/O layer that reads the record's raw files and builds
-  a ``ConvFinishSignals``. (Kept separate so the logic stays trivially testable.)
-
-Design rationale, per-category detection, and validation evidence live in
-``docs/metrics/conv_finish_submetrics.md``.
-"""
-
-import csv
-import json
-import re
-from dataclasses import dataclass, field
-from pathlib import Path
-from typing import Any
-
-from eva.metrics.processor import is_agent_timeout_on_user_turn
-from eva.models.results import MetricScore
-
-# Priority order: infra failures first (they invalidate the run and supersede apparent
-# behaviour), then model/pipeline-specific causes, then the residual catch-all.
-CATEGORY_PRIORITY = [
-    "tts_api_error",
-    "stt_api_error",
-    "llm_api_error",
-    "reasoning_too_long",
-    "reasoning_only",
-    "stt_empty_transcription",
-    "stt_missing_transcription",
-    "ended_with_user_interruption",
-    "vad_no_turn_detected",
-    "unknown_reason",
-]
-
-# Categories that indicate an environmental/infra failure — the agent was never actually
-# exercised, so the record should be treated as an invalid run, not an agent regression.
-INFRA_CATEGORIES = frozenset({"llm_api_error", "tts_api_error", "stt_api_error"})
-
-
-@dataclass
-class ConvFinishSignals:
-    """Everything the classifier needs, extracted once from a record's raw files.
-
-    Defaults describe a *clean* parent-failure baseline (nothing detected → ``unknown_reason``).
-    """
-
-    # --- parent gate -------------------------------------------------------
-    is_parent_failure: bool = True  # inactivity_timeout AND user was last audio speaker
-    is_cascade: bool = True  # CASCADE stack (vs S2S / AUDIO_LLM)
-    last_conv_message_role: str | None = "assistant"
-
-    # --- #1 reasoning_only / reasoning_too_long (agent_perf_stats last row) -----------
-    last_perf_response_empty: bool = False
-    last_perf_has_tool_call: bool = False
-    last_perf_reasoning: str = ""
-    last_perf_reasoning_tokens: int = 0  # hidden-reasoning models report tokens but no text
-    last_perf_stop_reason: str = ""  # 'length' ⇒ hit the token cap (reasoning_too_long)
-    num_llm_calls: int = 0
-
-    # --- #5 llm_api_error (logs.log litellm patterns) ----------------------
-    llm_api_error_terminal: bool = False
-    llm_error_type: str = ""
-    llm_error_excerpt: str = ""
-    num_llm_api_error_lines: int = 0
-
-    # --- #8 / #9 service errors (pipecat error frames) ---------------------
-    tts_service_error: bool = False
-    stt_service_error: bool = False
-    service_error_name: str = ""
-    service_error_excerpt: str = ""
-    num_service_error_frames: int = 0
-    assistant_audio_events: int = 1
-
-    # --- #2 / #3 stt no-transcription (logs.log) ---------------------------
-    vad_no_tx_warning_after_last_response: bool = False
-    empty_transcript_after_last_response: bool = False
-    num_vad_no_tx_warnings: int = 0
-
-    # --- #4 ended_with_user_interruption (logs.log) ------------------------
-    num_interruptions: int = 0
-    accepted_turn_before_last_interruption: str | None = None
-    response_after_last_interruption: bool = True
-    interruption_delay_after_turn_ms: int | None = None
-
-    # --- #7 vad_no_turn_detected (sim events vs pipecat VAD) ----------------
-    user_final_utterance_after_agent: bool = False
-    vad_onset_in_final_window: bool = True
-    user_final_words: str | None = None
-    assistant_still_speaking_before: bool = False
-
-    # extraction problems (missing files); still classifiable, noted in details
-    notes: list[str] = field(default_factory=list)
-
-
-@dataclass
-class Classification:
-    category: str | None  # None when the record is not a parent failure
-    details: dict[str, Any]
-
-
-def classify_conv_finish_failure(s: ConvFinishSignals) -> Classification:
-    """Return the single primary cause for a conv_finish failure (or ``None`` if not one)."""
-    if not s.is_parent_failure:
-        return Classification(None, {})
-
-    base: dict[str, Any] = {}
-    if s.notes:
-        base["notes"] = list(s.notes)
-
-    # 1-3. Infra / service errors (highest priority; mark invalid run).
-    if s.tts_service_error:
-        return Classification("tts_api_error", {**base, **_infra_details("tts", s)})
-    if s.stt_service_error:
-        return Classification("stt_api_error", {**base, **_infra_details("stt", s)})
-    if s.llm_api_error_terminal:
-        return Classification(
-            "llm_api_error",
-            {
-                **base,
-                "invalid_run": True,
-                "component": "llm",
-                "error_type": s.llm_error_type,
-                "error_excerpt": s.llm_error_excerpt,
-                "num_api_error_lines": s.num_llm_api_error_lines,
-                "responses_after_last_error": 0,
-            },
-        )
-
-    # 4. The model reasoned but produced no spoken answer / tool call. Reasoning is evidenced by
-    #    visible text OR a hidden-reasoning token count (gemini/gpt-5 report reasoning_tokens
-    #    without returning the thinking text). A truly empty generation (no text AND 0 tokens) is
-    #    NOT this — it falls through. Split by whether it hit the token cap:
-    #      - stop_reason == 'length' → `reasoning_too_long` (ran out of tokens mid-reasoning)
-    #      - otherwise             → `reasoning_only` (finished, but only reasoning came out; we
-    #                                 can't assume the answer was actually in the reasoning)
-    if (
-        s.last_perf_response_empty
-        and not s.last_perf_has_tool_call
-        and (s.last_perf_reasoning.strip() or s.last_perf_reasoning_tokens > 0)
-    ):
-        details = {
-            **base,
-            "final_response_empty": True,
-            "final_row_had_tool_call": False,
-            "reasoning_preview": s.last_perf_reasoning[:300],  # empty when reasoning is hidden
-            "reasoning_chars": len(s.last_perf_reasoning),
-            "reasoning_tokens": s.last_perf_reasoning_tokens,
-            "stop_reason": s.last_perf_stop_reason,
-            "num_llm_calls": s.num_llm_calls,
-        }
-        category = "reasoning_too_long" if s.last_perf_stop_reason == "length" else "reasoning_only"
-        return Classification(category, details)
-
-    # 5-6. STT gave no usable transcription for a turn VAD detected.
-    if s.is_cascade and s.last_conv_message_role == "assistant" and s.vad_no_tx_warning_after_last_response:
-        category = "stt_empty_transcription" if s.empty_transcript_after_last_response else "stt_missing_transcription"
-        return Classification(
-            category,
-            {
-                **base,
-                "vad_fired": True,
-                "final_user_turn_in_llm_context": False,
-                "user_turn_stopped_emitted": s.empty_transcript_after_last_response,
-                "user_said_ground_truth": s.user_final_words,
-                "num_vad_no_transcription_warnings": s.num_vad_no_tx_warnings,
-            },
-        )
-
-    # 7. Response cancelled by an interruption and never resumed.
-    if s.num_interruptions > 0 and s.accepted_turn_before_last_interruption and not s.response_after_last_interruption:
-        return Classification(
-            "ended_with_user_interruption",
-            {
-                **base,
-                "accepted_user_turn": s.accepted_turn_before_last_interruption,
-                "interruption_delay_after_turn_ms": s.interruption_delay_after_turn_ms,
-                "num_interruptions": s.num_interruptions,
-                "responses_after_last_interruption": 0,
-            },
-        )
-
-    # 8. VAD never fired for the user's final (emitted) utterance.
-    if (
-        s.user_final_utterance_after_agent
-        and not s.vad_onset_in_final_window
-        and s.last_conv_message_role == "assistant"
-    ):
-        return Classification(
-            "vad_no_turn_detected",
-            {
-                **base,
-                "user_said_ground_truth": s.user_final_words,
-                "assistant_still_speaking_before": s.assistant_still_speaking_before,
-                "vad_onsets_in_window": 0,
-            },
-        )
-
-    # 9. Residual — unexplained failure; candidate for a new sub-metric.
-    return Classification(
-        "unknown_reason",
-        {
-            **base,
-            "last_conversation_message_role": s.last_conv_message_role,
-            "num_interruptions": s.num_interruptions,
-            "final_user_turn_transcript": s.user_final_words,
-            "note": "no known sub-metric matched — candidate for a new sub-metric",
-        },
-    )
-
-
-# --- extraction (I/O layer) -------------------------------------------------
-
-csv.field_size_limit(10**7)  # agent_perf_stats rows embed full prompts
-
-_RE_RESP = re.compile(r"complete response: '(.*)'$")
-_RE_VADNOTX = re.compile(r"VAD fired but no previous transcription timestamp found")
-_RE_TXSTOP_EMPTY = re.compile(r"User turn stopped - complete transcript: ''")
-_RE_PROC = re.compile(r"Processing complete user turn: (.+)$")
-_RE_INTR = re.compile(r"Interruption received - cancelling ongoing")
-_RE_STILL = re.compile(r"Assistant still speaking - resetting inactivity")
-# Fatal LLM API-error signatures — allowlist, not a broad `litellm.*Error` match.
-_RE_LLM_ERR = re.compile(
-    r"MidStreamFallbackError|APIConnectionError|Retryable streaming error|RateLimitError|InternalServerError"
-)
-_RE_SERVICE = re.compile(r"([A-Za-z0-9]+(?:TTS|STT)Service)")
-_VAD_WINDOW_S = 1.5
-
-
-def _load_jsonl(path: Path) -> list[dict]:
-    out: list[dict] = []
-    if not path.exists():
-        return out
-    for line in path.read_text().splitlines():
-        line = line.strip()
-        if line:
-            try:
-                out.append(json.loads(line))
-            except json.JSONDecodeError:
-                continue
-    return out
-
-
-def extract_conv_finish_signals(context) -> ConvFinishSignals:  # noqa: ANN001 (MetricContext)
-    """Read a record's raw files and build the signals the classifier needs."""
-    s = ConvFinishSignals()
-    s.is_parent_failure = is_agent_timeout_on_user_turn(
-        context.conversation_ended_reason,
-        context.audio_timestamps_user_turns,
-        context.audio_timestamps_assistant_turns,
-    )
-    s.is_cascade = not context.is_audio_native
-    out = Path(context.output_dir) if context.output_dir else None
-    if out is None:
-        s.notes.append("no output_dir")
-        return s
-
-    # audit_log → last conversation message role
-    audit = out / "audit_log.json"
-    if audit.exists():
-        try:
-            cm = json.loads(audit.read_text()).get("conversation_messages", [])
-            s.last_conv_message_role = cm[-1]["role"] if cm else None
-        except (json.JSONDecodeError, KeyError):
-            s.notes.append("audit_log unreadable")
-
-    # agent_perf_stats.csv → last row (reasoning_only / reasoning_too_long signals)
-    perf = out / "agent_perf_stats.csv"
-    if perf.exists():
-        try:
-            rows = list(csv.DictReader(perf.open(newline="")))
-            s.num_llm_calls = len(rows)
-            if rows:
-                last = rows[-1]
-                s.last_perf_response_empty = not (last.get("response") or "").strip()
-                s.last_perf_has_tool_call = bool((last.get("tool_calls") or "").strip())
-                # The perf-stats writer wraps non-empty reasoning as f'"{content}"' and stores empty
-                # reasoning as an empty cell. Strip any wrapping quotes; both cases read as expected.
-                # (Older runs may still contain the literal '""' — stripping handles those too.)
-                s.last_perf_reasoning = (last.get("reasoning") or "").strip().strip('"').strip()
-                try:
-                    s.last_perf_reasoning_tokens = int(float(last.get("reasoning_tokens") or 0))
-                except (TypeError, ValueError):
-                    s.last_perf_reasoning_tokens = 0
-                s.last_perf_stop_reason = (last.get("stop_reason") or "").strip()
-        except (OSError, csv.Error):
-            s.notes.append("agent_perf_stats unreadable")
-
-    # user_simulator_events → ground-truth audio timeline + final words + assistant audio count
-    ev = _load_jsonl(out / "user_simulator_events.jsonl")
-    user_speech = [e for e in ev if e.get("type") == "user_speech"]
-    s.user_final_words = user_speech[-1]["data"]["text"] if user_speech else None
-    asst_ends = [
-        e["audio_timestamp"] for e in ev if e.get("event_type") == "audio_end" and e.get("user") == "assistant"
-    ]
-    s.assistant_audio_events = sum(
-        1 for e in ev if e.get("event_type") == "audio_start" and e.get("user") == "assistant"
-    )
-    u_starts = [
-        e["audio_timestamp"] for e in ev if e.get("event_type") == "audio_start" and e.get("user") == "simulated_user"
-    ]
-    u_ends = [
-        e["audio_timestamp"] for e in ev if e.get("event_type") == "audio_end" and e.get("user") == "simulated_user"
-    ]
-
-    # pipecat_logs → service-error frames + VAD onsets
-    pl = _load_jsonl(out / "pipecat_logs.jsonl")
-    for e in pl:
-        if e.get("type") == "error":
-            frame = str(e.get("data", {}).get("frame", ""))
-            m = _RE_SERVICE.search(frame)
-            if m:
-                s.num_service_error_frames += 1
-                svc = m.group(1)
-                if "TTS" in svc:
-                    s.tts_service_error = True
-                    s.service_error_name = svc
-                elif "STT" in svc:
-                    s.stt_service_error = True
-                    s.service_error_name = svc
-                s.service_error_excerpt = frame[:160]
-    vad_onsets = [e["timestamp"] / 1000.0 for e in pl if e.get("type") == "user_started_speaking"]
-
-    # #7: user's final emitted utterance after the agent, with no VAD onset in its window
-    if u_starts and asst_ends:
-        fu_start = max(u_starts)
-        fu_end = max([e for e in u_ends if e >= fu_start], default=fu_start + 1.0)
-        s.user_final_utterance_after_agent = fu_start >= max(asst_ends) - 0.5
-        s.vad_onset_in_final_window = any(fu_start - _VAD_WINDOW_S <= t <= fu_end + _VAD_WINDOW_S for t in vad_onsets)
-
-    # logs.log → response/interruption/STT/LLM-error line analysis
-    log_path = out / "logs.log"
-    if log_path.exists():
-        _extract_log_signals(log_path.read_text().splitlines(), s)
-    elif not (s.tts_service_error or s.stt_service_error or perf.exists()):
-        s.notes.append("no logs.log")
-
-    return s
-
-
-def _msg(line: str) -> str:
-    """The message portion of a log line ('… | msg'), or the whole line."""
-    return line.rsplit(" | ", 1)[-1]
-
-
-def _extract_log_signals(lines: list[str], s: ConvFinishSignals) -> None:
-    resp_idx = [i for i, ln in enumerate(lines) if (m := _RE_RESP.search(_msg(ln))) and m.group(1).strip()]
-    last_resp = resp_idx[-1] if resp_idx else -1
-    intr_idx = [i for i, ln in enumerate(lines) if _RE_INTR.search(ln)]
-    proc = [(i, m.group(1).strip()) for i, ln in enumerate(lines) if (m := _RE_PROC.search(_msg(ln)))]
-    err_idx = [i for i, ln in enumerate(lines) if _RE_LLM_ERR.search(ln)]
-    vadnotx_idx = [i for i, ln in enumerate(lines) if _RE_VADNOTX.search(ln)]
-    empty_tx_idx = [i for i, ln in enumerate(lines) if _RE_TXSTOP_EMPTY.search(ln)]
-    s.assistant_still_speaking_before = any(_RE_STILL.search(ln) for ln in lines)
-
-    # STT no-transcription: warning/empty-transcript after the last real assistant response
-    s.num_vad_no_tx_warnings = len(vadnotx_idx)
-    s.vad_no_tx_warning_after_last_response = any(i > last_resp for i in vadnotx_idx)
-    s.empty_transcript_after_last_response = any(i > last_resp for i in empty_tx_idx)
-
-    # LLM API error: fatal error with no non-empty response after the LAST error line
-    if err_idx:
-        last_err = err_idx[-1]
-        s.num_llm_api_error_lines = len(err_idx)
-        s.llm_api_error_terminal = not any(i > last_err for i in resp_idx)
-        if s.llm_api_error_terminal:
-            excerpt = _RE_LLM_ERR.search(lines[last_err])
-            s.llm_error_type = excerpt.group(0) if excerpt else ""
-            s.llm_error_excerpt = _msg(lines[last_err])[:160]
-
-    # Interruption: cancelled a real accepted turn, no response after the last interruption
-    if intr_idx:
-        last_intr = intr_idx[-1]
-        s.num_interruptions = len(intr_idx)
-        proc_before = [t for (i, t) in proc if i < last_intr]
-        s.accepted_turn_before_last_interruption = proc_before[-1] if proc_before else None
-        s.response_after_last_interruption = any(i > last_intr for i in resp_idx)
-
-
-# Final-turn input-characteristic heuristics — kept in sync with the analysis app
-# (EVABench `apps/analysis.py`: `_strip_annotations`, `_CONFIRM_START_RE`, `_spelled_out_signals`).
-# Prosody/annotation tags ([slow], [neutral], [user interrupts]) inflate word counts and hide the
-# acknowledgement lead, so strip them first.
-_ANNOTATION_RE = re.compile(r"\[[^\]]*\]")
-# Acknowledgement lead — mirrors EVABench `_CONFIRM_START_RE` (includes negations no/nope, tolerant
-# of leading [tags]); scope is "starts with a confirmation/acknowledgement".
-_ACKNOWLEDGEMENT_LEAD = re.compile(
-    r"^\s*(?:\[[^\]]*\]\s*)*\b(?:yes|no|nope|yep|yeah|sure|okay|ok|alright|correct|right)\b",
-    re.IGNORECASE,
-)
-# Spelled-entity signals (ported from EVABench `_spelled_out_signals`).
-# Single-letter / single-digit runs require ≥3 chars — 2 chars ("I T" = the word IT, "G A") isn't
-# spelling.
-_SEPS = r"[\s.,;:\-…]+"
-_SINGLE_LETTER_RUN_RE = re.compile(rf"(?:\b[a-zA-Z]\b{_SEPS}){{2,}}\b[a-zA-Z]\b")
-_SINGLE_DIGIT_RUN_RE = re.compile(rf"(?:\b\d\b{_SEPS}){{2,}}\b\d\b")
-_AS_IN_RE = re.compile(r"\b[a-zA-Z]\s+as\s+in\s+\w+", re.IGNORECASE)
-_CAPS_ALNUM_CODE_RE = re.compile(r"\b(?=[A-Z0-9]{3,})(?=.*[A-Z])(?=.*\d)[A-Z0-9]+\b")
-_CAPS_ACRONYM_RE = re.compile(r"\b[A-Z]{3,}\b")
-_DIGIT_WORD = r"(?:zero|one|two|three|four|five|six|seven|eight|nine|ten)"
-_SPOKEN_DIGIT_RUN_RE = re.compile(rf"(?:\b{_DIGIT_WORD}\b[^a-zA-Z]+){{2,}}\b{_DIGIT_WORD}\b", re.IGNORECASE)
-_NATO_WORDS = (
-    "alpha",
-    "bravo",
-    "charlie",
-    "delta",
-    "echo",
-    "foxtrot",
-    "golf",
-    "hotel",
-    "india",
-    "juliet",
-    "juliett",
-    "kilo",
-    "lima",
-    "mike",
-    "november",
-    "oscar",
-    "papa",
-    "quebec",
-    "romeo",
-    "sierra",
-    "tango",
-    "uniform",
-    "victor",
-    "whiskey",
-    "x-ray",
-    "xray",
-    "yankee",
-    "zulu",
-)
-_NATO_WORD_RE = re.compile(r"\b(?:" + "|".join(re.escape(w) for w in _NATO_WORDS) + r")\b", re.IGNORECASE)
-_NATO_SET = set(_NATO_WORDS)
-_DIGIT_WORD_SET = {"zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten"}
-# Length caps (annotation-stripped word counts). A genuine acknowledgement is short; a long sentence
-# that merely starts "yeah …" isn't one. A spelled entity must *dominate* the turn — so we cap the
-# surrounding non-spelled (prose) words rather than total words (real codes are long).
-_ACKNOWLEDGEMENT_MAX_WORDS = 6
-_SPELL_MAX_EXTRA_WORDS = 10
-
-
-def _strip_annotations(text: str) -> str:
-    return _ANNOTATION_RE.sub(" ", text or "")
-
-
-def _non_spell_word_count(text: str) -> int:
-    """Words that are NOT part of a spell-out (single letters, digit-words, 'dash', NATO, caps codes)."""
-    n = 0
-    for raw in text.split():
-        tok = raw.strip(".,!?;:'\"()[]…-")
-        if not tok:
-            continue
-        low = tok.lower()
-        is_spell_tok = (
-            (len(tok) == 1 and tok.isalpha())
-            or low in _DIGIT_WORD_SET
-            or low == "dash"
-            or low in _NATO_SET
-            or (tok.isupper() and len(tok) >= 3)  # caps acronym (VPN, HIPAA)
-            or (len(tok) >= 3 and tok.isalnum() and any(c.isupper() for c in tok) and any(c.isdigit() for c in tok))
-        )
-        if not is_spell_tok:
-            n += 1
-    return n
-
-
-def _looks_spelled(text: str) -> bool:
-    """Any spelled-out signal — single-letter/digit runs, NATO, 'X as in Y', caps codes/acronyms."""
-    return bool(
-        _SINGLE_LETTER_RUN_RE.search(text)
-        or _SINGLE_DIGIT_RUN_RE.search(text)
-        or _AS_IN_RE.search(text)
-        or _NATO_WORD_RE.search(text)
-        or _CAPS_ALNUM_CODE_RE.search(text)
-        or _CAPS_ACRONYM_RE.search(text)
-        or _SPOKEN_DIGIT_RUN_RE.search(text)
-    )
-
-
-def final_turn_input_flags(text: str | None) -> dict[str, bool]:
-    """Orthogonal input-characteristic flags for the user's final intended utterance.
-
-    These are NOT pipeline causes — they describe the *input* and cut across the cause partition
-    (a failure can be both `vad_no_turn_detected` and have an acknowledgement final turn). Used to
-    test whether short / acknowledgement / spelled-out final turns correlate with failures.
-    Heuristics mirror the analysis app (EVABench `apps/analysis.py`) so the two tools agree.
-
-    - ``short``: 1-2 words (< 3), counted **after stripping `[annotation]` tags**.
-    - ``acknowledgement``: leads with a confirmation/ack ("Yes, that is correct.", "Ok thanks",
-      "[neutral] Okay.", "No.") AND is at most ``_ACKNOWLEDGEMENT_MAX_WORDS`` words — a long sentence
-      that merely starts "yeah …" is not counted.
-    - ``spelled_entity``: letter/digit spell-out of an ID/code/name ("E M P eight nine …", NATO,
-      "V as in Victor", caps codes like EMP358) that **dominates** the turn (≤ ``_SPELL_MAX_EXTRA_WORDS``
-      non-spelled words) — a long sentence with a small spelled fragment is not counted. Single-char
-      runs need ≥3 chars, so 2-letter words like "IT" ("I T") don't count.
-    """
-    if not text or not text.strip():
-        return {"short": False, "acknowledgement": False, "spelled_entity": False}
-    stripped = _strip_annotations(text).strip()
-    word_count = len(stripped.split())
-    return {
-        "short": 0 < word_count < 3,
-        "acknowledgement": word_count <= _ACKNOWLEDGEMENT_MAX_WORDS and bool(_ACKNOWLEDGEMENT_LEAD.match(stripped)),
-        "spelled_entity": _looks_spelled(stripped) and _non_spell_word_count(stripped) <= _SPELL_MAX_EXTRA_WORDS,
-    }
-
-
-def build_final_turn_flag_sub_metrics(
-    user_final_text: str | None,
-    parent_name: str = "conversation_correctly_finished",
-) -> dict[str, "MetricScore"]:
-    """Binary flags (``1.0`` = present) for final-turn input characteristics — orthogonal to causes."""
-    flags = final_turn_input_flags(user_final_text)
-    subs: dict[str, MetricScore] = {}
-    for key in ("short", "acknowledgement", "spelled_entity"):
-        occurred = flags[key]
-        score = 1.0 if occurred else 0.0
-        sub_key = f"final_turn_{key}_rate"
-        subs[sub_key] = MetricScore(
-            name=f"{parent_name}.{sub_key}",
-            score=score,
-            normalized_score=score,
-            details={"final_user_text": user_final_text} if occurred else {},
-        )
-    return subs
-
-
-def build_conv_finish_sub_metrics(
-    classification: Classification,
-    parent_name: str = "conversation_correctly_finished",
-) -> dict[str, "MetricScore"]:
-    """Per-cause binary-flag sub-metrics (framework convention: ``1.0`` = this cause occurred).
-
-    One flag per category in ``CATEGORY_PRIORITY``, keyed ``<cause>_rate`` (the ``_rate`` suffix
-    signals lower-is-better to ``direction_for_sub_metric``). Emitted on every record (all 0.0
-    when ``classification.category`` is None, i.e. a clean finish), so the cross-record mean reads
-    as "fraction of all conversations with this cause" — an all-records denominator matching
-    faithfulness. The fired cause carries the evidence details.
-    """
-    subs: dict[str, MetricScore] = {}
-    for cause in CATEGORY_PRIORITY:
-        occurred = cause == classification.category
-        score = 1.0 if occurred else 0.0
-        sub_key = f"{cause}_rate"
-        subs[sub_key] = MetricScore(
-            name=f"{parent_name}.{sub_key}",
-            score=score,
-            normalized_score=score,
-            details=classification.details if occurred else {},
-        )
-    return subs
-
-
-def _infra_details(component: str, s: ConvFinishSignals) -> dict[str, Any]:
-    return {
-        "invalid_run": True,
-        "component": component,
-        "service": s.service_error_name,
-        "error_excerpt": s.service_error_excerpt,
-        "num_error_frames": s.num_service_error_frames,
-        "assistant_audio_events": s.assistant_audio_events,
-    }
diff --git a/src/eva/metrics/diagnostic/conversation_correctly_finished.py b/src/eva/metrics/diagnostic/conversation_correctly_finished.py
index f017d1c5..353e418a 100644
--- a/src/eva/metrics/diagnostic/conversation_correctly_finished.py
+++ b/src/eva/metrics/diagnostic/conversation_correctly_finished.py
@@ -1,16 +1,16 @@
 """Conversation-correctly-finished diagnostic metric."""
 
 from eva.metrics.base import CodeMetric, MetricContext
-from eva.metrics.diagnostic.conv_finish_classifier import (
+from eva.metrics.processor import is_agent_timeout_on_user_turn, last_audio_speaker
+from eva.metrics.registry import register_metric
+from eva.models.results import MetricScore
+from eva.utils.conversation_correctly_finished import (
     Classification,
     build_conv_finish_sub_metrics,
     build_final_turn_flag_sub_metrics,
     classify_conv_finish_failure,
     extract_conv_finish_signals,
 )
-from eva.metrics.processor import is_agent_timeout_on_user_turn, last_audio_speaker
-from eva.metrics.registry import register_metric
-from eva.models.results import MetricScore
 
 
 @register_metric
@@ -23,7 +23,7 @@ class ConversationCorrectlyFinishedMetric(CodeMetric):
     so their cross-record mean reads as "fraction of all conversations with this cause" —
     matching how faithfulness dimension rates aggregate (denominator = all records, not just
     failures). The orthogonal input-characteristic flags (``final_turn_*``) stay failure-only,
-    since they describe the unanswered final turn. See ``docs/metrics/conv_finish_submetrics.md``.
+    since they describe the unanswered final turn.
     """
 
     name = "conversation_correctly_finished"
diff --git a/src/eva/utils/conversation_correctly_finished/__init__.py b/src/eva/utils/conversation_correctly_finished/__init__.py
new file mode 100644
index 00000000..2731af4f
--- /dev/null
+++ b/src/eva/utils/conversation_correctly_finished/__init__.py
@@ -0,0 +1,27 @@
+"""Classifier for `conversation_correctly_finished` failures — one primary cause per record.
+
+Modules: ``signals`` (contract), ``classifier`` (extract → classify → build sub-metrics),
+``causes/*`` (per-cause detect + extract), ``final_turn`` (input-characteristic flags). Public API
+re-exported below.
+"""
+
+from eva.utils.conversation_correctly_finished.classifier import (
+    CATEGORY_PRIORITY,
+    build_conv_finish_sub_metrics,
+    build_final_turn_flag_sub_metrics,
+    classify_conv_finish_failure,
+    extract_conv_finish_signals,
+)
+from eva.utils.conversation_correctly_finished.final_turn import final_turn_input_flags
+from eva.utils.conversation_correctly_finished.signals import Classification, ConvFinishSignals
+
+__all__ = [
+    "CATEGORY_PRIORITY",
+    "Classification",
+    "ConvFinishSignals",
+    "build_conv_finish_sub_metrics",
+    "build_final_turn_flag_sub_metrics",
+    "classify_conv_finish_failure",
+    "extract_conv_finish_signals",
+    "final_turn_input_flags",
+]
diff --git a/src/eva/utils/conversation_correctly_finished/causes/__init__.py b/src/eva/utils/conversation_correctly_finished/causes/__init__.py
new file mode 100644
index 00000000..0438d744
--- /dev/null
+++ b/src/eva/utils/conversation_correctly_finished/causes/__init__.py
@@ -0,0 +1 @@
+"""Per-cause modules — each owns one cause-family's log/CSV/pipecat parsing and its ``detect_*`` logic."""
diff --git a/src/eva/utils/conversation_correctly_finished/causes/api_errors.py b/src/eva/utils/conversation_correctly_finished/causes/api_errors.py
new file mode 100644
index 00000000..abd02fce
--- /dev/null
+++ b/src/eva/utils/conversation_correctly_finished/causes/api_errors.py
@@ -0,0 +1,89 @@
+"""Causes: infra API errors — TTS/STT service frames and fatal LLM API errors.
+
+They invalidate the run, so the classifier ranks them ahead of any behavioral cause.
+"""
+
+import re
+from typing import Any
+
+from eva.utils.conversation_correctly_finished.signals import Classification, ConvFinishSignals
+from eva.utils.log_processing import parse_log_message
+
+# Fatal LLM API-error signatures — allowlist, not a broad `litellm.*Error` match.
+_RE_LLM_ERR = re.compile(
+    r"MidStreamFallbackError|APIConnectionError|Retryable streaming error|RateLimitError|InternalServerError"
+)
+_RE_SERVICE = re.compile(r"([A-Za-z0-9]+(?:TTS|STT)Service)")
+
+
+def extract_service_errors(pipecat_events: list[dict], s: ConvFinishSignals) -> None:
+    """Set tts/stt service-error flags from pipecat error frames."""
+    for e in pipecat_events:
+        if e.get("type") != "error":
+            continue
+        frame = str(e.get("data", {}).get("frame", ""))
+        m = _RE_SERVICE.search(frame)
+        if not m:
+            continue
+        s.num_service_error_frames += 1
+        svc = m.group(1)
+        if "TTS" in svc:
+            s.tts_service_error = True
+            s.service_error_name = svc
+        elif "STT" in svc:
+            s.stt_service_error = True
+            s.service_error_name = svc
+        s.service_error_excerpt = frame[:160]
+
+
+def extract_log_signals(lines: list[str], resp_idx: list[int], s: ConvFinishSignals) -> None:
+    """LLM API error: a fatal error with no non-empty response after the last error line."""
+    err_idx = [i for i, ln in enumerate(lines) if _RE_LLM_ERR.search(ln)]
+    if not err_idx:
+        return
+    last_err = err_idx[-1]
+    s.num_llm_api_error_lines = len(err_idx)
+    s.llm_api_error_terminal = not any(i > last_err for i in resp_idx)
+    if s.llm_api_error_terminal:
+        m = _RE_LLM_ERR.search(lines[last_err])
+        s.llm_error_type = m.group(0) if m else ""
+        s.llm_error_excerpt = parse_log_message(lines[last_err])[:160]
+
+
+def _infra_details(component: str, s: ConvFinishSignals) -> dict[str, Any]:
+    """Shared details block for the tts/stt infra-error classifications."""
+    return {
+        "invalid_run": True,
+        "component": component,
+        "service": s.service_error_name,
+        "error_excerpt": s.service_error_excerpt,
+        "num_error_frames": s.num_service_error_frames,
+        "assistant_audio_events": s.assistant_audio_events,
+    }
+
+
+def detect_service_error(s: ConvFinishSignals, base: dict[str, Any]) -> Classification | None:
+    """Infra: a TTS/STT service raised an error frame — the agent was never exercised."""
+    if s.tts_service_error:
+        return Classification("tts_api_error", {**base, **_infra_details("tts", s)})
+    if s.stt_service_error:
+        return Classification("stt_api_error", {**base, **_infra_details("stt", s)})
+    return None
+
+
+def detect_llm_api_error(s: ConvFinishSignals, base: dict[str, Any]) -> Classification | None:
+    """Infra: a fatal LLM API error with no successful response after it (invalid run)."""
+    if not s.llm_api_error_terminal:
+        return None
+    return Classification(
+        "llm_api_error",
+        {
+            **base,
+            "invalid_run": True,
+            "component": "llm",
+            "error_type": s.llm_error_type,
+            "error_excerpt": s.llm_error_excerpt,
+            "num_api_error_lines": s.num_llm_api_error_lines,
+            "responses_after_last_error": 0,
+        },
+    )
diff --git a/src/eva/utils/conversation_correctly_finished/causes/interruption.py b/src/eva/utils/conversation_correctly_finished/causes/interruption.py
new file mode 100644
index 00000000..93f2e980
--- /dev/null
+++ b/src/eva/utils/conversation_correctly_finished/causes/interruption.py
@@ -0,0 +1,40 @@
+"""Cause: a real accepted user turn was cancelled by an interruption and never resumed (``ended_with_user_interruption``)."""
+
+import re
+from typing import Any
+
+from eva.utils.conversation_correctly_finished.signals import Classification, ConvFinishSignals
+from eva.utils.log_processing import parse_log_message
+
+_RE_INTR = re.compile(r"Interruption received - cancelling ongoing")
+_RE_PROC = re.compile(r"Processing complete user turn: (.+)$")
+
+
+def extract_log_signals(lines: list[str], resp_idx: list[int], s: ConvFinishSignals) -> None:
+    """Interruption: cancelled a real accepted turn, with no response after the last interruption."""
+    intr_idx = [i for i, ln in enumerate(lines) if _RE_INTR.search(ln)]
+    if not intr_idx:
+        return
+    last_intr = intr_idx[-1]
+    s.num_interruptions = len(intr_idx)
+    proc = [(i, m.group(1).strip()) for i, ln in enumerate(lines) if (m := _RE_PROC.search(parse_log_message(ln)))]
+    proc_before = [t for (i, t) in proc if i < last_intr]
+    s.accepted_turn_before_last_interruption = proc_before[-1] if proc_before else None
+    s.response_after_last_interruption = any(i > last_intr for i in resp_idx)
+
+
+def detect_interruption(s: ConvFinishSignals, base: dict[str, Any]) -> Classification | None:
+    """A real accepted turn was cancelled by an interruption and never resumed."""
+    if not (
+        s.num_interruptions > 0 and s.accepted_turn_before_last_interruption and not s.response_after_last_interruption
+    ):
+        return None
+    return Classification(
+        "ended_with_user_interruption",
+        {
+            **base,
+            "accepted_user_turn": s.accepted_turn_before_last_interruption,
+            "num_interruptions": s.num_interruptions,
+            "responses_after_last_interruption": 0,
+        },
+    )
diff --git a/src/eva/utils/conversation_correctly_finished/causes/reasoning.py b/src/eva/utils/conversation_correctly_finished/causes/reasoning.py
new file mode 100644
index 00000000..e5646e7f
--- /dev/null
+++ b/src/eva/utils/conversation_correctly_finished/causes/reasoning.py
@@ -0,0 +1,62 @@
+"""Cause: the model reasoned but produced no spoken answer / tool call (``reasoning_only`` / ``reasoning_too_long``)."""
+
+import csv
+from pathlib import Path
+from typing import Any
+
+from eva.utils.conversation_correctly_finished.signals import Classification, ConvFinishSignals
+
+csv.field_size_limit(10**7)  # agent_perf_stats rows embed full prompts
+
+
+def extract_perf_stats(perf: Path, s: ConvFinishSignals) -> None:
+    """Reasoning signals from the last agent_perf_stats.csv row."""
+    if not perf.exists():
+        return
+    try:
+        rows = list(csv.DictReader(perf.open(newline="")))
+    except (OSError, csv.Error):
+        s.notes.append("agent_perf_stats unreadable")
+        return
+    s.num_llm_calls = len(rows)
+    if not rows:
+        return
+    last = rows[-1]
+    s.last_perf_response_empty = not (last.get("response") or "").strip()
+    s.last_perf_has_tool_call = bool((last.get("tool_calls") or "").strip())
+    # The perf-stats writer wraps non-empty reasoning as f'"{content}"' and stores empty reasoning
+    # as an empty cell. Strip any wrapping quotes; both cases read as expected. (Older runs may
+    # still contain the literal '""' — stripping handles those too.)
+    s.last_perf_reasoning = (last.get("reasoning") or "").strip().strip('"').strip()
+    try:
+        s.last_perf_reasoning_tokens = int(float(last.get("reasoning_tokens") or 0))
+    except (TypeError, ValueError):
+        s.last_perf_reasoning_tokens = 0
+    s.last_perf_stop_reason = (last.get("stop_reason") or "").strip()
+
+
+def detect_reasoning(s: ConvFinishSignals, base: dict[str, Any]) -> Classification | None:
+    """The model reasoned but produced no spoken answer / tool call.
+
+    Reasoning is evidenced by visible text OR a hidden-reasoning token count (gemini/gpt-5 report
+    ``reasoning_tokens`` without returning the thinking text). Split by whether it hit the token cap:
+    ``stop_reason == 'length'`` → ``reasoning_too_long`` (ran out of tokens mid-reasoning);
+    otherwise → ``reasoning_only`` (finished, but only reasoning came out).
+    """
+    reasoned = s.last_perf_reasoning.strip() or s.last_perf_reasoning_tokens > 0
+    if not (s.last_perf_response_empty and not s.last_perf_has_tool_call and reasoned):
+        return None
+    category = "reasoning_too_long" if s.last_perf_stop_reason == "length" else "reasoning_only"
+    return Classification(
+        category,
+        {
+            **base,
+            "final_response_empty": True,
+            "final_row_had_tool_call": False,
+            "reasoning_preview": s.last_perf_reasoning[:300],  # empty when reasoning is hidden
+            "reasoning_chars": len(s.last_perf_reasoning),
+            "reasoning_tokens": s.last_perf_reasoning_tokens,
+            "stop_reason": s.last_perf_stop_reason,
+            "num_llm_calls": s.num_llm_calls,
+        },
+    )
diff --git a/src/eva/utils/conversation_correctly_finished/causes/transcript_vad.py b/src/eva/utils/conversation_correctly_finished/causes/transcript_vad.py
new file mode 100644
index 00000000..7aa590ef
--- /dev/null
+++ b/src/eva/utils/conversation_correctly_finished/causes/transcript_vad.py
@@ -0,0 +1,87 @@
+"""Causes: the user spoke but the agent never got the turn.
+
+STT gave no usable transcript (``stt_missing_transcription`` / ``stt_empty_transcription``) or VAD
+never fired for the final utterance (``vad_no_turn_detected``).
+"""
+
+import re
+from typing import Any
+
+from eva.utils.conversation_correctly_finished.signals import Classification, ConvFinishSignals
+
+_RE_VADNOTX = re.compile(r"VAD fired but no previous transcription timestamp found")
+_RE_TXSTOP_EMPTY = re.compile(r"User turn stopped - complete transcript: ''")
+_RE_STILL = re.compile(r"Assistant still speaking - resetting inactivity")
+_VAD_WINDOW_S = 1.5
+
+
+def extract_vad_onsets(pipecat_events: list[dict]) -> list[float]:
+    """VAD onset timestamps (seconds) from pipecat ``user_started_speaking`` events."""
+    return [e["timestamp"] / 1000.0 for e in pipecat_events if e.get("type") == "user_started_speaking"]
+
+
+def detect_final_utterance_vad(
+    s: ConvFinishSignals,
+    u_starts: list[float],
+    u_ends: list[float],
+    asst_ends: list[float],
+    vad_onsets: list[float],
+) -> None:
+    """Whether the user's final emitted utterance came after the agent with no VAD onset in-window."""
+    if not (u_starts and asst_ends):
+        return
+    fu_start = max(u_starts)
+    fu_end = max([e for e in u_ends if e >= fu_start], default=fu_start + 1.0)
+    s.user_final_utterance_after_agent = fu_start >= max(asst_ends) - 0.5
+    s.vad_onset_in_final_window = any(fu_start - _VAD_WINDOW_S <= t <= fu_end + _VAD_WINDOW_S for t in vad_onsets)
+
+
+def extract_log_signals(lines: list[str], resp_idx: list[int], s: ConvFinishSignals) -> None:
+    """STT no-transcription: a VAD warning / empty transcript after the last real response.
+
+    Also records ``assistant_still_speaking_before`` (a VAD-cause signal read from the same lines).
+    """
+    s.assistant_still_speaking_before = any(_RE_STILL.search(ln) for ln in lines)
+    last_resp = resp_idx[-1] if resp_idx else -1
+    vadnotx_idx = [i for i, ln in enumerate(lines) if _RE_VADNOTX.search(ln)]
+    empty_tx_idx = [i for i, ln in enumerate(lines) if _RE_TXSTOP_EMPTY.search(ln)]
+    s.num_vad_no_tx_warnings = len(vadnotx_idx)
+    s.vad_no_tx_warning_after_last_response = any(i > last_resp for i in vadnotx_idx)
+    s.empty_transcript_after_last_response = any(i > last_resp for i in empty_tx_idx)
+
+
+def detect_stt_no_transcription(s: ConvFinishSignals, base: dict[str, Any]) -> Classification | None:
+    """STT gave no usable transcription for a turn VAD detected (empty vs missing transcript)."""
+    if not (s.is_cascade and s.last_conv_message_role == "assistant" and s.vad_no_tx_warning_after_last_response):
+        return None
+    category = "stt_empty_transcription" if s.empty_transcript_after_last_response else "stt_missing_transcription"
+    return Classification(
+        category,
+        {
+            **base,
+            "vad_fired": True,
+            "final_user_turn_in_llm_context": False,
+            "user_turn_stopped_emitted": s.empty_transcript_after_last_response,
+            "user_said_ground_truth": s.user_final_words,
+            "num_vad_no_transcription_warnings": s.num_vad_no_tx_warnings,
+        },
+    )
+
+
+def detect_vad_no_turn(s: ConvFinishSignals, base: dict[str, Any]) -> Classification | None:
+    """VAD never fired for the user's final (emitted) utterance after the agent finished."""
+    if not (
+        s.user_final_utterance_after_agent
+        and not s.vad_onset_in_final_window
+        and s.last_conv_message_role == "assistant"
+    ):
+        return None
+    return Classification(
+        "vad_no_turn_detected",
+        {
+            **base,
+            "user_said_ground_truth": s.user_final_words,
+            "assistant_still_speaking_before": s.assistant_still_speaking_before,
+            "vad_onsets_in_window": 0,
+        },
+    )
diff --git a/src/eva/utils/conversation_correctly_finished/classifier.py b/src/eva/utils/conversation_correctly_finished/classifier.py
new file mode 100644
index 00000000..b94ca5ba
--- /dev/null
+++ b/src/eva/utils/conversation_correctly_finished/classifier.py
@@ -0,0 +1,201 @@
+"""Core pipeline: extract signals from a record, classify the primary cause, build sub-metric flags.
+
+Three stages, top to bottom:
+
+1. **Extraction** (I/O) — read a record's raw files into a ``ConvFinishSignals``. The orchestrator
+   reads each file once, sets the shared fields, and hands pre-parsed data to each ``causes/*``
+   extractor (which owns that cause's signatures).
+2. **Classification** (pure logic, no I/O) — order the per-cause ``detect_*`` functions by priority
+   and return the single primary cause (or ``None`` when the record isn't a parent failure).
+3. **Sub-metric builders** — turn a ``Classification`` / final-turn text into ``MetricScore`` flags.
+"""
+
+import re
+from pathlib import Path
+from typing import Any
+
+from eva.metrics.processor import is_agent_timeout_on_user_turn
+from eva.models.results import MetricScore
+from eva.utils.conversation_correctly_finished.causes import api_errors, interruption, reasoning, transcript_vad
+from eva.utils.conversation_correctly_finished.final_turn import final_turn_input_flags
+from eva.utils.conversation_correctly_finished.signals import Classification, ConvFinishSignals
+from eva.utils.json_utils import load_jsonl
+from eva.utils.log_processing import load_audit_log, parse_log_message
+
+# Shared "the agent produced a real spoken response" signal — several causes ask "was there a
+# response after event X?", so its line index is computed once and passed to each extractor.
+_RE_RESP = re.compile(r"complete response: '(.*)'$")
+
+
+# --- extraction (I/O) -------------------------------------------------------
+
+
+def _extract_audit_log(audit: Path, s: ConvFinishSignals) -> None:
+    """Last conversation message role, from audit_log.json."""
+    if not audit.exists():
+        return
+    data = load_audit_log(audit)
+    if data is None:
+        s.notes.append("audit_log unreadable")
+        return
+    try:
+        cm = data.get("conversation_messages", [])
+        s.last_conv_message_role = cm[-1]["role"] if cm else None
+    except KeyError:
+        s.notes.append("audit_log unreadable")
+
+
+def _extract_sim_audio(out: Path, s: ConvFinishSignals) -> tuple[list[float], list[float], list[float]]:
+    """User's final words + assistant-audio count; returns (user_starts, user_ends, assistant_ends)."""
+    ev = load_jsonl(out / "user_simulator_events.jsonl")
+    user_speech = [e for e in ev if e.get("type") == "user_speech"]
+    s.user_final_words = user_speech[-1]["data"]["text"] if user_speech else None
+    s.assistant_audio_events = sum(
+        1 for e in ev if e.get("event_type") == "audio_start" and e.get("user") == "assistant"
+    )
+    asst_ends = [
+        e["audio_timestamp"] for e in ev if e.get("event_type") == "audio_end" and e.get("user") == "assistant"
+    ]
+    u_starts = [
+        e["audio_timestamp"] for e in ev if e.get("event_type") == "audio_start" and e.get("user") == "simulated_user"
+    ]
+    u_ends = [
+        e["audio_timestamp"] for e in ev if e.get("event_type") == "audio_end" and e.get("user") == "simulated_user"
+    ]
+    return u_starts, u_ends, asst_ends
+
+
+def _extract_log_signals(lines: list[str], s: ConvFinishSignals) -> None:
+    """Parse logs.log once into the shared ``resp_idx``, then fan out to per-cause extractors."""
+    resp_idx = [i for i, ln in enumerate(lines) if (m := _RE_RESP.search(parse_log_message(ln))) and m.group(1).strip()]
+    transcript_vad.extract_log_signals(lines, resp_idx, s)
+    api_errors.extract_log_signals(lines, resp_idx, s)
+    interruption.extract_log_signals(lines, resp_idx, s)
+
+
+def extract_conv_finish_signals(context) -> ConvFinishSignals:  # noqa: ANN001 (MetricContext)
+    """Read a record's raw files and build the signals the classifier needs."""
+    s = ConvFinishSignals()
+    s.is_parent_failure = is_agent_timeout_on_user_turn(
+        context.conversation_ended_reason,
+        context.audio_timestamps_user_turns,
+        context.audio_timestamps_assistant_turns,
+    )
+    s.is_cascade = not context.is_audio_native
+    out = Path(context.output_dir) if context.output_dir else None
+    if out is None:
+        s.notes.append("no output_dir")
+        return s
+
+    _extract_audit_log(out / "audit_log.json", s)
+    perf = out / "agent_perf_stats.csv"
+    reasoning.extract_perf_stats(perf, s)
+
+    u_starts, u_ends, asst_ends = _extract_sim_audio(out, s)
+    pipecat_events = load_jsonl(out / "pipecat_logs.jsonl")
+    api_errors.extract_service_errors(pipecat_events, s)
+    vad_onsets = transcript_vad.extract_vad_onsets(pipecat_events)
+    transcript_vad.detect_final_utterance_vad(s, u_starts, u_ends, asst_ends, vad_onsets)
+
+    log_path = out / "logs.log"
+    if log_path.exists():
+        _extract_log_signals(log_path.read_text().splitlines(), s)
+    elif not (s.tts_service_error or s.stt_service_error or perf.exists()):
+        s.notes.append("no logs.log")
+
+    return s
+
+
+# --- classification (pure logic, no I/O) ------------------------------------
+
+
+def _detect_unknown(s: ConvFinishSignals, base: dict[str, Any]) -> Classification:
+    """Residual — no known cause matched; a candidate for a new sub-metric."""
+    return Classification(
+        "unknown_reason",
+        {
+            **base,
+            "last_conversation_message_role": s.last_conv_message_role,
+            "num_interruptions": s.num_interruptions,
+            "final_user_turn_transcript": s.user_final_words,
+            "note": "no known sub-metric matched — candidate for a new sub-metric",
+        },
+    )
+
+
+# The detector partition, in priority order — the first detector to return a Classification wins.
+# Each detector is paired with the category names it can emit (also in priority order). Pairing
+# them here makes this the single source of truth: CATEGORY_PRIORITY is derived from it, so the
+# documented priority can never drift from the order detectors actually run in.
+_DETECTORS = (
+    (api_errors.detect_service_error, ("tts_api_error", "stt_api_error")),
+    (api_errors.detect_llm_api_error, ("llm_api_error",)),
+    (reasoning.detect_reasoning, ("reasoning_too_long", "reasoning_only")),
+    (transcript_vad.detect_stt_no_transcription, ("stt_empty_transcription", "stt_missing_transcription")),
+    (interruption.detect_interruption, ("ended_with_user_interruption",)),
+    (transcript_vad.detect_vad_no_turn, ("vad_no_turn_detected",)),
+)
+
+# Every category a failure can be assigned, in priority order: derived from _DETECTORS (infra
+# causes first — they invalidate the run and supersede apparent behavior) plus the residual
+# catch-all, which has no detector and fires iff nothing else did.
+CATEGORY_PRIORITY = [category for _, categories in _DETECTORS for category in categories] + ["unknown_reason"]
+
+
+def classify_conv_finish_failure(s: ConvFinishSignals) -> Classification:
+    """Return the single primary cause for a conv_finish failure (or ``None`` if not one)."""
+    if not s.is_parent_failure:
+        return Classification(None, {})
+    base: dict[str, Any] = {"notes": list(s.notes)} if s.notes else {}
+    for detect, _categories in _DETECTORS:
+        result = detect(s, base)
+        if result is not None:
+            return result
+    return _detect_unknown(s, base)
+
+
+# --- sub-metric builders ----------------------------------------------------
+
+
+def _flag_sub_metric(parent_name: str, sub_key: str, occurred: bool, details: dict[str, Any]) -> MetricScore:
+    """A binary-flag sub-metric (framework convention: ``1.0`` = the flag fired)."""
+    score = 1.0 if occurred else 0.0
+    return MetricScore(
+        name=f"{parent_name}.{sub_key}",
+        score=score,
+        normalized_score=score,
+        details=details if occurred else {},
+    )
+
+
+def build_final_turn_flag_sub_metrics(
+    user_final_text: str | None,
+    parent_name: str = "conversation_correctly_finished",
+) -> dict[str, MetricScore]:
+    """Binary flags (``1.0`` = present) for final-turn input characteristics — orthogonal to causes."""
+    flags = final_turn_input_flags(user_final_text)
+    details = {"final_user_text": user_final_text}
+    return {
+        f"final_turn_{key}_rate": _flag_sub_metric(parent_name, f"final_turn_{key}_rate", flags[key], details)
+        for key in ("short", "acknowledgement", "spelled_entity")
+    }
+
+
+def build_conv_finish_sub_metrics(
+    classification: Classification,
+    parent_name: str = "conversation_correctly_finished",
+) -> dict[str, MetricScore]:
+    """Per-cause binary-flag sub-metrics (framework convention: ``1.0`` = this cause occurred).
+
+    One flag per category in ``CATEGORY_PRIORITY``, keyed ``<cause>_rate`` (the ``_rate`` suffix
+    signals lower-is-better to ``direction_for_sub_metric``). Emitted on every record (all 0.0
+    when ``classification.category`` is None, i.e. a clean finish), so the cross-record mean reads
+    as "fraction of all conversations with this cause" — an all-records denominator matching
+    faithfulness. The fired cause carries the evidence details.
+    """
+    return {
+        f"{cause}_rate": _flag_sub_metric(
+            parent_name, f"{cause}_rate", cause == classification.category, classification.details
+        )
+        for cause in CATEGORY_PRIORITY
+    }
diff --git a/src/eva/utils/conversation_correctly_finished/final_turn.py b/src/eva/utils/conversation_correctly_finished/final_turn.py
new file mode 100644
index 00000000..06c9216f
--- /dev/null
+++ b/src/eva/utils/conversation_correctly_finished/final_turn.py
@@ -0,0 +1,132 @@
+"""Final-turn input-characteristic flags — pure functions of the user's final utterance text.
+
+Orthogonal to the pipeline causes: they describe the *input*, not a failure mode.
+"""
+
+import re
+
+_ANNOTATION_RE = re.compile(r"\[[^\]]*\]")
+# Acknowledgement lead — (includes negations no/nope, tolerant of leading [tags]);
+# scope is "starts with a confirmation/acknowledgement".
+_ACKNOWLEDGEMENT_LEAD = re.compile(
+    r"^\s*(?:\[[^\]]*\]\s*)*\b(?:yes|no|nope|yep|yeah|sure|okay|ok|alright|correct|right)\b",
+    re.IGNORECASE,
+)
+# Spelled-entity signals
+# Single-letter / single-digit runs require ≥3 chars — 2 chars ("I T" = the word IT, "G A") isn't spelling.
+_SEPS = r"[\s.,;:\-…]+"
+_SINGLE_LETTER_RUN_RE = re.compile(rf"(?:\b[a-zA-Z]\b{_SEPS}){{2,}}\b[a-zA-Z]\b")
+_SINGLE_DIGIT_RUN_RE = re.compile(rf"(?:\b\d\b{_SEPS}){{2,}}\b\d\b")
+_AS_IN_RE = re.compile(r"\b[a-zA-Z]\s+as\s+in\s+\w+", re.IGNORECASE)
+_CAPS_ALNUM_CODE_RE = re.compile(r"\b(?=[A-Z0-9]{3,})(?=.*[A-Z])(?=.*\d)[A-Z0-9]+\b")
+_CAPS_ACRONYM_RE = re.compile(r"\b[A-Z]{3,}\b")
+# Single source of truth for the spoken-digit vocabulary → regex fragment + membership set.
+_DIGIT_WORDS = ("zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten")
+_DIGIT_WORD = r"(?:" + "|".join(_DIGIT_WORDS) + r")"
+_DIGIT_WORD_SET = set(_DIGIT_WORDS)
+_SPOKEN_DIGIT_RUN_RE = re.compile(rf"(?:\b{_DIGIT_WORD}\b[^a-zA-Z]+){{2,}}\b{_DIGIT_WORD}\b", re.IGNORECASE)
+_NATO_WORDS = (
+    "alpha",
+    "bravo",
+    "charlie",
+    "delta",
+    "echo",
+    "foxtrot",
+    "golf",
+    "hotel",
+    "india",
+    "juliet",
+    "juliett",
+    "kilo",
+    "lima",
+    "mike",
+    "november",
+    "oscar",
+    "papa",
+    "quebec",
+    "romeo",
+    "sierra",
+    "tango",
+    "uniform",
+    "victor",
+    "whiskey",
+    "x-ray",
+    "xray",
+    "yankee",
+    "zulu",
+)
+_NATO_WORD_RE = re.compile(r"\b(?:" + "|".join(re.escape(w) for w in _NATO_WORDS) + r")\b", re.IGNORECASE)
+_NATO_SET = set(_NATO_WORDS)
+# Length caps (annotation-stripped word counts). A genuine acknowledgement is short; a long sentence
+# that merely starts "yeah …" isn't one. A spelled entity must *dominate* the turn — so we cap the
+# surrounding non-spelled (prose) words rather than total words (real codes are long).
+_ACKNOWLEDGEMENT_MAX_WORDS = 6
+_SPELL_MAX_EXTRA_WORDS = 10
+
+
+def _strip_annotations(text: str) -> str:
+    """Remove prosody/annotation tags like [slow], [neutral], [user interrupts]."""
+    return _ANNOTATION_RE.sub(" ", text or "")
+
+
+def _non_spell_word_count(text: str) -> int:
+    """Words that are NOT part of a spell-out (single letters, digit-words, 'dash', NATO, caps codes)."""
+    n = 0
+    for raw in text.split():
+        tok = raw.strip(".,!?;:'\"()[]…-")
+        if not tok:
+            continue
+        low = tok.lower()
+        is_spell_tok = (
+            (len(tok) == 1 and tok.isalpha())
+            or low in _DIGIT_WORD_SET
+            or low == "dash"
+            or low in _NATO_SET
+            or (tok.isupper() and len(tok) >= 3)  # caps acronym (VPN, HIPAA)
+            or (len(tok) >= 3 and tok.isalnum() and any(c.isupper() for c in tok) and any(c.isdigit() for c in tok))
+        )
+        if not is_spell_tok:
+            n += 1
+    return n
+
+
+def _looks_spelled(text: str) -> bool:
+    """Any spelled-out signal — single-letter/digit runs, NATO, 'X as in Y', caps codes/acronyms."""
+    return bool(
+        _SINGLE_LETTER_RUN_RE.search(text)
+        or _SINGLE_DIGIT_RUN_RE.search(text)
+        or _AS_IN_RE.search(text)
+        or _NATO_WORD_RE.search(text)
+        or _CAPS_ALNUM_CODE_RE.search(text)
+        or _CAPS_ACRONYM_RE.search(text)
+        or _SPOKEN_DIGIT_RUN_RE.search(text)
+    )
+
+
+def final_turn_input_flags(text: str | None) -> dict[str, bool]:
+    """Orthogonal input-characteristic flags for the user's final intended utterance.
+
+    Used to test whether short / acknowledgement / spelled-out final turns correlate with failures.
+    Heuristics mirror the analysis app (EVABench `apps/analysis.py`) so the two tools agree, and are
+    **English-only** (the ack lead, spoken-digit words, and "X as in Y" cues are English phrasing;
+    single-letter/digit runs, caps codes, and NATO are largely language-neutral). Prosody/annotation
+    tags ([slow], [neutral], …) inflate word counts, so they're stripped first.
+
+    - ``short``: 1-2 words (< 3), counted **after stripping `[annotation]` tags**.
+    - ``acknowledgement``: leads with a confirmation/ack ("Yes, that is correct.", "Ok thanks",
+      "[neutral] Okay.", "No.") AND is at most ``_ACKNOWLEDGEMENT_MAX_WORDS`` words — a long sentence
+      that merely starts "yeah …" is not counted.
+    - ``spelled_entity``: letter/digit spell-out of an ID/code/name ("E M P eight nine …", NATO,
+      "V as in Victor", caps codes like EMP358) that **dominates** the turn (≤ ``_SPELL_MAX_EXTRA_WORDS``
+      non-spelled words) — a long sentence with a small spelled fragment is not counted. Single-char
+      runs need ≥3 chars, so 2-letter words like "IT" ("I T") don't count.
+    """
+    if not text or not text.strip():
+        return {"short": False, "acknowledgement": False, "spelled_entity": False}
+    stripped = _strip_annotations(text).strip()
+    word_count = len(stripped.split())
+    return {
+        "short": 0 < word_count < 3,
+        "acknowledgement": word_count <= _ACKNOWLEDGEMENT_MAX_WORDS and bool(_ACKNOWLEDGEMENT_LEAD.match(stripped)),
+        "spelled_entity": _looks_spelled(stripped) and _non_spell_word_count(stripped) <= _SPELL_MAX_EXTRA_WORDS,
+    }
diff --git a/src/eva/utils/conversation_correctly_finished/signals.py b/src/eva/utils/conversation_correctly_finished/signals.py
new file mode 100644
index 00000000..606f1a33
--- /dev/null
+++ b/src/eva/utils/conversation_correctly_finished/signals.py
@@ -0,0 +1,64 @@
+"""Shared data contract: ``ConvFinishSignals`` (signals extracted from a record) and ``Classification`` (the result)."""
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class ConvFinishSignals:
+    """Everything the classifier needs, extracted once from a record's raw files.
+
+    Defaults describe a *clean* parent-failure baseline (nothing detected → ``unknown_reason``).
+    """
+
+    # --- parent gate -------------------------------------------------------
+    is_parent_failure: bool = True  # inactivity_timeout AND user was last audio speaker
+    is_cascade: bool = True  # CASCADE stack (vs S2S / AUDIO_LLM)
+    last_conv_message_role: str | None = "assistant"
+
+    # --- reasoning_only / reasoning_too_long (agent_perf_stats last row) ----
+    last_perf_response_empty: bool = False
+    last_perf_has_tool_call: bool = False
+    last_perf_reasoning: str = ""
+    last_perf_reasoning_tokens: int = 0  # hidden-reasoning models report tokens but no text
+    last_perf_stop_reason: str = ""  # 'length' ⇒ hit the token cap (reasoning_too_long)
+    num_llm_calls: int = 0
+
+    # --- llm_api_error (logs.log litellm patterns) -------------------------
+    llm_api_error_terminal: bool = False
+    llm_error_type: str = ""
+    llm_error_excerpt: str = ""
+    num_llm_api_error_lines: int = 0
+
+    # --- tts / stt service errors (pipecat error frames) -------------------
+    tts_service_error: bool = False
+    stt_service_error: bool = False
+    service_error_name: str = ""
+    service_error_excerpt: str = ""
+    num_service_error_frames: int = 0
+    assistant_audio_events: int = 1
+
+    # --- stt no-transcription (logs.log) -----------------------------------
+    vad_no_tx_warning_after_last_response: bool = False
+    empty_transcript_after_last_response: bool = False
+    num_vad_no_tx_warnings: int = 0
+
+    # --- ended_with_user_interruption (logs.log) ---------------------------
+    num_interruptions: int = 0
+    accepted_turn_before_last_interruption: str | None = None
+    response_after_last_interruption: bool = True
+
+    # --- vad_no_turn_detected (sim events vs pipecat VAD) ------------------
+    user_final_utterance_after_agent: bool = False
+    vad_onset_in_final_window: bool = True
+    user_final_words: str | None = None
+    assistant_still_speaking_before: bool = False
+
+    # extraction problems (missing files); still classifiable, noted in details
+    notes: list[str] = field(default_factory=list)
+
+
+@dataclass
+class Classification:
+    category: str | None  # None when the record is not a parent failure
+    details: dict[str, Any]
diff --git a/src/eva/utils/json_utils.py b/src/eva/utils/json_utils.py
index 0f4ed74a..90ec45f2 100644
--- a/src/eva/utils/json_utils.py
+++ b/src/eva/utils/json_utils.py
@@ -2,6 +2,7 @@
 import logging
 import re
 from collections.abc import Generator
+from pathlib import Path
 
 logger = logging.getLogger(__name__)
 
@@ -53,3 +54,22 @@ def extract_and_load_json(text: str) -> dict | list | None:
     if json_object is None:
         logger.warning(f"Error extracting JSON from text: {text}")
     return json_object
+
+
+def load_jsonl(path: Path) -> list[dict]:
+    """Parse a JSONL file into dicts, skipping blank/malformed lines; empty if absent.
+
+    For known-format ``.jsonl`` logs (not free-form LLM output), so ``json.loads`` per line is the
+    right tool — use ``extract_and_load_json`` only for LLM-generated JSON.
+    """
+    out: list[dict] = []
+    if not path.exists():
+        return out
+    for line in path.read_text().splitlines():
+        line = line.strip()
+        if line:
+            try:
+                out.append(json.loads(line))
+            except json.JSONDecodeError:
+                continue
+    return out
diff --git a/src/eva/utils/log_processing.py b/src/eva/utils/log_processing.py
index 23235cc1..102e8386 100644
--- a/src/eva/utils/log_processing.py
+++ b/src/eva/utils/log_processing.py
@@ -2,10 +2,28 @@
 
 import copy
 import itertools
+import json
 from enum import StrEnum
+from pathlib import Path
 from typing import Any
 
 
+def parse_log_message(line: str) -> str:
+    """The message portion of a ``logs.log`` line ('… | msg'), or the whole line."""
+    return line.rsplit(" | ", 1)[-1]
+
+
+def load_audit_log(path: Path) -> dict | None:
+    """Parse ``audit_log.json`` into a dict; ``None`` if missing, unparseable, or not a JSON object."""
+    if not path.exists():
+        return None
+    try:
+        data = json.loads(path.read_text())
+    except json.JSONDecodeError:
+        return None
+    return data if isinstance(data, dict) else None
+
+
 class AnnotationLabel(StrEnum):
     """Annotation labels inserted into transcript text to mark interruption events."""
 
diff --git a/tests/fixtures/metric_signatures.json b/tests/fixtures/metric_signatures.json
index 2eacb778..9e20b166 100644
--- a/tests/fixtures/metric_signatures.json
+++ b/tests/fixtures/metric_signatures.json
@@ -14,7 +14,7 @@
   "ConversationCorrectlyFinishedMetric": {
     "name": "conversation_correctly_finished",
     "prompt_hash": null,
-    "source_hash": "cee94b5782ac",
+    "source_hash": "9cf0a1d26e80",
     "version": "v0.2"
   },
   "ConversationProgressionJudgeMetric": {
diff --git a/tests/unit/metrics/test_conv_finish_extraction.py b/tests/unit/metrics/test_conv_finish_extraction.py
index 81cefdfe..45f4bfdf 100644
--- a/tests/unit/metrics/test_conv_finish_extraction.py
+++ b/tests/unit/metrics/test_conv_finish_extraction.py
@@ -2,7 +2,7 @@
 
 import json
 
-from eva.metrics.diagnostic.conv_finish_classifier import (
+from eva.utils.conversation_correctly_finished import (
     classify_conv_finish_failure,
     extract_conv_finish_signals,
 )
diff --git a/tests/unit/metrics/test_conv_finish_submetrics.py b/tests/unit/metrics/test_conv_finish_submetrics.py
index 1f658e19..3c5af9c0 100644
--- a/tests/unit/metrics/test_conv_finish_submetrics.py
+++ b/tests/unit/metrics/test_conv_finish_submetrics.py
@@ -33,7 +33,7 @@ def _fail_ctx(tmp_path, **overrides):
 
 @pytest.mark.asyncio
 async def test_success_emits_zero_cause_rates(metric):
-    from eva.metrics.diagnostic.conv_finish_classifier import CATEGORY_PRIORITY
+    from eva.utils.conversation_correctly_finished import CATEGORY_PRIORITY
 
     ctx = make_metric_context(
         conversation_ended_reason="goodbye",
@@ -65,7 +65,7 @@ async def test_failure_reasoning_only_sub_metric_fires(metric, tmp_path):
     assert subs["stt_empty_transcription_rate"].score == 0.0
     assert subs["unknown_reason_rate"].score == 0.0
     # exactly one *cause* flag is hot (input-characteristic flags are orthogonal, excluded here)
-    from eva.metrics.diagnostic.conv_finish_classifier import CATEGORY_PRIORITY
+    from eva.utils.conversation_correctly_finished import CATEGORY_PRIORITY
 
     assert sum(subs[f"{c}_rate"].score for c in CATEGORY_PRIORITY) == 1.0
     # fired sub-metric name is namespaced under the parent
diff --git a/tests/unit/metrics/test_conv_finish_classifier.py b/tests/unit/metrics/test_conversation_finish_classifier.py
similarity index 98%
rename from tests/unit/metrics/test_conv_finish_classifier.py
rename to tests/unit/metrics/test_conversation_finish_classifier.py
index ee6edd1c..439f804d 100644
--- a/tests/unit/metrics/test_conv_finish_classifier.py
+++ b/tests/unit/metrics/test_conversation_finish_classifier.py
@@ -2,7 +2,7 @@
 
 import pytest
 
-from eva.metrics.diagnostic.conv_finish_classifier import (
+from eva.utils.conversation_correctly_finished import (
     ConvFinishSignals,
     classify_conv_finish_failure,
 )
@@ -215,7 +215,7 @@ def test_infra_details_flag_invalid_run():
 
 
 # --- final-turn input flags (orthogonal to the cause classifier) ------------
-from eva.metrics.diagnostic.conv_finish_classifier import final_turn_input_flags  # noqa: E402
+from eva.utils.conversation_correctly_finished import final_turn_input_flags  # noqa: E402
 
 
 @pytest.mark.parametrize(

From 2f92cf7ee00403bc6139990d918e232e00942675 Mon Sep 17 00:00:00 2001
From: Gabrielle Gauthier-Melancon <gabrielle.gm@servicenow.com>
Date: Thu, 23 Jul 2026 16:18:46 -0400
Subject: [PATCH 08/10] Improve documentation

---
 .../conversation_correctly_finished.md        | 33 +++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/docs/metrics/conversation_correctly_finished.md b/docs/metrics/conversation_correctly_finished.md
index 3d15dfe4..1b650459 100644
--- a/docs/metrics/conversation_correctly_finished.md
+++ b/docs/metrics/conversation_correctly_finished.md
@@ -53,6 +53,38 @@ Uses the following MetricContext fields:
 }
 ```
 
+## Diagnostic Sub-Metrics
+
+When a record fails (score 0.0), a deterministic classifier assigns **exactly one primary cause** for why the agent went silent, emitted as sub-metrics keyed `conversation_correctly_finished.<cause>_rate`.
+
+**General logic:**
+- **One cause per failure.** Causes are checked in a fixed priority order (infra errors first, since they invalidate the run) and the first match wins — causes never double-count. `unknown_reason` is the residual when nothing else matches.
+- **Score convention (`_rate`)**: `1.0` = this cause occurred, `0.0` = it did not (lower is better). Every cause flag is emitted on every record (all `0.0` on a clean finish), so the cross-record mean reads as "fraction of conversations with this cause".
+- **Signals come from the raw record files** (`agent_perf_stats.csv`, `logs.log`, `pipecat_logs.jsonl`, `user_simulator_events.jsonl`, `audit_log.json`). A missing file just means that cause can't fire — it's noted in `details.notes`, never an error.
+
+**Causes** (in priority order):
+
+| Cause | Detected when |
+|-------|---------------|
+| `tts_api_error` | `pipecat_logs.jsonl` error frame naming a `*TTSService` (infra → `invalid_run`) |
+| `stt_api_error` | `pipecat_logs.jsonl` error frame naming a `*STTService` (infra → `invalid_run`) |
+| `llm_api_error` | fatal LLM error line in `logs.log` (allowlist) with no response after it (infra → `invalid_run`) |
+| `reasoning_too_long` | last `agent_perf_stats.csv` row: empty `response`, no tool call, model reasoned, `stop_reason == length` (ran out of tokens mid-reasoning) |
+| `reasoning_only` | same, but `stop_reason != length` — reasoned (visible text **or** `reasoning_tokens > 0`) yet emitted no answer |
+| `stt_empty_transcription` | cascade, last message from assistant, `VAD fired but no transcription` after the last response, plus a `User turn stopped - complete transcript: ''` line (turn closed empty) |
+| `stt_missing_transcription` | same, but **no** `User turn stopped` line (turn never closed) |
+| `ended_with_user_interruption` | an accepted non-empty user turn, then an interruption, with no response after it |
+| `vad_no_turn_detected` | the user's final utterance came after the agent, but no VAD onset fired in its ±1.5s window |
+| `unknown_reason` | none of the above (residual; a discovery queue for new causes) |
+
+**Input-characteristic flags** are also emitted on failures. These are *orthogonal* — they describe the user's final utterance, not a cause, so they overlap each other and the causes and do **not** sum to 1.0.
+
+| Flag (`…_rate`) | Fires when |
+|-----------------|-----------|
+| `final_turn_short` | 1–2 words, after stripping `[annotation]` tags |
+| `final_turn_acknowledgement` | leads with `yes/no/ok/sure/…` **and** ≤ 6 words |
+| `final_turn_spelled_entity` | a spelled ID/code/name (letter/digit runs ≥3 chars, NATO, "X as in Y", caps codes) that dominates the turn |
+
 ## Related Metrics
 
 - [conversation_valid_end.md](conversation_valid_end.md) - Validates the conversation ended via the `end_call` tool (simulator-side quality gate).
@@ -66,3 +98,4 @@ Uses the following MetricContext fields:
 - **Category**: `diagnostic`
 - **`exclude_from_pass_at_k`**: `True`
 - **Configuration**: None (deterministic)
+- **Sub-metric classifier**: `src/eva/utils/conversation_correctly_finished/` — `classifier.py` (extract raw files into signals → fixed-priority cause selection → build sub-metric flags), `causes/*` (per-cause detect + extract), `final_turn.py` (input-characteristic flags).

From b385411b679497315ba08be4dce3966e9859031d Mon Sep 17 00:00:00 2001
From: Gabrielle Gauthier-Melancon <gabrielle.gm@servicenow.com>
Date: Fri, 24 Jul 2026 15:07:34 -0400
Subject: [PATCH 09/10] Improve response speed

---
 src/eva/metrics/diagnostic/response_speed.py | 122 +++++++++++--------
 tests/fixtures/metric_signatures.json        |   2 +-
 tests/unit/metrics/test_response_speed.py    | 105 +++++++++++-----
 3 files changed, 144 insertions(+), 85 deletions(-)

diff --git a/src/eva/metrics/diagnostic/response_speed.py b/src/eva/metrics/diagnostic/response_speed.py
index 920aad38..c70b6633 100644
--- a/src/eva/metrics/diagnostic/response_speed.py
+++ b/src/eva/metrics/diagnostic/response_speed.py
@@ -10,6 +10,7 @@
 from eva.metrics.base import CodeMetric, MetricContext
 from eva.metrics.registry import register_metric
 from eva.models.results import MetricScore
+from eva.utils.log_processing import load_audit_log
 
 
 def _load_component_latencies(output_dir: str) -> dict[str, dict]:
@@ -36,65 +37,47 @@ def _load_component_latencies(output_dir: str) -> dict[str, dict]:
     return latencies
 
 
-def _load_audit_stats(output_dir: str) -> dict | None:
-    """Read audit_log.json once and compute LLM-call / tool-call stats.
+def _load_inference_tool_counts(output_dir: str) -> list[int] | None:
+    """Read audit_log.json once and return per-inference tool-call counts.
 
-    - ``num_llm_calls``: number of LLM completions (``llm_prompts`` entries)
-    - ``num_turns``: user turns (``transcript`` entries with ``message_type == "user"``)
-    - ``num_tool_calls``: total tool calls across all completions
-      (sum of each completion's ``response_message.tool_calls`` length)
-    - ``calls_with_tool_calls`` / ``parallel_tool_call_completions``: completions
-      emitting >=1 / >1 tool call (the latter reveals parallel/batched tool calls)
+    An *inference* is one model round-trip (one ``llm_prompts`` entry): the
+    agent may run several within a single user turn as it loops through the
+    reason-act cycle. Returns a list where element ``i`` is the number of tool
+    calls emitted by the i-th inference; the caller derives all aggregate counts
+    and rates from it.
 
-    Uses the same LLM-call and turn definitions as the cross-run scan so numbers
-    are comparable. Returns a stats dict, or None if the audit log is
-    missing/unreadable, has no user turns, or has no ``llm_prompts`` — the last
-    case covers models (e.g. GPT realtime / speech-to-speech) that don't log
-    prompt-level calls, for which these stats are undefined and must be omitted
-    rather than reported as zero.
+    Uses the same inference definition as the cross-run scan so numbers are
+    comparable. Returns None if the audit log is missing/unreadable or has no
+    ``llm_prompts``.
     """
-    audit_path = Path(output_dir) / "audit_log.json"
-    if not audit_path.exists():
-        return None
-
-    try:
-        audit = json.loads(audit_path.read_text())
-    except Exception:
+    audit = load_audit_log(Path(output_dir) / "audit_log.json")
+    if audit is None:
         return None
 
     prompts = audit.get("llm_prompts")
     prompts = prompts if isinstance(prompts, list) else []
-    num_calls = len(prompts)
-    num_turns = sum(1 for e in audit.get("transcript", []) if isinstance(e, dict) and e.get("message_type") == "user")
-    if num_turns <= 0 or num_calls <= 0:
+    if not prompts:
         return None
 
-    per_call_tools = []
+    per_inference_tools = []
     for c in prompts:
         rm = c.get("response_message") if isinstance(c, dict) else None
         tool_calls = rm.get("tool_calls") if isinstance(rm, dict) else None
-        per_call_tools.append(len(tool_calls) if isinstance(tool_calls, list) else 0)
-    num_tool_calls = sum(per_call_tools)
+        per_inference_tools.append(len(tool_calls) if isinstance(tool_calls, list) else 0)
 
-    return {
-        "num_llm_calls": num_calls,
-        "num_turns": num_turns,
-        "llm_calls_per_turn": round(num_calls / num_turns, 3),
-        "num_tool_calls": num_tool_calls,
-        "tools_per_llm_call": round(num_tool_calls / num_calls, 3),
-        "calls_with_tool_calls": sum(1 for n in per_call_tools if n >= 1),
-        "parallel_tool_call_completions": sum(1 for n in per_call_tools if n > 1),
-        "max_tools_per_call": max(per_call_tools) if per_call_tools else 0,
-    }
+    return per_inference_tools
+
+
+def _tool_call_turn_ids(context: MetricContext) -> set:
+    """Return the set of turn_ids that contain at least one tool call."""
+    return {entry["turn_id"] for entry in (context.conversation_trace or []) if entry.get("type") == "tool_call"}
 
 
 def _split_by_tool_calls(
     context: MetricContext,
 ) -> tuple[list[float], list[float]]:
     """Partition per_turn_latency values into (with_tool_calls, no_tool_calls)."""
-    tool_call_turn_ids = {
-        entry["turn_id"] for entry in (context.conversation_trace or []) if entry.get("type") == "tool_call"
-    }
+    tool_call_turn_ids = _tool_call_turn_ids(context)
 
     with_tool = [v for k, v in context.latency_assistant_turns.items() if k in tool_call_turn_ids]
     no_tool = [v for k, v in context.latency_assistant_turns.items() if k not in tool_call_turn_ids]
@@ -181,22 +164,55 @@ async def compute(self, context: MetricContext) -> MetricScore:
                         details=stats,
                     )
 
-            # LLM-call / tool-call efficiency diagnostics from audit_log.json.
-            audit_stats = _load_audit_stats(context.output_dir)
-            if audit_stats is not None:
-                # LLM completions per user turn (tool-loop / model efficiency).
-                sub_metrics["llm_calls_per_turn"] = MetricScore(
-                    name=f"{self.name}.llm_calls_per_turn",
-                    score=audit_stats["llm_calls_per_turn"],
+            # Inference / tool-call efficiency diagnostics from audit_log.json.
+            # Rates count tool-calling inferences only (no-tool inferences excluded).
+            # Each sub-metric carries only the counts that explain its own score.
+            per_inference_tools = _load_inference_tool_counts(context.output_dir)
+            if per_inference_tools is not None:
+                num_inferences_with_tool_calls = sum(1 for n in per_inference_tools if n >= 1)
+
+                # Tool calls per tool-using turn: when the model uses tools in a turn,
+                # how many tool calls land in that turn (across all its inferences).
+                num_turns_with_tool_calls = len(_tool_call_turn_ids(context))
+                sub_metrics["num_tool_calls_per_tool_turn"] = MetricScore(
+                    name=f"{self.name}.num_tool_calls_per_tool_turn",
+                    score=(
+                        round(sum(per_inference_tools) / num_turns_with_tool_calls, 3)
+                        if num_turns_with_tool_calls
+                        else 0.0
+                    ),
+                    normalized_score=None,
+                    details={
+                        "num_tool_calls": sum(per_inference_tools),
+                        "num_turns_with_tool_calls": num_turns_with_tool_calls,
+                    },
+                )
+                # Mean tool calls per tool-calling inference — the model's tool-call
+                # parallelism (1.0 = always sequential, >1.0 = emits parallel tool calls).
+                sub_metrics["tool_call_parallelism"] = MetricScore(
+                    name=f"{self.name}.tool_call_parallelism",
+                    score=(
+                        round(sum(per_inference_tools) / num_inferences_with_tool_calls, 3)
+                        if num_inferences_with_tool_calls
+                        else 0.0
+                    ),
                     normalized_score=None,
-                    details=audit_stats,
+                    details={
+                        "num_tool_calls": sum(per_inference_tools),
+                        "num_inferences_with_tool_calls": num_inferences_with_tool_calls,
+                        "num_inferences_with_parallel_tool_calls": sum(1 for n in per_inference_tools if n > 1),
+                        "max_tools_per_inference": max(per_inference_tools),
+                    },
                 )
-                # Tool calls per LLM completion (reveals sequential vs batched tool calls).
-                sub_metrics["tools_per_llm_call"] = MetricScore(
-                    name=f"{self.name}.tools_per_llm_call",
-                    score=audit_stats["tools_per_llm_call"],
+                # Fraction of inferences that call >=1 tool.
+                sub_metrics["tool_inference_rate"] = MetricScore(
+                    name=f"{self.name}.tool_inference_rate",
+                    score=round(num_inferences_with_tool_calls / len(per_inference_tools), 3),
                     normalized_score=None,
-                    details=audit_stats,
+                    details={
+                        "num_inferences_with_tool_calls": num_inferences_with_tool_calls,
+                        "num_inferences": len(per_inference_tools),
+                    },
                 )
 
             # Add per-component latency sub_metrics from result.json.
diff --git a/tests/fixtures/metric_signatures.json b/tests/fixtures/metric_signatures.json
index 1f00ea10..7b51f26d 100644
--- a/tests/fixtures/metric_signatures.json
+++ b/tests/fixtures/metric_signatures.json
@@ -44,7 +44,7 @@
   "ResponseSpeedMetric": {
     "name": "response_speed",
     "prompt_hash": null,
-    "source_hash": "101a7600f080",
+    "source_hash": "d83f7a24792a",
     "version": "v0.3"
   },
   "STTWERMetric": {
diff --git a/tests/unit/metrics/test_response_speed.py b/tests/unit/metrics/test_response_speed.py
index 8a4c69a5..059873a4 100644
--- a/tests/unit/metrics/test_response_speed.py
+++ b/tests/unit/metrics/test_response_speed.py
@@ -9,16 +9,16 @@
 from .conftest import make_metric_context
 
 
-def _write_audit_log(output_dir, n_llm_calls: int, n_user_turns: int, tools_per_call=None) -> None:
-    """Write a minimal audit_log.json with the given call/turn/tool counts.
+def _write_audit_log(output_dir, n_inferences: int, n_user_turns: int, tools_per_inference=None) -> None:
+    """Write a minimal audit_log.json with the given inference/turn/tool counts.
 
-    tools_per_call: optional list giving the number of tool calls emitted by each
-    completion (defaults to no tool calls).
+    tools_per_inference: optional list giving the number of tool calls emitted by
+    each inference (defaults to no tool calls).
     """
-    tools_per_call = tools_per_call or [0] * n_llm_calls
+    tools_per_inference = tools_per_inference or [0] * n_inferences
     prompts = [
-        {"latency_ms": 100.0, "response_message": {"tool_calls": [{} for _ in range(tools_per_call[i])]}}
-        for i in range(n_llm_calls)
+        {"latency_ms": 100.0, "response_message": {"tool_calls": [{} for _ in range(tools_per_inference[i])]}}
+        for i in range(n_inferences)
     ]
     audit = {
         "llm_prompts": prompts,
@@ -194,12 +194,16 @@ async def test_tool_call_breakdown_filters_invalid_latencies(self):
         assert with_tc.details["num_turns"] == 2  # only 5.0 and 3.0 pass the filter
 
     @pytest.mark.asyncio
-    async def test_llm_calls_per_turn_sub_metric(self, tmp_path):
-        """llm_calls_per_turn sub-metric = total llm calls / user turns from audit_log.json."""
-        _write_audit_log(tmp_path, n_llm_calls=6, n_user_turns=3)
+    async def test_tool_calls_per_tool_turn_sub_metric(self, tmp_path):
+        """num_tool_calls_per_tool_turn = total tool calls / turns that contain tool calls."""
+        # audit: 4 tool calls total across inferences.
+        _write_audit_log(tmp_path, n_inferences=4, n_user_turns=4, tools_per_inference=[1, 2, 1, 0])
+        # trace: tool calls occur in 2 distinct turns → 4 tool calls / 2 tool-turns = 2.0.
+        trace = _make_trace(tool_call_turn_ids={2, 4}, all_turn_ids={1, 2, 3, 4})
         metric = ResponseSpeedMetric()
         ctx = make_metric_context(
-            latency_assistant_turns={1: 1.0, 2: 2.0, 3: 3.0},
+            latency_assistant_turns={1: 1.0, 2: 2.0, 3: 3.0, 4: 4.0},
+            conversation_trace=trace,
             output_dir=str(tmp_path),
         )
 
@@ -207,16 +211,16 @@ async def test_llm_calls_per_turn_sub_metric(self, tmp_path):
 
         assert result.error is None
         assert result.sub_metrics is not None
-        cpt = result.sub_metrics["llm_calls_per_turn"]
-        assert cpt.score == pytest.approx(2.0)
-        assert cpt.details["num_llm_calls"] == 6
-        assert cpt.details["num_turns"] == 3
+        tct = result.sub_metrics["num_tool_calls_per_tool_turn"]
+        assert tct.score == pytest.approx(2.0)
+        assert tct.details["num_tool_calls"] == 4
+        assert tct.details["num_turns_with_tool_calls"] == 2
 
     @pytest.mark.asyncio
-    async def test_tools_per_llm_call_sub_metric(self, tmp_path):
-        """tools_per_llm_call = total tool calls / llm completions, with batching detail."""
-        # 4 completions: 2 with 1 tool, 1 with 2 tools (parallel), 1 with none → 4 tools / 4 calls.
-        _write_audit_log(tmp_path, n_llm_calls=4, n_user_turns=2, tools_per_call=[1, 2, 1, 0])
+    async def test_tool_call_parallelism_sub_metric(self, tmp_path):
+        """tool_call_parallelism = tool calls / tool-calling inferences (no-tool excluded)."""
+        # 4 inferences: 2 with 1 tool, 1 with 2 tools (parallel), 1 with none → 4 tools / 3 tool-inferences.
+        _write_audit_log(tmp_path, n_inferences=4, n_user_turns=2, tools_per_inference=[1, 2, 1, 0])
         metric = ResponseSpeedMetric()
         ctx = make_metric_context(
             latency_assistant_turns={1: 1.0, 2: 2.0},
@@ -226,18 +230,55 @@ async def test_tools_per_llm_call_sub_metric(self, tmp_path):
         result = await metric.compute(ctx)
 
         assert result.error is None
-        tpc = result.sub_metrics["tools_per_llm_call"]
-        assert tpc.score == pytest.approx(1.0)  # 4 tools / 4 calls
+        tpc = result.sub_metrics["tool_call_parallelism"]
+        assert tpc.score == pytest.approx(round(4 / 3, 3))  # 4 tools / 3 tool-calling inferences
         assert tpc.details["num_tool_calls"] == 4
-        assert tpc.details["calls_with_tool_calls"] == 3
-        assert tpc.details["parallel_tool_call_completions"] == 1
-        assert tpc.details["max_tools_per_call"] == 2
+        assert tpc.details["num_inferences_with_tool_calls"] == 3
+        assert tpc.details["num_inferences_with_parallel_tool_calls"] == 1
+        assert tpc.details["max_tools_per_inference"] == 2
+
+    @pytest.mark.asyncio
+    async def test_tool_inference_rate_sub_metric(self, tmp_path):
+        """tool_inference_rate = tool-calling inferences / total inferences."""
+        # 4 inferences, 3 of which call a tool → rate 0.75.
+        _write_audit_log(tmp_path, n_inferences=4, n_user_turns=2, tools_per_inference=[1, 2, 1, 0])
+        metric = ResponseSpeedMetric()
+        ctx = make_metric_context(
+            latency_assistant_turns={1: 1.0, 2: 2.0},
+            output_dir=str(tmp_path),
+        )
+
+        result = await metric.compute(ctx)
+
+        assert result.error is None
+        rate = result.sub_metrics["tool_inference_rate"]
+        assert rate.score == pytest.approx(0.75)
+        assert rate.details["num_inferences_with_tool_calls"] == 3
+        assert rate.details["num_inferences"] == 4
+
+    @pytest.mark.asyncio
+    async def test_no_tool_calling_inferences(self, tmp_path):
+        """When no inference calls a tool, rates are 0 and don't divide by zero."""
+        _write_audit_log(tmp_path, n_inferences=3, n_user_turns=3)  # all inferences have 0 tools
+        metric = ResponseSpeedMetric()
+        ctx = make_metric_context(
+            latency_assistant_turns={1: 1.0, 2: 2.0, 3: 3.0},
+            output_dir=str(tmp_path),
+        )
+
+        result = await metric.compute(ctx)
+
+        assert result.error is None
+        assert result.sub_metrics is not None
+        assert result.sub_metrics["num_tool_calls_per_tool_turn"].score == pytest.approx(0.0)
+        assert result.sub_metrics["tool_call_parallelism"].score == pytest.approx(0.0)
+        assert result.sub_metrics["tool_inference_rate"].score == pytest.approx(0.0)
 
     @pytest.mark.asyncio
-    async def test_call_sub_metrics_absent_without_llm_prompts(self, tmp_path):
+    async def test_inference_sub_metrics_absent_without_llm_prompts(self, tmp_path):
         """Models without llm_prompts (e.g. GPT realtime / S2S) omit the sub-metrics."""
         # audit_log with user turns but empty llm_prompts (no prompt-level instrumentation).
-        _write_audit_log(tmp_path, n_llm_calls=0, n_user_turns=3)
+        _write_audit_log(tmp_path, n_inferences=0, n_user_turns=3)
         metric = ResponseSpeedMetric()
         ctx = make_metric_context(
             latency_assistant_turns={1: 1.0, 2: 2.0, 3: 3.0},
@@ -249,19 +290,21 @@ async def test_call_sub_metrics_absent_without_llm_prompts(self, tmp_path):
         assert result.error is None
         # response_speed still computes latency stats, but the llm_prompts-based
         # sub-metrics must be absent (aggregating a fabricated 0.0 would be wrong).
-        assert "llm_calls_per_turn" not in (result.sub_metrics or {})
-        assert "tools_per_llm_call" not in (result.sub_metrics or {})
+        assert "num_tool_calls_per_tool_turn" not in (result.sub_metrics or {})
+        assert "tool_call_parallelism" not in (result.sub_metrics or {})
+        assert "tool_inference_rate" not in (result.sub_metrics or {})
 
     @pytest.mark.asyncio
-    async def test_llm_calls_per_turn_absent_without_audit_log(self):
-        """No audit_log.json → llm_calls_per_turn sub-metric is omitted (no error)."""
+    async def test_inference_sub_metrics_absent_without_audit_log(self):
+        """No audit_log.json → inference sub-metrics are omitted (no error)."""
         metric = ResponseSpeedMetric()
         ctx = make_metric_context(latency_assistant_turns={1: 1.0, 2: 2.0})
 
         result = await metric.compute(ctx)
 
         assert result.error is None
-        assert "llm_calls_per_turn" not in (result.sub_metrics or {})
+        assert "num_tool_calls_per_tool_turn" not in (result.sub_metrics or {})
+        assert "tool_inference_rate" not in (result.sub_metrics or {})
 
     @pytest.mark.asyncio
     async def test_with_and_no_tool_split_is_exhaustive(self):

From 9b7f2d70677b08b832c2b70da2a968a6562a9e8f Mon Sep 17 00:00:00 2001
From: Gabrielle Gauthier-Melancon <gabrielle.gm@servicenow.com>
Date: Fri, 24 Jul 2026 15:08:50 -0400
Subject: [PATCH 10/10] Bump versions again

---
 .../metrics/diagnostic/conversation_correctly_finished.py | 2 +-
 src/eva/metrics/diagnostic/response_speed.py              | 2 +-
 tests/fixtures/metric_signatures.json                     | 8 ++++----
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/eva/metrics/diagnostic/conversation_correctly_finished.py b/src/eva/metrics/diagnostic/conversation_correctly_finished.py
index 353e418a..ff1a48cf 100644
--- a/src/eva/metrics/diagnostic/conversation_correctly_finished.py
+++ b/src/eva/metrics/diagnostic/conversation_correctly_finished.py
@@ -27,7 +27,7 @@ class ConversationCorrectlyFinishedMetric(CodeMetric):
     """
 
     name = "conversation_correctly_finished"
-    version = "v0.2"
+    version = "v0.3"
     description = "Diagnostic metric: 0.0 when agent failed to respond to the user's final turn"
     category = "diagnostic"
     exclude_from_pass_at_k = True
diff --git a/src/eva/metrics/diagnostic/response_speed.py b/src/eva/metrics/diagnostic/response_speed.py
index c70b6633..1af7a0b0 100644
--- a/src/eva/metrics/diagnostic/response_speed.py
+++ b/src/eva/metrics/diagnostic/response_speed.py
@@ -122,7 +122,7 @@ class ResponseSpeedMetric(CodeMetric):
     description = "Diagnostic metric: latency between user utterance end and assistant response start"
     exclude_from_pass_at_k = True
     higher_is_better = False  # Score is latency in seconds — lower is better.
-    version = "v0.3"
+    version = "v0.4"
 
     async def compute(self, context: MetricContext) -> MetricScore:
         try:
diff --git a/tests/fixtures/metric_signatures.json b/tests/fixtures/metric_signatures.json
index 7b51f26d..64f9ae00 100644
--- a/tests/fixtures/metric_signatures.json
+++ b/tests/fixtures/metric_signatures.json
@@ -14,8 +14,8 @@
   "ConversationCorrectlyFinishedMetric": {
     "name": "conversation_correctly_finished",
     "prompt_hash": null,
-    "source_hash": "9cf0a1d26e80",
-    "version": "v0.2"
+    "source_hash": "da7a915516e1",
+    "version": "v0.3"
   },
   "ConversationProgressionJudgeMetric": {
     "name": "conversation_progression",
@@ -44,8 +44,8 @@
   "ResponseSpeedMetric": {
     "name": "response_speed",
     "prompt_hash": null,
-    "source_hash": "d83f7a24792a",
-    "version": "v0.3"
+    "source_hash": "3b0926903322",
+    "version": "v0.4"
   },
   "STTWERMetric": {
     "name": "stt_wer",