fix(reports): JUnit CI-gate review fixes + CE027 env-var lint

Code-review fixes to the JUnit XML CI gate, plus a new doc/config lint
rule and a producer→consumer parity test.

reports_junit.py:
- Load task.json for dataset rows: task_id "<suite>/<row_id>" is a real
  nested dir, so validate it as a contained relative path (_is_safe_relpath)
  instead of rejecting any '/'. Rejects absolute/backslash/'..'/Windows-drive
  values; the resolve()-containment check remains the backstop.
- Guard the root <testsuites> time against NaN/inf like per-testcase time
  via a shared _time_attr helper; also degrade (not crash) on a
  pathologically large integer duration that overflows float()/'.3f'.
- Render informational (gating=False) criteria as [INFO], not [FAIL]/[PASS],
  mirroring reports.py; only an explicit JSON `false` is informational
  (null / non-bool fails safe to gating).

Docs: fix CODER_EVAL_API_BACKEND -> API_BACKEND (the real Settings env
name; the CODER_EVAL_-prefixed spelling was silently dropped by
Settings extra="ignore", selecting no backend).

CE027 (tests/lint/doc_env_parity.py): new rule flagging framework-prefixed
env-var assignments in README/action.yml/docs that no Settings field/alias
or src/ consumer backs — the class that produced the API_BACKEND bug.
Assignment-scoped with a hardened boundary and a real-consumer src scan to
avoid false positives.

Tests: dataset nested-load, nested '..'/drive rejection, informational
[INFO] (passing + null-gating), root/huge-int time degradation, and a
producer→consumer parity test running real eval_result_to_task_dict output
through generate_junit_xml.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: uipreliga

README.md

@@ -145,7 +145,7 @@ it can't leak into later steps). Set whatever the run needs, Anthropic or not:
     tasks: tests/tasks/**/*.yaml
     minimum-task-score: "0.8"   # fail the build if any task scores below 0.8
     env: |
-      CODER_EVAL_API_BACKEND=bedrock
+      API_BACKEND=bedrock
       AWS_BEARER_TOKEN_BEDROCK=${{ secrets.BEDROCK_TOKEN }}
 ```
 

action.yml

@@ -52,7 +52,7 @@ inputs:
       the coder-eval process only (scoped to the run step; NOT written to
       $GITHUB_ENV, so nothing leaks into later job steps). This is the sole
       channel for credentials and backend config: set ANTHROPIC_API_KEY,
-      CODER_EVAL_API_BACKEND, model vars, EVALBOARD_*, plugin paths, etc. Names
+      API_BACKEND, model vars, EVALBOARD_*, plugin paths, etc. Names
       must match ^[A-Za-z_][A-Za-z0-9_]*$; blank lines and `#` comments are
       ignored. Wire values from repository secrets (secrets.MY_KEY) — never
       inline a secret literal.

src/coder_eval/reports_junit.py

@@ -24,7 +24,7 @@
 import math
 import re
 import xml.etree.ElementTree as ET
-from pathlib import Path, PurePosixPath
+from pathlib import Path, PurePosixPath, PureWindowsPath
 from typing import Any, Literal
 
 from .evaluation.judge_context import truncate
@@ -99,13 +99,60 @@ def _is_safe_component(value: str) -> bool:
     """True when ``value`` is usable as a single, contained path segment.
 
     ``run.json`` rows are untyped and may be blob-pulled from elsewhere, so a
-    crafted ``variant_id``/``task_id`` must not steer the lookup outside the run
-    directory (an absolute value would discard ``run_dir`` entirely, and ``..``
-    would walk up).
+    crafted ``variant_id`` must not steer the lookup outside the run directory
+    (an absolute value would discard ``run_dir`` entirely, and ``..`` would walk
+    up).
     """
     return bool(value) and value not in {".", ".."} and "/" not in value and "\\" not in value
 
 
+def _is_safe_relpath(value: str) -> bool:
+    """True when ``value`` is a contained relative path — possibly *nested*.
+
+    Unlike :func:`_is_safe_component`, an internal ``/`` is allowed: dataset
+    expansion rewrites a row's ``task_id`` to ``"<suite>/<row_id>"``
+    (``task_loader.expand_dataset``) and the on-disk layout is correspondingly
+    nested (``run_dir/<variant>/<suite>/<row_id>/<NN>/task.json``, see
+    ``path_utils.build_task_run_dir``). Rejecting the ``/`` would degrade every
+    dataset-derived row (e.g. the activation suite) to a status-only body.
+
+    Only an absolute path, a Windows drive/UNC prefix, or a ``.``/``..``
+    component could escape ``run_dir``; all are rejected here, and the
+    ``resolve()``-containment check in :func:`_load_task_json` is the
+    belt-and-braces backstop (symlinks included).
+    """
+    # A Windows drive-qualified value (``C:/x``) reads as a plain relative path
+    # on POSIX but is absolute on Windows, so reject it explicitly rather than
+    # leaning on the resolve-containment backstop alone.
+    if not value or value.startswith("/") or "\\" in value or PureWindowsPath(value).drive:
+        return False
+    parts = PurePosixPath(value).parts
+    return bool(parts) and all(p not in {".", ".."} for p in parts)
+
+
+def _time_attr(value: Any) -> str:
+    """Serialize a duration as a JUnit ``time`` attribute string.
+
+    The value must be a finite, non-negative number; NaN/inf/negative/non-numeric
+    all fall back to ``"0.000"``. NaN/inf would otherwise serialize as
+    ``"nan"``/``"inf"`` and make the document invalid for JUnit ingesters. Shared
+    by the per-testcase time and the root ``<testsuites>`` time so both are
+    guarded identically.
+
+    Rows are untyped, so ``value`` may be a pathologically large JSON integer
+    (hundreds of digits) that overflows on the ``float()`` conversion / ``.3f``
+    format — the ``OverflowError`` is caught and degraded rather than aborting
+    the whole report, matching this module's degrade-don't-crash contract.
+    """
+    if isinstance(value, bool) or not isinstance(value, int | float) or value < 0:
+        return "0.000"
+    try:
+        as_float = float(value)
+    except (OverflowError, ValueError):
+        return "0.000"
+    return f"{as_float:.3f}" if math.isfinite(as_float) else "0.000"
+
+
 def _load_task_json(run_dir: Path, row: dict[str, Any], variant: str) -> dict[str, Any] | None:
     """Best-effort load of a failed row's ``task.json`` as a plain dict.
 
@@ -116,7 +163,7 @@ def _load_task_json(run_dir: Path, row: dict[str, Any], variant: str) -> dict[st
     ``None`` so the caller falls back to a status-only body.
     """
     task_id = str(row.get("task_id", "<unknown>"))
-    if not _is_safe_component(variant) or not _is_safe_component(task_id):
+    if not _is_safe_component(variant) or not _is_safe_relpath(task_id):
         return None
 
     replicate_index = row.get("replicate_index")
@@ -162,11 +209,23 @@ def _criteria_body(row: dict[str, Any], run_dir: Path, variant: str) -> str:
             description = str(crit.get("description", ""))
             score = crit.get("score")
             threshold = crit.get("pass_threshold")
+            # Only an explicit JSON ``false`` marks an informational criterion; a
+            # missing key or a schema-skewed value (null / non-bool) fails safe
+            # to gating — matching CriterionResult.gating's default and this
+            # module's isinstance-guarded, degrade-don't-crash reads of untyped
+            # rows. Informational criteria are excluded from the score/gate, so
+            # they are labelled [INFO] regardless of pass/fail and never rendered
+            # as the failure cause (mirrors reports.py `_compute_suite_rollup`'s
+            # `if not cr.gating: continue`).
+            informational = crit.get("gating", True) is False
+            score_str = f"{score:.2f}" if isinstance(score, int | float) else str(score)
+            if informational:
+                lines.append(f"[INFO] {ctype}: score {score_str} — {description}")
+                continue
             passed = isinstance(score, int | float) and isinstance(threshold, int | float) and score >= threshold
             if passed:
                 lines.append(f"[PASS] {ctype}: {description}")
                 continue
-            score_str = f"{score:.2f}" if isinstance(score, int | float) else str(score)
             thr_str = f"{threshold:.2f}" if isinstance(threshold, int | float) else str(threshold)
             lines.append(f"[FAIL] {ctype}: score {score_str} < threshold {thr_str} — {description}")
             detail = crit.get("details") or crit.get("error")
@@ -193,16 +252,7 @@ def _task_case(row: dict[str, Any], run_dir: Path) -> ET.Element:
     task_path = row.get("task_path")
     classname = (Path(task_path).stem or variant) if isinstance(task_path, str) and task_path else variant
 
-    # time must be a finite, non-negative float: NaN/inf would serialize as
-    # "nan"/"inf" and make the report invalid for JUnit ingesters.
-    duration = row.get("duration")
-    valid_duration = (
-        isinstance(duration, int | float)
-        and not isinstance(duration, bool)
-        and math.isfinite(duration)
-        and duration >= 0
-    )
-    time_str = f"{duration:.3f}" if valid_duration else "0.000"
+    time_str = _time_attr(row.get("duration"))
 
     case = ET.Element(
         "testcase",
@@ -332,7 +382,10 @@ def generate_junit_xml(run_dir: Path) -> str:
     summary = RunSummary.model_validate_json(run_json.read_text(encoding="utf-8"))
 
     root = ET.Element("testsuites", {"name": _xml_safe(summary.run_id)})
-    root.set("time", f"{summary.total_duration_seconds:.3f}")
+    # Guard the root time identically to per-testcase time: a corrupt/blob-pulled
+    # run.json can carry a NaN/inf total_duration_seconds (RunSummary has no
+    # finite validator), which would emit an invalid time="nan".
+    root.set("time", _time_attr(summary.total_duration_seconds))
 
     # Group task rows by variant, preserving first-seen order.
     grouped: dict[str, list[dict[str, Any]]] = {}

tests/lint/doc_env_parity.py

@@ -0,0 +1,125 @@
+"""CE027 — documented framework env vars must be backed by a real consumer.
+
+``coder_eval.config.Settings`` sets no ``env_prefix`` and uses ``extra="ignore"``,
+so a documented env var whose name does not match a ``Settings`` field (or one of
+its ``AliasChoices``) is **silently dropped** at runtime with zero signal — the
+exact failure mode behind the ``CODER_EVAL_API_BACKEND`` doc bug (the real field
+is ``API_BACKEND``, so the ``CODER_EVAL_``-prefixed spelling selected no backend
+and the run fell back to Direct Anthropic).
+
+This rule scans the doc/config surfaces (``README.md``, ``action.yml``,
+``docs/**``) for env-var **assignments** (``NAME=value`` — the copy-pasteable,
+dangerous form) carrying a **framework-owned prefix** and flags any whose name is
+neither a ``Settings`` env name/alias nor referenced anywhere in ``src/`` — the
+framework also reads a handful of vars directly via ``os.getenv`` (e.g.
+``CODER_EVAL_SKILLS_DIR``, ``CODEX_BASE_URL``, ``CODER_EVAL_IN_CONTAINER``), and
+those are legitimately documentable.
+
+Scope note: only *assignments* are checked, not bare prose mentions. Prose
+scanning is too false-positive-prone (markdown links like
+``CODEX_AGENT_GUIDE.md``, secret RHS references like ``secrets.BEDROCK_TOKEN``,
+regex-pattern examples like ``API_KEY = "…"``), and the assignment form is the
+one users actually copy into a workflow, so it carries the real risk.
+
+It is intentionally NOT a ``BaseRule`` registered in ``tests/lint/runner.py``:
+that runner is AST-only and walks ``.py`` files, whereas this rule reasons over
+Markdown/YAML doc surfaces. It is wired as a dedicated test in
+``tests/test_custom_lint.py::TestCE027DocEnvVarParity``.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+from pydantic import AliasChoices
+
+
+# Framework-owned env-var name prefixes. A documented token starting with one of
+# these is owned by coder-eval and MUST be consumed by it. Deliberately EXCLUDES
+# broad third-party namespaces (AWS_, ANTHROPIC_, GEMINI_, GITHUB_, EVALBOARD_,
+# PLUGIN_) whose vars are consumed by SDKs / CI, not necessarily via Settings, so
+# scanning them would produce false positives on legitimately-external names.
+FRAMEWORK_ENV_PREFIXES: tuple[str, ...] = (
+    "CODER_EVAL_",
+    "API_",
+    "BEDROCK_",
+    "CODEX_",
+    "ANTIGRAVITY_",
+    "TELEMETRY_",
+)
+
+_PREFIX_ALT = "|".join(FRAMEWORK_ENV_PREFIXES)
+
+# A framework-prefixed env-var ASSIGNMENT in doc/config text: ``NAME=`` where NAME
+# carries a framework prefix. The negative lookbehind rejects a name embedded in
+# a larger token — attribute access (``secrets.BEDROCK_TOKEN``), a hyphenated
+# token (``X-API_KEY=``), a path/URL segment (``dir/API_X=``, ``http://API_Y=``),
+# or a Windows path (``C:\\API_Z=``). The single ``=`` (not ``==``) with no space
+# before it also rejects ``API_KEY = "…"`` regex-pattern examples.
+_ENV_ASSIGNMENT = re.compile(r"(?<![\w./:\\-])((?:" + _PREFIX_ALT + r")[A-Z0-9_]*[A-Z0-9])=(?!=)")
+
+# How the framework actually *consumes* an env var, so a documented assignment is
+# only "backed" if some module reads it: a direct ``os.getenv("NAME")`` /
+# ``os.environ["NAME"]`` / ``os.environ.get("NAME")`` read, or the NAME side of an
+# inline ``"NAME=VALUE"`` literal handed to a child process (e.g. docker
+# ``--env NAME=1``). This is stricter than "any uppercase literal" so an unrelated
+# constant that merely spells a var name cannot silently mask a documented-but-
+# unconsumed assignment.
+_SRC_ENV_READ = re.compile(r"""(?:getenv\(\s*|environ(?:\.get\(\s*|\[\s*))['"]([A-Z][A-Z0-9_]{2,})['"]""")
+_SRC_ENV_VALUE = re.compile(r"""['"]([A-Z][A-Z0-9_]{2,})=[^'"]*['"]""")
+
+
+def settings_env_names() -> set[str]:
+    """Uppercased env names Settings actually reads: field names + AliasChoices."""
+    from coder_eval.config import Settings
+
+    names: set[str] = set()
+    for field_name, field in Settings.model_fields.items():
+        names.add(field_name.upper())
+        alias = field.validation_alias
+        if isinstance(alias, AliasChoices):
+            names.update(str(c).upper() for c in alias.choices if isinstance(c, str))
+        elif isinstance(alias, str):
+            names.add(alias.upper())
+    return names
+
+
+def src_env_literals(src_root: Path) -> set[str]:
+    """Env-var names ``src/`` actually consumes: direct ``os.getenv``/``os.environ``
+    reads plus the NAME side of inline ``"NAME=VALUE"`` child-process literals."""
+    names: set[str] = set()
+    for py in src_root.rglob("*.py"):
+        text = py.read_text(encoding="utf-8")
+        names.update(_SRC_ENV_READ.findall(text))
+        names.update(_SRC_ENV_VALUE.findall(text))
+    return names
+
+
+def scan_doc_env_assignments(text: str) -> set[str]:
+    """Framework-prefixed env-var names *assigned* (``NAME=…``) in a doc file."""
+    return set(_ENV_ASSIGNMENT.findall(text))
+
+
+def find_unbacked_env_vars(doc_paths: list[Path], src_root: Path) -> dict[str, list[str]]:
+    """Map each doc path to the framework-prefixed env vars it assigns that
+    nothing in the framework consumes (would be silently dropped at runtime)."""
+    valid = settings_env_names() | src_env_literals(src_root)
+    findings: dict[str, list[str]] = {}
+    for path in doc_paths:
+        if not path.is_file():
+            continue
+        unbacked = sorted(t for t in scan_doc_env_assignments(path.read_text(encoding="utf-8")) if t not in valid)
+        if unbacked:
+            findings[str(path)] = unbacked
+    return findings
+
+
+def default_doc_paths(repo_root: Path) -> list[Path]:
+    """The doc/config surfaces CE027 scans: README, the published action, docs/**."""
+    paths = [repo_root / "README.md", repo_root / "action.yml"]
+    docs = repo_root / "docs"
+    if docs.is_dir():
+        for suffix in ("*.md", "*.yaml", "*.yml"):
+            paths.extend(sorted(docs.rglob(suffix)))
+    return paths

tests/test_custom_lint.py

@@ -630,3 +630,80 @@ def test_compliant_class_not_flagged(self):
     def test_exempt_modules_not_scoped(self):
         for path in ("src/coder_eval/models/results.py", "src/coder_eval/models/telemetry.py"):
             assert not self._run(self._VIOLATING, path)
+
+
+@pytest.mark.lint
+class TestCE027DocEnvVarParity:
+    """CE027 — documented framework env-var assignments must be backed by a
+    real consumer (a Settings field/alias or an os.getenv read in src/).
+
+    `Settings` sets no `env_prefix` and uses `extra="ignore"`, so a documented
+    `NAME=value` whose name matches no field is silently dropped at runtime — the
+    `CODER_EVAL_API_BACKEND` (real field: `API_BACKEND`) doc bug. Scans real
+    Markdown/YAML surfaces, so it lives here rather than in the AST-only runner.
+    """
+
+    REPO_ROOT = Path(__file__).parent.parent
+
+    def test_repo_docs_have_no_unbacked_env_vars(self):
+        from tests.lint.doc_env_parity import default_doc_paths, find_unbacked_env_vars
+
+        findings = find_unbacked_env_vars(default_doc_paths(self.REPO_ROOT), self.REPO_ROOT / "src")
+        assert not findings, (
+            "\nDocumented framework env-var assignment(s) that no Settings field/alias or "
+            "src/ reference backs — they are silently dropped at runtime (Settings uses "
+            'extra="ignore"). Fix the spelling to a real env name, or add the consumer:\n\n'
+            + "\n".join(f"  {path}: {', '.join(names)}" for path, names in sorted(findings.items()))
+        )
+
+    def test_catches_the_coder_eval_api_backend_shadow(self):
+        # The exact regression: a CODER_EVAL_-prefixed spelling of a real field.
+        from tests.lint.doc_env_parity import scan_doc_env_assignments, settings_env_names, src_env_literals
+
+        valid = settings_env_names() | src_env_literals(self.REPO_ROOT / "src")
+        found = scan_doc_env_assignments("      CODER_EVAL_API_BACKEND=bedrock\n")
+        assert "CODER_EVAL_API_BACKEND" in found
+        assert "CODER_EVAL_API_BACKEND" not in valid  # would be flagged
+
+    def test_real_backend_field_is_backed(self):
+        from tests.lint.doc_env_parity import settings_env_names
+
+        assert "API_BACKEND" in settings_env_names()
+
+    @pytest.mark.parametrize(
+        "line",
+        [
+            "AWS_BEARER_TOKEN_BEDROCK=${{ secrets.BEDROCK_TOKEN }}",  # secret RHS, not an assignment of BEDROCK_TOKEN
+            "[Codex Agent Guide](docs/CODEX_AGENT_GUIDE.md)",  # markdown link, not an assignment
+            'pattern: "^API_KEY = \\"\\\\w+\\""',  # regex example with a space before '='
+            "the CODEX_MODEL setting selects the model",  # bare prose mention, not an assignment
+            "X-API_KEY=v",  # hyphenated token — prefix embedded, not a standalone name
+            "dir/API_THING=v",  # path segment — prefix embedded
+            "http://API_HOST=1",  # URL segment — prefix embedded
+        ],
+    )
+    def test_non_assignment_shapes_are_not_flagged(self, line: str):
+        from tests.lint.doc_env_parity import scan_doc_env_assignments, settings_env_names, src_env_literals
+
+        valid = settings_env_names() | src_env_literals(self.REPO_ROOT / "src")
+        unbacked = [t for t in scan_doc_env_assignments(line) if t not in valid]
+        assert unbacked == [], f"false positive on non-assignment shape: {unbacked}"
+
+    def test_name_side_of_env_value_literal_counts_as_backed(self):
+        # `--env CODER_EVAL_IN_CONTAINER=1` in src makes the doc assignment backed.
+        from tests.lint.doc_env_parity import src_env_literals
+
+        names = src_env_literals(self.REPO_ROOT / "src")
+        assert "CODER_EVAL_IN_CONTAINER" in names
+
+    def test_src_scan_requires_a_real_consumer_not_any_literal(self, tmp_path: Path):
+        # A bare uppercase constant that no code reads must NOT count as "backed",
+        # or it could silently mask a documented-but-unconsumed assignment.
+        from tests.lint.doc_env_parity import src_env_literals
+
+        (tmp_path / "m.py").write_text(
+            'CONST = "CODER_EVAL_BOGUS"\nx = os.getenv("CODER_EVAL_REAL")\n', encoding="utf-8"
+        )
+        names = src_env_literals(tmp_path)
+        assert "CODER_EVAL_REAL" in names  # a genuine os.getenv read is backed
+        assert "CODER_EVAL_BOGUS" not in names  # a bare constant is not