From 971bcbb5ec89e5f033015c9efc67b221e7512ef0 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sun, 2 Aug 2026 22:55:53 -0600 Subject: [PATCH] feat(benchmarks): import the SWE-agent CodeTraceBench family Add the sweagent_traj CodeTracer user skill (one step per trajectory[] entry, blanks included; 106/108 verified rows reproduce the annotated step_count), route the SWE-agent family through prepare-bulk.py, and record the import results. The 2 mismatched rows publish function-call style trajectories whose annotated view condensed the run; they fail the step-count gate instead of importing misaligned labels. --- .../codetracebench-oht2-20260801/README.md | 11 +- .../prepare-bulk.py | 9 +- .../skills/sweagent_traj/SKILL.md | 56 +++++++++ .../skills/sweagent_traj/parser.py | 117 ++++++++++++++++++ 4 files changed, 186 insertions(+), 7 deletions(-) create mode 100644 benchmarks/trace-analysis/codetracebench-oht2-20260801/skills/sweagent_traj/SKILL.md create mode 100644 benchmarks/trace-analysis/codetracebench-oht2-20260801/skills/sweagent_traj/parser.py diff --git a/benchmarks/trace-analysis/codetracebench-oht2-20260801/README.md b/benchmarks/trace-analysis/codetracebench-oht2-20260801/README.md index 1283aa2a..db3e95c1 100644 --- a/benchmarks/trace-analysis/codetracebench-oht2-20260801/README.md +++ b/benchmarks/trace-analysis/codetracebench-oht2-20260801/README.md @@ -1,6 +1,6 @@ -# CodeTraceBench OpenHands + Terminus2 Bulk Import Recipe +# CodeTraceBench OpenHands + Terminus2 + SWE-agent Bulk Import Recipe -This directory turns the verified CodeTraceBench OpenHands and Terminus2 rows into traces-importable inputs. +This directory turns the verified CodeTraceBench OpenHands, Terminus2, and SWE-agent rows into traces-importable inputs. It extends the mini-SWE recipe in [`../codetracebench-glm52-20260730/`](../codetracebench-glm52-20260730/) from a 32-row single-shot preparation to a per-row fault-tolerant bulk pipeline. ## Provenance @@ -14,7 +14,7 @@ It extends the mini-SWE recipe in [`../codetracebench-glm52-20260730/`](../codet ## Why local skills exist -Upstream CodeTracer ships `openhands` and `terminus2` seed parsers, but neither reproduces the CodeTraceBench annotation's step numbering, and the swe_raw OpenHands layout has no upstream parser at any commit. +Upstream CodeTracer ships `openhands` and `terminus2` seed parsers, but neither reproduces the CodeTraceBench annotation's step numbering, and the swe_raw OpenHands and SWE-agent layouts have no upstream parser at any commit (checked at the pinned revision, which is also repository HEAD). A trajectory whose steps.json disagrees with the annotation's step ids silently misaligns every label, so each published layout gets a parser whose enumeration was validated against the manifest `step_count` for every verified row: | Skill | Layout | Convention | Count evidence | @@ -22,6 +22,7 @@ A trajectory whose steps.json disagrees with the annotation's step ids silently | [`skills/openhands_completions`](./skills/openhands_completions/SKILL.md) | swe_raw per-call LiteLLM logs | non-finish assistant tool calls in the fullest call view | 313/313 | | [`skills/openhands_sessions`](./skills/openhands_sessions/SKILL.md) | session event streams | action events with a cause-paired observation, any action type | 183/199 | | [`skills/terminus2_commands`](./skills/terminus2_commands/SKILL.md) | episode logs | one step per `commands[]` entry across episodes | 220/222 | +| [`skills/sweagent_traj`](./skills/sweagent_traj/SKILL.md) | classic SWE-agent `.traj` files | one step per `trajectory[]` entry, blanks included | 106/108 | The skills follow the upstream `SKILL.md` + `parser.py` contract and load through `SkillPool(user_dir=...)`, so CodeTracer detects them exactly like seed skills. The upstream seed parsers stay untouched; family membership in `prepare-bulk.py` routes every row through the annotation-faithful skill and treats any other detection as a `wrong-normalizer` failure. @@ -57,7 +58,7 @@ traces import-codetracebench "$WORK/ctb-oht2-labels-openhands.json" \ --concurrency 8 ``` -`--family Terminus2` runs the same way. +`--family Terminus2` and `--family SWE-agent` run the same way. Archives cache in `$WORK//archives`, so reruns skip completed downloads. `$WORK//prepare-bulk-receipt.json` records every row's status, normalizer, step count, hashes, and failure reason. @@ -67,6 +68,8 @@ Archives cache in `$WORK//archives`, so reruns skip completed downloads. | --- | ---: | ---: | ---: | --- | | OpenHands | 520 | 496 | 24 | 8 `no-artifact` (manifest rows without archives), 16 `step-count-mismatch` (published session views disagree with the annotated count) | | Terminus2 | 222 | 220 | 2 | 2 `no-normalizer` (archives containing only empty directories) | +| SWE-agent (2026-08-02) | 108 | 106 | 2 | 2 `step-count-mismatch` (function-call-style trajectories whose annotated view condensed the run) | Passing OpenHands rows split 313 `openhands_completions` + 183 `openhands_sessions`. The 16 OpenHands `step-count-mismatch` rows are condensed or partial session streams where no published view (flat export, events, event_cache) reproduces the annotated step count; importing any of them would misalign labels, so they stay out. +The 2 SWE-agent `step-count-mismatch` rows publish trajectories with 45/48 and 92/94 empty `response` fields whose annotated views (26 and 54 steps) match no published view of the 48- and 94-entry arrays, so they stay out for the same reason. diff --git a/benchmarks/trace-analysis/codetracebench-oht2-20260801/prepare-bulk.py b/benchmarks/trace-analysis/codetracebench-oht2-20260801/prepare-bulk.py index 486e0ec1..51bea2c0 100644 --- a/benchmarks/trace-analysis/codetracebench-oht2-20260801/prepare-bulk.py +++ b/benchmarks/trace-analysis/codetracebench-oht2-20260801/prepare-bulk.py @@ -44,11 +44,14 @@ # Manifest `agent` value -> CodeTracer skills allowed to normalize the row. # swe_raw OpenHands trials publish LiteLLM completion logs, handled by the # openhands_completions user skill; terminal-bench trials use the upstream -# openhands sessions skill. A row that detects as any skill outside its -# family set is a faithfulness failure, not a fallback. mini-SWE rows keep -# the original prepare.py staging path and are out of scope here. +# openhands sessions skill. SWE-agent trials publish classic .traj files, +# handled by the sweagent_traj user skill (no upstream commit ships a +# SWE-agent parser). A row that detects as any skill outside its family set +# is a faithfulness failure, not a fallback. mini-SWE rows keep the +# original prepare.py staging path and are out of scope here. FAMILY_SKILLS = { "OpenHands": ("openhands_sessions", "openhands_completions"), + "SWE-agent": ("sweagent_traj",), "Terminus2": ("terminus2_commands",), } diff --git a/benchmarks/trace-analysis/codetracebench-oht2-20260801/skills/sweagent_traj/SKILL.md b/benchmarks/trace-analysis/codetracebench-oht2-20260801/skills/sweagent_traj/SKILL.md new file mode 100644 index 00000000..1d30ec1d --- /dev/null +++ b/benchmarks/trace-analysis/codetracebench-oht2-20260801/skills/sweagent_traj/SKILL.md @@ -0,0 +1,56 @@ +--- +name: sweagent_traj +description: > + Parse classic SWE-agent .traj trajectories: one step per entry of the + trajectory array, in order, matching the CodeTraceBench annotation + convention. No upstream CodeTracer commit ships a SWE-agent parser. +fingerprints: + - "*.traj" +priority: 60 +metadata: + version: "1.0" + source: agent-eval codetracebench-oht2-20260801 +--- + +# SWE-agent .traj Parser + +## Directory Layout + +``` +trial_dir/ + .traj # JSON: environment, trajectory[], history[], info, context + .context.json # edited/viewed file context (not steps) + .patch # final submission patch + report.json # evaluation outcome +``` + +The run directory must hold exactly one `*.traj` file; zero or several +refuse detection instead of guessing. + +## Step Convention + +- One step per `trajectory[]` entry, in array order. Every entry counts, + including entries whose `action` is blank. +- The action is the entry's raw `action`; actions that are blank after + trimming render as their JSON literal so the step survives non-empty + action gates losslessly. +- The entry's `observation` passes through unchanged (empty strings stay + empty strings); a non-string, non-null observation is a parse error. +- The entry's `thought` becomes `thinking` when it is a non-blank string. +- `action_ref` points at `.traj#trajectory[i]` and embeds the raw + entry JSON (action, observation, response, state, thought), so the full + source record stays recoverable per step; `observation_ref` stays null + because it would duplicate the same entry. +- The task description is the first user `history[]` message that is not + the interface demonstration (marker: `--- DEMONSTRATION ---`). Both + published prompt templates (`` upload and `ISSUE:` text) + sit in that message on all 108 verified rows. + +This enumeration reproduces the CodeTraceBench annotation `step_count` on +106 of 108 verified SWE-agent rows at dataset revision +`aa213b84ffb6690fc37ca15766d6ca174ec36d4d`. The other 2 rows +(`keras-team__keras-19775`, `mui__material-ui-11451`) publish a +function-call-style trajectory (45/48 and 92/94 entries with empty +`response`) whose annotated view condensed the run (26 vs 48, 54 vs 94 +entries); no published view reproduces those counts, so they fail the +step-count gate loudly instead of importing misaligned labels. diff --git a/benchmarks/trace-analysis/codetracebench-oht2-20260801/skills/sweagent_traj/parser.py b/benchmarks/trace-analysis/codetracebench-oht2-20260801/skills/sweagent_traj/parser.py new file mode 100644 index 00000000..7643b4e8 --- /dev/null +++ b/benchmarks/trace-analysis/codetracebench-oht2-20260801/skills/sweagent_traj/parser.py @@ -0,0 +1,117 @@ +"""SWE-agent .traj parser: one step per `trajectory[]` entry, in order. + +CodeTraceBench annotates classic SWE-agent runs against the .traj file's +`trajectory` array: every entry is one step, including entries whose action +is blank. This enumeration reproduces the CodeTraceBench `step_count` on +106/108 verified SWE-agent rows; the other 2 archives publish a +function-call-style trajectory (92-94% empty `response` fields) whose +annotated view condensed the run, so no published view matches and those +rows must fail step-count checks rather than import misaligned labels. + +The run directory holds exactly one `.traj`; ambiguity (zero or +several .traj files) refuses detection instead of guessing. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from codetracer.models import FileRef, NormalizedTrajectory, StepRecord + +_DEMONSTRATION_MARKER = "--- DEMONSTRATION ---" + + +class SweAgentTrajParser: + format_id = "sweagent_traj" + + def can_parse(self, run_dir: Path) -> bool: + traj_path = _single_traj_file(run_dir) + if traj_path is None: + return False + try: + data = json.loads(traj_path.read_text(encoding="utf-8", errors="replace")) + except ValueError: + return False + return isinstance(data, dict) and isinstance(data.get("trajectory"), list) + + def parse(self, run_dir: Path) -> NormalizedTrajectory: + traj_path = _single_traj_file(run_dir) + if traj_path is None: + raise ValueError(f"expected exactly one .traj file in {run_dir}") + data = json.loads(traj_path.read_text(encoding="utf-8", errors="replace")) + trajectory = data.get("trajectory") + if not isinstance(trajectory, list): + raise ValueError(f"{traj_path.name} has no trajectory array") + steps = [ + _step(index, entry, traj_path) + for index, entry in enumerate(trajectory) + ] + return NormalizedTrajectory( + steps=steps, + task_description=_task_text(data), + metadata={"format": self.format_id, "run_dir": str(run_dir), "traj": traj_path.name}, + ) + + +def _single_traj_file(run_dir: Path) -> Path | None: + if not run_dir.is_dir(): + return None + candidates = sorted(p for p in run_dir.glob("*.traj") if p.is_file()) + return candidates[0] if len(candidates) == 1 else None + + +def _step(index: int, entry: Any, traj_path: Path) -> StepRecord: + label = f"{traj_path.name}#trajectory[{index}]" + if not isinstance(entry, dict): + raise ValueError(f"{label} is not an object") + observation = entry.get("observation") + if observation is not None and not isinstance(observation, str): + raise ValueError(f"{label}.observation is neither a string nor null") + thought = entry.get("thought") + thinking = thought if isinstance(thought, str) and thought.strip() else None + return StepRecord( + step_id=index + 1, + action=_action_text(entry.get("action")), + observation=observation, + thinking=thinking, + action_ref=FileRef( + path=label, + line_start=1, + line_end=1, + content=json.dumps(entry, ensure_ascii=False), + ), + # The raw entry in action_ref already carries the observation's + # source; a second ref would only duplicate it. + observation_ref=None, + ) + + +def _action_text(raw: Any) -> str: + text = raw if isinstance(raw, str) else json.dumps(raw, ensure_ascii=False) + # Blank actions (empty send, model returned no command) are annotated + # steps; render them as their JSON literal so the step survives + # non-empty action gates losslessly. + if not text.strip(): + text = json.dumps(text, ensure_ascii=False) + return text + + +def _task_text(data: dict[str, Any]) -> str: + """First user history message that is not the interface demonstration.""" + history = data.get("history") + if not isinstance(history, list): + return "" + for message in history: + if not isinstance(message, dict) or message.get("role") != "user": + continue + content = message.get("content") + if not isinstance(content, str) or _DEMONSTRATION_MARKER in content: + continue + if content.strip(): + return content.strip() + return "" + + +parser = SweAgentTrajParser()