Skip to content

Add JSONL filter for codex output #13

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
juhgiyo merged 1 commit into from
Mar 17, 2026
+133 −73
Merged

Add JSONL filter for codex output #13

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 53 additions & 2 deletions codex/wrapper/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,11 @@
// to "codex" CLI flags. It does NOT manage processes, sessions, or any
// infrastructure — that is handled by WrapperRuntime in StrawPot core.
//
// Subcommands: setup, build
// Subcommands: setup, build, filter
package main

import (
"bufio"
"encoding/json"
"fmt"
"io/fs"
Expand All @@ -28,6 +29,8 @@ func main() {
cmdSetup()
case "build":
cmdBuild(os.Args[2:])
case "filter":
cmdFilter()
default:
fmt.Fprintf(os.Stderr, "Unknown subcommand: %s\n", os.Args[1])
os.Exit(1)
Expand Down Expand Up @@ -66,6 +69,30 @@ func cmdSetup() {
}
}

// ---------------------------------------------------------------------------
// filter — reads Codex JSONL from stdin, emits only agent_message text
// ---------------------------------------------------------------------------

func cmdFilter() {
scanner := bufio.NewScanner(os.Stdin)
scanner.Buffer(make([]byte, 1024*1024), 1024*1024) // 1 MB lines
for scanner.Scan() {
var event struct {
Type string `json:"type"`
Item struct {
Type string `json:"type"`
Text string `json:"text"`
} `json:"item"`
}
if err := json.Unmarshal(scanner.Bytes(), &event); err != nil {
continue
}
if event.Type == "item.completed" && event.Item.Type == "agent_message" {
fmt.Println(event.Item.Text)
}
}
}

// ---------------------------------------------------------------------------
// build
// ---------------------------------------------------------------------------
Expand Down Expand Up @@ -271,9 +298,19 @@ func cmdBuild(args []string) {
}
}

// Resolve wrapper binary path for the filter pipe.
wrapperBin, err := os.Executable()
if err != nil {
fmt.Fprintf(os.Stderr, "Failed to resolve wrapper path: %v\n", err)
os.Exit(1)
}

// Build shell command that pipes codex JSONL through the filter.
shellCmd := shellJoin(cmd) + " | " + shellEscape(wrapperBin) + " filter"

// Output JSON
result := map[string]interface{}{
"cmd": cmd,
"cmd": []string{"sh", "-c", shellCmd},
"cwd": ba.WorkingDir,
}

Expand All @@ -283,3 +320,17 @@ func cmdBuild(args []string) {
os.Exit(1)
}
}

// shellEscape wraps a string in single quotes for sh.
func shellEscape(s string) string {
return "'" + strings.ReplaceAll(s, "'", "'\"'\"'") + "'"
}

// shellJoin quotes and joins args for sh -c.
func shellJoin(args []string) string {
escaped := make([]string, len(args))
for i, a := range args {
escaped[i] = shellEscape(a)
}
return strings.Join(escaped, " ")
}
151 changes: 80 additions & 71 deletions codex/wrapper/main_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
)

Expand Down Expand Up @@ -96,12 +97,19 @@ func TestCmdBuild_MinimalArgs(t *testing.T) {
t.Fatal("Missing 'cmd' in output")
}

if len(cmd) < 2 || cmd[0] != "codex" || cmd[1] != "exec" {
t.Errorf("cmd should start with [codex, exec], got %v", cmd[:2])
// cmd is now ["sh", "-c", "<shell pipeline>"]
if len(cmd) != 3 || cmd[0] != "sh" || cmd[1] != "-c" {
t.Fatalf("cmd should be [sh, -c, <shell>], got %v", cmd)
}
assertContains(t, cmd, "--add-dir")
assertContains(t, cmd, wsDir)
assertSequence(t, cmd, "-C", "/project")

shell := cmd[2].(string)
assertShellContains(t, shell, "'codex' 'exec'")
assertShellContains(t, shell, "--json")
assertShellContains(t, shell, "--add-dir")
assertShellContains(t, shell, wsDir)
assertShellContains(t, shell, "'-C' '/project'")
assertShellContains(t, shell, "| ")
assertShellContains(t, shell, "filter")

if result["cwd"] != "/project" {
t.Errorf("cwd = %v, want /project", result["cwd"])
Expand Down Expand Up @@ -132,13 +140,9 @@ func TestCmdBuild_WithTaskAndModel(t *testing.T) {
t.Fatalf("Failed to parse JSON output: %v", err)
}

cmd := result["cmd"].([]interface{})

// Task is positional after "codex exec --json"
if len(cmd) < 4 || cmd[3] != "fix the bug" {
t.Errorf("cmd[3] = %v, want %q", cmd[3], "fix the bug")
}
assertSequence(t, cmd, "-m", "gpt-5.2-codex")
shell := result["cmd"].([]interface{})[2].(string)
assertShellContains(t, shell, "'fix the bug'")
assertShellContains(t, shell, "'-m' 'gpt-5.2-codex'")
}

func TestCmdBuild_PromptFile(t *testing.T) {
Expand All @@ -151,7 +155,7 @@ func TestCmdBuild_PromptFile(t *testing.T) {
"--memory-prompt", "Use Go",
}

output := captureBuildOutput(t, args)
captureBuildOutput(t, args)

content, err := os.ReadFile(filepath.Join(wsDir, "prompt.md"))
if err != nil {
Expand All @@ -163,26 +167,13 @@ func TestCmdBuild_PromptFile(t *testing.T) {
t.Errorf("prompt.md = %q, want %q", string(content), expected)
}

// Verify -c model_instructions_file is in cmd
// Verify -c model_instructions_file is in the shell command
output := captureBuildOutput(t, args)
var result map[string]interface{}
json.Unmarshal(output, &result)
cmd := result["cmd"].([]interface{})
assertContains(t, cmd, "-c")

// Find the model_instructions_file config value
found := false
for i, v := range cmd {
if v == "-c" && i+1 < len(cmd) {
val, ok := cmd[i+1].(string)
if ok && len(val) > 25 && val[:25] == "model_instructions_file=\"" {
found = true
break
}
}
}
if !found {
t.Errorf("cmd %v does not contain -c model_instructions_file=...", cmd)
}
shell := result["cmd"].([]interface{})[2].(string)
assertShellContains(t, shell, "'-c'")
assertShellContains(t, shell, "model_instructions_file=")
}

func TestCmdBuild_NoPromptFlag_WhenEmpty(t *testing.T) {
Expand All @@ -197,16 +188,11 @@ func TestCmdBuild_NoPromptFlag_WhenEmpty(t *testing.T) {

var result map[string]interface{}
json.Unmarshal(output, &result)
cmd := result["cmd"].([]interface{})

// Should NOT contain -c model_instructions_file when no prompts provided
for i, v := range cmd {
if v == "-c" && i+1 < len(cmd) {
val, ok := cmd[i+1].(string)
if ok && len(val) >= 25 && val[:25] == "model_instructions_file=\"" {
t.Error("cmd should NOT contain -c model_instructions_file when no prompts provided")
}
}
shell := result["cmd"].([]interface{})[2].(string)

// Should NOT contain model_instructions_file when no prompts provided
if strings.Contains(shell, "model_instructions_file=") {
t.Error("shell cmd should NOT contain model_instructions_file when no prompts provided")
}
}

Expand Down Expand Up @@ -299,9 +285,8 @@ func TestCmdBuild_SandboxMode(t *testing.T) {

var result map[string]interface{}
json.Unmarshal(output, &result)

cmd := result["cmd"].([]interface{})
assertSequence(t, cmd, "--sandbox", "workspace-write")
shell := result["cmd"].([]interface{})[2].(string)
assertShellContains(t, shell, "'--sandbox' 'workspace-write'")
}

func TestCmdBuild_DangerouslyBypass_Default(t *testing.T) {
Expand All @@ -316,9 +301,8 @@ func TestCmdBuild_DangerouslyBypass_Default(t *testing.T) {

var result map[string]interface{}
json.Unmarshal(output, &result)

cmd := result["cmd"].([]interface{})
assertContains(t, cmd, "--dangerously-bypass-approvals-and-sandbox")
shell := result["cmd"].([]interface{})[2].(string)
assertShellContains(t, shell, "--dangerously-bypass-approvals-and-sandbox")
}

func TestCmdBuild_DangerouslyBypass_ExplicitTrue(t *testing.T) {
Expand All @@ -334,9 +318,8 @@ func TestCmdBuild_DangerouslyBypass_ExplicitTrue(t *testing.T) {

var result map[string]interface{}
json.Unmarshal(output, &result)

cmd := result["cmd"].([]interface{})
assertContains(t, cmd, "--dangerously-bypass-approvals-and-sandbox")
shell := result["cmd"].([]interface{})[2].(string)
assertShellContains(t, shell, "--dangerously-bypass-approvals-and-sandbox")
}

func TestCmdBuild_DangerouslyBypass_Disabled(t *testing.T) {
Expand All @@ -352,12 +335,51 @@ func TestCmdBuild_DangerouslyBypass_Disabled(t *testing.T) {

var result map[string]interface{}
json.Unmarshal(output, &result)
shell := result["cmd"].([]interface{})[2].(string)
if strings.Contains(shell, "--dangerously-bypass-approvals-and-sandbox") {
t.Error("shell cmd should NOT contain --dangerously-bypass-approvals-and-sandbox when disabled")
}
}

cmd := result["cmd"].([]interface{})
for _, v := range cmd {
if v == "--dangerously-bypass-approvals-and-sandbox" {
t.Error("cmd should NOT contain --dangerously-bypass-approvals-and-sandbox when disabled")
}
func TestCmdFilter(t *testing.T) {
// Simulate JSONL input with mixed event types
input := strings.Join([]string{
`{"type":"thread.started","thread_id":"abc"}`,
`{"type":"turn.started"}`,
`{"type":"item.completed","item":{"type":"reasoning","text":"thinking..."}}`,
`{"type":"item.completed","item":{"type":"command_execution","command":"ls"}}`,
`{"type":"item.completed","item":{"type":"agent_message","text":"Hello, here is the result."}}`,
`{"type":"turn.completed","usage":{"input_tokens":100}}`,
}, "\n")

// Pipe input through cmdFilter
oldStdin := os.Stdin
oldStdout := os.Stdout

inR, inW, _ := os.Pipe()
outR, outW, _ := os.Pipe()
os.Stdin = inR
os.Stdout = outW

go func() {
inW.WriteString(input)
inW.Close()
}()

cmdFilter()

outW.Close()
os.Stdin = oldStdin
os.Stdout = oldStdout

var buf [4096]byte
n, _ := outR.Read(buf[:])
outR.Close()

got := strings.TrimSpace(string(buf[:n]))
want := "Hello, here is the result."
if got != want {
t.Errorf("filter output = %q, want %q", got, want)
}
}

Expand Down Expand Up @@ -386,22 +408,9 @@ func captureBuildOutput(t *testing.T, args []string) []byte {
return buf[:n]
}

func assertContains(t *testing.T, slice []interface{}, val string) {
t.Helper()
for _, v := range slice {
if v == val {
return
}
}
t.Errorf("cmd %v does not contain %q", slice, val)
}

func assertSequence(t *testing.T, slice []interface{}, key, val string) {
func assertShellContains(t *testing.T, shell, substr string) {
t.Helper()
for i, v := range slice {
if v == key && i+1 < len(slice) && slice[i+1] == val {
return
}
if !strings.Contains(shell, substr) {
t.Errorf("shell cmd %q does not contain %q", shell, substr)
}
t.Errorf("cmd %v does not contain %q %q in sequence", slice, key, val)
}
Loading