Skip to content

Commit 101b919

Browse files
committed
SDK spec llm span test runner
1 parent 10f6ecc commit 101b919

File tree

13 files changed

+2767
-0
lines changed

13 files changed

+2767
-0
lines changed

sdk-spec-impl/build.gradle

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
plugins {
2+
id 'java'
3+
id 'application'
4+
id 'com.github.johnrengelman.shadow' version '8.1.1'
5+
}
6+
7+
java {
8+
toolchain {
9+
languageVersion = JavaLanguageVersion.of(17)
10+
}
11+
}
12+
13+
repositories {
14+
mavenCentral()
15+
mavenLocal()
16+
}
17+
18+
dependencies {
19+
// Braintrust SDK (local project dependencies)
20+
implementation project(':braintrust-sdk')
21+
implementation project(':braintrust-sdk:instrumentation:openai_2_8_0')
22+
implementation project(':braintrust-sdk:instrumentation:anthropic_2_2_0')
23+
implementation project(':braintrust-sdk:instrumentation:genai_1_18_0')
24+
implementation project(':braintrust-sdk:instrumentation:langchain_1_8_0')
25+
26+
// Jackson for JSON processing
27+
implementation 'com.fasterxml.jackson.core:jackson-databind:2.16.1'
28+
implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.16.1'
29+
30+
// OpenAI SDK
31+
implementation 'com.openai:openai-java:2.8.1'
32+
33+
// Anthropic SDK
34+
implementation 'com.anthropic:anthropic-java:2.10.0'
35+
36+
// Gemini SDK
37+
implementation 'org.springframework.ai:spring-ai-google-genai:1.1.0'
38+
39+
// LangChain4j
40+
implementation 'dev.langchain4j:langchain4j:1.9.1'
41+
implementation 'dev.langchain4j:langchain4j-http-client:1.9.1'
42+
implementation 'dev.langchain4j:langchain4j-open-ai:1.9.1'
43+
44+
// OpenTelemetry
45+
implementation 'io.opentelemetry:opentelemetry-api:1.54.1'
46+
47+
// Logging
48+
implementation 'org.slf4j:slf4j-api:2.0.17'
49+
runtimeOnly 'org.slf4j:slf4j-simple:2.0.17'
50+
51+
// YAML parsing for spec files
52+
implementation 'org.yaml:snakeyaml:2.3'
53+
54+
// Test dependencies
55+
testImplementation(testFixtures(project(":test-harness")))
56+
testImplementation "org.junit.jupiter:junit-jupiter:${rootProject.ext.junitVersion}"
57+
testImplementation "org.junit.jupiter:junit-jupiter-params:${rootProject.ext.junitVersion}"
58+
testImplementation "io.opentelemetry:opentelemetry-sdk:${rootProject.ext.otelVersion}"
59+
testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
60+
}
61+
62+
application {
63+
mainClass = 'dev.braintrust.btx.BtxServer'
64+
}
65+
66+
// Configure shadow JAR
67+
shadowJar {
68+
archiveBaseName.set('btx-server')
69+
archiveClassifier.set('')
70+
archiveVersion.set('')
71+
manifest {
72+
attributes 'Main-Class': 'dev.braintrust.btx.BtxServer'
73+
}
74+
// Merge service files for OpenTelemetry SPI
75+
mergeServiceFiles()
76+
}
77+
78+
// Make build task also create the shadow JAR
79+
build.dependsOn shadowJar
80+
81+
// Configure run task to properly pass arguments
82+
run {
83+
standardInput = System.in
84+
if (project.hasProperty('args')) {
85+
args project.property('args').split('\\s+')
86+
}
87+
}
88+
89+
test {
90+
useJUnitPlatform()
91+
workingDir = rootProject.projectDir
92+
testLogging {
93+
events "passed", "skipped", "failed"
94+
showStandardStreams = true
95+
exceptionFormat "full"
96+
}
97+
}
Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
# BTX Server Health Check Script
5+
# Usage: ./scripts/health-check-btx-server <runner-directory>
6+
# Example: ./scripts/health-check-btx-server ./src/btx/runners/java
7+
#
8+
# Exits with:
9+
# 0 - Server started successfully and health check passed
10+
# 1 - Server failed to start or health check failed
11+
12+
if [ $# -ne 1 ]; then
13+
echo "Usage: $0 <runner-directory>" >&2
14+
echo "Example: $0 ./src/btx/runners/java" >&2
15+
exit 1
16+
fi
17+
18+
RUNNER_DIR="$1"
19+
START_SCRIPT="$RUNNER_DIR/start.sh"
20+
21+
# Validate runner directory
22+
if [ ! -d "$RUNNER_DIR" ]; then
23+
echo "Error: Runner directory does not exist: $RUNNER_DIR" >&2
24+
exit 1
25+
fi
26+
27+
if [ ! -f "$START_SCRIPT" ]; then
28+
echo "Error: start.sh not found in $RUNNER_DIR" >&2
29+
exit 1
30+
fi
31+
32+
if [ ! -x "$START_SCRIPT" ]; then
33+
echo "Error: start.sh is not executable: $START_SCRIPT" >&2
34+
exit 1
35+
fi
36+
37+
# Find a random open port
38+
find_open_port() {
39+
python3 -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()'
40+
}
41+
42+
PORT=$(find_open_port)
43+
HEALTH_URL="http://localhost:$PORT/health"
44+
MAX_WAIT=30 # Maximum seconds to wait for health check
45+
POLL_INTERVAL=0.5 # Seconds between health check attempts
46+
47+
echo "Starting BTX server on port $PORT..."
48+
49+
# Start the server in background, redirecting output to temp files
50+
TMPDIR=$(mktemp -d)
51+
LOG_FILE="$TMPDIR/btx-server.log"
52+
PID_FILE="$TMPDIR/btx-server.pid"
53+
54+
# Cleanup function
55+
cleanup() {
56+
if [ -f "$PID_FILE" ]; then
57+
PID=$(cat "$PID_FILE")
58+
if kill -0 "$PID" 2>/dev/null; then
59+
echo "Stopping server (PID: $PID)..."
60+
kill "$PID" 2>/dev/null || true
61+
# Give it a moment to shut down gracefully
62+
sleep 0.5
63+
# Force kill if still running
64+
if kill -0 "$PID" 2>/dev/null; then
65+
kill -9 "$PID" 2>/dev/null || true
66+
fi
67+
fi
68+
fi
69+
rm -rf "$TMPDIR"
70+
}
71+
72+
trap cleanup EXIT
73+
74+
# Start the server
75+
"$START_SCRIPT" "$PORT" > "$LOG_FILE" 2>&1 &
76+
SERVER_PID=$!
77+
echo $SERVER_PID > "$PID_FILE"
78+
79+
echo "Server started with PID: $SERVER_PID"
80+
echo "Log file: $LOG_FILE"
81+
82+
# Wait for server to be healthy
83+
echo "Waiting for health check at $HEALTH_URL..."
84+
ATTEMPTS=0
85+
MAX_ATTEMPTS=$((MAX_WAIT * 2)) # Poll twice per second
86+
87+
while [ $ATTEMPTS -lt $MAX_ATTEMPTS ]; do
88+
# Check if process is still alive
89+
if ! kill -0 $SERVER_PID 2>/dev/null; then
90+
echo "Error: Server process died unexpectedly" >&2
91+
echo "Last 20 lines of log:" >&2
92+
tail -20 "$LOG_FILE" >&2
93+
exit 1
94+
fi
95+
96+
# Try health check
97+
if curl -sf "$HEALTH_URL" > /dev/null 2>&1; then
98+
RESPONSE=$(curl -s "$HEALTH_URL")
99+
echo "Health check passed!"
100+
echo "Response: $RESPONSE"
101+
exit 0
102+
fi
103+
104+
sleep $POLL_INTERVAL
105+
ATTEMPTS=$((ATTEMPTS + 1))
106+
107+
# Show progress every 10 attempts (5 seconds)
108+
if [ $((ATTEMPTS % 10)) -eq 0 ]; then
109+
ELAPSED=$((ATTEMPTS / 2))
110+
echo "Still waiting... (${ELAPSED}s elapsed)"
111+
fi
112+
done
113+
114+
# Timeout reached
115+
echo "Error: Health check timed out after ${MAX_WAIT}s" >&2
116+
echo "Last 20 lines of log:" >&2
117+
tail -20 "$LOG_FILE" >&2
118+
exit 1

0 commit comments

Comments
 (0)