Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions py/noxfile.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ def _pinned_python_version():
"autoevals",
"braintrust_core",
"litellm",
"openrouter",
"opentelemetry-api",
"opentelemetry-sdk",
"opentelemetry-exporter-otlp-proto-http",
Expand Down Expand Up @@ -100,6 +101,7 @@ def _pinned_python_version():
GENAI_VERSIONS = (LATEST,)
DSPY_VERSIONS = (LATEST,)
GOOGLE_ADK_VERSIONS = (LATEST, "1.14.1")
OPENROUTER_VERSIONS = (LATEST, "0.6.0")
# temporalio 1.19.0+ requires Python >= 3.10; skip Python 3.9 entirely
TEMPORAL_VERSIONS = (LATEST, "1.20.0", "1.19.0")
PYTEST_VERSIONS = (LATEST, "8.4.2")
Expand Down Expand Up @@ -210,6 +212,7 @@ def test_openai(session, version):
# openai-agents requires Python >= 3.10
_install(session, "openai-agents")
_run_tests(session, f"{WRAPPER_DIR}/test_openai.py")
_run_tests(session, f"{WRAPPER_DIR}/test_openai_openrouter_gateway.py")
_run_core_tests(session)


Expand All @@ -224,11 +227,12 @@ def test_openai_http2_streaming(session):


@nox.session()
def test_openrouter(session):
"""Test wrap_openai with OpenRouter. Requires OPENROUTER_API_KEY env var."""
@nox.parametrize("version", OPENROUTER_VERSIONS, ids=OPENROUTER_VERSIONS)
def test_openrouter(session, version):
"""Test the native OpenRouter SDK integration."""
_install_test_deps(session)
_install(session, "openai")
_run_tests(session, f"{WRAPPER_DIR}/test_openrouter.py")
_install(session, "openrouter", version)
_run_tests(session, f"{INTEGRATION_DIR}/openrouter/test_openrouter.py")


@nox.session()
Expand Down
1 change: 1 addition & 0 deletions py/requirements-optional.txt
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,4 @@ google-adk==1.14.1
dspy==3.1.3
langsmith==0.7.12
litellm==1.82.0
openrouter==0.7.11
3 changes: 3 additions & 0 deletions py/src/braintrust/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,9 @@ def is_equal(expected, output):
from .integrations.anthropic import (
wrap_anthropic, # noqa: F401 # type: ignore[reportUnusedImport]
)
from .integrations.openrouter import (
wrap_openrouter, # noqa: F401 # type: ignore[reportUnusedImport]
)
from .logger import *
from .logger import (
_internal_get_global_state, # noqa: F401 # type: ignore[reportUnusedImport]
Expand Down
5 changes: 5 additions & 0 deletions py/src/braintrust/auto.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
AnthropicIntegration,
ClaudeAgentSDKIntegration,
GoogleGenAIIntegration,
OpenRouterIntegration,
)


Expand All @@ -39,6 +40,7 @@ def auto_instrument(
litellm: bool = True,
pydantic_ai: bool = True,
google_genai: bool = True,
openrouter: bool = True,
agno: bool = True,
claude_agent_sdk: bool = True,
dspy: bool = True,
Expand All @@ -59,6 +61,7 @@ def auto_instrument(
litellm: Enable LiteLLM instrumentation (default: True)
pydantic_ai: Enable Pydantic AI instrumentation (default: True)
google_genai: Enable Google GenAI instrumentation (default: True)
openrouter: Enable OpenRouter instrumentation (default: True)
agno: Enable Agno instrumentation (default: True)
claude_agent_sdk: Enable Claude Agent SDK instrumentation (default: True)
dspy: Enable DSPy instrumentation (default: True)
Expand Down Expand Up @@ -120,6 +123,8 @@ def auto_instrument(
results["pydantic_ai"] = _instrument_pydantic_ai()
if google_genai:
results["google_genai"] = _instrument_integration(GoogleGenAIIntegration)
if openrouter:
results["openrouter"] = _instrument_integration(OpenRouterIntegration)
if agno:
results["agno"] = _instrument_integration(AgnoIntegration)
if claude_agent_sdk:
Expand Down
2 changes: 2 additions & 0 deletions py/src/braintrust/integrations/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from .anthropic import AnthropicIntegration
from .claude_agent_sdk import ClaudeAgentSDKIntegration
from .google_genai import GoogleGenAIIntegration
from .openrouter import OpenRouterIntegration


__all__ = [
Expand All @@ -11,4 +12,5 @@
"AnthropicIntegration",
"ClaudeAgentSDKIntegration",
"GoogleGenAIIntegration",
"OpenRouterIntegration",
]
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
"""Test auto_instrument for OpenRouter."""

import os

import openrouter
from braintrust.auto import auto_instrument
from braintrust.wrappers.test_utils import autoinstrument_test_context


results = auto_instrument()
assert results.get("openrouter") == True

results2 = auto_instrument()
assert results2.get("openrouter") == True

with autoinstrument_test_context("test_auto_openrouter") as memory_logger:
client = openrouter.OpenRouter(api_key=os.environ.get("OPENROUTER_API_KEY"))
response = client.chat.send(
model="openai/gpt-4o-mini",
messages=[{"role": "user", "content": "What is 2+2? Reply with just the number."}],
max_tokens=10,
)
assert "4" in response.choices[0].message.content

spans = memory_logger.pop()
assert len(spans) == 1, f"Expected 1 span, got {len(spans)}"
span = spans[0]
assert span["metadata"]["provider"] == "openai"
assert span["metadata"]["model"] == "gpt-4o-mini"
assert "4" in span["output"][0]["message"]["content"]

print("SUCCESS")
10 changes: 10 additions & 0 deletions py/src/braintrust/integrations/openrouter/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
"""Braintrust integration for the OpenRouter Python SDK."""

from .integration import OpenRouterIntegration
from .tracing import wrap_openrouter


__all__ = [
"OpenRouterIntegration",
"wrap_openrouter",
]
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
interactions:
- request:
body: '{"messages":[{"role":"user","content":"What is 2+2? Reply with just the
number."}],"model":"openai/gpt-4o-mini","max_tokens":10.0,"stream":false,"temperature":1.0,"top_p":1.0}'
headers:
Accept:
- application/json
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
Content-Length:
- '175'
Host:
- openrouter.ai
content-type:
- application/json
user-agent:
- speakeasy-sdk/python 0.7.11 2.768.0 1.0.0 openrouter
method: POST
uri: https://openrouter.ai/api/v1/chat/completions
response:
body:
string: "\n \n{\"id\":\"gen-1774558315-QxPl7aBABQW8bOKxzdxL\",\"object\":\"chat.completion\",\"created\":1774558315,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_f85b8886b6\",\"choices\":[{\"index\":0,\"logprobs\":null,\"finish_reason\":\"stop\",\"native_finish_reason\":\"stop\",\"message\":{\"role\":\"assistant\",\"content\":\"4\",\"refusal\":null,\"reasoning\":null}}],\"usage\":{\"prompt_tokens\":20,\"completion_tokens\":1,\"total_tokens\":21,\"cost\":0.0000036,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"cache_write_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":0.0000036,\"upstream_inference_prompt_cost\":0.000003,\"upstream_inference_completions_cost\":6e-7},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"image_tokens\":0,\"audio_tokens\":0}}}"
headers:
Access-Control-Allow-Origin:
- '*'
CF-RAY:
- 9e2909425a3b053a-SJC
Connection:
- keep-alive
Content-Type:
- application/json
Date:
- Thu, 26 Mar 2026 20:51:56 GMT
Permissions-Policy:
- payment=(self "https://checkout.stripe.com" "https://connect-js.stripe.com"
"https://js.stripe.com" "https://*.js.stripe.com" "https://hooks.stripe.com")
Referrer-Policy:
- no-referrer, strict-origin-when-cross-origin
Server:
- cloudflare
Transfer-Encoding:
- chunked
X-Content-Type-Options:
- nosniff
content-length:
- '785'
status:
code: 200
message: OK
version: 1
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
interactions:
- request:
body: '{"messages":[{"role":"user","content":"What is 3+3? Reply with just the
number."}],"model":"openai/gpt-4o-mini","max_tokens":10.0,"stream":false,"temperature":1.0,"top_p":1.0}'
headers:
Accept:
- application/json
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
Content-Length:
- '175'
Host:
- openrouter.ai
content-type:
- application/json
user-agent:
- speakeasy-sdk/python 0.7.11 2.768.0 1.0.0 openrouter
method: POST
uri: https://openrouter.ai/api/v1/chat/completions
response:
body:
string: "\n \n{\"id\":\"gen-1774558313-OLdiGZAnrgirxGutvM1d\",\"object\":\"chat.completion\",\"created\":1774558313,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_f85b8886b6\",\"choices\":[{\"index\":0,\"logprobs\":null,\"finish_reason\":\"stop\",\"native_finish_reason\":\"stop\",\"message\":{\"role\":\"assistant\",\"content\":\"6\",\"refusal\":null,\"reasoning\":null}}],\"usage\":{\"prompt_tokens\":20,\"completion_tokens\":1,\"total_tokens\":21,\"cost\":0.0000036,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"cache_write_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":0.0000036,\"upstream_inference_prompt_cost\":0.000003,\"upstream_inference_completions_cost\":6e-7},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"image_tokens\":0,\"audio_tokens\":0}}}"
headers:
Access-Control-Allow-Origin:
- '*'
CF-RAY:
- 9e2909316898c52e-SJC
Connection:
- keep-alive
Content-Type:
- application/json
Date:
- Thu, 26 Mar 2026 20:51:53 GMT
Permissions-Policy:
- payment=(self "https://checkout.stripe.com" "https://connect-js.stripe.com"
"https://js.stripe.com" "https://*.js.stripe.com" "https://hooks.stripe.com")
Referrer-Policy:
- no-referrer, strict-origin-when-cross-origin
Server:
- cloudflare
Transfer-Encoding:
- chunked
X-Content-Type-Options:
- nosniff
content-length:
- '785'
status:
code: 200
message: OK
version: 1
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
interactions:
- request:
body: '{"messages":[{"role":"user","content":"What is 5+5? Reply with just the
number."}],"model":"openai/gpt-4o-mini","max_tokens":10.0,"stream":true,"temperature":1.0,"top_p":1.0}'
headers:
Accept:
- text/event-stream
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
Content-Length:
- '174'
Host:
- openrouter.ai
content-type:
- application/json
user-agent:
- speakeasy-sdk/python 0.7.11 2.768.0 1.0.0 openrouter
method: POST
uri: https://openrouter.ai/api/v1/chat/completions
response:
body:
string: ': OPENROUTER PROCESSING


data: {"id":"gen-1774558312-UWJRiYAjZxsKV2WOReFJ","object":"chat.completion.chunk","created":1774558312,"model":"openai/gpt-4o-mini","provider":"OpenAI","system_fingerprint":"fp_f85b8886b6","choices":[{"index":0,"delta":{"content":"10","role":"assistant"},"finish_reason":null,"native_finish_reason":null}]}


data: {"id":"gen-1774558312-UWJRiYAjZxsKV2WOReFJ","object":"chat.completion.chunk","created":1774558312,"model":"openai/gpt-4o-mini","provider":"OpenAI","system_fingerprint":"fp_f85b8886b6","choices":[{"index":0,"delta":{"content":"","role":"assistant"},"finish_reason":"stop","native_finish_reason":"stop"}]}


data: {"id":"gen-1774558312-UWJRiYAjZxsKV2WOReFJ","object":"chat.completion.chunk","created":1774558312,"model":"openai/gpt-4o-mini","provider":"OpenAI","system_fingerprint":"fp_f85b8886b6","choices":[{"index":0,"delta":{"content":"","role":"assistant"},"finish_reason":"stop","native_finish_reason":"stop"}],"usage":{"prompt_tokens":20,"completion_tokens":1,"total_tokens":21,"cost":0.0000036,"is_byok":false,"prompt_tokens_details":{"cached_tokens":0,"cache_write_tokens":0,"audio_tokens":0,"video_tokens":0},"cost_details":{"upstream_inference_cost":0.0000036,"upstream_inference_prompt_cost":0.000003,"upstream_inference_completions_cost":6e-7},"completion_tokens_details":{"reasoning_tokens":0,"image_tokens":0,"audio_tokens":0}}}


data: [DONE]


'
headers:
Access-Control-Allow-Origin:
- '*'
CF-RAY:
- 9e29092c8dbc552b-SJC
Cache-Control:
- no-cache
Connection:
- keep-alive
Content-Type:
- text/event-stream
Date:
- Thu, 26 Mar 2026 20:51:52 GMT
Permissions-Policy:
- payment=(self "https://checkout.stripe.com" "https://connect-js.stripe.com"
"https://js.stripe.com" "https://*.js.stripe.com" "https://hooks.stripe.com")
Referrer-Policy:
- no-referrer, strict-origin-when-cross-origin
Server:
- cloudflare
Transfer-Encoding:
- chunked
X-Content-Type-Options:
- nosniff
status:
code: 200
message: OK
version: 1
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
interactions:
- request:
body: '{"messages":[{"role":"user","content":"What is 2+2? Reply with just the
number."}],"model":"openai/gpt-4o-mini","max_tokens":10.0,"stream":false,"temperature":1.0,"top_p":1.0}'
headers:
Accept:
- application/json
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
Content-Length:
- '175'
Host:
- openrouter.ai
content-type:
- application/json
user-agent:
- speakeasy-sdk/python 0.7.11 2.768.0 1.0.0 openrouter
method: POST
uri: https://openrouter.ai/api/v1/chat/completions
response:
body:
string: "\n \n{\"id\":\"gen-1774558311-aXJHnsIUyX9yfP5jKa4u\",\"object\":\"chat.completion\",\"created\":1774558311,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_f85b8886b6\",\"choices\":[{\"index\":0,\"logprobs\":null,\"finish_reason\":\"stop\",\"native_finish_reason\":\"stop\",\"message\":{\"role\":\"assistant\",\"content\":\"4\",\"refusal\":null,\"reasoning\":null}}],\"usage\":{\"prompt_tokens\":20,\"completion_tokens\":1,\"total_tokens\":21,\"cost\":0.0000036,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"cache_write_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":0.0000036,\"upstream_inference_prompt_cost\":0.000003,\"upstream_inference_completions_cost\":6e-7},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"image_tokens\":0,\"audio_tokens\":0}}}"
headers:
Access-Control-Allow-Origin:
- '*'
CF-RAY:
- 9e290926cb7af591-SJC
Connection:
- keep-alive
Content-Type:
- application/json
Date:
- Thu, 26 Mar 2026 20:51:52 GMT
Permissions-Policy:
- payment=(self "https://checkout.stripe.com" "https://connect-js.stripe.com"
"https://js.stripe.com" "https://*.js.stripe.com" "https://hooks.stripe.com")
Referrer-Policy:
- no-referrer, strict-origin-when-cross-origin
Server:
- cloudflare
Transfer-Encoding:
- chunked
X-Content-Type-Options:
- nosniff
content-length:
- '785'
status:
code: 200
message: OK
version: 1

Large diffs are not rendered by default.

Loading
Loading