Skip to content

Commit 7d19ada

Browse files
authored
feat(tracing): emit token usage on spans for SGP billing (#458)
1 parent 9245b70 commit 7d19ada

18 files changed

Lines changed: 871 additions & 468 deletions

File tree

examples/tutorials/10_async/00_base/030_tracing/README.md

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,39 @@ await adk.tracing.end_span(
4141

4242
Spans create a hierarchical view of agent execution, making it easy to see which operations take time and where errors occur.
4343

44+
## Token Usage & Cost Tracking
45+
46+
Token usage on spans is what the backend bills from, and it reads two shapes:
47+
48+
- **Per-turn aggregate**`span.data["usage"]` + `span.data["cost_usd"]`. Emit at most
49+
once per turn, holding that turn's own usage (not a session-cumulative total). When a
50+
trace has an aggregate, the backend keeps it and de-dups all per-call spans against it.
51+
- **Per-call detail**`span.output["usage"]`. Optional; the SDK's LLM adapters
52+
(litellm, OpenAI Agents SDK, LangGraph) emit this automatically. Summed only when no
53+
aggregate exists in the trace.
54+
55+
Record the turn rollup with `adk.tracing.turn_span()` instead of hand-writing usage keys.
56+
It accepts the harness `TurnUsage` that every turn adapter reports (`LangGraphTurn.usage()`,
57+
`run_turn(...).usage`, `ClaudeCodeTurn.usage()`, ...), cost included:
58+
59+
```python
60+
async with adk.tracing.turn_span(
61+
trace_id=task.id,
62+
name="turn",
63+
input={"prompt": prompt},
64+
task_id=task.id,
65+
) as turn:
66+
result = await run_turn(...)
67+
turn.output = {"response": result.final_output}
68+
turn.record_usage(result.usage) # TurnUsage; cost_usd stamped automatically
69+
```
70+
71+
**Never put usage on both a rollup span's `output` and its per-call children's
72+
`output`** — that double-counts. `turn_span` writes the aggregate to `data`, so child
73+
spans stay safe to emit. Recognized token keys: `input_tokens`/`prompt_tokens`,
74+
`output_tokens`/`completion_tokens`, `cached_input_tokens`/`cached_tokens`,
75+
`reasoning_tokens`; cost is `cost_usd`.
76+
4477
## When to Use
4578
- Debugging complex agent behaviors
4679
- Performance optimization and bottleneck identification

src/agentex/lib/adk/__init__.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
from agentex.lib.adk._modules.state import StateModule
2828
from agentex.lib.adk._modules.streaming import StreamingModule
2929
from agentex.lib.adk._modules.tasks import TasksModule
30-
from agentex.lib.adk._modules.tracing import TracingModule
30+
from agentex.lib.adk._modules.tracing import TracingModule, TurnSpan
3131

3232
# Unified harness surface (AGX1-375)
3333
from agentex.lib.core.harness import (
@@ -66,6 +66,7 @@
6666
"tracing",
6767
"events",
6868
"agent_task_tracker",
69+
"TurnSpan",
6970
# Checkpointing / LangGraph
7071
"create_checkpointer",
7172
"stream_langgraph_events",

src/agentex/lib/adk/_modules/tracing.py

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
TracingActivityName,
2121
)
2222
from agentex.lib.core.tracing.tracer import AsyncTracer
23+
from agentex.lib.core.harness.types import TurnUsage
2324
from agentex.types.span import Span
2425
from agentex.lib.utils.logging import make_logger
2526
from agentex.lib.utils.model_utils import BaseModel
@@ -30,6 +31,21 @@
3031
DEFAULT_RETRY_POLICY = RetryPolicy(maximum_attempts=1)
3132
TEMPORAL_SPAN_ACTIVITY_DROPPED_METRIC = "agentex.tracing.temporal_span_activity.dropped"
3233

34+
# Token key spellings the backend accepts when billing usage from spans.
35+
RECOGNIZED_USAGE_KEYS = frozenset(
36+
{
37+
"input_tokens",
38+
"prompt_tokens",
39+
"output_tokens",
40+
"completion_tokens",
41+
"cached_input_tokens",
42+
"cached_tokens",
43+
"reasoning_tokens",
44+
"total_tokens",
45+
"cost_usd",
46+
}
47+
)
48+
3349

3450
def _record_temporal_span_activity_dropped(event_type: str) -> None:
3551
try:
@@ -42,6 +58,80 @@ def _record_temporal_span_activity_dropped(event_type: str) -> None:
4258
pass
4359

4460

61+
class TurnSpan:
62+
"""Handle for a turn-level (rollup) span, yielded by ``TracingModule.turn_span``.
63+
64+
Encapsulates the billing contract so agents cannot double-count usage:
65+
the turn's aggregate usage goes to ``span.data["usage"]`` (+
66+
``span.data["cost_usd"]``) via :meth:`record_usage`. The backend keeps the
67+
aggregate and de-dups any per-call ``output["usage"]`` children against it.
68+
Never hand-write usage into ``output`` on a rollup span — that is the
69+
double-count bug this helper exists to prevent.
70+
71+
All methods no-op when tracing is disabled (``span`` is None), so agent
72+
code needs no ``if span:`` guards.
73+
"""
74+
75+
def __init__(self, span: Span | None):
76+
self.span = span
77+
78+
def record_usage(
79+
self,
80+
usage: TurnUsage | dict[str, Any] | None = None,
81+
cost_usd: float | None = None,
82+
) -> None:
83+
"""Record the turn's aggregate usage on the span's ``data``.
84+
85+
Pass the harness ``TurnUsage`` (e.g. ``LangGraphTurn.usage()`` or
86+
``run_turn(...).usage``) — its ``cost_usd`` is stamped automatically —
87+
or a plain dict with backend-recognized token spellings
88+
(``prompt_tokens``/``completion_tokens`` also work). An explicit
89+
``cost_usd`` argument overrides any cost carried by ``usage``. The
90+
usage must be this turn's own tokens, not a session-cumulative total.
91+
"""
92+
if self.span is None:
93+
return
94+
95+
blob: dict[str, Any]
96+
if isinstance(usage, TurnUsage):
97+
blob = usage.model_dump(exclude_none=True)
98+
# cost lives beside the blob as data["cost_usd"], not inside it
99+
blob_cost = blob.pop("cost_usd", None)
100+
if cost_usd is None:
101+
cost_usd = blob_cost
102+
elif usage is not None:
103+
blob = dict(usage)
104+
if not any(key in RECOGNIZED_USAGE_KEYS for key in blob):
105+
logger.warning(
106+
"TurnSpan.record_usage: usage has no recognized token keys and will "
107+
f"not be billed. Got keys {sorted(blob)}; expected any of "
108+
f"{sorted(RECOGNIZED_USAGE_KEYS)}."
109+
)
110+
else:
111+
blob = {}
112+
113+
if self.span.data is not None and not isinstance(self.span.data, dict):
114+
logger.warning(
115+
f"TurnSpan.record_usage: span.data is {type(self.span.data).__name__} "
116+
"(expected dict or None); existing data will be replaced."
117+
)
118+
data = self.span.data if isinstance(self.span.data, dict) else {}
119+
if blob:
120+
data["usage"] = blob
121+
if cost_usd is not None:
122+
data["cost_usd"] = cost_usd
123+
self.span.data = data
124+
125+
@property
126+
def output(self) -> Any:
127+
return self.span.output if self.span is not None else None
128+
129+
@output.setter
130+
def output(self, value: Any) -> None:
131+
if self.span is not None:
132+
self.span.output = value
133+
134+
45135
class TracingModule:
46136
"""
47137
Module for managing tracing and span operations in Agentex.
@@ -156,6 +246,47 @@ async def span(
156246
retry_policy=retry_policy,
157247
)
158248

249+
@asynccontextmanager
250+
async def turn_span(
251+
self,
252+
trace_id: str,
253+
name: str,
254+
input: list[Any] | dict[str, Any] | BaseModel | None = None,
255+
data: list[Any] | dict[str, Any] | BaseModel | None = None,
256+
parent_id: str | None = None,
257+
task_id: str | None = None,
258+
start_to_close_timeout: timedelta = timedelta(seconds=5),
259+
heartbeat_timeout: timedelta = timedelta(seconds=5),
260+
retry_policy: RetryPolicy = DEFAULT_RETRY_POLICY,
261+
) -> AsyncGenerator[TurnSpan, None]:
262+
"""Span for one agent turn, with usage recorded as the billable aggregate.
263+
264+
Same lifecycle as :meth:`span`, but yields a :class:`TurnSpan` whose
265+
``record_usage(usage=..., cost_usd=...)`` writes the turn's rollup
266+
usage to ``span.data`` — the shape the backend bills once per turn.
267+
Per-call child spans (LLM adapters) may still carry
268+
``output["usage"]``; the backend de-dups them against this aggregate.
269+
270+
Example (with a harness turn, e.g. ``LangGraphTurn`` / ``run_turn``)::
271+
272+
async with adk.tracing.turn_span(trace_id=task.id, name="turn", input={...}, task_id=task.id) as turn:
273+
result = await run_turn(...)
274+
turn.output = {"response": result.final_output}
275+
turn.record_usage(result.usage) # TurnUsage, cost_usd included
276+
"""
277+
async with self.span(
278+
trace_id=trace_id,
279+
name=name,
280+
input=input,
281+
data=data,
282+
parent_id=parent_id,
283+
task_id=task_id,
284+
start_to_close_timeout=start_to_close_timeout,
285+
heartbeat_timeout=heartbeat_timeout,
286+
retry_policy=retry_policy,
287+
) as span:
288+
yield TurnSpan(span)
289+
159290
async def start_span(
160291
self,
161292
trace_id: str,

src/agentex/lib/core/services/adk/providers/litellm.py

Lines changed: 35 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,20 @@
2525
logger = logging.make_logger(__name__)
2626

2727

28+
def _stream_kwargs_with_usage(llm_config: LLMConfig) -> dict:
29+
"""Completion kwargs with usage reporting enabled on the final stream chunk.
30+
31+
litellm only reports usage for streaming calls when
32+
``stream_options={"include_usage": True}`` is set; default it on so usage
33+
reaches the span. Callers can still opt out by explicitly passing
34+
``stream_options={"include_usage": False}``.
35+
"""
36+
kwargs = llm_config.model_dump()
37+
stream_options = kwargs.get("stream_options") or {}
38+
kwargs["stream_options"] = {"include_usage": True, **stream_options}
39+
return kwargs
40+
41+
2842
class LiteLLMService:
2943
def __init__(
3044
self,
@@ -121,7 +135,11 @@ async def chat_completion_auto_send(
121135

122136
if span:
123137
if streaming_context.task_message:
124-
span.output = streaming_context.task_message.model_dump()
138+
output = streaming_context.task_message.model_dump()
139+
# Per-call usage for billing; deduped against any turn aggregate
140+
if completion.usage is not None:
141+
output["usage"] = completion.usage.model_dump()
142+
span.output = output
125143
return streaming_context.task_message if streaming_context.task_message else None
126144

127145
async def chat_completion_stream(
@@ -150,18 +168,21 @@ async def chat_completion_stream(
150168
if self.llm_gateway is None:
151169
raise ValueError("LLM Gateway is not set")
152170

171+
completion_kwargs = _stream_kwargs_with_usage(llm_config)
153172
trace = self.tracer.trace(trace_id)
154173
async with trace.span(
155174
parent_id=parent_span_id,
156175
name="chat_completion_stream",
157-
input=llm_config.model_dump(),
176+
input=completion_kwargs,
158177
) as span:
159178
# Direct streaming outside temporal - yield each chunk as it comes
160179
chunks: list[Completion] = []
161-
async for chunk in self.llm_gateway.acompletion_stream(**llm_config.model_dump()):
180+
async for chunk in self.llm_gateway.acompletion_stream(**completion_kwargs):
162181
chunks.append(chunk)
163182
yield chunk
164183
if span:
184+
# The usage-bearing final chunk survives concat, so the dumped
185+
# completion carries usage for billing
165186
span.output = concat_completion_chunks(chunks).model_dump()
166187

167188
async def chat_completion_stream_auto_send(
@@ -190,11 +211,12 @@ async def chat_completion_stream_auto_send(
190211
if not llm_config.stream:
191212
llm_config.stream = True
192213

214+
completion_kwargs = _stream_kwargs_with_usage(llm_config)
193215
trace = self.tracer.trace(trace_id)
194216
async with trace.span(
195217
parent_id=parent_span_id,
196218
name="chat_completion_stream_auto_send",
197-
input=llm_config.model_dump(),
219+
input=completion_kwargs,
198220
) as span:
199221
# Use streaming context manager
200222
async with self.streaming_service.streaming_task_message_context(
@@ -208,8 +230,11 @@ async def chat_completion_stream_auto_send(
208230
) as streaming_context:
209231
# Get the streaming response
210232
chunks = []
211-
async for response in self.llm_gateway.acompletion_stream(**llm_config.model_dump()):
233+
async for response in self.llm_gateway.acompletion_stream(**completion_kwargs):
212234
heartbeat_if_in_workflow("chat completion streaming")
235+
# Store every chunk for final message assembly, including
236+
# the usage-only final chunk, which has no choices
237+
chunks.append(response)
213238
if response.choices and len(response.choices) > 0 and response.choices[0].delta:
214239
delta = response.choices[0].delta.content
215240
if delta:
@@ -223,9 +248,6 @@ async def chat_completion_stream_auto_send(
223248
)
224249
heartbeat_if_in_workflow("content chunk streamed")
225250

226-
# Store the chunk for final message assembly
227-
chunks.append(response)
228-
229251
# Update the final message content
230252
complete_message = concat_completion_chunks(chunks)
231253
if complete_message and complete_message.choices and complete_message.choices[0].message:
@@ -246,6 +268,10 @@ async def chat_completion_stream_auto_send(
246268

247269
if span:
248270
if streaming_context.task_message:
249-
span.output = streaming_context.task_message.model_dump()
271+
output = streaming_context.task_message.model_dump()
272+
# Per-call usage for billing; deduped against any turn aggregate
273+
if complete_message.usage is not None:
274+
output["usage"] = complete_message.usage.model_dump()
275+
span.output = output
250276

251277
return streaming_context.task_message if streaming_context.task_message else None

src/agentex/lib/core/temporal/plugins/__init__.py

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111
Example:
1212
>>> from agentex.lib.core.temporal.plugins.openai_agents import (
1313
... TemporalStreamingModelProvider,
14-
... TemporalTracingModelProvider,
1514
... ContextInterceptor,
1615
... )
1716
>>> from temporalio.contrib.openai_agents import OpenAIAgentsPlugin, ModelActivityParameters
@@ -37,7 +36,6 @@
3736
ContextInterceptor,
3837
TemporalStreamingHooks,
3938
TemporalStreamingModel,
40-
TemporalTracingModelProvider,
4139
TemporalStreamingModelProvider,
4240
streaming_task_id,
4341
streaming_trace_id,
@@ -48,11 +46,10 @@
4846
__all__ = [
4947
"TemporalStreamingModel",
5048
"TemporalStreamingModelProvider",
51-
"TemporalTracingModelProvider",
5249
"ContextInterceptor",
5350
"streaming_task_id",
5451
"streaming_trace_id",
5552
"streaming_parent_span_id",
5653
"TemporalStreamingHooks",
5754
"stream_lifecycle_content",
58-
]
55+
]

src/agentex/lib/core/temporal/plugins/openai_agents/__init__.py

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -61,9 +61,6 @@
6161
from agentex.lib.core.temporal.plugins.openai_agents.hooks.activities import (
6262
stream_lifecycle_content,
6363
)
64-
from agentex.lib.core.temporal.plugins.openai_agents.models.temporal_tracing_model import (
65-
TemporalTracingModelProvider,
66-
)
6764
from agentex.lib.core.temporal.plugins.openai_agents.models.temporal_streaming_model import (
6865
TemporalStreamingModel,
6966
TemporalStreamingModelProvider,
@@ -78,7 +75,6 @@
7875
__all__ = [
7976
"TemporalStreamingModel",
8077
"TemporalStreamingModelProvider",
81-
"TemporalTracingModelProvider",
8278
"ContextInterceptor",
8379
"streaming_task_id",
8480
"streaming_trace_id",

src/agentex/lib/core/temporal/plugins/openai_agents/models/__init__.py

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,6 @@
44
capabilities to standard OpenAI models when running in Temporal workflows/activities.
55
"""
66

7-
from agentex.lib.core.temporal.plugins.openai_agents.models.temporal_tracing_model import (
8-
TemporalTracingModelProvider,
9-
TemporalTracingResponsesModel,
10-
TemporalTracingChatCompletionsModel,
11-
)
127
from agentex.lib.core.temporal.plugins.openai_agents.models.temporal_streaming_model import (
138
TemporalStreamingModel,
149
TemporalStreamingModelProvider,
@@ -17,7 +12,4 @@
1712
__all__ = [
1813
"TemporalStreamingModel",
1914
"TemporalStreamingModelProvider",
20-
"TemporalTracingModelProvider",
21-
"TemporalTracingResponsesModel",
22-
"TemporalTracingChatCompletionsModel",
23-
]
15+
]

0 commit comments

Comments
 (0)