diff --git a/public/fi_verify.py b/public/fi_verify.py new file mode 100644 index 00000000..b0b27f27 --- /dev/null +++ b/public/fi_verify.py @@ -0,0 +1,246 @@ +"""fi_verify - checks a Future AGI integration against what the collector really received. + + python fi_verify.py preflight closes G1, before any code is touched + python fi_verify.py check all ten gates, after one real request + +Python 3.10+. No dependency beyond the OpenTelemetry SDK. +""" +import json, os, sys, time, urllib.error, urllib.request + +EP = os.getenv("FI_ENDPOINT", "https://api.futureagi.com/tracer/v1/traces") +SPANS = os.getenv("FI_VERIFY_FILE", ".fi_verify/spans.jsonl") +DIR = os.path.dirname(SPANS) or "." + +# Two spellings are read for each: the Future AGI SDK writes the first, a plain +# OpenTelemetry setup writes the second. +KIND = ("gen_ai.span.kind", "fi.span.kind", "openinference.span.kind") +IN = ("input.value", "gen_ai.input.messages") +OUT = ("output.value", "gen_ai.output.messages") +MODEL = ("gen_ai.request.model", "gen_ai.response.model", "llm.model_name") +TOKENS = ("gen_ai.usage.input_tokens", "llm.token_count.prompt") +COST = ("gen_ai.cost.total", "llm.cost.total") +SESSION = ("session.id", "fi.session.id") +USER = ("user.id", "fi.user.id") + + +def first(attrs, names): + for n in names: + if attrs.get(n) not in (None, "", [], {}): + return attrs[n] + return None + + +def put(kind, ok, detail): + os.makedirs(DIR, exist_ok=True) + p = os.path.join(DIR, kind + ".json") + json.dump({"ok": bool(ok), "detail": detail, "at": int(time.time())}, open(p, "w"), default=str) + + +def get(kind): + try: + return json.load(open(os.path.join(DIR, kind + ".json"))) + except Exception: + return {"ok": False, "detail": "no " + kind + " receipt"} + + +def clip(raw): # a 404 answers with an HTML page; one line of it is plenty + t = " ".join(raw.decode("utf-8", "replace").split()) + return (t[:110] + "...") if len(t) > 110 else t + + +def send(key, secret, project, auth=True, slash=False): + now = int(time.time()) + span = {"traceId": "4bf92f3577b34da6a3ce929d0e0e4736", "spanId": "00f067aa0ba902b7", + "name": "futureagi.preflight", "kind": 1, + "startTimeUnixNano": str(now) + "000000000", + "endTimeUnixNano": str(now + 1) + "000000000"} + res = [{"key": "project_name", "value": {"stringValue": project}}, + {"key": "project_type", "value": {"stringValue": "observe"}}] + body = {"resourceSpans": [{"resource": {"attributes": res}, + "scopeSpans": [{"spans": [span]}]}]} + head = {"Content-Type": "application/json"} + if auth: + head["X-Api-Key"] = key + head["X-Secret-Key"] = secret + req = urllib.request.Request(EP + ("/" if slash else ""), + data=json.dumps(body).encode(), headers=head) + try: + r = urllib.request.urlopen(req, timeout=30) + return r.status, clip(r.read()) + except urllib.error.HTTPError as e: + return e.code, clip(e.read()) + except Exception as e: + return 0, type(e).__name__ + ": " + str(e) + + +def preflight(): + """One real send plus three broken controls. Writes the receipt check() reads as G1.""" + key, secret = os.getenv("FI_API_KEY"), os.getenv("FI_SECRET_KEY") + project = os.getenv("FI_PROJECT_NAME") or "preflight" + if not key or not secret: + put("preflight", False, "FI_API_KEY and FI_SECRET_KEY are not both set") + return False, ["FI_API_KEY and FI_SECRET_KEY are not both set."] + code, body = send(key, secret, project) + out = ["keys HTTP " + str(code) + " " + body] + if code != 200: + c, b = send(key, secret, project, auth=False) + out.append("no headers HTTP " + str(c) + " " + b) + out.append(WHY.get(code, "unexpected status; the body above is the collector's.")) + put("preflight", False, {"http": code, "body": body}) + return False, out + for label, k, kw in (("wrong key ", key[:-4] + "0000", {}), + ("no headers ", key, {"auth": False}), + ("trailing slash", key, {"slash": True})): + c, b = send(k, secret, project, **kw) + out.append(label + " HTTP " + str(c) + " " + b) + if c == 200: + put("preflight", False, {"accepted_control": label.strip()}) + out.append("a deliberately broken send was accepted, so nothing here is proven.") + return False, out + out.append("accepted with the keys, refused every broken variant.") + put("preflight", True, {"http": 200, "project": project, "endpoint": EP}) + return True, out + + +WHY = {401: "the keys reached the collector and were refused: wrong keys, or keys from" + " another environment. The headerless send above answering 'missing" + " credentials' shows the headers do arrive, so this is the values, not a proxy.", + 400: "the keys are fine and the payload is not. The body names the field.", + 404: "wrong path. It ends /tracer/v1/traces, with no trailing slash.", + 0: "the endpoint was not reachable at all. Proxy, firewall or DNS."} + + +def attach(provider, path=None): + """Copy spans to a local file, and record what the collector said about the real batch. + + The provider drops its processors on the first add_span_processor call, which would + remove the exporter register() installed: delivery stops, offline gates keep passing. + A local file shows a span's shape, never its arrival, so the exporter is wrapped too. + """ + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + path = path or SPANS + os.makedirs(os.path.dirname(path) or ".", exist_ok=True) + open(path, "w").close() + for proc in getattr(provider._active_span_processor, "_span_processors", ()): + if getattr(proc, "span_exporter", None) is not None: + _watch(proc.span_exporter) + if getattr(provider, "_default_processor", False): + provider._default_processor = False # keep the exporter register() installed + provider.add_span_processor(SimpleSpanProcessor(_Tee(path))) + return path + + +def _watch(exporter): + real = exporter.export + tally = {"accepted": 0, "refused": 0, "spans": 0, "by": type(exporter).__name__} + + def export(spans): + result = real(spans) + ok = str(result).endswith("SUCCESS") + tally["accepted" if ok else "refused"] += 1 + tally["spans"] += len(spans) if ok else 0 + put("delivery", tally["accepted"] > 0 and tally["refused"] == 0, tally) + return result + + exporter.export = export + + +class _Tee: # writes exported spans to a local file, verification only + def __init__(self, path): + self.path = path + + def export(self, spans): + from opentelemetry.sdk.trace.export import SpanExportResult + rows = [self.row(s) for s in spans] + with open(self.path, "a") as fh: + fh.write("".join(json.dumps(r, default=str) + "\n" for r in rows)) + return SpanExportResult.SUCCESS + + def row(self, s): + c = s.get_span_context() + return {"name": s.name, "trace_id": format(c.trace_id, "032x"), + "span_id": format(c.span_id, "016x"), + "parent_id": format(s.parent.span_id, "016x") if s.parent else None, + "attrs": dict(s.attributes or {}), + "resource": dict(s.resource.attributes or {})} + + def shutdown(self): + return None + + def force_flush(self, timeout_millis=30000): + return True + + +def check(path=None, require_user=True): + """The ten gates. Returns (green, rows). Green means all ten, never nine.""" + path = path or SPANS + if not os.path.exists(path) or os.path.getsize(path) == 0: + return False, [("G0", False, "no spans captured, so the traced path never ran")] + spans = [json.loads(l) for l in open(path) if l.strip()] + res = spans[0]["resource"] + ids = set(s["span_id"] for s in spans) + traces = set(s["trace_id"] for s in spans) + roots = [s for s in spans if not s["parent_id"]] + orphans = [s for s in spans if s["parent_id"] and s["parent_id"] not in ids] + llm = [s for s in spans if str(first(s["attrs"], KIND) or "").upper() == "LLM"] + untyped = [s["name"] for s in spans if not first(s["attrs"], KIND)] + root = roots[0] if roots else {"attrs": {}} + sess = set(first(s["attrs"], SESSION) for s in spans) + users = set(first(s["attrs"], USER) for s in spans) + thin = [s["name"] for s in llm if not (first(s["attrs"], IN) and first(s["attrs"], OUT))] + nomodel = [s["name"] for s in llm if not first(s["attrs"], MODEL)] + cost = first(root["attrs"], COST) + counted = llm and all(first(s["attrs"], TOKENS) is not None for s in llm) + rolled = first(root["attrs"], TOKENS) is not None + # G1 is read back from two receipts, never assumed. Without it the other nine only say + # the spans are well formed on this machine, which is not an integration. + pre, deliver = get("preflight"), get("delivery") + keys = [v for k, v in os.environ.items() if len(v) > 15 + and any(t in k.upper() for t in ("KEY", "TOKEN", "SECRET", "PASSWORD"))] + leaked = sorted(set(s["name"] for s in spans + if any(x in json.dumps(s["attrs"], default=str) for x in keys))) + rows = [ + ("G1", pre["ok"] and deliver["ok"], + "preflight " + ok_or(pre) + ", delivery " + ok_or(deliver)), + ("G2", bool(res.get("project_name")), + "project_name=" + repr(res.get("project_name")) + + " project_type=" + repr(res.get("project_type"))), + ("G3", len(roots) == 1 and len(traces) == 1 and not orphans, + "%d spans, %d trace(s), %d root(s), %d orphan(s)" + % (len(spans), len(traces), len(roots), len(orphans))), + ("G4", bool(llm) and not untyped, + "%d LLM span(s), %d untyped %s" % (len(llm), len(untyped), untyped[:3] or "")), + ("G5", bool(llm) and not thin, + "prompt and completion on every LLM span" if not thin else "empty on " + str(thin[:3])), + ("G6", len(sess) == 1 and None not in sess, "session.id=" + str(sorted(map(str, sess)))), + ("G7", (not require_user) or None not in users, + "user.id=" + str(sorted(map(str, users)))), + ("G8", bool(llm) and not nomodel, + "model on every LLM span" if not nomodel else "missing on " + str(nomodel[:3])), + # Cost is what the trace list is read for and the one most often missing. It is not + # computed for you, and a zero is not a cost, so a zero does not pass. + ("G9", bool(counted and rolled and isinstance(cost, (int, float)) and cost > 0), + "tokens and cost on LLM spans, rolled up onto the root; root cost=" + str(cost)), + ("G10", not leaked, "no credential in any span attribute" if not leaked + else "CREDENTIAL ON SPAN " + str(leaked)), + ] + return all(ok for _, ok, _ in rows), rows + + +def ok_or(receipt): + return "ok" if receipt["ok"] else str(receipt["detail"])[:50] + + +if __name__ == "__main__": + if (sys.argv[1:2] or ["check"])[0] == "preflight": + good, said = preflight() + print("\n" + "\n".join(" " + l for l in said)) + print("\n " + ("PASS" if good else "FAIL") + " G1 keys and route") + sys.exit(0 if good else 1) + green, rows = check() + print() + for gid, ok, msg in rows: + print(" %s %-4s %s" % ("PASS" if ok else "FAIL", gid, msg)) + print("\n Future AGI integrated\n GREEN LIGHT achieved" if green + else "\n NOT GREEN. The FAIL rows name what is missing.") + sys.exit(0 if green else 1) diff --git a/public/images/docs/cookbook-instrument-and-verify/session.png b/public/images/docs/cookbook-instrument-and-verify/session.png new file mode 100644 index 00000000..fa5e5a41 Binary files /dev/null and b/public/images/docs/cookbook-instrument-and-verify/session.png differ diff --git a/public/images/docs/cookbook-instrument-and-verify/trace-list.png b/public/images/docs/cookbook-instrument-and-verify/trace-list.png new file mode 100644 index 00000000..f945b88c Binary files /dev/null and b/public/images/docs/cookbook-instrument-and-verify/trace-list.png differ diff --git a/public/images/docs/cookbook-instrument-and-verify/trace-tree.png b/public/images/docs/cookbook-instrument-and-verify/trace-tree.png new file mode 100644 index 00000000..20a602f4 Binary files /dev/null and b/public/images/docs/cookbook-instrument-and-verify/trace-tree.png differ diff --git a/src/lib/navigation.ts b/src/lib/navigation.ts index 90c690bc..34433ae6 100644 --- a/src/lib/navigation.ts +++ b/src/lib/navigation.ts @@ -885,6 +885,7 @@ export const tabNavigation: NavTab[] = [ { title: 'Observability', items: [ + { title: 'Instrument an Existing App and Verify It End to End', href: '/docs/cookbook/quickstart/instrument-and-verify' }, { title: 'Manual Tracing: Add Custom Spans to Any Application', href: '/docs/cookbook/quickstart/manual-tracing' }, { title: 'Session-Based Observability for Multi-Turn Conversations', href: '/docs/cookbook/quickstart/session-observability' }, { title: 'Monitoring & Alerts: Track LLM Performance and Set Quality Thresholds', href: '/docs/cookbook/quickstart/monitoring-alerts' }, diff --git a/src/pages/docs/cookbook/quickstart/instrument-and-verify.mdx b/src/pages/docs/cookbook/quickstart/instrument-and-verify.mdx new file mode 100644 index 00000000..03a2d319 --- /dev/null +++ b/src/pages/docs/cookbook/quickstart/instrument-and-verify.mdx @@ -0,0 +1,668 @@ +--- +title: "Instrument an Existing App and Verify It End to End" +description: "Add tracing to an app that has none, then prove it worked with ten machine-checked gates that either pass or name exactly what is missing." +--- + + +Adding tracing is the easy half. Knowing it worked is the half that gets skipped, because a trace that arrives looking fine can still be missing cost, sessions, users, or half its spans. This guide instruments an app that has none, then runs `fi_verify.py`, which checks ten gates against the spans your app really produced and exits 0 or names the gate that failed. + + +
+Open in Colab +GitHub +
+ +| Time | Difficulty | Package | +|------|-----------|---------| +| 20 min | Beginner | `fi-instrumentation-otel` | + + +- FutureAGI account → [app.futureagi.com](https://app.futureagi.com) +- API keys: `FI_API_KEY` and `FI_SECRET_KEY` (see [Get your API keys](/docs/admin-settings)) +- An app that makes at least one real LLM call, with an entry point you can run once +- Python 3.10+ to run `fi_verify.py`. The listings below are Python, but the app you are tracing can be in any language: see [If your app is not Python](#if-your-app-is-not-python) + + + +Handing this to a coding agent? Point it at this page and say: *follow this end to end to GREEN LIGHT, and tell me the one thing you need from me.* Step 1 downloads the checker, and Step 5 decides the result, so the agent never has to claim success on your behalf. + + +## Install + +Everything new lands in one directory. Existing files get a dependency, some configuration, one call at the entry point, and one around the model call. + +``` +your-repo/ +├── observability/futureagi/ +│ ├── setup.py # provider, exporter, instrumentor G1 G2 +│ ├── fi_verify.py # the checker, downloaded below all ten +│ ├── futureagi_spans.py # the attribute helpers G4 G5 G10 +│ └── futureagi_rollup.py # per-span cost, summed onto the root G8 G9 +├── app/main.py # + one scope at the entry point G3 G6 G7 +├── requirements.txt # + fi-instrumentation-otel +└── .gitignore # + .fi_verify/ +``` + +If your repository ships as a package, put `observability/futureagi/` under your own package root instead of the top level. + +Every step below offers two tracks, the FutureAGI SDK and plain OpenTelemetry. **Pick one and stay on it from here to the end.** The two write files of the same name that are not interchangeable, so every listing names its own track on the first line. + + + + +```bash +# FutureAGI SDK track +pip install fi-instrumentation-otel # not fi-instrumentation. Python 3.10+ +pip install traceai-openai # one per framework in use: -openai-agents, -anthropic, + # -langchain, -llamaindex, -crewai, -litellm, -bedrock, ... +``` + + + + +```bash +# OpenTelemetry track +pip install opentelemetry-sdk opentelemetry-exporter-otlp-proto-http +``` + + + + +Then the checker. One file, no dependency beyond the OpenTelemetry SDK, and Step 1 runs it. + +```bash +mkdir -p observability/futureagi +curl -fsSL https://docs.futureagi.com/fi_verify.py -o observability/futureagi/fi_verify.py +shasum -a 256 observability/futureagi/fi_verify.py +# 9b331742bc4143a46ae22ee0e8a1c8487d70cc242f689491cc20ca94a336ecd0 +``` + +```bash +export FI_API_KEY="your-api-key" +export FI_SECRET_KEY="your-secret-key" +export FI_PROJECT_NAME="my-app" +``` + +## What a verified integration means + +These failures all produce a trace list that looks populated, and reading the dashboard cannot tell them apart from a correct integration: + +- Cost and token columns are empty, because nothing computes cost for you +- One request appears as three traces, and each piece looks valid on its own +- Sessions and users are blank, so conversations do not group and per-customer spend cannot be read +- Evals report nothing, which looks identical to an eval that found no problems +- Nothing arrives at all, and the client stays healthy because the collector refused the batch quietly + +So the result is decided by a checker rather than by looking. Ten gates, each tied to the step that closes it: + +| Gate | Holds when | Step | +|---|---|---| +| G1 | the keys and route are accepted, and the collector took the real batch | 1 | +| G2 | `project_name` and `project_type` are on the resource | 2 | +| G3 | one request is one trace, one root, no orphans | 2 | +| G6 | one `session.id`, identical across the trace | 3 | +| G7 | `user.id` is present | 3 | +| G4 | every span is typed, and at least one is an LLM span | 4 | +| G5 | prompt and completion on every LLM span | 4 | +| G8 | the model name is on every LLM span | 4 | +| G9 | tokens and cost, rolled up onto the root | 4 | +| G10 | no credential in any span attribute | 4 | + +Nine gates read a local capture of your spans. G1 reads two receipts written during the run, because a capture says nothing about whether anything arrived. + + +Nothing substitutes for `fi_verify.py`. A checker that skips G1 reports success on keys the collector refused, because every other gate reads a local capture that a broken integration still writes perfectly. If the download is blocked, ask for the file rather than writing your own. + + +## Tutorial + + + + + +Three values: `FI_API_KEY` and `FI_SECRET_KEY` from the console, and `FI_PROJECT_NAME`, the name this app appears under. Tracing never needs your model provider key. + +```bash +python observability/futureagi/fi_verify.py preflight +``` + +`preflight` sends one real span, then three deliberately broken variants. If any broken variant is accepted, nothing is proven and it fails. + +| Answer | Means | +|---|---| +| `200` | keys, route and payload all valid. Go to Step 2 | +| `401 authentication failed` | the keys arrived and were refused: wrong keys, or keys from another environment | +| `401 missing credentials` | the `X-Api-Key` and `X-Secret-Key` headers never arrived: unset, or a proxy strips them | +| `400 no project_name` | keys fine, payload not. It belongs on the resource, not the span | +| `404` | wrong path. It ends `/tracer/v1/traces`, with no trailing slash | + + +You can start before the keys arrive. They usually sit with whoever owns the account rather than the engineer integrating, so ask for `FI_API_KEY` and `FI_SECRET_KEY` by name and say they belong in the environment. Every other step is built meanwhile, and only this gate waits. + + + + + + +Tokens, cost and latency roll up to the root, and both the trace list and every trace-scoped eval read the root. A request that arrives in pieces makes all three wrong, so get the shape right before anything else. + +Three things break a trace into pieces, and nothing else does: + +**A thread hand-off, a pool, a background task.** Copy the context in the caller and run the work inside it: `c = contextvars.copy_context()`, then `c.run(fn, ...)` in the worker. Captured on the far side it captures nothing, and `attach` alone restores the parent while dropping the Step 3 scope. Nothing crosses a message broker, so inject the W3C `traceparent` into the message and extract it in the task, which [Distributed Tracing](/docs/cookbook/quickstart/distributed-tracing) covers in full. + +**A stream or generator.** The first chunk nests and the rest do not, so open the model span inside the generator and close it in a `finally`. + +**No entry point at all**: a job, a consumer, a CLI, a frozen runtime like Lambda. The most common of the three. Open a `CHAIN` root by hand and flush before returning, because a batch sends on a timer and the sandbox freezes first. + + + + +```python +# observability/futureagi/setup.py FutureAGI SDK track +import os, sys +from opentelemetry import trace +from fi_instrumentation import FITracer, register +from fi_instrumentation.fi_types import ProjectType +from traceai_openai import OpenAIInstrumentor + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # for fi_verify + +# Import once at process start, AFTER whatever loads your .env and before any model call. +# Imported earlier the keys are not there yet, and only G1 says so. +tracer_provider = None +tracer = trace.get_tracer(__name__) # a no-op tracer, so no key means no spans, not a crash + +if os.getenv("FI_API_KEY") and os.getenv("FI_SECRET_KEY"): + tracer_provider = register( + project_name=os.getenv("FI_PROJECT_NAME", "my-app"), + project_type=ProjectType.OBSERVE, + set_global_tracer_provider=True, + ) + OpenAIInstrumentor().instrument(tracer_provider=tracer_provider) + # FITracer, not get_tracer(): a plain Tracer drops session.id and user.id, failing G6 and G7 + tracer = FITracer(tracer_provider.get_tracer(__name__)) + if os.getenv("FI_VERIFY") == "1": + import fi_verify; fi_verify.attach(tracer_provider) +``` + +`register()` puts `project_name` and `project_type` on the resource for you, which is G2. + + + + +```python +# observability/futureagi/setup.py OpenTelemetry track +import os, sys, contextvars +from opentelemetry import trace +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider, SpanProcessor +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # for fi_verify + +# Import this AFTER whatever loads your .env, or the keys are not there yet and only G1 says so. +# On the RESOURCE: without it the collector answers 400 and the client still looks healthy +resource = Resource.create({"project_name": os.getenv("FI_PROJECT_NAME", "my-app"), + "project_type": "observe"}) +provider = TracerProvider(resource=resource) + +if os.getenv("FI_API_KEY") and os.getenv("FI_SECRET_KEY"): + provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter( + endpoint="https://api.futureagi.com/tracer/v1/traces", + headers={"X-Api-Key": os.getenv("FI_API_KEY"), + "X-Secret-Key": os.getenv("FI_SECRET_KEY")}))) + +# Step 3 sets this once at the edge; every span picks it up here, so no call site remembers it +_scope = contextvars.ContextVar("fi_scope", default={}) + +class FiScope(SpanProcessor): # the base class no-ops the other three methods + def on_start(self, span, parent_context=None): + for k, v in _scope.get().items(): span.set_attribute(k, v) + +provider.add_span_processor(FiScope()) +trace.set_tracer_provider(provider) +tracer = trace.get_tracer(__name__) + +if os.getenv("FI_VERIFY") == "1": + import fi_verify; fi_verify.attach(provider) +``` + +`OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`, `OTEL_EXPORTER_OTLP_TRACES_HEADERS` and `OTEL_RESOURCE_ATTRIBUTES` are an alternative to the module above. + + + + + + + + +A session groups traces into a conversation you can read in order, and a user id lets you read cost and quality per customer. One scope, opened where the request enters, so every span inside inherits it. + + + + +```python +# your entry point: the scope outside, the root inside FutureAGI SDK track +from fi_instrumentation import using_attributes +from observability.futureagi.setup import tracer + +# The scope goes OUTSIDE the root. Opened within it, it reaches the children and misses +# the root itself, and G6 and G7 read every span in the trace. +def handle(message): # an HTTP route, a consumer, a job, a CLI run + with using_attributes(session_id=conversation_id, user_id=account_id, + tags=["prod"], metadata={"tenant": tenant}): + with tracer.start_as_current_span("orders.reprice") as root: + root.set_attribute("gen_ai.span.kind", "CHAIN") + root.set_attribute("input.value", message.body) + answer = run_the_work(message) # every model call nests under this + root.set_attribute("output.value", answer) + return answer +``` + + + + +```python +# your entry point: the scope outside, the root inside OpenTelemetry track +from observability.futureagi.setup import tracer, _scope + +# Set here and nowhere else. FiScope copies it onto every span in the request, root included. +def handle(message): # an HTTP route, a consumer, a job, a CLI run + token = _scope.set({"session.id": conversation_id, "user.id": account_id}) + try: + with tracer.start_as_current_span("orders.reprice") as root: + root.set_attribute("gen_ai.span.kind", "CHAIN") + root.set_attribute("input.value", message.body) + answer = run_the_work(message) # every model call nests under this + root.set_attribute("output.value", answer) + return answer + finally: + _scope.reset(token) +``` + + + + + +Three mistakes leave both fields empty. A new id on every span is worse than no session at all, because it looks correct. A scope opened inside the root reaches the children and misses the root, and both gates read every span. A scope lost at a thread or stream boundary is empty on exactly the spans that used to be orphans, which means Step 2 is not finished. Use an internal account id, never an email. + + + + + + +A field is empty in the product because the attribute behind it was never sent. None of these are filled in for you on our side, cost included, and five of the ten gates depend on this step. + +| Field that stays empty | Attribute that fills it | Written by | +|---|---|---| +| the span typed in the tree | `gen_ai.span.kind` | you, every span | +| prompt and completion, every eval binding | `input.value`, `output.value` | instrumentor or you | +| tokens and cost in the trace list | `gen_ai.usage.input_tokens`, `.output_tokens`, `gen_ai.cost.total` on the root | you | +| model and provider filters | `gen_ai.request.model`, `gen_ai.provider.name` | instrumentor, except on a stream | +| session grouping, user analytics | `session.id`, `user.id` | you, at the edge | + +The instrumentor writes every LLM key for the calls it wraps. The root, a local `TOOL` or `RETRIEVER` span, and cost all fall outside it. A hand-rolled client has no instrumentor at all, so its LLM span is written the same way, by hand, with the model, the messages and the counts taken off the response object. The kind is the bare name in capitals: `CHAIN` for the root, `LLM` for one model call, then ten more listed in the [instrumentor reference](/docs/tracing/auto). + +**Cost is not calculated for you. You send it.** From the token counts and the published rate, onto the root, which is what the trace list reads. It is billed per LLM span as each one ends and added up there, because an ended span is read-only and cannot carry the number itself. Rates come from the environment, so a price change is configuration rather than a release: + +```bash +export FI_MODEL_RATES='{"gpt-4o-mini": [0.15, 0.60]}' # {"": [in, out]} per 1M +``` + + +A streamed call is missing two of these. Pass `stream_options={"include_usage": True}` and the token counts arrive on the final chunk, which is G9; without it the provider sends none at all. The model name never arrives either, and the instrumentor's span is never current in your code, so it has to be set as the span opens, which is G8. Off a stream both arrive on their own. + + + + + +```python +# observability/futureagi/futureagi_spans.py FutureAGI SDK track +from contextlib import contextmanager +from observability.futureagi.futureagi_rollup import MODEL + +@contextmanager # wrap the client call. The instrumentor writes the rest +def llm_call(model): + t = MODEL.set(model) + try: yield + finally: MODEL.reset(t) + +# a local TOOL or RETRIEVER span, and the root: three lines each +with tracer.start_as_current_span("lookup_price") as span: + span.set_attribute("gen_ai.span.kind", "TOOL") + span.set_attribute("gen_ai.tool.name", "lookup_price") + span.set_attribute("output.value", json.dumps(catalogue.price(sku))) +``` + +```python +# observability/futureagi/futureagi_rollup.py FutureAGI SDK track +import json, os, contextvars +from opentelemetry import context as otel_context +from opentelemetry.sdk.trace import SpanProcessor +from observability.futureagi.setup import tracer_provider as P # None until the keys are set + +RATES = json.loads(os.getenv("FI_MODEL_RATES", "{}")) # {"": [in, out]} per 1M +MODEL = contextvars.ContextVar("fi_model", default=None) # what llm_call is about to call +RUN = otel_context.create_key("fi_run") # rides the OpenTelemetry context, so a copied +M = "gen_ai.request.model" # context carries it over a thread hand-off too. +T = ["gen_ai.usage.input_tokens", "gen_ai.usage.output_tokens", + "gen_ai.usage.total_tokens", "gen_ai.cost.total"] + +class RollUp(SpanProcessor): + def on_start(self, span, parent_context=None): # the instrumentor never leaves its LLM + if MODEL.get(): span.set_attribute(M, MODEL.get()) # span current in your code, so + def on_end(self, span): # the name goes on as the span opens + run, a = otel_context.get_value(RUN), span.attributes or {} + if run is not None and T[0] in a: # tokens land on LLM spans, which end first + i, o = RATES.get(a.get(M), [0, 0]) # the only place a cost is + a = dict(a, **{T[3]: (a[T[0]] * i + a.get(T[1], 0) * o) / 1e6}) # ever computed + for k in T: run[k] = run.get(k, 0) + a.get(k, 0) + +# register() leaves _default_processor set, and the first add_span_processor call on that +# provider discards the exporter it installed. Clearing it first is what keeps delivery. +if P: P._default_processor = False; P.add_span_processor(RollUp()) +``` + + + + +```python +# observability/futureagi/futureagi_spans.py OpenTelemetry track +# Every key comes from here. One module, so a name can be misspelled only once. +import json +from contextlib import contextmanager + +def _set(span, attrs): # None is never written: an empty attribute + for k, v in attrs.items(): # reads as a missing one + if v is not None: span.set_attribute(k, v if isinstance(v, (str, bool, int, float)) + else json.dumps(v, default=str)) + +@contextmanager +def span_of(t, name, kind, opening, alias=None): # one shape for all twelve kinds + with t.start_as_current_span(name) as span: + _set(span, {"gen_ai.span.kind": kind, **opening}) + yield lambda out=None, extra=None: _set(span, {"output.value": out, **(extra or {}), + **({alias: out} if alias else {})}) +``` + +```python +# observability/futureagi/futureagi_rollup.py OpenTelemetry track +import json, os +from opentelemetry import context as otel_context +from opentelemetry.sdk.trace import SpanProcessor +from observability.futureagi.setup import provider as P # the provider Step 2 built + +RATES = json.loads(os.getenv("FI_MODEL_RATES", "{}")) # {"": [in, out]} per 1M +RUN = otel_context.create_key("fi_run") # rides the OpenTelemetry context, so a copied +M = "gen_ai.request.model" # context carries it over a thread hand-off too. +T = ["gen_ai.usage.input_tokens", "gen_ai.usage.output_tokens", + "gen_ai.usage.total_tokens", "gen_ai.cost.total"] + +class RollUp(SpanProcessor): + def on_end(self, span): # the base class no-ops the other three methods + run, a = otel_context.get_value(RUN), span.attributes or {} + if run is not None and T[0] in a: # tokens land on LLM spans, which end first + i, o = RATES.get(a.get(M), [0, 0]) # the only place a cost is + a = dict(a, **{T[3]: (a[T[0]] * i + a.get(T[1], 0) * o) / 1e6}) # ever computed + for k in T: run[k] = run.get(k, 0) + a.get(k, 0) + +P.add_span_processor(RollUp()) +``` + + + + +The roll-up needs a per-request accumulator, opened at the entry point rather than inside the module above: + +```python +# your entry point, around the root from Step 3 +run = {k: 0 for k in T} # mutated in place, so a copied context shares it +token = otel_context.attach(otel_context.set_value(RUN, run)) +try: + with tracer.start_as_current_span("orders.reprice") as root: + answer = plan_and_run(order_id) + run[T[2]] = run.get(T[2]) or run.get(T[0], 0) + run.get(T[1], 0) # only if the + for k in T: root.set_attribute(k, run.get(k, 0)) # provider sent none +finally: + otel_context.detach(token) +``` + +A reasoning model bills for tokens in neither bucket, so carry the total the provider sent rather than adding the two up. Clip `input.value` at 8000 characters and redact it for G10: a serialised config object and a prompt that already carries a key are the two ways a credential reaches a span. + + + + + +Three commands. It exits `0`, or it names the gate that failed and why. Run it again after every change. + +```bash +printf '\n.fi_verify/\n' >> .gitignore # echo would join the last line +export FI_VERIFY=1 + +python observability/futureagi/fi_verify.py preflight # keys, route, three broken controls +python -m your_app # one real request, your entry point +python observability/futureagi/fi_verify.py check # ten gates: receipts, then spans +``` + +A passing run: + +``` + PASS G1 preflight ok, delivery ok + PASS G2 project_name='my-app' project_type='observe' + PASS G3 7 spans, 1 trace(s), 1 root(s), 0 orphan(s) + PASS G4 3 LLM span(s), 0 untyped + PASS G5 prompt and completion on every LLM span + PASS G6 session.id=['c-4182'] + PASS G7 user.id=['acct-993'] + PASS G8 model on every LLM span + PASS G9 tokens and cost on LLM spans, rolled up onto the root; root cost=0.0002694 + PASS G10 no credential in any span attribute + + Future AGI integrated + GREEN LIGHT achieved +``` + +Each `FAIL` row names one gate and one cause. If the same gate fails twice for the same reason, the problem is outside your codebase. + +`ai-evaluation is not installed, please install it to trace protect` on stderr is expected and affects no gate. It is the optional Protect package, which tracing does not need. + + + + + +A correct trace is what lets you answer quality questions. An eval can only grade what it is bound to, and you bind every variable yourself. One with nothing bound reports nothing, which looks the same as one that found no problems. + +Evals are configured in the console, never in application code. Binding is manual per eval task: a scope, a template, then each variable pointed at an attribute path in your own data. + +- Anything reading the request binds at **Traces** scope against the root, where `input.value` is the real question. On an instrumented LLM span it holds only the first message, and the full turn is under `gen_ai.input.messages.*`. +- Output-only evals bind at **Spans** scope to `output.value`, which G5 already proved is there. + +Read your own attributes out of `.fi_verify/spans.jsonl` first, then pick from the [eval catalogue](/docs/evaluation/builtin) and attach them with [Setup evals](/docs/observe/guides/setup-evals). Finish by naming five, each satisfiable by an attribute already in the captured file: the eval, the variable, the attribute path, and one line on why it fits. + + + + + +## The same six steps on a real repository + +Everything above was run against [`openai/openai-agents-python`](https://github.com/openai/openai-agents-python), on its `examples/customer_service` airline support agent. Nothing in that repository was written for this guide, which is the point: it is a normal app with normal problems. + +It is a fair target because it has all four of them at once. It emits no trace your platform can read. A single message fans out across a triage agent, a handoff, a specialist agent and a local tool, so Step 2 has real work to do. `main.py` is an interactive REPL, so there is no entry point that runs once, which is Step 2's third case. And it carries a conversation id and a passenger context already, so Step 3 has a real session and a real user to attach rather than invented ones. + +```bash +git clone --depth 1 https://github.com/openai/openai-agents-python +cd openai-agents-python +pip install openai-agents fi-instrumentation-otel traceai-openai-agents +``` + +`observability/futureagi/` is the four files from Step 2 and Step 4 on the FutureAGI SDK track, with `traceai-openai-agents` as the instrumentor. The one file added outside it is the entry point the repository does not have. No business logic was edited, and `main.py` was imported, not modified. + +```python +# examples/customer_service/run_traced.py +"""One customer message through the airline support agent, traced end to end. + +main.py is an interactive REPL, so the repository has no entry point that runs once. This +is that entry point: the roll-up and the scope outside, the CHAIN root inside, a flush +before returning. Run it twice with the same FI_SESSION_ID to see a session of two traces. +""" +import asyncio, os, sys, uuid + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) + +from observability.futureagi.setup import tracer, tracer_provider +from observability.futureagi import futureagi_rollup as R +from fi_instrumentation import using_attributes +from opentelemetry import context as otel_context + +from agents import Runner, RunConfig, set_default_openai_api +from examples.customer_service.main import AirlineAgentContext, triage_agent + +if os.getenv("OPENAI_BASE_URL"): # an OpenAI-compatible endpoint that is not OpenAI + set_default_openai_api("chat_completions") # only OpenAI serves the Responses API + +MODEL = os.getenv("AGENT_MODEL", "gpt-4o-mini") + + +async def handle(question, session_id, user_id): + run = {k: 0 for k in R.T} # mutated in place, so a copied context shares it + token = otel_context.attach(otel_context.set_value(R.RUN, run)) + try: + with using_attributes(session_id=session_id, user_id=user_id, + tags=["prod"], metadata={"channel": "web"}): + with tracer.start_as_current_span("support.turn") as root: + root.set_attribute("gen_ai.span.kind", "CHAIN") + root.set_attribute("input.value", question[:8000]) + root.set_attribute(R.M, MODEL) # the Model column reads the root, not the + # LLM spans the instrumentor writes it on + result = await Runner.run( + triage_agent, [{"content": question, "role": "user"}], + context=AirlineAgentContext(), run_config=RunConfig(model=MODEL)) + answer = str(result.final_output) + root.set_attribute("output.value", answer[:8000]) + run[R.T[2]] = run.get(R.T[2]) or run.get(R.T[0], 0) + run.get(R.T[1], 0) + for k in R.T: root.set_attribute(k, run.get(k, 0)) + return result, answer + finally: + otel_context.detach(token) + + +async def main(): + question = " ".join(sys.argv[1:]) + session_id = os.getenv("FI_SESSION_ID") or "conv_" + uuid.uuid4().hex[:12] + user_id = os.getenv("ACCOUNT_ID", "acct_10427") + + result, answer = await handle(question, session_id, user_id) + print(f"{result.last_agent.name}: {answer}") + + if tracer_provider: tracer_provider.force_flush() # a batch sends on a timer + + +if __name__ == "__main__": + asyncio.run(main()) +``` + +```bash +export FI_VERIFY=1 FI_SESSION_ID=conv_94ac3e74fcc4 + +python observability/futureagi/fi_verify.py preflight +python examples/customer_service/run_traced.py "How much baggage am I allowed to bring on the plane?" +python observability/futureagi/fi_verify.py check +``` + +``` + PASS G1 preflight ok, delivery ok + PASS G2 project_name='support-agent-quickstart' project_type='observe' + PASS G3 13 spans, 1 trace(s), 1 root(s), 0 orphan(s) + PASS G4 3 LLM span(s), 0 untyped + PASS G5 prompt and completion on every LLM span + PASS G6 session.id=['conv_94ac3e74fcc4'] + PASS G7 user.id=['acct_10427'] + PASS G8 model on every LLM span + PASS G9 tokens and cost on LLM spans, rolled up onto the root; root cost=0.00108604 + PASS G10 no credential in any span attribute + + Future AGI integrated + GREEN LIGHT achieved +``` + +Run it a second time with the same `FI_SESSION_ID` and a second message, and the two turns join one session. + +### What that produced + +The trace list reads the root and nothing else, which is why Step 4 puts the totals there. Latency, tokens and status arrive on their own; cost and the model do not. + +The FutureAGI trace list showing two support.turn traces with input, output, latency, tokens, total cost and model + +Inside a trace, the instrumentor typed the agents and the model calls, and the handoff. `faq_lookup_tool` is a local function the instrumentor cannot see, so its `TOOL` span is the three lines from Step 4. The attributes panel is the same list Step 4 sends, read back off a real span. + +The trace tree for one support turn, from the support.turn root through the triage agent, the handoff, the FAQ agent, the tool span and three LLM spans, with the attributes panel showing session.id, user.id and cost + +Because `session.id` was set once at the edge, both turns group without either turn knowing about the other. + +The FutureAGI sessions view showing one session with its first and last message, duration, total cost, three traces and the user id + + +The model on these captures reads `llama-3.3-70b-versatile` because the run pointed at an OpenAI-compatible endpoint that is not OpenAI. Set `AGENT_MODEL` to whatever you use. Nothing in the integration changes with the provider, and `FI_MODEL_RATES` is where its published rate goes. + + +### The evals bound to it + +Each of these binds only to an attribute the run above already carries, read out of `.fi_verify/spans.jsonl`. + +| Eval | Scope | Bound to | +|---|---|---| +| Task Completion | Traces | `input.value` and `output.value` on the root | +| Evaluate Function Calling | Spans | the tool call arguments on the first LLM span | +| Detect Hallucination | Traces | `input.value` and `output.value`, catching an answer the tool never returned | +| Instruction Adherence | Traces | `input.value` and `output.value` against the agent's own instructions | +| PII Detection | Spans | `output.value`, which G5 already proved is on every LLM span | + +On an instrumented LLM span, `input.value` holds only the first message and the full turn sits under `gen_ai.input.messages.*`. Bind anything that reads the request at Traces scope against the root, where `input.value` is the customer's actual question. + +## If your app is not Python + +The listings translate line for line, and everything reaches the same endpoint over plain OpenTelemetry. Carry across: + +- Both headers, `X-Api-Key` and `X-Secret-Key`, on `https://api.futureagi.com/tracer/v1/traces` +- `project_name` and `project_type` on the **resource**, not the span +- One root per unit of work +- The scope from Step 3, set at the edge +- The Step 4 attribute keys, byte for byte +- The roll-up, because cost is never computed server side + +Spring Boot also needs `management.tracing.sampling.probability=1.0`. First-party SDKs and the full framework list are in the [instrumentor reference](/docs/tracing/auto). + +**The checker still runs.** `preflight` and `check` are plain Python with no dependency at all, so they work next to an app in any language. Only `fi_verify.attach()` is Python-bound, because it installs a processor inside your process. Without it, write the capture yourself: one JSON object per line in `.fi_verify/spans.jsonl`, each with `name`, `trace_id`, `span_id`, `parent_id` (null on the root), `attrs`, and `resource`. That is roughly ten lines in any OpenTelemetry span exporter, and `check` reads it the same way either way. + +## What you built + + +An integration whose correctness is decided by a machine rather than by looking at a dashboard. `fi_verify.py check` exits 0 only when all ten gates pass, never nine. + + +- Proved the keys, the route and the payload before a line of code depended on them, with three broken controls to show the check is real +- One request arriving as one trace with one root, so tokens, cost and latency roll up somewhere the trace list can read +- `session.id` and `user.id` on every span including the root, set once at the edge instead of remembered at each call site +- Every span typed, prompt and completion on every LLM span, and the model name present even on streamed calls +- Cost computed per LLM span from the published rate and summed onto the root, with rates in the environment so a price change is not a release +- A capture in `.fi_verify/spans.jsonl` you can read your real attribute paths out of before binding a single eval + +Future AGI is integrated, and you can show that rather than assert it. + +## Next steps + + + + Diagnose an app that sends nothing + + + Keep one trace across a service boundary + + + Setup guides for every framework + + + Attach the evals from Step 6 + +