diff --git a/pr-assets/Benchmark.pdf b/pr-assets/Benchmark.pdf new file mode 100644 index 00000000..1a9b3d6b Binary files /dev/null and b/pr-assets/Benchmark.pdf differ diff --git a/pr-assets/Observe-Docs-Analysis.pdf b/pr-assets/Observe-Docs-Analysis.pdf new file mode 100644 index 00000000..166ddb2d Binary files /dev/null and b/pr-assets/Observe-Docs-Analysis.pdf differ diff --git a/pr-assets/Scorecard-v4.pdf b/pr-assets/Scorecard-v4.pdf new file mode 100644 index 00000000..020aa784 Binary files /dev/null and b/pr-assets/Scorecard-v4.pdf differ diff --git a/public/images/docs/observe/1.png b/public/images/docs/observe/1.png deleted file mode 100644 index 4cf8d841..00000000 Binary files a/public/images/docs/observe/1.png and /dev/null differ diff --git a/public/images/docs/observe/2.png b/public/images/docs/observe/2.png deleted file mode 100644 index e5b66386..00000000 Binary files a/public/images/docs/observe/2.png and /dev/null differ diff --git a/public/images/docs/observe/3.png b/public/images/docs/observe/3.png deleted file mode 100644 index 94a4ada7..00000000 Binary files a/public/images/docs/observe/3.png and /dev/null differ diff --git a/public/images/docs/observe/4.png b/public/images/docs/observe/4.png deleted file mode 100644 index 25ba68de..00000000 Binary files a/public/images/docs/observe/4.png and /dev/null differ diff --git a/public/images/docs/observe/5.png b/public/images/docs/observe/5.png deleted file mode 100644 index 598c228e..00000000 Binary files a/public/images/docs/observe/5.png and /dev/null differ diff --git a/public/images/docs/observe/5.webp b/public/images/docs/observe/5.webp deleted file mode 100644 index d4577521..00000000 Binary files a/public/images/docs/observe/5.webp and /dev/null differ diff --git a/public/images/docs/observe/alerts-create.png b/public/images/docs/observe/alerts-create.png new file mode 100644 index 00000000..92ed57d4 Binary files /dev/null and b/public/images/docs/observe/alerts-create.png differ diff --git a/public/images/docs/observe/alerts-create.webp b/public/images/docs/observe/alerts-create.webp new file mode 100644 index 00000000..c3b34b03 Binary files /dev/null and b/public/images/docs/observe/alerts-create.webp differ diff --git a/public/images/docs/observe/alerts-overview.png b/public/images/docs/observe/alerts-overview.png new file mode 100644 index 00000000..8732d1b5 Binary files /dev/null and b/public/images/docs/observe/alerts-overview.png differ diff --git a/public/images/docs/observe/alerts-overview.webp b/public/images/docs/observe/alerts-overview.webp new file mode 100644 index 00000000..618ad48c Binary files /dev/null and b/public/images/docs/observe/alerts-overview.webp differ diff --git a/public/images/docs/observe/dashboard-add-widget.png b/public/images/docs/observe/dashboard-add-widget.png new file mode 100644 index 00000000..816fa177 Binary files /dev/null and b/public/images/docs/observe/dashboard-add-widget.png differ diff --git a/public/images/docs/observe/dashboard-add-widget.webp b/public/images/docs/observe/dashboard-add-widget.webp new file mode 100644 index 00000000..495f0d65 Binary files /dev/null and b/public/images/docs/observe/dashboard-add-widget.webp differ diff --git a/public/images/docs/observe/dashboard-configure-metric.webp b/public/images/docs/observe/dashboard-configure-metric.webp new file mode 100644 index 00000000..2ba85b42 Binary files /dev/null and b/public/images/docs/observe/dashboard-configure-metric.webp differ diff --git a/public/images/docs/observe/dashboard-overview.png b/public/images/docs/observe/dashboard-overview.png new file mode 100644 index 00000000..0ef53489 Binary files /dev/null and b/public/images/docs/observe/dashboard-overview.png differ diff --git a/public/images/docs/observe/dashboard-overview.webp b/public/images/docs/observe/dashboard-overview.webp new file mode 100644 index 00000000..6df319da Binary files /dev/null and b/public/images/docs/observe/dashboard-overview.webp differ diff --git a/public/images/docs/observe/dashboard-populated.png b/public/images/docs/observe/dashboard-populated.png new file mode 100644 index 00000000..1c990723 Binary files /dev/null and b/public/images/docs/observe/dashboard-populated.png differ diff --git a/public/images/docs/observe/dashboard-populated.webp b/public/images/docs/observe/dashboard-populated.webp new file mode 100644 index 00000000..7154473f Binary files /dev/null and b/public/images/docs/observe/dashboard-populated.webp differ diff --git a/public/images/docs/observe/evals-create.png b/public/images/docs/observe/evals-create.png new file mode 100644 index 00000000..c616928d Binary files /dev/null and b/public/images/docs/observe/evals-create.png differ diff --git a/public/images/docs/observe/evals-create.webp b/public/images/docs/observe/evals-create.webp new file mode 100644 index 00000000..26fab78a Binary files /dev/null and b/public/images/docs/observe/evals-create.webp differ diff --git a/public/images/docs/observe/evals-overview.png b/public/images/docs/observe/evals-overview.png new file mode 100644 index 00000000..f45b9633 Binary files /dev/null and b/public/images/docs/observe/evals-overview.png differ diff --git a/public/images/docs/observe/evals-overview.webp b/public/images/docs/observe/evals-overview.webp new file mode 100644 index 00000000..fe423838 Binary files /dev/null and b/public/images/docs/observe/evals-overview.webp differ diff --git a/public/images/docs/observe/evals-span-tab.webp b/public/images/docs/observe/evals-span-tab.webp new file mode 100644 index 00000000..70372e67 Binary files /dev/null and b/public/images/docs/observe/evals-span-tab.webp differ diff --git a/public/images/docs/observe/llm-tracing-agent-graph.png b/public/images/docs/observe/llm-tracing-agent-graph.png new file mode 100644 index 00000000..a9587b75 Binary files /dev/null and b/public/images/docs/observe/llm-tracing-agent-graph.png differ diff --git a/public/images/docs/observe/llm-tracing-agent-path.png b/public/images/docs/observe/llm-tracing-agent-path.png new file mode 100644 index 00000000..7e7ffbba Binary files /dev/null and b/public/images/docs/observe/llm-tracing-agent-path.png differ diff --git a/public/images/docs/observe/llm-tracing-bulk-actions.png b/public/images/docs/observe/llm-tracing-bulk-actions.png new file mode 100644 index 00000000..55d0ed9d Binary files /dev/null and b/public/images/docs/observe/llm-tracing-bulk-actions.png differ diff --git a/public/images/docs/observe/llm-tracing-bulk-actions.webp b/public/images/docs/observe/llm-tracing-bulk-actions.webp new file mode 100644 index 00000000..203817dd Binary files /dev/null and b/public/images/docs/observe/llm-tracing-bulk-actions.webp differ diff --git a/public/images/docs/observe/llm-tracing-date-range.png b/public/images/docs/observe/llm-tracing-date-range.png new file mode 100644 index 00000000..5067316d Binary files /dev/null and b/public/images/docs/observe/llm-tracing-date-range.png differ diff --git a/public/images/docs/observe/llm-tracing-date-range.webp b/public/images/docs/observe/llm-tracing-date-range.webp new file mode 100644 index 00000000..3a1fbc81 Binary files /dev/null and b/public/images/docs/observe/llm-tracing-date-range.webp differ diff --git a/public/images/docs/observe/llm-tracing-detail-drawer-crop.webp b/public/images/docs/observe/llm-tracing-detail-drawer-crop.webp new file mode 100644 index 00000000..a76e912b Binary files /dev/null and b/public/images/docs/observe/llm-tracing-detail-drawer-crop.webp differ diff --git a/public/images/docs/observe/llm-tracing-detail-drawer.png b/public/images/docs/observe/llm-tracing-detail-drawer.png new file mode 100644 index 00000000..2378c742 Binary files /dev/null and b/public/images/docs/observe/llm-tracing-detail-drawer.png differ diff --git a/public/images/docs/observe/llm-tracing-detail-drawer.webp b/public/images/docs/observe/llm-tracing-detail-drawer.webp new file mode 100644 index 00000000..980e23bd Binary files /dev/null and b/public/images/docs/observe/llm-tracing-detail-drawer.webp differ diff --git a/public/images/docs/observe/llm-tracing-display.png b/public/images/docs/observe/llm-tracing-display.png new file mode 100644 index 00000000..9ea95e6a Binary files /dev/null and b/public/images/docs/observe/llm-tracing-display.png differ diff --git a/public/images/docs/observe/llm-tracing-display.webp b/public/images/docs/observe/llm-tracing-display.webp new file mode 100644 index 00000000..ae32fa6b Binary files /dev/null and b/public/images/docs/observe/llm-tracing-display.webp differ diff --git a/public/images/docs/observe/llm-tracing-filter-crop.webp b/public/images/docs/observe/llm-tracing-filter-crop.webp new file mode 100644 index 00000000..f2115a39 Binary files /dev/null and b/public/images/docs/observe/llm-tracing-filter-crop.webp differ diff --git a/public/images/docs/observe/llm-tracing-filter.png b/public/images/docs/observe/llm-tracing-filter.png new file mode 100644 index 00000000..c67fd6db Binary files /dev/null and b/public/images/docs/observe/llm-tracing-filter.png differ diff --git a/public/images/docs/observe/llm-tracing-filter.webp b/public/images/docs/observe/llm-tracing-filter.webp new file mode 100644 index 00000000..73b72116 Binary files /dev/null and b/public/images/docs/observe/llm-tracing-filter.webp differ diff --git a/public/images/docs/observe/llm-tracing-overview.png b/public/images/docs/observe/llm-tracing-overview.png new file mode 100644 index 00000000..b8538b3f Binary files /dev/null and b/public/images/docs/observe/llm-tracing-overview.png differ diff --git a/public/images/docs/observe/llm-tracing-overview.webp b/public/images/docs/observe/llm-tracing-overview.webp new file mode 100644 index 00000000..81015fc8 Binary files /dev/null and b/public/images/docs/observe/llm-tracing-overview.webp differ diff --git a/public/images/docs/observe/llm-tracing-sessions-tab.png b/public/images/docs/observe/llm-tracing-sessions-tab.png new file mode 100644 index 00000000..48ffa061 Binary files /dev/null and b/public/images/docs/observe/llm-tracing-sessions-tab.png differ diff --git a/public/images/docs/observe/llm-tracing-users-tab.png b/public/images/docs/observe/llm-tracing-users-tab.png new file mode 100644 index 00000000..e5ea255e Binary files /dev/null and b/public/images/docs/observe/llm-tracing-users-tab.png differ diff --git a/public/images/docs/observe/llm-tracing-voice-detail.png b/public/images/docs/observe/llm-tracing-voice-detail.png new file mode 100644 index 00000000..de0f1079 Binary files /dev/null and b/public/images/docs/observe/llm-tracing-voice-detail.png differ diff --git a/public/images/docs/observe/llm-tracing-voice-overview.png b/public/images/docs/observe/llm-tracing-voice-overview.png new file mode 100644 index 00000000..cf935b6a Binary files /dev/null and b/public/images/docs/observe/llm-tracing-voice-overview.png differ diff --git a/public/images/docs/observe/sessions-bulk-actions.png b/public/images/docs/observe/sessions-bulk-actions.png new file mode 100644 index 00000000..b301ecc5 Binary files /dev/null and b/public/images/docs/observe/sessions-bulk-actions.png differ diff --git a/public/images/docs/observe/sessions-date-range.png b/public/images/docs/observe/sessions-date-range.png new file mode 100644 index 00000000..9cc71673 Binary files /dev/null and b/public/images/docs/observe/sessions-date-range.png differ diff --git a/public/images/docs/observe/sessions-detail.png b/public/images/docs/observe/sessions-detail.png new file mode 100644 index 00000000..a5e08cbd Binary files /dev/null and b/public/images/docs/observe/sessions-detail.png differ diff --git a/public/images/docs/observe/sessions-display.png b/public/images/docs/observe/sessions-display.png new file mode 100644 index 00000000..2c3750f5 Binary files /dev/null and b/public/images/docs/observe/sessions-display.png differ diff --git a/public/images/docs/observe/sessions-filter.png b/public/images/docs/observe/sessions-filter.png new file mode 100644 index 00000000..a90eda0d Binary files /dev/null and b/public/images/docs/observe/sessions-filter.png differ diff --git a/public/images/docs/observe/sessions-overview.png b/public/images/docs/observe/sessions-overview.png new file mode 100644 index 00000000..9143d4cf Binary files /dev/null and b/public/images/docs/observe/sessions-overview.png differ diff --git a/public/images/docs/observe/sessions-overview.webp b/public/images/docs/observe/sessions-overview.webp new file mode 100644 index 00000000..bfc4285e Binary files /dev/null and b/public/images/docs/observe/sessions-overview.webp differ diff --git a/public/images/docs/observe/sessions-replay-config.png b/public/images/docs/observe/sessions-replay-config.png new file mode 100644 index 00000000..16ce3464 Binary files /dev/null and b/public/images/docs/observe/sessions-replay-config.png differ diff --git a/public/images/docs/observe/troubleshooting-alerts-not-firing.webp b/public/images/docs/observe/troubleshooting-alerts-not-firing.webp new file mode 100644 index 00000000..6ebec9fc Binary files /dev/null and b/public/images/docs/observe/troubleshooting-alerts-not-firing.webp differ diff --git a/public/images/docs/observe/troubleshooting-dashboard-numbers.webp b/public/images/docs/observe/troubleshooting-dashboard-numbers.webp new file mode 100644 index 00000000..5713ba2a Binary files /dev/null and b/public/images/docs/observe/troubleshooting-dashboard-numbers.webp differ diff --git a/public/images/docs/observe/troubleshooting-missing-attributes.webp b/public/images/docs/observe/troubleshooting-missing-attributes.webp new file mode 100644 index 00000000..ce20ae26 Binary files /dev/null and b/public/images/docs/observe/troubleshooting-missing-attributes.webp differ diff --git a/public/images/docs/observe/users-date-range.png b/public/images/docs/observe/users-date-range.png new file mode 100644 index 00000000..1d5a0e0a Binary files /dev/null and b/public/images/docs/observe/users-date-range.png differ diff --git a/public/images/docs/observe/users-detail.png b/public/images/docs/observe/users-detail.png new file mode 100644 index 00000000..c88cf233 Binary files /dev/null and b/public/images/docs/observe/users-detail.png differ diff --git a/public/images/docs/observe/users-detail.webp b/public/images/docs/observe/users-detail.webp new file mode 100644 index 00000000..54106965 Binary files /dev/null and b/public/images/docs/observe/users-detail.webp differ diff --git a/public/images/docs/observe/users-display.png b/public/images/docs/observe/users-display.png new file mode 100644 index 00000000..adbffb56 Binary files /dev/null and b/public/images/docs/observe/users-display.png differ diff --git a/public/images/docs/observe/users-filter.png b/public/images/docs/observe/users-filter.png new file mode 100644 index 00000000..f488c9b7 Binary files /dev/null and b/public/images/docs/observe/users-filter.png differ diff --git a/public/images/docs/observe/users-overview.png b/public/images/docs/observe/users-overview.png new file mode 100644 index 00000000..e1f11a11 Binary files /dev/null and b/public/images/docs/observe/users-overview.png differ diff --git a/public/images/docs/observe/users-overview.webp b/public/images/docs/observe/users-overview.webp new file mode 100644 index 00000000..0a2c365e Binary files /dev/null and b/public/images/docs/observe/users-overview.webp differ diff --git a/public/images/docs/observe/voice-agent-definitions.png b/public/images/docs/observe/voice-agent-definitions.png new file mode 100644 index 00000000..33bb2d4a Binary files /dev/null and b/public/images/docs/observe/voice-agent-definitions.png differ diff --git a/public/images/docs/observe/voice-agent-definitions.webp b/public/images/docs/observe/voice-agent-definitions.webp new file mode 100644 index 00000000..85a46a86 Binary files /dev/null and b/public/images/docs/observe/voice-agent-definitions.webp differ diff --git a/public/images/docs/observe/voice-call-detail.png b/public/images/docs/observe/voice-call-detail.png new file mode 100644 index 00000000..ffc058e0 Binary files /dev/null and b/public/images/docs/observe/voice-call-detail.png differ diff --git a/public/images/docs/observe/voice-call-detail.webp b/public/images/docs/observe/voice-call-detail.webp new file mode 100644 index 00000000..ae571fce Binary files /dev/null and b/public/images/docs/observe/voice-call-detail.webp differ diff --git a/public/images/docs/observe/voice-create-form.png b/public/images/docs/observe/voice-create-form.png new file mode 100644 index 00000000..6aa525bf Binary files /dev/null and b/public/images/docs/observe/voice-create-form.png differ diff --git a/public/images/docs/observe/voice-create-form.webp b/public/images/docs/observe/voice-create-form.webp new file mode 100644 index 00000000..acfb4c38 Binary files /dev/null and b/public/images/docs/observe/voice-create-form.webp differ diff --git a/public/images/docs/observe/voice-projects-list.png b/public/images/docs/observe/voice-projects-list.png new file mode 100644 index 00000000..23d19398 Binary files /dev/null and b/public/images/docs/observe/voice-projects-list.png differ diff --git a/public/images/docs/observe/voice-tracing-overview.png b/public/images/docs/observe/voice-tracing-overview.png new file mode 100644 index 00000000..c94d0d73 Binary files /dev/null and b/public/images/docs/observe/voice-tracing-overview.png differ diff --git a/public/images/docs/observe/voice-tracing-overview.webp b/public/images/docs/observe/voice-tracing-overview.webp new file mode 100644 index 00000000..3759c6d2 Binary files /dev/null and b/public/images/docs/observe/voice-tracing-overview.webp differ diff --git a/src/components/Sidebar.astro b/src/components/Sidebar.astro index 2df4e379..cc85bbfc 100644 --- a/src/components/Sidebar.astro +++ b/src/components/Sidebar.astro @@ -150,18 +150,18 @@ const isApiTab = activeTab?.tab === 'API'; function inferApiMethod(title: string): { method: string; css: string } | null { const t = title.toLowerCase(); - if (/\b(delete|remove)\b/.test(t)) { - return { method: 'DELETE', css: 'api-method-delete' }; - } - if (/\b(update|edit|apply|restore)\b/.test(t)) { - return { method: 'PATCH', css: 'api-method-patch' }; - } - if (/\b(list|get|retrieve|health|find|export|progress|analytics|agreement|compare|stats|summary|voices|tts|aggregat\w*)\b/.test(t)) { + if (/\b(list|get|retrieve|health|find|export|progress|analytics|agreement|compare|stats|summary|voices|tts)\b/.test(t)) { return { method: 'GET', css: 'api-method-get' }; } if (/\b(create|add|generate|execute|submit|assign|bulk|complete|skip|release|pause|unpause|check|upload|start|duplicate|fetch|run|rerun|cancel|clone|merge)\b/.test(t)) { return { method: 'POST', css: 'api-method-post' }; } + if (/\b(delete|remove)\b/.test(t)) { + return { method: 'DEL', css: 'api-method-delete' }; + } + if (/\b(update|edit|apply|restore)\b/.test(t)) { + return { method: 'PATCH', css: 'api-method-patch' }; + } return null; } --- diff --git a/src/components/docs/CodeGroup.astro b/src/components/docs/CodeGroup.astro index 7193f41a..95d82d7c 100644 --- a/src/components/docs/CodeGroup.astro +++ b/src/components/docs/CodeGroup.astro @@ -40,6 +40,11 @@ const id = `code-group-${Math.random().toString(36).slice(2, 9)}`; + + diff --git a/src/lib/api-navigation.ts b/src/lib/api-navigation.ts index 67eea59f..f5ae3e9d 100644 --- a/src/lib/api-navigation.ts +++ b/src/lib/api-navigation.ts @@ -61,6 +61,11 @@ export const apiNavigation: ApiNavGroup[] = [ "title": "Add columns to a scenario", "href": "/docs/api/scenarios/addcolumns", "method": "POST" + }, + { + "title": "Add empty rows to a scenario", + "href": "/docs/api/scenarios/addemptyrowstodataset", + "method": "POST" } ] }, @@ -182,6 +187,11 @@ export const apiNavigation: ApiNavGroup[] = [ "href": "/docs/api/run-tests/executeruntest", "method": "POST" }, + { + "title": "Update test run components", + "href": "/docs/api/run-tests/updatetestcomponents", + "method": "PATCH" + }, { "title": "Get test executions", "href": "/docs/api/run-tests/gettestexecutions", @@ -192,6 +202,11 @@ export const apiNavigation: ApiNavGroup[] = [ "href": "/docs/api/run-tests/gettestscenarios", "method": "GET" }, + { + "title": "Get call executions for a test run", + "href": "/docs/api/run-tests/getcallexecutions", + "method": "GET" + }, { "title": "Get evaluation summary", "href": "/docs/api/run-tests/getevalsummary", @@ -279,8 +294,7 @@ export const apiNavigation: ApiNavGroup[] = [ { "title": "Delete Eval Task", "href": "/docs/api/eval-tasks/delete-eval-task", "method": "DELETE" }, { "title": "Bulk Delete Eval Tasks", "href": "/docs/api/eval-tasks/bulk-delete-eval-tasks", "method": "POST" }, { "title": "Pause Eval Task", "href": "/docs/api/eval-tasks/pause-eval-task", "method": "POST" }, - { "title": "Unpause Eval Task", "href": "/docs/api/eval-tasks/unpause-eval-task", "method": "POST" }, - { "title": "Eval Task Aggregations", "href": "/docs/api/eval-tasks/eval-task-aggregations", "method": "GET" } + { "title": "Unpause Eval Task", "href": "/docs/api/eval-tasks/unpause-eval-task", "method": "POST" } ] }, { diff --git a/src/lib/navigation.ts b/src/lib/navigation.ts index 059c31d7..a6d1e91e 100644 --- a/src/lib/navigation.ts +++ b/src/lib/navigation.ts @@ -344,113 +344,148 @@ export const tabNavigation: NavTab[] = [ ] }, { - group: 'Observability', + group: 'Observe', icon: 'eye', items: [ - { title: 'Overview', href: '/docs/observe' }, + { title: 'Get started', href: '/docs/observe' }, + { title: 'Quick start', href: '/docs/observe/quickstart' }, { title: 'Concepts', items: [ - { title: 'Understanding Observability', href: '/docs/tracing/concepts' }, - { title: 'What are Traces?', href: '/docs/tracing/concepts/traces' }, - { title: 'What are Spans?', href: '/docs/tracing/concepts/spans' }, - { title: 'What is OpenTelemetry?', href: '/docs/tracing/concepts/otel' }, - { title: 'What is traceAI?', href: '/docs/tracing/concepts/traceai' }, + { title: 'Observability model', href: '/docs/observe/concepts/observability-model' }, + { title: 'Traces', href: '/docs/observe/concepts/traces' }, + { title: 'Spans', href: '/docs/observe/concepts/spans' }, + { title: 'Sessions and users', href: '/docs/observe/concepts/sessions-and-users' }, + { title: 'OpenTelemetry', href: '/docs/observe/concepts/otel' }, + { title: 'traceAI SDK', href: '/docs/observe/concepts/traceai' }, ] }, { title: 'Features', items: [ - { title: 'Set Up Observability', href: '/docs/observe/features/quickstart' }, - { title: 'Run Evals on Traces', href: '/docs/observe/features/evals' }, + { title: 'Trace explorer', href: '/docs/observe/features/llm-tracing' }, { title: 'Sessions', href: '/docs/observe/features/session' }, { title: 'Users', href: '/docs/observe/features/users' }, - { title: 'Alerts & Monitors', href: '/docs/observe/features/alerts' }, - { title: 'Voice Observability', href: '/docs/observe/features/voice' }, + { title: 'Tags', href: '/docs/observe/features/tags' }, { title: 'Dashboards', href: '/docs/observe/features/dashboard' }, - { - title: 'Manual Tracing', - items: [ - { title: 'Set Up Tracing', href: '/docs/observe/features/manual-tracing/set-up-tracing' }, - { title: 'Instrument with traceAI Helpers', href: '/docs/observe/features/manual-tracing/instrument-with-traceai-helpers' }, - { title: 'Get Current Tracer and Span', href: '/docs/observe/features/manual-tracing/get-current-span-context' }, - { title: 'Enriching Spans with Attributes, Metadata, and Tags', href: '/docs/observe/features/manual-tracing/add-attributes-metadata-tags' }, - { title: 'Logging Prompt Templates & Variables', href: '/docs/observe/features/manual-tracing/log-prompt-templates' }, - { title: 'Events, Exceptions, and Status', href: '/docs/observe/features/manual-tracing/add-events-exceptions-status' }, - { title: 'Set Session ID and User ID', href: '/docs/observe/features/manual-tracing/set-session-user-id' }, - { title: 'Tool Spans Creation', href: '/docs/observe/features/manual-tracing/create-tool-spans' }, - { title: 'Mask Span Attributes', href: '/docs/observe/features/manual-tracing/mask-span-attributes' }, - { title: 'Advanced Tracing (OTEL)', href: '/docs/observe/features/manual-tracing/advanced-tracing-examples' }, - { title: 'FI Semantic Conventions', href: '/docs/observe/features/manual-tracing/semantic-conventions' }, - { title: 'In-line Evaluations', href: '/docs/observe/features/manual-tracing/in-line-evals' }, - { title: 'Adding Annotations to your Spans', href: '/docs/observe/features/manual-tracing/annotating-using-api' }, - { title: 'Langfuse Integration', href: '/docs/observe/features/manual-tracing/langfuse-integration' }, - ] - }, + { title: 'Alerts and monitors', href: '/docs/observe/features/alerts' }, + { title: 'Trace evals', href: '/docs/observe/features/evals' }, + { title: 'Voice observability', href: '/docs/observe/features/voice' }, + { title: 'Manual tracing', href: '/docs/traceai/manual-instrumentation/set-up-tracing' }, ] }, { - title: 'Integration', + title: 'Reference', + items: [ + { title: 'Filter syntax', href: '/docs/observe/reference/trace-filter-syntax' }, + { title: 'Dashboard metrics', href: '/docs/observe/reference/dashboard-metric-definitions' }, + { title: 'Export and endpoints', href: '/docs/observe/reference/export-formats' }, + ] + }, + { + title: 'Troubleshooting', + items: [ + { title: 'No traces appear', href: '/docs/observe/troubleshooting/no-traces-appearing' }, + { title: 'Spans and attributes', href: '/docs/observe/troubleshooting/missing-attributes' }, + { title: 'Dashboard numbers', href: '/docs/observe/troubleshooting/dashboard-numbers-look-wrong' }, + { title: 'Alerts not firing', href: '/docs/observe/troubleshooting/alerts-did-not-fire' }, + ] + }, + ] + }, + { + group: 'traceAI', + icon: 'code', + items: [ + { title: 'Get started', href: '/docs/traceai' }, + { title: 'Quick start', href: '/docs/traceai/quickstart' }, + { + title: 'Concepts', items: [ - { title: 'Overview', href: '/docs/tracing/auto' }, + { title: 'OpenTelemetry', href: '/docs/observe/concepts/otel' }, + { title: 'traceAI SDK', href: '/docs/observe/concepts/traceai' }, + ] + }, + { + title: 'Manual instrumentation', + items: [ + { title: 'Set up tracing', href: '/docs/traceai/manual-instrumentation/set-up-tracing' }, + { title: 'Instrument with traceAI helpers', href: '/docs/traceai/manual-instrumentation/instrument-with-traceai-helpers' }, + { title: 'Get the current span', href: '/docs/traceai/manual-instrumentation/get-current-span-context' }, + { title: 'Add attributes and metadata', href: '/docs/traceai/manual-instrumentation/add-attributes-metadata-tags' }, + { title: 'Log prompt templates', href: '/docs/traceai/manual-instrumentation/log-prompt-templates' }, + { title: 'Events and exceptions', href: '/docs/traceai/manual-instrumentation/add-events-exceptions-status' }, + { title: 'Set session and user', href: '/docs/traceai/manual-instrumentation/set-session-user-id' }, + { title: 'Create tool spans', href: '/docs/traceai/manual-instrumentation/create-tool-spans' }, + { title: 'Mask span attributes', href: '/docs/traceai/manual-instrumentation/mask-span-attributes' }, + { title: 'Advanced tracing', href: '/docs/traceai/manual-instrumentation/advanced-tracing-examples' }, + { title: 'Inline evals', href: '/docs/traceai/manual-instrumentation/in-line-evals' }, + { title: 'Annotate spans', href: '/docs/traceai/manual-instrumentation/annotating-using-api' }, + { title: 'Import from Langfuse', href: '/docs/traceai/manual-instrumentation/langfuse-integration' }, + ] + }, + { + title: 'Framework integrations', + items: [ + { title: 'Overview', href: '/docs/traceai/auto' }, { - title: 'LLM Providers', + title: 'LLM providers', items: [ - { title: 'OpenAI', href: '/docs/tracing/auto/openai' }, - { title: 'Anthropic', href: '/docs/tracing/auto/anthropic' }, - { title: 'AWS Bedrock', href: '/docs/tracing/auto/bedrock' }, - { title: 'Vertex AI', href: '/docs/tracing/auto/vertexai' }, - { title: 'Google GenAI', href: '/docs/tracing/auto/google_genai' }, - { title: 'Google ADK', href: '/docs/tracing/auto/google_adk' }, - { title: 'Groq', href: '/docs/tracing/auto/groq' }, - { title: 'MistralAI', href: '/docs/tracing/auto/mistralai' }, - { title: 'Together AI', href: '/docs/tracing/auto/togetherai' }, - { title: 'Ollama', href: '/docs/tracing/auto/ollama' }, - { title: 'Portkey', href: '/docs/tracing/auto/portkey' }, + { title: 'OpenAI', href: '/docs/traceai/auto/openai' }, + { title: 'Anthropic', href: '/docs/traceai/auto/anthropic' }, + { title: 'AWS Bedrock', href: '/docs/traceai/auto/bedrock' }, + { title: 'Vertex AI', href: '/docs/traceai/auto/vertexai' }, + { title: 'Google GenAI', href: '/docs/traceai/auto/google_genai' }, + { title: 'Google ADK', href: '/docs/traceai/auto/google_adk' }, + { title: 'Groq', href: '/docs/traceai/auto/groq' }, + { title: 'MistralAI', href: '/docs/traceai/auto/mistralai' }, + { title: 'Together AI', href: '/docs/traceai/auto/togetherai' }, + { title: 'Ollama', href: '/docs/traceai/auto/ollama' }, + { title: 'Portkey', href: '/docs/traceai/auto/portkey' }, ] }, { - title: 'Frameworks & Agents', + title: 'Frameworks and agents', items: [ - { title: 'LangChain', href: '/docs/tracing/auto/langchain' }, - { title: 'LangGraph', href: '/docs/tracing/auto/langgraph' }, - { title: 'LlamaIndex', href: '/docs/tracing/auto/llamaindex' }, - { title: 'LlamaIndex Workflows', href: '/docs/tracing/auto/llamaindex-workflows' }, - { title: 'LiteLLM', href: '/docs/tracing/auto/litellm' }, - { title: 'CrewAI', href: '/docs/tracing/auto/crewai' }, - { title: 'AutoGen', href: '/docs/tracing/auto/autogen' }, - { title: 'Haystack', href: '/docs/tracing/auto/haystack' }, - { title: 'DSPy', href: '/docs/tracing/auto/dspy' }, - { title: 'OpenAI Agents', href: '/docs/tracing/auto/openai_agents' }, - { title: 'Smol Agents', href: '/docs/tracing/auto/smol_agents' }, - { title: 'Instructor', href: '/docs/tracing/auto/instructor' }, - { title: 'PromptFlow', href: '/docs/tracing/auto/promptflow' }, - { title: 'Guardrails', href: '/docs/tracing/auto/guardrails' }, - { title: 'MCP', href: '/docs/tracing/auto/mcp' }, - { title: 'Mastra', href: '/docs/tracing/auto/mastra' }, - { title: 'Vercel AI SDK', href: '/docs/tracing/auto/vercel' }, + { title: 'LangChain', href: '/docs/traceai/auto/langchain' }, + { title: 'LangGraph', href: '/docs/traceai/auto/langgraph' }, + { title: 'LlamaIndex', href: '/docs/traceai/auto/llamaindex' }, + { title: 'LlamaIndex Workflows', href: '/docs/traceai/auto/llamaindex-workflows' }, + { title: 'LiteLLM', href: '/docs/traceai/auto/litellm' }, + { title: 'CrewAI', href: '/docs/traceai/auto/crewai' }, + { title: 'AutoGen', href: '/docs/traceai/auto/autogen' }, + { title: 'Haystack', href: '/docs/traceai/auto/haystack' }, + { title: 'DSPy', href: '/docs/traceai/auto/dspy' }, + { title: 'OpenAI Agents', href: '/docs/traceai/auto/openai_agents' }, + { title: 'Smol Agents', href: '/docs/traceai/auto/smol_agents' }, + { title: 'Instructor', href: '/docs/traceai/auto/instructor' }, + { title: 'PromptFlow', href: '/docs/traceai/auto/promptflow' }, + { title: 'Guardrails', href: '/docs/traceai/auto/guardrails' }, + { title: 'MCP', href: '/docs/traceai/auto/mcp' }, + { title: 'Mastra', href: '/docs/traceai/auto/mastra' }, + { title: 'Vercel AI SDK', href: '/docs/traceai/auto/vercel' }, ] }, { - title: 'Voice & Realtime', + title: 'Voice and realtime', items: [ - { title: 'LiveKit', href: '/docs/tracing/auto/livekit' }, - { title: 'Pipecat', href: '/docs/tracing/auto/pipecat' }, + { title: 'LiveKit', href: '/docs/traceai/auto/livekit' }, + { title: 'Pipecat', href: '/docs/traceai/auto/pipecat' }, ] }, { title: 'Java', items: [ - { title: 'Overview', href: '/docs/tracing/auto/java' }, - { title: 'Spring Boot', href: '/docs/tracing/auto/spring-boot' }, - { title: 'OpenAI', href: '/docs/tracing/auto/java/openai' }, - { title: 'Anthropic', href: '/docs/tracing/auto/java/anthropic' }, - { title: 'AWS Bedrock', href: '/docs/tracing/auto/java/bedrock' }, - { title: 'Cohere', href: '/docs/tracing/auto/java/cohere' }, - { title: 'Pinecone', href: '/docs/tracing/auto/java/pinecone' }, - { title: 'LLM Providers', href: '/docs/tracing/auto/java/llm-providers' }, - { title: 'Vector Databases', href: '/docs/tracing/auto/java/vector-databases' }, - { title: 'Frameworks', href: '/docs/tracing/auto/java/frameworks' }, + { title: 'Overview', href: '/docs/traceai/auto/java' }, + { title: 'Spring Boot', href: '/docs/traceai/auto/spring-boot' }, + { title: 'OpenAI', href: '/docs/traceai/auto/java/openai' }, + { title: 'Anthropic', href: '/docs/traceai/auto/java/anthropic' }, + { title: 'AWS Bedrock', href: '/docs/traceai/auto/java/bedrock' }, + { title: 'Cohere', href: '/docs/traceai/auto/java/cohere' }, + { title: 'Pinecone', href: '/docs/traceai/auto/java/pinecone' }, + { title: 'LLM Providers', href: '/docs/traceai/auto/java/llm-providers' }, + { title: 'Vector Databases', href: '/docs/traceai/auto/java/vector-databases' }, + { title: 'Frameworks', href: '/docs/traceai/auto/java/frameworks' }, ] }, { @@ -461,6 +496,18 @@ export const tabNavigation: NavTab[] = [ }, ] }, + { + title: 'Reference', + items: [ + { title: 'Semantic conventions', href: '/docs/traceai/manual-instrumentation/semantic-conventions' }, + ] + }, + { + title: 'Troubleshooting', + items: [ + { title: 'Spans not exported', href: '/docs/traceai/troubleshooting/spans-not-exported' }, + ] + }, ] }, { @@ -1011,7 +1058,6 @@ export const tabNavigation: NavTab[] = [ { title: 'Bulk Delete Eval Tasks', href: '/docs/api/eval-tasks/bulk-delete-eval-tasks' }, { title: 'Pause Eval Task', href: '/docs/api/eval-tasks/pause-eval-task' }, { title: 'Unpause Eval Task', href: '/docs/api/eval-tasks/unpause-eval-task' }, - { title: 'Eval Task Aggregations', href: '/docs/api/eval-tasks/eval-task-aggregations' }, ] }, { @@ -1048,6 +1094,7 @@ export const tabNavigation: NavTab[] = [ { title: 'Delete Scenario', href: '/docs/api/scenarios/deletescenario' }, { title: 'Add Rows with AI', href: '/docs/api/scenarios/addscenariorowswithai' }, { title: 'Add Columns', href: '/docs/api/scenarios/addcolumns' }, + { title: 'Add Empty Rows', href: '/docs/api/scenarios/addemptyrowstodataset' }, ] }, { @@ -1088,8 +1135,10 @@ export const tabNavigation: NavTab[] = [ { title: 'Get Test Run Details', href: '/docs/api/run-tests/getruntestdetails' }, { title: 'Delete Test Run', href: '/docs/api/run-tests/deleteruntest' }, { title: 'Execute Run Test', href: '/docs/api/run-tests/executeruntest' }, + { title: 'Update Components', href: '/docs/api/run-tests/updatetestcomponents' }, { title: 'Get Test Executions', href: '/docs/api/run-tests/gettestexecutions' }, { title: 'Get Test Scenarios', href: '/docs/api/run-tests/gettestscenarios' }, + { title: 'Get Call Executions', href: '/docs/api/run-tests/getcallexecutions' }, { title: 'Get Eval Summary', href: '/docs/api/run-tests/getevalsummary' }, { title: 'Compare Eval Summaries', href: '/docs/api/run-tests/compareevalsummaries' }, { title: 'Add Eval Configs', href: '/docs/api/run-tests/addevalconfigs' }, diff --git a/src/lib/redirects.ts b/src/lib/redirects.ts index 93b7acbb..51a921bf 100644 --- a/src/lib/redirects.ts +++ b/src/lib/redirects.ts @@ -1,12 +1,13 @@ // Auto-generated redirect map: old Mintlify URLs → new docs URLs // 275 redirects from futureagi.mintlify.app export const redirectMap: Record = { + '/docs/observe/getting-started': '/docs/observe', '/docs/observe/features/annotation-queue-using-sdk': '/docs/annotations/sdk/annotation-queue-using-sdk', '/docs/observe/voice/set-up': '/docs/observe/features/voice', '/docs/quickstart/installation': '/docs/installation', - '/docs/observability': '/docs/tracing/auto', - '/docs/tracing': '/docs/tracing/auto', - '/docs/tracing/auto-overview': '/docs/tracing/auto', + '/docs/observability': '/docs/traceai/auto', + '/docs/tracing': '/docs/traceai/auto', + '/docs/tracing/auto-overview': '/docs/traceai/auto', '/docs/evaluation/builtin/eval-context-retrieval': '/docs/evaluation/builtin', '/docs/evaluation/features/groups': '/docs/evaluation', '/docs/optimization/optimizers/overview': '/docs/optimization', @@ -14,17 +15,17 @@ export const redirectMap: Record = { '/docs/knowledge-base/concept': '/docs/knowledge-base/concepts/concept', '/docs/prompt-workbench': '/docs/prompt', '/docs/prompt-workbench/sdk': '/docs/prompt/features/sdk', - '/docs/tracing/manual/log-prompt-templates': '/docs/observe/features/manual-tracing/log-prompt-templates', - '/docs/tracing/manual/in-line-evals': '/docs/observe/features/manual-tracing/in-line-evals', + '/docs/tracing/manual/log-prompt-templates': '/docs/traceai/manual-instrumentation/log-prompt-templates', + '/docs/tracing/manual/in-line-evals': '/docs/traceai/manual-instrumentation/in-line-evals', '/docs/simulation/set-up/scenarios': '/docs/simulation/concepts/scenarios', '/docs/simulation/set-up/agent-definition': '/docs/simulation/concepts/agent-definition', - '/docs/tracing/concepts/components': '/docs/tracing/concepts', - '/docs/tracing/manual/add-attributes-metadata-tags': '/docs/observe/features/manual-tracing/add-attributes-metadata-tags', - '/docs/tracing/manual/add-events-exceptions-status': '/docs/observe/features/manual-tracing/add-events-exceptions-status', - '/docs/tracing/manual/advanced-tracing-examples': '/docs/observe/features/manual-tracing/advanced-tracing-examples', - '/docs/tracing/manual/langfuse-integration': '/docs/observe/features/manual-tracing/langfuse-integration', - '/docs/tracing/manual/set-session-user-id': '/docs/observe/features/manual-tracing/set-session-user-id', - '/docs/tracing/manual/set-up-tracing': '/docs/observe/features/manual-tracing/set-up-tracing', + '/docs/tracing/concepts/components': '/docs/observe/concepts/observability-model', + '/docs/tracing/manual/add-attributes-metadata-tags': '/docs/traceai/manual-instrumentation/add-attributes-metadata-tags', + '/docs/tracing/manual/add-events-exceptions-status': '/docs/traceai/manual-instrumentation/add-events-exceptions-status', + '/docs/tracing/manual/advanced-tracing-examples': '/docs/traceai/manual-instrumentation/advanced-tracing-examples', + '/docs/tracing/manual/langfuse-integration': '/docs/traceai/manual-instrumentation/langfuse-integration', + '/docs/tracing/manual/set-session-user-id': '/docs/traceai/manual-instrumentation/set-session-user-id', + '/docs/tracing/manual/set-up-tracing': '/docs/traceai/manual-instrumentation/set-up-tracing', '/docs/prism': '/docs/command-center', '/docs/prism/concepts/api-reference': '/docs/command-center/concepts/api-reference', '/docs/prism/concepts/configuration': '/docs/command-center/concepts/configuration', @@ -48,12 +49,13 @@ export const redirectMap: Record = { '/api-reference/prompt-workbench/get-prompt-version-by-name': '/docs/api', '/api-reference/run-tests/create-a-new-test-run': '/docs/api/run-tests/createruntest', '/api-reference/run-tests/execute-a-test-run': '/docs/api/run-tests/executeruntest', + '/api-reference/scenarios/add-empty-rows-to-a-scenario': '/docs/api/scenarios/addemptyrowstodataset', '/api-reference/scenarios/add-rows-to-a-scenario-using-ai': '/docs/api/scenarios/addscenariorowswithai', '/api-reference/scenarios/edit-a-scenario': '/docs/api/scenarios/editscenario', '/api-reference/scenarios/generate-or-create-a-scenario': '/docs/api/scenarios/createscenario', '/cookbook/ai-evaluation/autoeval': '/docs/cookbook', '/cookbook/ai-evaluation/feedback-loop': '/docs/cookbook', - '/cookbook/ai-evaluation/guardrails': '/docs/tracing/auto/guardrails', + '/cookbook/ai-evaluation/guardrails': '/docs/traceai/auto/guardrails', '/cookbook/ai-evaluation/llm-judge': '/docs/cookbook', '/cookbook/ai-evaluation/local-metrics': '/docs/cookbook', '/cookbook/ai-evaluation/multimodal-judge': '/docs/cookbook', @@ -163,20 +165,20 @@ export const redirectMap: Record = { '/future-agi/get-started/knowledge-base/how-to/create-kb-using-sdk': '/docs/knowledge-base/features/sdk', '/future-agi/get-started/knowledge-base/how-to/create-kb-using-ui': '/docs/knowledge-base/features/ui', '/future-agi/get-started/knowledge-base/overview': '/docs/knowledge-base', - '/future-agi/get-started/observability/manual-tracing/add-attributes-metadata-tags': '/docs/observe/features/manual-tracing/add-attributes-metadata-tags', - '/future-agi/get-started/observability/manual-tracing/add-events-exceptions-status': '/docs/observe/features/manual-tracing/add-events-exceptions-status', - '/future-agi/get-started/observability/manual-tracing/advanced-tracing-examples': '/docs/observe/features/manual-tracing/advanced-tracing-examples', - '/future-agi/get-started/observability/manual-tracing/annotating-using-api': '/docs/observe/features/manual-tracing/annotating-using-api', - '/future-agi/get-started/observability/manual-tracing/create-tool-spans': '/docs/observe/features/manual-tracing/create-tool-spans', - '/future-agi/get-started/observability/manual-tracing/get-current-span-context': '/docs/observe/features/manual-tracing/get-current-span-context', - '/future-agi/get-started/observability/manual-tracing/in-line-evals': '/docs/observe/features/manual-tracing/in-line-evals', - '/future-agi/get-started/observability/manual-tracing/instrument-with-traceai-helpers': '/docs/observe/features/manual-tracing/instrument-with-traceai-helpers', - '/future-agi/get-started/observability/manual-tracing/langfuse-intergation': '/docs/observe/features/manual-tracing/langfuse-integration', - '/future-agi/get-started/observability/manual-tracing/log-prompt-templates': '/docs/observe/features/manual-tracing/log-prompt-templates', - '/future-agi/get-started/observability/manual-tracing/mask-span-attributes': '/docs/observe/features/manual-tracing/mask-span-attributes', - '/future-agi/get-started/observability/manual-tracing/semantic-conventions': '/docs/observe/features/manual-tracing/semantic-conventions', - '/future-agi/get-started/observability/manual-tracing/set-session-user-id': '/docs/observe/features/manual-tracing/set-session-user-id', - '/future-agi/get-started/observability/manual-tracing/set-up-tracing': '/docs/observe/features/manual-tracing/set-up-tracing', + '/future-agi/get-started/observability/manual-tracing/add-attributes-metadata-tags': '/docs/traceai/manual-instrumentation/add-attributes-metadata-tags', + '/future-agi/get-started/observability/manual-tracing/add-events-exceptions-status': '/docs/traceai/manual-instrumentation/add-events-exceptions-status', + '/future-agi/get-started/observability/manual-tracing/advanced-tracing-examples': '/docs/traceai/manual-instrumentation/advanced-tracing-examples', + '/future-agi/get-started/observability/manual-tracing/annotating-using-api': '/docs/traceai/manual-instrumentation/annotating-using-api', + '/future-agi/get-started/observability/manual-tracing/create-tool-spans': '/docs/traceai/manual-instrumentation/create-tool-spans', + '/future-agi/get-started/observability/manual-tracing/get-current-span-context': '/docs/traceai/manual-instrumentation/get-current-span-context', + '/future-agi/get-started/observability/manual-tracing/in-line-evals': '/docs/traceai/manual-instrumentation/in-line-evals', + '/future-agi/get-started/observability/manual-tracing/instrument-with-traceai-helpers': '/docs/traceai/manual-instrumentation/instrument-with-traceai-helpers', + '/future-agi/get-started/observability/manual-tracing/langfuse-intergation': '/docs/traceai/manual-instrumentation/langfuse-integration', + '/future-agi/get-started/observability/manual-tracing/log-prompt-templates': '/docs/traceai/manual-instrumentation/log-prompt-templates', + '/future-agi/get-started/observability/manual-tracing/mask-span-attributes': '/docs/traceai/manual-instrumentation/mask-span-attributes', + '/future-agi/get-started/observability/manual-tracing/semantic-conventions': '/docs/traceai/manual-instrumentation/semantic-conventions', + '/future-agi/get-started/observability/manual-tracing/set-session-user-id': '/docs/traceai/manual-instrumentation/set-session-user-id', + '/future-agi/get-started/observability/manual-tracing/set-up-tracing': '/docs/traceai/manual-instrumentation/set-up-tracing', '/future-agi/get-started/optimization/dataset-optimization': '/docs/cookbook/quickstart/dataset-optimization', '/future-agi/get-started/optimization/how-to/using-python-sdk': '/docs/optimization/features/using-python-sdk', '/future-agi/get-started/optimization/optimizers/bayesian-search': '/docs/optimization/optimizers/bayesian-search', @@ -193,24 +195,24 @@ export const redirectMap: Record = { '/future-agi/get-started/protect/overview': '/docs/protect', '/future-agi/get-started/prototype/evals': '/docs/prototype/features/evals', '/future-agi/get-started/prototype/overview': '/docs/prototype', - '/future-agi/get-started/prototype/quickstart': '/docs/observe/features/quickstart', + '/future-agi/get-started/prototype/quickstart': '/docs/observe/quickstart', '/future-agi/get-started/prototype/winner': '/docs/prototype/features/choose-winner', - '/future-agi/products/observability/auto-instrumentation/overview': '/docs/tracing/auto', - '/future-agi/products/observability/concept/core-components': '/docs/tracing/concepts', - '/future-agi/products/observability/concept/otel': '/docs/tracing/concepts/otel', - '/future-agi/products/observability/concept/overview': '/docs/tracing/concepts', - '/future-agi/products/observability/concept/spans': '/docs/tracing/concepts/spans', - '/future-agi/products/observability/concept/traceai': '/docs/tracing/concepts/traceai', - '/future-agi/products/observability/concept/traces': '/docs/tracing/concepts/traces', - '/future-agi/products/observability/overview': '/docs/tracing/auto', + '/future-agi/products/observability/auto-instrumentation/overview': '/docs/traceai/auto', + '/future-agi/products/observability/concept/core-components': '/docs/observe/concepts/observability-model', + '/future-agi/products/observability/concept/otel': '/docs/observe/concepts/otel', + '/future-agi/products/observability/concept/overview': '/docs/observe/concepts/observability-model', + '/future-agi/products/observability/concept/spans': '/docs/observe/concepts/spans', + '/future-agi/products/observability/concept/traceai': '/docs/observe/concepts/traceai', + '/future-agi/products/observability/concept/traces': '/docs/observe/concepts/traces', + '/future-agi/products/observability/overview': '/docs/traceai/auto', '/future-agi/products/observe/alerts-and-monitors': '/docs/observe/features/alerts', '/future-agi/products/observe/evals': '/docs/observe/features/evals', '/future-agi/products/observe/overview': '/docs/observe', - '/future-agi/products/observe/quickstart': '/docs/observe/features/quickstart', + '/future-agi/products/observe/quickstart': '/docs/observe/quickstart', '/future-agi/products/observe/session': '/docs/observe/features/session', '/future-agi/products/observe/users': '/docs/observe/features/users', '/future-agi/products/observe/voice/overview': '/docs/observe/features/voice', - '/future-agi/products/observe/voice/quickstart': '/docs/observe/features/quickstart', + '/future-agi/products/observe/voice/quickstart': '/docs/observe/quickstart', '/home': '/docs', '/integrations/anthropic': '/docs/integrations/traceai/anthropic', '/integrations/autogen': '/docs/integrations/traceai/autogen', @@ -306,4 +308,67 @@ export const redirectMap: Record = { '/sdk-reference/python-sdk-client': '/docs/sdk', '/sdk-reference/testcase': '/docs/sdk/testcase', '/sdk-reference/tracing': '/docs/sdk/tracing', + // === traceAI/Observe revamp moves (Phase 2) === + '/docs/observe/features/quickstart': '/docs/observe/quickstart', + '/docs/observe/features/manual-tracing/add-attributes-metadata-tags': '/docs/traceai/manual-instrumentation/add-attributes-metadata-tags', + '/docs/observe/features/manual-tracing/add-events-exceptions-status': '/docs/traceai/manual-instrumentation/add-events-exceptions-status', + '/docs/observe/features/manual-tracing/advanced-tracing-examples': '/docs/traceai/manual-instrumentation/advanced-tracing-examples', + '/docs/observe/features/manual-tracing/annotating-using-api': '/docs/traceai/manual-instrumentation/annotating-using-api', + '/docs/observe/features/manual-tracing/create-tool-spans': '/docs/traceai/manual-instrumentation/create-tool-spans', + '/docs/observe/features/manual-tracing/get-current-span-context': '/docs/traceai/manual-instrumentation/get-current-span-context', + '/docs/observe/features/manual-tracing/in-line-evals': '/docs/traceai/manual-instrumentation/in-line-evals', + '/docs/observe/features/manual-tracing/instrument-with-traceai-helpers': '/docs/traceai/manual-instrumentation/instrument-with-traceai-helpers', + '/docs/observe/features/manual-tracing/langfuse-integration': '/docs/traceai/manual-instrumentation/langfuse-integration', + '/docs/observe/features/manual-tracing/log-prompt-templates': '/docs/traceai/manual-instrumentation/log-prompt-templates', + '/docs/observe/features/manual-tracing/mask-span-attributes': '/docs/traceai/manual-instrumentation/mask-span-attributes', + '/docs/observe/features/manual-tracing/semantic-conventions': '/docs/traceai/manual-instrumentation/semantic-conventions', + '/docs/observe/features/manual-tracing/set-session-user-id': '/docs/traceai/manual-instrumentation/set-session-user-id', + '/docs/observe/features/manual-tracing/set-up-tracing': '/docs/traceai/manual-instrumentation/set-up-tracing', + '/docs/tracing/auto': '/docs/traceai/auto', + '/docs/tracing/auto/anthropic': '/docs/traceai/auto/anthropic', + '/docs/tracing/auto/autogen': '/docs/traceai/auto/autogen', + '/docs/tracing/auto/bedrock': '/docs/traceai/auto/bedrock', + '/docs/tracing/auto/crewai': '/docs/traceai/auto/crewai', + '/docs/tracing/auto/dspy': '/docs/traceai/auto/dspy', + '/docs/tracing/auto/google_adk': '/docs/traceai/auto/google_adk', + '/docs/tracing/auto/google_genai': '/docs/traceai/auto/google_genai', + '/docs/tracing/auto/groq': '/docs/traceai/auto/groq', + '/docs/tracing/auto/guardrails': '/docs/traceai/auto/guardrails', + '/docs/tracing/auto/haystack': '/docs/traceai/auto/haystack', + '/docs/tracing/auto/instructor': '/docs/traceai/auto/instructor', + '/docs/tracing/auto/java': '/docs/traceai/auto/java', + '/docs/tracing/auto/java/anthropic': '/docs/traceai/auto/java/anthropic', + '/docs/tracing/auto/java/bedrock': '/docs/traceai/auto/java/bedrock', + '/docs/tracing/auto/java/cohere': '/docs/traceai/auto/java/cohere', + '/docs/tracing/auto/java/frameworks': '/docs/traceai/auto/java/frameworks', + '/docs/tracing/auto/java/llm-providers': '/docs/traceai/auto/java/llm-providers', + '/docs/tracing/auto/java/openai': '/docs/traceai/auto/java/openai', + '/docs/tracing/auto/java/pinecone': '/docs/traceai/auto/java/pinecone', + '/docs/tracing/auto/java/vector-databases': '/docs/traceai/auto/java/vector-databases', + '/docs/tracing/auto/langchain': '/docs/traceai/auto/langchain', + '/docs/tracing/auto/langgraph': '/docs/traceai/auto/langgraph', + '/docs/tracing/auto/litellm': '/docs/traceai/auto/litellm', + '/docs/tracing/auto/livekit': '/docs/traceai/auto/livekit', + '/docs/tracing/auto/llamaindex': '/docs/traceai/auto/llamaindex', + '/docs/tracing/auto/llamaindex-workflows': '/docs/traceai/auto/llamaindex-workflows', + '/docs/tracing/auto/mastra': '/docs/traceai/auto/mastra', + '/docs/tracing/auto/mcp': '/docs/traceai/auto/mcp', + '/docs/tracing/auto/mistralai': '/docs/traceai/auto/mistralai', + '/docs/tracing/auto/ollama': '/docs/traceai/auto/ollama', + '/docs/tracing/auto/openai': '/docs/traceai/auto/openai', + '/docs/tracing/auto/openai_agents': '/docs/traceai/auto/openai_agents', + '/docs/tracing/auto/pipecat': '/docs/traceai/auto/pipecat', + '/docs/tracing/auto/portkey': '/docs/traceai/auto/portkey', + '/docs/tracing/auto/promptflow': '/docs/traceai/auto/promptflow', + '/docs/tracing/auto/smol_agents': '/docs/traceai/auto/smol_agents', + '/docs/tracing/auto/spring-boot': '/docs/traceai/auto/spring-boot', + '/docs/tracing/auto/togetherai': '/docs/traceai/auto/togetherai', + '/docs/tracing/auto/vercel': '/docs/traceai/auto/vercel', + '/docs/tracing/auto/vertexai': '/docs/traceai/auto/vertexai', + '/docs/tracing/concepts': '/docs/observe/concepts/observability-model', + '/docs/tracing/concepts/otel': '/docs/observe/concepts/otel', + '/docs/tracing/concepts/sessions-and-users': '/docs/observe/concepts/sessions-and-users', + '/docs/tracing/concepts/spans': '/docs/observe/concepts/spans', + '/docs/tracing/concepts/traceai': '/docs/observe/concepts/traceai', + '/docs/tracing/concepts/traces': '/docs/observe/concepts/traces', }; diff --git a/src/pages/docs/admin-settings/integrations.mdx b/src/pages/docs/admin-settings/integrations.mdx index 7dd4a86b..01e34600 100644 --- a/src/pages/docs/admin-settings/integrations.mdx +++ b/src/pages/docs/admin-settings/integrations.mdx @@ -65,4 +65,4 @@ When connecting Datadog, select your site: ## Next Steps - [Workspace Management](/docs/admin-settings/workspace-management) - Organize projects and teams -- [Observability](/docs/tracing/auto) - Monitor your AI applications +- [Observability](/docs/traceai/auto) - Monitor your AI applications diff --git a/src/pages/docs/api/agent-definitions/createagentdefinition.mdx b/src/pages/docs/api/agent-definitions/createagentdefinition.mdx index 267f8176..92127c83 100644 --- a/src/pages/docs/api/agent-definitions/createagentdefinition.mdx +++ b/src/pages/docs/api/agent-definitions/createagentdefinition.mdx @@ -1,5 +1,5 @@ --- -title: "Create Agent Definition" +title: "Create Agent Definition — API" description: "Create a new agent definition and its initial version. Accepts agent type, name, provider, API key, and system prompt. Returns the created agent object." --- diff --git a/src/pages/docs/api/agent-definitions/deleteagentdefinitions.mdx b/src/pages/docs/api/agent-definitions/deleteagentdefinitions.mdx index c1c19d65..c357b872 100644 --- a/src/pages/docs/api/agent-definitions/deleteagentdefinitions.mdx +++ b/src/pages/docs/api/agent-definitions/deleteagentdefinitions.mdx @@ -1,5 +1,5 @@ --- -title: "Delete Agent Definitions" +title: "Delete Agent Definitions — API" description: "Bulk soft-delete one or more agent definitions and all associated versions. Accepts a list of agent UUIDs and returns a count of agents updated." --- diff --git a/src/pages/docs/api/agent-definitions/fetchassistantfromprovider.mdx b/src/pages/docs/api/agent-definitions/fetchassistantfromprovider.mdx index 280e7bc3..960ebdf6 100644 --- a/src/pages/docs/api/agent-definitions/fetchassistantfromprovider.mdx +++ b/src/pages/docs/api/agent-definitions/fetchassistantfromprovider.mdx @@ -1,5 +1,5 @@ --- -title: "Fetch Assistant from Provider" +title: "Fetch Assistant from Provider — API" description: "Fetch an assistant's name and system prompt from an external voice provider such as Vapi. Requires assistant ID, API key, and provider name." --- diff --git a/src/pages/docs/api/agent-definitions/getagentdefinition.mdx b/src/pages/docs/api/agent-definitions/getagentdefinition.mdx index 60ab5b64..1eff520b 100644 --- a/src/pages/docs/api/agent-definitions/getagentdefinition.mdx +++ b/src/pages/docs/api/agent-definitions/getagentdefinition.mdx @@ -1,5 +1,5 @@ --- -title: "Get Agent Definition" +title: "Get Agent Definition — API" description: "Retrieve a specific agent definition by UUID, including its full version history, provider details, and configuration snapshot." --- diff --git a/src/pages/docs/api/agent-definitions/listagentdefinitions.mdx b/src/pages/docs/api/agent-definitions/listagentdefinitions.mdx index 95ee761c..a398609a 100644 --- a/src/pages/docs/api/agent-definitions/listagentdefinitions.mdx +++ b/src/pages/docs/api/agent-definitions/listagentdefinitions.mdx @@ -1,5 +1,5 @@ --- -title: "List Agent Definitions" +title: "List Agent Definitions — API" description: "Return a paginated list of agent definitions. Filter by agent type, search by name or assistant ID, and pin a specific agent to the top of results." --- diff --git a/src/pages/docs/api/agent-versions/createagentversion.mdx b/src/pages/docs/api/agent-versions/createagentversion.mdx index a98a3fca..af1a1002 100644 --- a/src/pages/docs/api/agent-versions/createagentversion.mdx +++ b/src/pages/docs/api/agent-versions/createagentversion.mdx @@ -1,5 +1,5 @@ --- -title: "Create Agent Version" +title: "Create Agent Version — API" description: "Create a new version of an agent definition. Accepts agent name, language, system prompt, commit message, and provider config. Returns the new version object." --- diff --git a/src/pages/docs/api/agent-versions/getagentversion.mdx b/src/pages/docs/api/agent-versions/getagentversion.mdx index be1201a5..a3ae81ad 100644 --- a/src/pages/docs/api/agent-versions/getagentversion.mdx +++ b/src/pages/docs/api/agent-versions/getagentversion.mdx @@ -1,5 +1,5 @@ --- -title: "Get Agent Version" +title: "Get Agent Version — API" description: "Retrieve a specific agent version by agent UUID and version UUID, including its full configuration snapshot, prompt, and provider settings." --- diff --git a/src/pages/docs/api/agent-versions/getversioncallexecutions.mdx b/src/pages/docs/api/agent-versions/getversioncallexecutions.mdx index 76a62c2d..e726601e 100644 --- a/src/pages/docs/api/agent-versions/getversioncallexecutions.mdx +++ b/src/pages/docs/api/agent-versions/getversioncallexecutions.mdx @@ -1,5 +1,5 @@ --- -title: "Get Version Call Executions" +title: "Get Version Call Executions — API" description: "Retrieve paginated call executions and their evaluation results for a specific agent version. Supports limit and page query parameters." --- diff --git a/src/pages/docs/api/agent-versions/getversionevalsummary.mdx b/src/pages/docs/api/agent-versions/getversionevalsummary.mdx index 858d440b..d9c954fd 100644 --- a/src/pages/docs/api/agent-versions/getversionevalsummary.mdx +++ b/src/pages/docs/api/agent-versions/getversionevalsummary.mdx @@ -1,5 +1,5 @@ --- -title: "Get Version Eval Summary" +title: "Get Version Eval Summary — API" description: "Retrieve aggregated evaluation summary statistics for an agent version, including pass rate, fail count, and error count per eval metric." --- diff --git a/src/pages/docs/api/agent-versions/listagentversions.mdx b/src/pages/docs/api/agent-versions/listagentversions.mdx index e2f02c0f..1f972f7e 100644 --- a/src/pages/docs/api/agent-versions/listagentversions.mdx +++ b/src/pages/docs/api/agent-versions/listagentversions.mdx @@ -1,5 +1,5 @@ --- -title: "List Agent Versions" +title: "List Agent Versions — API" description: "Retrieve a paginated list of all versions for a given agent definition. Supports limit and page parameters. Returns version metadata and commit history." --- diff --git a/src/pages/docs/api/annotations/bulk/bulk-annotate-spans.mdx b/src/pages/docs/api/annotations/bulk/bulk-annotate-spans.mdx index 2e8f8ecd..5acf4d29 100644 --- a/src/pages/docs/api/annotations/bulk/bulk-annotate-spans.mdx +++ b/src/pages/docs/api/annotations/bulk/bulk-annotate-spans.mdx @@ -1,5 +1,5 @@ --- -title: "Bulk Annotate Spans" +title: "Bulk Annotate Spans — API" description: "Submit label scores and notes for multiple observation spans in one request. Accepts an array of span IDs with their annotation values and score sources." --- diff --git a/src/pages/docs/api/annotations/items/add-items.mdx b/src/pages/docs/api/annotations/items/add-items.mdx index ad44e941..db49aa4a 100644 --- a/src/pages/docs/api/annotations/items/add-items.mdx +++ b/src/pages/docs/api/annotations/items/add-items.mdx @@ -1,5 +1,5 @@ --- -title: "Add Items to Annotation Queue" +title: "Add Items to Annotation Queue — API" description: "Add one or more source items (traces, spans) to an annotation queue in bulk. Accepts queue UUID and an array of source type and source ID pairs." --- diff --git a/src/pages/docs/api/annotations/items/assign-items.mdx b/src/pages/docs/api/annotations/items/assign-items.mdx index 7e9cc671..a3d95e89 100644 --- a/src/pages/docs/api/annotations/items/assign-items.mdx +++ b/src/pages/docs/api/annotations/items/assign-items.mdx @@ -1,5 +1,5 @@ --- -title: "Assign Queue Items to Annotator" +title: "Assign Queue Items to Annotator — API" description: "Assign one or more annotation queue items to specific annotators. Accepts queue UUID, a list of item UUIDs, and a list of user UUIDs to assign." --- diff --git a/src/pages/docs/api/annotations/items/bulk-remove-items.mdx b/src/pages/docs/api/annotations/items/bulk-remove-items.mdx index 72d753fb..c9a2cbdc 100644 --- a/src/pages/docs/api/annotations/items/bulk-remove-items.mdx +++ b/src/pages/docs/api/annotations/items/bulk-remove-items.mdx @@ -1,5 +1,5 @@ --- -title: "Bulk Remove Queue Items" +title: "Bulk Remove Queue Items — API" description: "Remove multiple items from an annotation queue in a single request. Accepts the queue UUID and an array of item UUIDs to delete from the queue." --- diff --git a/src/pages/docs/api/annotations/items/complete-item.mdx b/src/pages/docs/api/annotations/items/complete-item.mdx index 93a25e0a..5c72bd9b 100644 --- a/src/pages/docs/api/annotations/items/complete-item.mdx +++ b/src/pages/docs/api/annotations/items/complete-item.mdx @@ -1,5 +1,5 @@ --- -title: "Complete Annotation Queue Item" +title: "Complete Annotation Queue Item — API" description: "Mark an annotation queue item as completed. Returns the completed item ID and, optionally, the next pending item available for annotation." --- diff --git a/src/pages/docs/api/annotations/items/get-annotate-detail.mdx b/src/pages/docs/api/annotations/items/get-annotate-detail.mdx index 29579242..c1123941 100644 --- a/src/pages/docs/api/annotations/items/get-annotate-detail.mdx +++ b/src/pages/docs/api/annotations/items/get-annotate-detail.mdx @@ -1,5 +1,5 @@ --- -title: "Get Annotation Item Detail" +title: "Get Annotation Item Detail — API" description: "Retrieve a queue item with full source content, queue metadata, labels, existing annotations, and navigation pointers for the annotation interface." --- diff --git a/src/pages/docs/api/annotations/items/get-item-annotations.mdx b/src/pages/docs/api/annotations/items/get-item-annotations.mdx index 92d84af6..f383d94e 100644 --- a/src/pages/docs/api/annotations/items/get-item-annotations.mdx +++ b/src/pages/docs/api/annotations/items/get-item-annotations.mdx @@ -1,5 +1,5 @@ --- -title: "Get Queue Item Annotations" +title: "Get Queue Item Annotations — API" description: "Retrieve all annotation scores submitted for a specific queue item, including label values, score source, annotator ID, and timestamps." --- diff --git a/src/pages/docs/api/annotations/items/get-next-item.mdx b/src/pages/docs/api/annotations/items/get-next-item.mdx index da43e8bd..5b5d5e3d 100644 --- a/src/pages/docs/api/annotations/items/get-next-item.mdx +++ b/src/pages/docs/api/annotations/items/get-next-item.mdx @@ -1,5 +1,5 @@ --- -title: "Get Next Annotation Queue Item" +title: "Get Next Annotation Queue Item — API" description: "Retrieve the next available queue item for the current user to annotate. Returns item ID, source type, status, and assignment info." --- diff --git a/src/pages/docs/api/annotations/items/list-items.mdx b/src/pages/docs/api/annotations/items/list-items.mdx index 07116365..01b8f135 100644 --- a/src/pages/docs/api/annotations/items/list-items.mdx +++ b/src/pages/docs/api/annotations/items/list-items.mdx @@ -1,5 +1,5 @@ --- -title: "List Annotation Queue Items" +title: "List Annotation Queue Items — API" description: "List items in an annotation queue with pagination. Filter by status, source type, or assigned user. Returns item details and annotation progress." --- diff --git a/src/pages/docs/api/annotations/items/release-item.mdx b/src/pages/docs/api/annotations/items/release-item.mdx index c7b1c11b..8041e549 100644 --- a/src/pages/docs/api/annotations/items/release-item.mdx +++ b/src/pages/docs/api/annotations/items/release-item.mdx @@ -1,5 +1,5 @@ --- -title: "Release Annotation Queue Item" +title: "Release Annotation Queue Item — API" description: "Release a reserved annotation queue item so it becomes available for reassignment to another annotator. Returns a boolean release confirmation." --- diff --git a/src/pages/docs/api/annotations/items/skip-item.mdx b/src/pages/docs/api/annotations/items/skip-item.mdx index a69a6563..1c9c4ed2 100644 --- a/src/pages/docs/api/annotations/items/skip-item.mdx +++ b/src/pages/docs/api/annotations/items/skip-item.mdx @@ -1,5 +1,5 @@ --- -title: "Skip Annotation Queue Item" +title: "Skip Annotation Queue Item — API" description: "Mark an annotation queue item as skipped by the current user. Returns the skipped item ID and the next available pending item for annotation." --- diff --git a/src/pages/docs/api/annotations/items/submit-annotations.mdx b/src/pages/docs/api/annotations/items/submit-annotations.mdx index 0bc7f958..c00d124b 100644 --- a/src/pages/docs/api/annotations/items/submit-annotations.mdx +++ b/src/pages/docs/api/annotations/items/submit-annotations.mdx @@ -1,5 +1,5 @@ --- -title: "Submit Item Annotations" +title: "Submit Item Annotations — API" description: "Submit label scores and reviewer notes for an annotation queue item. Accepts queue UUID, item UUID, an array of annotation values, and optional notes." --- diff --git a/src/pages/docs/api/annotations/labels/create-label.mdx b/src/pages/docs/api/annotations/labels/create-label.mdx index d66b872a..d6e8fd89 100644 --- a/src/pages/docs/api/annotations/labels/create-label.mdx +++ b/src/pages/docs/api/annotations/labels/create-label.mdx @@ -1,5 +1,5 @@ --- -title: "Create Annotation Label" +title: "Create Annotation Label — API" description: "Create a new annotation label with a name, type (star, binary, categorical), description, settings, and optional notes. Returns the created label object." --- diff --git a/src/pages/docs/api/annotations/labels/delete-label.mdx b/src/pages/docs/api/annotations/labels/delete-label.mdx index b59d8888..a70413b0 100644 --- a/src/pages/docs/api/annotations/labels/delete-label.mdx +++ b/src/pages/docs/api/annotations/labels/delete-label.mdx @@ -1,5 +1,5 @@ --- -title: "Delete Annotation Label" +title: "Delete Annotation Label — API" description: "Soft-delete an annotation label by UUID. Deleted labels are hidden but recoverable via the restore endpoint. Returns HTTP 204 on success." --- diff --git a/src/pages/docs/api/annotations/labels/get-label.mdx b/src/pages/docs/api/annotations/labels/get-label.mdx index 5bc6da29..92797671 100644 --- a/src/pages/docs/api/annotations/labels/get-label.mdx +++ b/src/pages/docs/api/annotations/labels/get-label.mdx @@ -1,5 +1,5 @@ --- -title: "Get Annotation Label" +title: "Get Annotation Label — API" description: "Retrieve a specific annotation label by UUID, including its type, settings, description, project scope, and allow-notes configuration." --- diff --git a/src/pages/docs/api/annotations/labels/list-labels.mdx b/src/pages/docs/api/annotations/labels/list-labels.mdx index a9ac7796..3109bc03 100644 --- a/src/pages/docs/api/annotations/labels/list-labels.mdx +++ b/src/pages/docs/api/annotations/labels/list-labels.mdx @@ -1,5 +1,5 @@ --- -title: "List Annotation Labels" +title: "List Annotation Labels — API" description: "List annotation labels with optional filters for type, name search, project, dataset scope, and usage count. Returns a paginated array of label objects." --- diff --git a/src/pages/docs/api/annotations/labels/restore-label.mdx b/src/pages/docs/api/annotations/labels/restore-label.mdx index 6b4d8f94..2c5a3c5b 100644 --- a/src/pages/docs/api/annotations/labels/restore-label.mdx +++ b/src/pages/docs/api/annotations/labels/restore-label.mdx @@ -1,5 +1,5 @@ --- -title: "Restore Annotation Label" +title: "Restore Annotation Label — API" description: "Restore a previously soft-deleted annotation label by UUID. Returns the full restored label object with its original type, settings, and configuration." --- diff --git a/src/pages/docs/api/annotations/labels/update-label.mdx b/src/pages/docs/api/annotations/labels/update-label.mdx index eb7c5f11..4c0ae7f4 100644 --- a/src/pages/docs/api/annotations/labels/update-label.mdx +++ b/src/pages/docs/api/annotations/labels/update-label.mdx @@ -1,5 +1,5 @@ --- -title: "Update Annotation Label" +title: "Update Annotation Label — API" description: "Update an existing annotation label's name, description, type, or settings by UUID. Returns the updated label object with all current field values." --- diff --git a/src/pages/docs/api/annotations/queues/add-label.mdx b/src/pages/docs/api/annotations/queues/add-label.mdx index 05abc59c..4c007996 100644 --- a/src/pages/docs/api/annotations/queues/add-label.mdx +++ b/src/pages/docs/api/annotations/queues/add-label.mdx @@ -1,5 +1,5 @@ --- -title: "Add Label to Annotation Queue" +title: "Add Label to Annotation Queue — API" description: "Attach an existing annotation label to a queue by queue UUID and label UUID. Returns the updated queue-label association with ordering and required flag." --- diff --git a/src/pages/docs/api/annotations/queues/create-queue.mdx b/src/pages/docs/api/annotations/queues/create-queue.mdx index 14ab6b8e..8d1e60fb 100644 --- a/src/pages/docs/api/annotations/queues/create-queue.mdx +++ b/src/pages/docs/api/annotations/queues/create-queue.mdx @@ -1,5 +1,5 @@ --- -title: "Create Annotation Queue" +title: "Create Annotation Queue — API" description: "Create an annotation queue with name, assignment strategy, annotator list, labels, and review settings. Returns the fully configured queue object." --- diff --git a/src/pages/docs/api/annotations/queues/delete-queue.mdx b/src/pages/docs/api/annotations/queues/delete-queue.mdx index 4021a8b6..29330b21 100644 --- a/src/pages/docs/api/annotations/queues/delete-queue.mdx +++ b/src/pages/docs/api/annotations/queues/delete-queue.mdx @@ -1,5 +1,5 @@ --- -title: "Delete Annotation Queue" +title: "Delete Annotation Queue — API" description: "Soft-delete an annotation queue by UUID. The queue and its items are hidden but not permanently removed. Returns a deleted boolean confirmation." --- diff --git a/src/pages/docs/api/annotations/queues/export-to-dataset.mdx b/src/pages/docs/api/annotations/queues/export-to-dataset.mdx index 5aabb90f..5a7b094a 100644 --- a/src/pages/docs/api/annotations/queues/export-to-dataset.mdx +++ b/src/pages/docs/api/annotations/queues/export-to-dataset.mdx @@ -1,5 +1,5 @@ --- -title: "Export Queue to Dataset" +title: "Export Queue to Dataset — API" description: "Export completed annotation queue items into a Future AGI dataset. Accepts queue UUID, target dataset ID, and optional status filter for selective export." --- diff --git a/src/pages/docs/api/annotations/queues/export.mdx b/src/pages/docs/api/annotations/queues/export.mdx index ba3f5ef6..c4a02fe2 100644 --- a/src/pages/docs/api/annotations/queues/export.mdx +++ b/src/pages/docs/api/annotations/queues/export.mdx @@ -1,5 +1,5 @@ --- -title: "Export Annotation Queue" +title: "Export Annotation Queue — API" description: "Export annotation queue items and their scores as JSON or CSV. Filter by item status. Returns source type, annotations, annotator names, and timestamps." --- diff --git a/src/pages/docs/api/annotations/queues/find-queues-for-source.mdx b/src/pages/docs/api/annotations/queues/find-queues-for-source.mdx index e84bf2e6..dc3948c4 100644 --- a/src/pages/docs/api/annotations/queues/find-queues-for-source.mdx +++ b/src/pages/docs/api/annotations/queues/find-queues-for-source.mdx @@ -1,5 +1,5 @@ --- -title: "Find Queues for Source" +title: "Find Queues for Source — API" description: "Find all annotation queues containing a specific source item. Accepts source type, source UUID, or a JSON array of sources for multi-source lookup." --- diff --git a/src/pages/docs/api/annotations/queues/get-agreement.mdx b/src/pages/docs/api/annotations/queues/get-agreement.mdx index cdb7a826..e376dd74 100644 --- a/src/pages/docs/api/annotations/queues/get-agreement.mdx +++ b/src/pages/docs/api/annotations/queues/get-agreement.mdx @@ -1,5 +1,5 @@ --- -title: "Get Inter-Annotator Agreement" +title: "Get Inter-Annotator Agreement — API" description: "Retrieve inter-annotator agreement scores for a queue, including overall agreement percentage and per-label breakdown for quality assurance analysis." --- diff --git a/src/pages/docs/api/annotations/queues/get-analytics.mdx b/src/pages/docs/api/annotations/queues/get-analytics.mdx index c9676b97..18e52ad9 100644 --- a/src/pages/docs/api/annotations/queues/get-analytics.mdx +++ b/src/pages/docs/api/annotations/queues/get-analytics.mdx @@ -1,5 +1,5 @@ --- -title: "Get Annotation Queue Analytics" +title: "Get Annotation Queue Analytics — API" description: "Retrieve detailed analytics for a queue including daily throughput, annotator performance, label score distribution, and item status breakdown." --- diff --git a/src/pages/docs/api/annotations/queues/get-or-create-default.mdx b/src/pages/docs/api/annotations/queues/get-or-create-default.mdx index b936d282..d23a7ba1 100644 --- a/src/pages/docs/api/annotations/queues/get-or-create-default.mdx +++ b/src/pages/docs/api/annotations/queues/get-or-create-default.mdx @@ -1,5 +1,5 @@ --- -title: "Get or Create Default Queue" +title: "Get or Create Default Queue — API" description: "Get the default annotation queue for a project, dataset, or agent definition, automatically creating one if it does not yet exist. Returns queue and labels." --- diff --git a/src/pages/docs/api/annotations/queues/get-progress.mdx b/src/pages/docs/api/annotations/queues/get-progress.mdx index aa25c28c..d0d7cc0a 100644 --- a/src/pages/docs/api/annotations/queues/get-progress.mdx +++ b/src/pages/docs/api/annotations/queues/get-progress.mdx @@ -1,5 +1,5 @@ --- -title: "Get Annotation Queue Progress" +title: "Get Annotation Queue Progress — API" description: "Retrieve progress statistics for an annotation queue, including total, pending, in-progress, completed, and skipped counts with per-annotator breakdowns." --- diff --git a/src/pages/docs/api/annotations/queues/get-queue.mdx b/src/pages/docs/api/annotations/queues/get-queue.mdx index 71ec6ed0..5c662338 100644 --- a/src/pages/docs/api/annotations/queues/get-queue.mdx +++ b/src/pages/docs/api/annotations/queues/get-queue.mdx @@ -1,5 +1,5 @@ --- -title: "Get Annotation Queue" +title: "Get Annotation Queue — API" description: "Retrieve full details of an annotation queue by UUID, including assignment strategy, labels, annotators, review settings, and creation metadata." --- diff --git a/src/pages/docs/api/annotations/queues/list-queues.mdx b/src/pages/docs/api/annotations/queues/list-queues.mdx index ec2f95c7..e1fff1a8 100644 --- a/src/pages/docs/api/annotations/queues/list-queues.mdx +++ b/src/pages/docs/api/annotations/queues/list-queues.mdx @@ -1,5 +1,5 @@ --- -title: "List Annotation Queues" +title: "List Annotation Queues — API" description: "List annotation queues with optional filters for status and name search. Supports pagination and optionally includes item counts per queue." --- diff --git a/src/pages/docs/api/annotations/queues/remove-label.mdx b/src/pages/docs/api/annotations/queues/remove-label.mdx index 5b0dd169..27618bfb 100644 --- a/src/pages/docs/api/annotations/queues/remove-label.mdx +++ b/src/pages/docs/api/annotations/queues/remove-label.mdx @@ -1,5 +1,5 @@ --- -title: "Remove Label from Queue" +title: "Remove Label from Queue — API" description: "Detach an annotation label from a queue by queue UUID and label UUID. Removing a label hides it from future annotation sessions for that queue." --- diff --git a/src/pages/docs/api/annotations/queues/update-queue.mdx b/src/pages/docs/api/annotations/queues/update-queue.mdx index b9cdb876..f15be116 100644 --- a/src/pages/docs/api/annotations/queues/update-queue.mdx +++ b/src/pages/docs/api/annotations/queues/update-queue.mdx @@ -1,5 +1,5 @@ --- -title: "Update Annotation Queue" +title: "Update Annotation Queue — API" description: "Update an annotation queue's name, description, instructions, assignment strategy, or review settings by UUID. Returns the updated queue configuration." --- diff --git a/src/pages/docs/api/annotations/queues/update-status.mdx b/src/pages/docs/api/annotations/queues/update-status.mdx index ed6974e7..6d90880c 100644 --- a/src/pages/docs/api/annotations/queues/update-status.mdx +++ b/src/pages/docs/api/annotations/queues/update-status.mdx @@ -1,5 +1,5 @@ --- -title: "Update Annotation Queue Status" +title: "Update Annotation Queue Status — API" description: "Transition an annotation queue to a new status (draft, active, paused, or completed). Accepts queue UUID and target status. Returns the updated queue object." --- diff --git a/src/pages/docs/api/annotations/scores/bulk-create-scores.mdx b/src/pages/docs/api/annotations/scores/bulk-create-scores.mdx index 4eb02c17..4d4adc87 100644 --- a/src/pages/docs/api/annotations/scores/bulk-create-scores.mdx +++ b/src/pages/docs/api/annotations/scores/bulk-create-scores.mdx @@ -1,5 +1,5 @@ --- -title: "Bulk Create Annotation Scores" +title: "Bulk Create Annotation Scores — API" description: "Create multiple annotation scores on a single source in one request. Accepts source type, source UUID, and an array of label ID, value, and score source." --- diff --git a/src/pages/docs/api/annotations/scores/create-score.mdx b/src/pages/docs/api/annotations/scores/create-score.mdx index e05f47a9..c20fee4e 100644 --- a/src/pages/docs/api/annotations/scores/create-score.mdx +++ b/src/pages/docs/api/annotations/scores/create-score.mdx @@ -1,5 +1,5 @@ --- -title: "Create Annotation Score" +title: "Create Annotation Score — API" description: "Create a single annotation score on a trace, span, or session. Accepts source type, source UUID, label UUID, score value, and score source type." --- diff --git a/src/pages/docs/api/annotations/scores/delete-score.mdx b/src/pages/docs/api/annotations/scores/delete-score.mdx index 1cf6440a..e113d82c 100644 --- a/src/pages/docs/api/annotations/scores/delete-score.mdx +++ b/src/pages/docs/api/annotations/scores/delete-score.mdx @@ -1,5 +1,5 @@ --- -title: "Delete Annotation Score" +title: "Delete Annotation Score — API" description: "Soft-delete an annotation score by UUID. Only the score creator or an org admin can delete. Returns HTTP 204 on success." --- diff --git a/src/pages/docs/api/annotations/scores/get-scores-for-source.mdx b/src/pages/docs/api/annotations/scores/get-scores-for-source.mdx index 91c4b7e7..e8021c3c 100644 --- a/src/pages/docs/api/annotations/scores/get-scores-for-source.mdx +++ b/src/pages/docs/api/annotations/scores/get-scores-for-source.mdx @@ -1,5 +1,5 @@ --- -title: "Get Scores for Source" +title: "Get Scores for Source — API" description: "Retrieve all annotation scores for a specific source (trace, span, generation, or session). Filter by label UUID or annotator UUID to narrow results." --- diff --git a/src/pages/docs/api/annotations/scores/list-scores.mdx b/src/pages/docs/api/annotations/scores/list-scores.mdx index 40c3f727..1b84bf42 100644 --- a/src/pages/docs/api/annotations/scores/list-scores.mdx +++ b/src/pages/docs/api/annotations/scores/list-scores.mdx @@ -1,5 +1,5 @@ --- -title: "List Annotation Scores" +title: "List Annotation Scores — API" description: "List annotation scores with optional filters for source type, source UUID, label, and annotator. Supports pagination via page and page_size parameters." --- diff --git a/src/pages/docs/api/custom-eval-configs/check-config-exists.mdx b/src/pages/docs/api/custom-eval-configs/check-config-exists.mdx index 7273d1a5..c10d1220 100644 --- a/src/pages/docs/api/custom-eval-configs/check-config-exists.mdx +++ b/src/pages/docs/api/custom-eval-configs/check-config-exists.mdx @@ -1,5 +1,5 @@ --- -title: "Check Eval Config Exists" +title: "Check Eval Config Exists — API" description: "Check if a custom eval config with the same name and field mapping already exists in a project before creating a duplicate." --- diff --git a/src/pages/docs/api/custom-eval-configs/create-custom-eval-config.mdx b/src/pages/docs/api/custom-eval-configs/create-custom-eval-config.mdx index ad8ff607..175c1109 100644 --- a/src/pages/docs/api/custom-eval-configs/create-custom-eval-config.mdx +++ b/src/pages/docs/api/custom-eval-configs/create-custom-eval-config.mdx @@ -1,5 +1,5 @@ --- -title: "Create Custom Eval Config" +title: "Create Custom Eval Config — API" description: "Create a new custom eval config for a project by specifying an eval template, name, field mapping, and optional config settings." --- diff --git a/src/pages/docs/api/custom-eval-configs/delete-custom-eval-config.mdx b/src/pages/docs/api/custom-eval-configs/delete-custom-eval-config.mdx index 68bcedd8..062ad844 100644 --- a/src/pages/docs/api/custom-eval-configs/delete-custom-eval-config.mdx +++ b/src/pages/docs/api/custom-eval-configs/delete-custom-eval-config.mdx @@ -1,5 +1,5 @@ --- -title: "Delete Custom Eval Config" +title: "Delete Custom Eval Config — API" description: "Soft-delete a custom eval config by UUID. Returns 204 No Content on success. The config is deactivated and no longer applied." --- diff --git a/src/pages/docs/api/custom-eval-configs/get-custom-eval-config.mdx b/src/pages/docs/api/custom-eval-configs/get-custom-eval-config.mdx index 7655acfc..fb7f4016 100644 --- a/src/pages/docs/api/custom-eval-configs/get-custom-eval-config.mdx +++ b/src/pages/docs/api/custom-eval-configs/get-custom-eval-config.mdx @@ -1,5 +1,5 @@ --- -title: "Get Custom Eval Config" +title: "Get Custom Eval Config — API" description: "Retrieve a specific custom eval config by UUID, returning its template, name, project, field mapping, and current configuration." --- diff --git a/src/pages/docs/api/custom-eval-configs/list-configs-filtered.mdx b/src/pages/docs/api/custom-eval-configs/list-configs-filtered.mdx index 687a0ac0..fc30f0a0 100644 --- a/src/pages/docs/api/custom-eval-configs/list-configs-filtered.mdx +++ b/src/pages/docs/api/custom-eval-configs/list-configs-filtered.mdx @@ -1,5 +1,5 @@ --- -title: "List Custom Eval Configs" +title: "List Custom Eval Configs — API" description: "List all custom eval configs, with optional filtering by project UUID or eval task UUID to narrow results." --- diff --git a/src/pages/docs/api/custom-eval-configs/update-custom-eval-config.mdx b/src/pages/docs/api/custom-eval-configs/update-custom-eval-config.mdx index 593a1697..1a983f63 100644 --- a/src/pages/docs/api/custom-eval-configs/update-custom-eval-config.mdx +++ b/src/pages/docs/api/custom-eval-configs/update-custom-eval-config.mdx @@ -1,5 +1,5 @@ --- -title: "Update Custom Eval Config" +title: "Update Custom Eval Config — API" description: "Partially update an existing custom eval config by UUID. Supports patching name, mapping, config, or template fields individually." --- diff --git a/src/pages/docs/api/dataset-evals/add-dataset-eval.mdx b/src/pages/docs/api/dataset-evals/add-dataset-eval.mdx index 5eaa08cb..2d84e4e2 100644 --- a/src/pages/docs/api/dataset-evals/add-dataset-eval.mdx +++ b/src/pages/docs/api/dataset-evals/add-dataset-eval.mdx @@ -1,5 +1,5 @@ --- -title: "Add Dataset Eval" +title: "Add Dataset Eval — API" description: "Add an evaluation to a dataset by selecting a template and configuring the input/output key mapping. Returns the created eval record." --- diff --git a/src/pages/docs/api/dataset-evals/create-custom-eval-template.mdx b/src/pages/docs/api/dataset-evals/create-custom-eval-template.mdx index caf36e86..7ff2ba66 100644 --- a/src/pages/docs/api/dataset-evals/create-custom-eval-template.mdx +++ b/src/pages/docs/api/dataset-evals/create-custom-eval-template.mdx @@ -1,5 +1,5 @@ --- -title: "Create Custom Eval Template" +title: "Create Custom Eval Template — API" description: "Create a reusable custom eval template with criteria, output type (e.g. Pass/Fail), required keys, and model configuration." --- diff --git a/src/pages/docs/api/dataset-evals/delete-dataset-eval.mdx b/src/pages/docs/api/dataset-evals/delete-dataset-eval.mdx index 667ecc22..7f6980ee 100644 --- a/src/pages/docs/api/dataset-evals/delete-dataset-eval.mdx +++ b/src/pages/docs/api/dataset-evals/delete-dataset-eval.mdx @@ -1,5 +1,5 @@ --- -title: "Delete Dataset Eval" +title: "Delete Dataset Eval — API" description: "Remove an evaluation from a dataset by eval ID. Optionally delete the associated eval column and its data from the dataset." --- diff --git a/src/pages/docs/api/dataset-evals/edit-and-run-eval.mdx b/src/pages/docs/api/dataset-evals/edit-and-run-eval.mdx index a887f7fe..fdbc23be 100644 --- a/src/pages/docs/api/dataset-evals/edit-and-run-eval.mdx +++ b/src/pages/docs/api/dataset-evals/edit-and-run-eval.mdx @@ -1,5 +1,5 @@ --- -title: "Edit and Run Dataset Eval" +title: "Edit and Run Dataset Eval — API" description: "Update an evaluation's config and key mapping, then optionally re-run it across all dataset rows to refresh scores." --- diff --git a/src/pages/docs/api/dataset-evals/get-eval-structure.mdx b/src/pages/docs/api/dataset-evals/get-eval-structure.mdx index 107f5846..b6625b8c 100644 --- a/src/pages/docs/api/dataset-evals/get-eval-structure.mdx +++ b/src/pages/docs/api/dataset-evals/get-eval-structure.mdx @@ -1,5 +1,5 @@ --- -title: "Get Eval Structure" +title: "Get Eval Structure — API" description: "Retrieve the configuration structure of an eval by ID and type (preset, user, or previously configured), including required keys and mapping." --- diff --git a/src/pages/docs/api/dataset-evals/get-eval-template-names.mdx b/src/pages/docs/api/dataset-evals/get-eval-template-names.mdx index a67ff283..5a2ea902 100644 --- a/src/pages/docs/api/dataset-evals/get-eval-template-names.mdx +++ b/src/pages/docs/api/dataset-evals/get-eval-template-names.mdx @@ -1,5 +1,5 @@ --- -title: "Get Eval Template Names" +title: "Get Eval Template Names — API" description: "Search and retrieve evaluation template names available in your organization. Supports text search to filter results by name." --- diff --git a/src/pages/docs/api/dataset-evals/list-dataset-evals.mdx b/src/pages/docs/api/dataset-evals/list-dataset-evals.mdx index 4831dde5..ab4ac8c4 100644 --- a/src/pages/docs/api/dataset-evals/list-dataset-evals.mdx +++ b/src/pages/docs/api/dataset-evals/list-dataset-evals.mdx @@ -1,5 +1,5 @@ --- -title: "List Dataset Evals" +title: "List Dataset Evals — API" description: "List available evaluations for a dataset. Filter by name, category (built-in or user), type, or tags to find the right eval." --- diff --git a/src/pages/docs/api/dataset-evals/start-evals-process.mdx b/src/pages/docs/api/dataset-evals/start-evals-process.mdx index 1aef1491..99599b5c 100644 --- a/src/pages/docs/api/dataset-evals/start-evals-process.mdx +++ b/src/pages/docs/api/dataset-evals/start-evals-process.mdx @@ -1,5 +1,5 @@ --- -title: "Start Evals Process" +title: "Start Evals Process — API" description: "Trigger one or more evaluations to run across a dataset by submitting user eval IDs. Scores are computed for every row." --- diff --git a/src/pages/docs/api/datasets/add-as-new.mdx b/src/pages/docs/api/datasets/add-as-new.mdx index 94e2d6ec..1393374e 100644 --- a/src/pages/docs/api/datasets/add-as-new.mdx +++ b/src/pages/docs/api/datasets/add-as-new.mdx @@ -1,5 +1,5 @@ --- -title: "Add as New Dataset" +title: "Add as New Dataset — API" description: "Create a new dataset from selected columns of an existing dataset or experiment, with a custom column mapping and dataset name." --- diff --git a/src/pages/docs/api/datasets/add-empty-rows.mdx b/src/pages/docs/api/datasets/add-empty-rows.mdx index 2fc448ce..c997637e 100644 --- a/src/pages/docs/api/datasets/add-empty-rows.mdx +++ b/src/pages/docs/api/datasets/add-empty-rows.mdx @@ -1,5 +1,5 @@ --- -title: "Add Empty Rows to Dataset" +title: "Add Empty Rows to Dataset — API" description: "Append a specified number of empty rows to an existing dataset by ID. Returns a confirmation message on success." --- diff --git a/src/pages/docs/api/datasets/add-rows-from-existing.mdx b/src/pages/docs/api/datasets/add-rows-from-existing.mdx index 8edf3313..3c08a71d 100644 --- a/src/pages/docs/api/datasets/add-rows-from-existing.mdx +++ b/src/pages/docs/api/datasets/add-rows-from-existing.mdx @@ -1,5 +1,5 @@ --- -title: "Add Rows from Existing Dataset" +title: "Add Rows from Existing Dataset — API" description: "Copy rows from a source dataset into a target dataset using a column mapping to align fields between the two datasets." --- diff --git a/src/pages/docs/api/datasets/add-rows-from-file.mdx b/src/pages/docs/api/datasets/add-rows-from-file.mdx index bde882a7..de9bbdcd 100644 --- a/src/pages/docs/api/datasets/add-rows-from-file.mdx +++ b/src/pages/docs/api/datasets/add-rows-from-file.mdx @@ -1,5 +1,5 @@ --- -title: "Add Rows from File" +title: "Add Rows from File — API" description: "Append rows to an existing dataset by uploading a CSV or JSONL file as multipart form data with the target dataset ID." --- diff --git a/src/pages/docs/api/datasets/add-rows-from-huggingface.mdx b/src/pages/docs/api/datasets/add-rows-from-huggingface.mdx index 78e8f015..2739353c 100644 --- a/src/pages/docs/api/datasets/add-rows-from-huggingface.mdx +++ b/src/pages/docs/api/datasets/add-rows-from-huggingface.mdx @@ -1,5 +1,5 @@ --- -title: "Add Rows from HuggingFace" +title: "Add Rows from HuggingFace — API" description: "Import rows from a HuggingFace dataset into an existing dataset. Specify dataset name, config, split, and number of rows to import." --- diff --git a/src/pages/docs/api/datasets/analytics/annotation-summary.mdx b/src/pages/docs/api/datasets/analytics/annotation-summary.mdx index 8b4173f3..70ef2db4 100644 --- a/src/pages/docs/api/datasets/analytics/annotation-summary.mdx +++ b/src/pages/docs/api/datasets/analytics/annotation-summary.mdx @@ -1,5 +1,5 @@ --- -title: "Dataset Annotation Summary" +title: "Dataset Annotation Summary — API" description: "Get annotation statistics for a dataset, including total annotations, label distributions, and per-annotator contribution breakdown." --- diff --git a/src/pages/docs/api/datasets/analytics/eval-stats.mdx b/src/pages/docs/api/datasets/analytics/eval-stats.mdx index b8bee353..42231773 100644 --- a/src/pages/docs/api/datasets/analytics/eval-stats.mdx +++ b/src/pages/docs/api/datasets/analytics/eval-stats.mdx @@ -1,5 +1,5 @@ --- -title: "Dataset Eval Stats" +title: "Dataset Eval Stats — API" description: "Retrieve evaluation statistics for a dataset by column, including template name, metric count, and average score per eval column." --- diff --git a/src/pages/docs/api/datasets/analytics/explanation-summary.mdx b/src/pages/docs/api/datasets/analytics/explanation-summary.mdx index 2c5e911c..d61a3ec8 100644 --- a/src/pages/docs/api/datasets/analytics/explanation-summary.mdx +++ b/src/pages/docs/api/datasets/analytics/explanation-summary.mdx @@ -1,5 +1,5 @@ --- -title: "Dataset Explanation Summary" +title: "Dataset Explanation Summary — API" description: "Get an AI-generated natural language summary of a dataset's content and patterns, returned as a structured explanation object." --- diff --git a/src/pages/docs/api/datasets/analytics/run-prompt-stats.mdx b/src/pages/docs/api/datasets/analytics/run-prompt-stats.mdx index 9e4321a7..4795db48 100644 --- a/src/pages/docs/api/datasets/analytics/run-prompt-stats.mdx +++ b/src/pages/docs/api/datasets/analytics/run-prompt-stats.mdx @@ -1,5 +1,5 @@ --- -title: "Run Prompt Stats" +title: "Run Prompt Stats — API" description: "Get aggregated statistics for run prompt columns in a dataset, including average token usage, cost, and per-prompt-ID breakdowns." --- diff --git a/src/pages/docs/api/datasets/clone-dataset.mdx b/src/pages/docs/api/datasets/clone-dataset.mdx index af8f30b3..13cdc12f 100644 --- a/src/pages/docs/api/datasets/clone-dataset.mdx +++ b/src/pages/docs/api/datasets/clone-dataset.mdx @@ -1,5 +1,5 @@ --- -title: "Clone Dataset" +title: "Clone Dataset — API" description: "Create a full copy of an existing dataset, preserving all columns and rows, under a new name specified in the request body." --- diff --git a/src/pages/docs/api/datasets/columns/add-columns.mdx b/src/pages/docs/api/datasets/columns/add-columns.mdx index 249cfd53..0e062957 100644 --- a/src/pages/docs/api/datasets/columns/add-columns.mdx +++ b/src/pages/docs/api/datasets/columns/add-columns.mdx @@ -1,5 +1,5 @@ --- -title: "Add Columns to Dataset" +title: "Add Columns to Dataset — API" description: "Add one or more columns to a dataset in a single request. Specify each column's name and data type in the request body." --- diff --git a/src/pages/docs/api/datasets/columns/add-multiple-static-columns.mdx b/src/pages/docs/api/datasets/columns/add-multiple-static-columns.mdx index aee3f02e..b75d40f0 100644 --- a/src/pages/docs/api/datasets/columns/add-multiple-static-columns.mdx +++ b/src/pages/docs/api/datasets/columns/add-multiple-static-columns.mdx @@ -1,5 +1,5 @@ --- -title: "Add Multiple Static Columns" +title: "Add Multiple Static Columns — API" description: "Add multiple static columns to a dataset in one request. Each column requires a name and data type such as text or number." --- diff --git a/src/pages/docs/api/datasets/columns/add-static-column.mdx b/src/pages/docs/api/datasets/columns/add-static-column.mdx index 307565ef..f22dd07d 100644 --- a/src/pages/docs/api/datasets/columns/add-static-column.mdx +++ b/src/pages/docs/api/datasets/columns/add-static-column.mdx @@ -1,5 +1,5 @@ --- -title: "Add Static Column to Dataset" +title: "Add Static Column to Dataset — API" description: "Add a single static column to an existing dataset by specifying the column name and data type. Returns the new column's metadata." --- diff --git a/src/pages/docs/api/datasets/columns/delete-column.mdx b/src/pages/docs/api/datasets/columns/delete-column.mdx index 30fd4e04..11cd6b0b 100644 --- a/src/pages/docs/api/datasets/columns/delete-column.mdx +++ b/src/pages/docs/api/datasets/columns/delete-column.mdx @@ -1,5 +1,5 @@ --- -title: "Delete Dataset Column" +title: "Delete Dataset Column — API" description: "Permanently delete a column and all its cell data from a dataset. Requires both the dataset UUID and column UUID as path params." --- diff --git a/src/pages/docs/api/datasets/columns/get-column-config.mdx b/src/pages/docs/api/datasets/columns/get-column-config.mdx index 128af03b..39324634 100644 --- a/src/pages/docs/api/datasets/columns/get-column-config.mdx +++ b/src/pages/docs/api/datasets/columns/get-column-config.mdx @@ -1,5 +1,5 @@ --- -title: "Get Dataset Column Config" +title: "Get Dataset Column Config — API" description: "Retrieve configuration details for a specific dataset column by UUID, including model, messages, and source-specific settings." --- diff --git a/src/pages/docs/api/datasets/columns/get-column-details.mdx b/src/pages/docs/api/datasets/columns/get-column-details.mdx index db862881..cf6b4f9d 100644 --- a/src/pages/docs/api/datasets/columns/get-column-details.mdx +++ b/src/pages/docs/api/datasets/columns/get-column-details.mdx @@ -1,5 +1,5 @@ --- -title: "Get Dataset Column Details" +title: "Get Dataset Column Details — API" description: "Retrieve column metadata for a dataset including names, types, and IDs. Filter by source type or include run prompt columns." --- diff --git a/src/pages/docs/api/datasets/columns/update-column-name.mdx b/src/pages/docs/api/datasets/columns/update-column-name.mdx index baaf0195..48474db6 100644 --- a/src/pages/docs/api/datasets/columns/update-column-name.mdx +++ b/src/pages/docs/api/datasets/columns/update-column-name.mdx @@ -1,5 +1,5 @@ --- -title: "Update Dataset Column Name" +title: "Update Dataset Column Name — API" description: "Rename an existing column in a dataset by providing the new column name. Requires dataset UUID and column UUID as path params." --- diff --git a/src/pages/docs/api/datasets/columns/update-column-type.mdx b/src/pages/docs/api/datasets/columns/update-column-type.mdx index a77ab3f7..da7af4de 100644 --- a/src/pages/docs/api/datasets/columns/update-column-type.mdx +++ b/src/pages/docs/api/datasets/columns/update-column-type.mdx @@ -1,5 +1,5 @@ --- -title: "Update Dataset Column Type" +title: "Update Dataset Column Type — API" description: "Change the data type of an existing dataset column. Use the preview flag to dry-run the conversion before committing the change." --- diff --git a/src/pages/docs/api/datasets/create-dataset-from-huggingface.mdx b/src/pages/docs/api/datasets/create-dataset-from-huggingface.mdx index 9794dd81..73c8ea89 100644 --- a/src/pages/docs/api/datasets/create-dataset-from-huggingface.mdx +++ b/src/pages/docs/api/datasets/create-dataset-from-huggingface.mdx @@ -1,5 +1,5 @@ --- -title: "Create Dataset from HuggingFace" +title: "Create Dataset from HuggingFace — API" description: "Create a new dataset by importing rows from a HuggingFace dataset. Specify dataset name, config, split, and number of rows." --- diff --git a/src/pages/docs/api/datasets/create-dataset.mdx b/src/pages/docs/api/datasets/create-dataset.mdx index 56407b14..e1cc027e 100644 --- a/src/pages/docs/api/datasets/create-dataset.mdx +++ b/src/pages/docs/api/datasets/create-dataset.mdx @@ -1,5 +1,5 @@ --- -title: "Create Dataset" +title: "Create Dataset — API" description: "Create a new dataset with a specified name, number of rows, and number of columns. Returns the created dataset ID and metadata." --- diff --git a/src/pages/docs/api/datasets/create-empty-dataset.mdx b/src/pages/docs/api/datasets/create-empty-dataset.mdx index b6df774c..bda833c9 100644 --- a/src/pages/docs/api/datasets/create-empty-dataset.mdx +++ b/src/pages/docs/api/datasets/create-empty-dataset.mdx @@ -1,5 +1,5 @@ --- -title: "Create Empty Dataset" +title: "Create Empty Dataset — API" description: "Create a new empty dataset with a given name, model type, and optional pre-created blank rows. Returns the new dataset ID." --- diff --git a/src/pages/docs/api/datasets/delete-dataset.mdx b/src/pages/docs/api/datasets/delete-dataset.mdx index eb499dbd..9e84409e 100644 --- a/src/pages/docs/api/datasets/delete-dataset.mdx +++ b/src/pages/docs/api/datasets/delete-dataset.mdx @@ -1,5 +1,5 @@ --- -title: "Delete Dataset" +title: "Delete Dataset — API" description: "Permanently delete one or more datasets by passing an array of dataset UUIDs. Returns a count of successfully deleted datasets." --- diff --git a/src/pages/docs/api/datasets/delete-rows.mdx b/src/pages/docs/api/datasets/delete-rows.mdx index 4e448d69..42431c44 100644 --- a/src/pages/docs/api/datasets/delete-rows.mdx +++ b/src/pages/docs/api/datasets/delete-rows.mdx @@ -1,5 +1,5 @@ --- -title: "Delete Dataset Rows" +title: "Delete Dataset Rows — API" description: "Delete one or more rows from a dataset by row UUIDs, or delete all rows at once using the selected_all_rows flag." --- diff --git a/src/pages/docs/api/datasets/duplicate-dataset.mdx b/src/pages/docs/api/datasets/duplicate-dataset.mdx index f1afd2f4..c73c55a1 100644 --- a/src/pages/docs/api/datasets/duplicate-dataset.mdx +++ b/src/pages/docs/api/datasets/duplicate-dataset.mdx @@ -1,5 +1,5 @@ --- -title: "Duplicate Dataset" +title: "Duplicate Dataset — API" description: "Create a new dataset from selected rows of an existing dataset. Supports duplicating all rows or a specific subset by row IDs." --- diff --git a/src/pages/docs/api/datasets/duplicate-rows.mdx b/src/pages/docs/api/datasets/duplicate-rows.mdx index b2a41cc3..966c09ba 100644 --- a/src/pages/docs/api/datasets/duplicate-rows.mdx +++ b/src/pages/docs/api/datasets/duplicate-rows.mdx @@ -1,5 +1,5 @@ --- -title: "Duplicate Dataset Rows" +title: "Duplicate Dataset Rows — API" description: "Create one or more copies of specific rows within a dataset by providing row UUIDs and the number of copies to produce." --- diff --git a/src/pages/docs/api/datasets/list-datasets.mdx b/src/pages/docs/api/datasets/list-datasets.mdx index d60d1f82..c63abfe1 100644 --- a/src/pages/docs/api/datasets/list-datasets.mdx +++ b/src/pages/docs/api/datasets/list-datasets.mdx @@ -1,5 +1,5 @@ --- -title: "List Datasets" +title: "List Datasets — API" description: "Retrieve a paginated list of datasets in your organization. Supports page, page_size, name search, and sort query parameters." --- diff --git a/src/pages/docs/api/datasets/merge-dataset.mdx b/src/pages/docs/api/datasets/merge-dataset.mdx index df76ef08..422efb23 100644 --- a/src/pages/docs/api/datasets/merge-dataset.mdx +++ b/src/pages/docs/api/datasets/merge-dataset.mdx @@ -1,5 +1,5 @@ --- -title: "Merge Dataset" +title: "Merge Dataset — API" description: "Merge rows from a source dataset into a target dataset. Choose to merge all rows or a selected subset using the request body." --- diff --git a/src/pages/docs/api/datasets/run-prompt/add-run-prompt-column.mdx b/src/pages/docs/api/datasets/run-prompt/add-run-prompt-column.mdx index 266fed5b..0a197005 100644 --- a/src/pages/docs/api/datasets/run-prompt/add-run-prompt-column.mdx +++ b/src/pages/docs/api/datasets/run-prompt/add-run-prompt-column.mdx @@ -1,5 +1,5 @@ --- -title: "Add Run Prompt Column" +title: "Add Run Prompt Column — API" description: "Add a run prompt column to a dataset that generates LLM responses per row. Configure model, messages, and output format in the body." --- diff --git a/src/pages/docs/api/datasets/run-prompt/edit-run-prompt-column.mdx b/src/pages/docs/api/datasets/run-prompt/edit-run-prompt-column.mdx index a280adbc..040f0be0 100644 --- a/src/pages/docs/api/datasets/run-prompt/edit-run-prompt-column.mdx +++ b/src/pages/docs/api/datasets/run-prompt/edit-run-prompt-column.mdx @@ -1,5 +1,5 @@ --- -title: "Edit Run Prompt Column" +title: "Edit Run Prompt Column — API" description: "Update the model, messages, or output config of an existing run prompt column and re-execute it across all dataset rows." --- diff --git a/src/pages/docs/api/datasets/run-prompt/get-column-values.mdx b/src/pages/docs/api/datasets/run-prompt/get-column-values.mdx index a9dec016..51a46c6f 100644 --- a/src/pages/docs/api/datasets/run-prompt/get-column-values.mdx +++ b/src/pages/docs/api/datasets/run-prompt/get-column-values.mdx @@ -1,5 +1,5 @@ --- -title: "Get Run Prompt Column Values" +title: "Get Run Prompt Column Values — API" description: "Retrieve sample cell values from specified dataset columns, mapped by placeholder name, for use in prompt preview and testing." --- diff --git a/src/pages/docs/api/datasets/run-prompt/get-model-voices.mdx b/src/pages/docs/api/datasets/run-prompt/get-model-voices.mdx index 901ef06a..f3193c9b 100644 --- a/src/pages/docs/api/datasets/run-prompt/get-model-voices.mdx +++ b/src/pages/docs/api/datasets/run-prompt/get-model-voices.mdx @@ -1,5 +1,5 @@ --- -title: "Get Model Voices" +title: "Get Model Voices — API" description: "Retrieve the available voice options for a specific model's audio output. Pass the model name as a required query parameter." --- diff --git a/src/pages/docs/api/datasets/run-prompt/retrieve-run-prompt-column-config.mdx b/src/pages/docs/api/datasets/run-prompt/retrieve-run-prompt-column-config.mdx index 85a0bbc8..3715958a 100644 --- a/src/pages/docs/api/datasets/run-prompt/retrieve-run-prompt-column-config.mdx +++ b/src/pages/docs/api/datasets/run-prompt/retrieve-run-prompt-column-config.mdx @@ -1,5 +1,5 @@ --- -title: "Get Run Prompt Column Config" +title: "Get Run Prompt Column Config — API" description: "Retrieve the full configuration of an existing run prompt column by column UUID, including model, messages, and output settings." --- diff --git a/src/pages/docs/api/datasets/run-prompt/retrieve-run-prompt-options.mdx b/src/pages/docs/api/datasets/run-prompt/retrieve-run-prompt-options.mdx index 0355d081..dc24ecb6 100644 --- a/src/pages/docs/api/datasets/run-prompt/retrieve-run-prompt-options.mdx +++ b/src/pages/docs/api/datasets/run-prompt/retrieve-run-prompt-options.mdx @@ -1,5 +1,5 @@ --- -title: "Get Run Prompt Options" +title: "Get Run Prompt Options — API" description: "Retrieve available models, tools, output formats, and tool choices for configuring a run prompt column in a dataset." --- diff --git a/src/pages/docs/api/datasets/run-prompt/tts-voices.mdx b/src/pages/docs/api/datasets/run-prompt/tts-voices.mdx index d01493b9..c998b437 100644 --- a/src/pages/docs/api/datasets/run-prompt/tts-voices.mdx +++ b/src/pages/docs/api/datasets/run-prompt/tts-voices.mdx @@ -1,5 +1,5 @@ --- -title: "List TTS Voices" +title: "List TTS Voices — API" description: "List custom text-to-speech voices configured for your organization, including voice ID, provider, and display name for each entry." --- diff --git a/src/pages/docs/api/datasets/update-cell-value.mdx b/src/pages/docs/api/datasets/update-cell-value.mdx index 8bf41418..7d02bc69 100644 --- a/src/pages/docs/api/datasets/update-cell-value.mdx +++ b/src/pages/docs/api/datasets/update-cell-value.mdx @@ -1,5 +1,5 @@ --- -title: "Update Dataset Cell Value" +title: "Update Dataset Cell Value — API" description: "Update the value of a specific cell in a dataset by providing the dataset ID, row ID, column ID, and the new cell value." --- diff --git a/src/pages/docs/api/datasets/update-dataset.mdx b/src/pages/docs/api/datasets/update-dataset.mdx index e4235155..79f8f9b0 100644 --- a/src/pages/docs/api/datasets/update-dataset.mdx +++ b/src/pages/docs/api/datasets/update-dataset.mdx @@ -1,5 +1,5 @@ --- -title: "Update Dataset" +title: "Update Dataset — API" description: "Update properties of an existing dataset such as its name. Pass the dataset UUID as a path parameter and the new values in the body." --- diff --git a/src/pages/docs/api/datasets/upload-dataset.mdx b/src/pages/docs/api/datasets/upload-dataset.mdx index d8aa48ae..2e98eb61 100644 --- a/src/pages/docs/api/datasets/upload-dataset.mdx +++ b/src/pages/docs/api/datasets/upload-dataset.mdx @@ -1,5 +1,5 @@ --- -title: "Upload Dataset from File" +title: "Upload Dataset from File — API" description: "Create a new dataset by uploading a local CSV or JSONL file as multipart form data. Returns the new dataset ID and row count." --- diff --git a/src/pages/docs/api/eval-tasks/bulk-delete-eval-tasks.mdx b/src/pages/docs/api/eval-tasks/bulk-delete-eval-tasks.mdx index 410ce9db..eb0fff51 100644 --- a/src/pages/docs/api/eval-tasks/bulk-delete-eval-tasks.mdx +++ b/src/pages/docs/api/eval-tasks/bulk-delete-eval-tasks.mdx @@ -1,5 +1,5 @@ --- -title: "Bulk Delete Eval Tasks" +title: "Bulk Delete Eval Tasks — API" description: "Soft-delete multiple eval tasks in one request. Accepts an array of eval task UUIDs; tasks must be in a non-running state. Returns success status." --- diff --git a/src/pages/docs/api/eval-tasks/create-eval-task.mdx b/src/pages/docs/api/eval-tasks/create-eval-task.mdx index 04109cd2..7694d4b7 100644 --- a/src/pages/docs/api/eval-tasks/create-eval-task.mdx +++ b/src/pages/docs/api/eval-tasks/create-eval-task.mdx @@ -1,5 +1,5 @@ --- -title: "Create Eval Task" +title: "Create Eval Task — API" description: "Create an eval task for a project. Set eval configs, sampling rate, run type (continuous or historical), span filters, and spans limit. Returns the new task UUID." --- diff --git a/src/pages/docs/api/eval-tasks/delete-eval-task.mdx b/src/pages/docs/api/eval-tasks/delete-eval-task.mdx index 9e474717..ad0f3276 100644 --- a/src/pages/docs/api/eval-tasks/delete-eval-task.mdx +++ b/src/pages/docs/api/eval-tasks/delete-eval-task.mdx @@ -1,5 +1,5 @@ --- -title: "Delete Eval Task" +title: "Delete Eval Task — API" description: "Soft-delete a single eval task by UUID. The task must not be in running state. Returns 204 No Content on success." --- diff --git a/src/pages/docs/api/eval-tasks/eval-task-aggregations.mdx b/src/pages/docs/api/eval-tasks/eval-task-aggregations.mdx deleted file mode 100644 index 9ed98b6e..00000000 --- a/src/pages/docs/api/eval-tasks/eval-task-aggregations.mdx +++ /dev/null @@ -1,135 +0,0 @@ ---- -title: "Eval Task Aggregations" -description: "Aggregate eval-task results as per-eval rollups, per-span pivots, or both." ---- - - - - - - Your Future AGI API key used to authenticate requests. You can find and manage your API keys in the [Dashboard](https://app.futureagi.com) under Settings. - - - Your Future AGI secret key, used alongside the API key for request authentication. This is generated when you create an API key in the [Dashboard](https://app.futureagi.com). - - - - - - The eval task whose runs should be aggregated. - - - When `true`, the response includes the `eval_aggregation` object — one rollup per `CustomEvalConfig` that ran in the task, keyed by eval name. Defaults to `false`. At least one of `eval_aggregation` or `span_aggregation` must be `true`. - - - When `true`, the response includes the `span_aggregation` object — one entry per span the task evaluated, keyed by `span_id`, with the raw value of every eval that touched it. Defaults to `false`. At least one of `eval_aggregation` or `span_aggregation` must be `true`. - - - Inclusive lower bound on the **span's `created_at`** — only eval runs whose linked span was created at or after this instant are aggregated. When omitted, no lower bound is applied. - - - Inclusive upper bound on the **span's `created_at`** — only eval runs whose linked span was created at or before this instant are aggregated. When omitted, no upper bound is applied. - - - - - UUID of the eval task that was aggregated. Echoed back from the request. - - - Per-eval rollup. Present only when `eval_aggregation=true`. Keys are `CustomEvalConfig` names; values are one rollup object per eval. - - UUID of the eval config. - Eval config name (same as the parent key). - Normalised output type for the eval: `percentage`, `pass_fail`, or `deterministic`. Drives the shape of `aggregated_score`. - - The eval-level rollup. Shape depends on `output_type`: -
• **`percentage`** — `number` (4-dp average across non-error runs, e.g. `0.7421`). -
• **`pass_fail`** — `number` (pass rate as `0–100` with 2 dp, e.g. `87.5`). -
• **`deterministic`** — `object` mapping each observed choice to its occurrence percentage `0–100` with 2 dp, e.g. `{"positive": 62.5, "neutral": 25.0}`. Only choices that actually appeared in the data are included. -
`null` when no aggregatable rows exist (all errors / empty). - - - - - - Per-span pivot. Present only when `span_aggregation=true`. Outer keys are `span_id` (one per span the task evaluated); inner keys are eval names; inner values are one entry per eval that touched the span. - - UUID of the eval config. - Eval config name. - Normalised output type for the eval: `percentage`, `pass_fail`, or `deterministic`. Drives the shape of `value`. - - The raw per-row eval result — **no averaging**. Shape depends on `output_type`: -
• **`percentage`** — `number` (e.g. `0.82`). -
• **`pass_fail`** — `boolean`. -
• **`deterministic`** — `array` of choice strings (e.g. `["positive"]`). -
When the same `(span, eval)` pair has multiple runs (re-runs), the latest by `created_at` wins. - - - - - - - Soft-deleted eval runs are skipped in both aggregations so the rollups reflect the user's current view of the data. - - Both `eval_aggregation` and `span_aggregation` only include span-linked eval runs — session-target eval runs (where there is no underlying span) are excluded from both rollups, regardless of whether a date range is supplied. - - `start_date` and `end_date` filter on the **span's creation time** (`observation_span.created_at`), not on when the eval ran. The aggregation results therefore reflect only those spans that were created in the supplied window — eval runs against spans created outside the window are dropped from both rollups. When neither parameter is supplied, every span linked to the eval task is included. - - - - - `eval_task_id` is missing, or no eval task with that ID exists in the caller's organization. - - - Invalid or missing API credentials. - - - Unexpected server error. - - diff --git a/src/pages/docs/api/eval-tasks/get-eval-task.mdx b/src/pages/docs/api/eval-tasks/get-eval-task.mdx index 9c8eb375..f7186c74 100644 --- a/src/pages/docs/api/eval-tasks/get-eval-task.mdx +++ b/src/pages/docs/api/eval-tasks/get-eval-task.mdx @@ -1,5 +1,5 @@ --- -title: "Get Eval Task" +title: "Get Eval Task — API" description: "Retrieve a single eval task by UUID. Returns status, sampling rate, run type, attached eval configs, filters, failed spans, and timestamps." --- diff --git a/src/pages/docs/api/eval-tasks/list-eval-tasks-filtered.mdx b/src/pages/docs/api/eval-tasks/list-eval-tasks-filtered.mdx index ed757ef3..8dfbc6b3 100644 --- a/src/pages/docs/api/eval-tasks/list-eval-tasks-filtered.mdx +++ b/src/pages/docs/api/eval-tasks/list-eval-tasks-filtered.mdx @@ -1,5 +1,5 @@ --- -title: "List Eval Tasks" +title: "List Eval Tasks — API" description: "Retrieve a paginated list of eval tasks. Filter by project UUID or name. Returns task status, sampling rate, eval count, and last run timestamp." --- diff --git a/src/pages/docs/api/eval-tasks/pause-eval-task.mdx b/src/pages/docs/api/eval-tasks/pause-eval-task.mdx index e5b13a3a..d0b59ddf 100644 --- a/src/pages/docs/api/eval-tasks/pause-eval-task.mdx +++ b/src/pages/docs/api/eval-tasks/pause-eval-task.mdx @@ -1,5 +1,5 @@ --- -title: "Pause Eval Task" +title: "Pause Eval Task — API" description: "Pause a running eval task by UUID. Task must be in running state. Returns a confirmation status and message on success." --- diff --git a/src/pages/docs/api/eval-tasks/unpause-eval-task.mdx b/src/pages/docs/api/eval-tasks/unpause-eval-task.mdx index 26cfa9ed..2e917147 100644 --- a/src/pages/docs/api/eval-tasks/unpause-eval-task.mdx +++ b/src/pages/docs/api/eval-tasks/unpause-eval-task.mdx @@ -1,5 +1,5 @@ --- -title: "Unpause Eval Task" +title: "Unpause Eval Task — API" description: "Resume a paused eval task by UUID. Task must be in paused state; status resets to pending on resume. Returns confirmation status and message." --- diff --git a/src/pages/docs/api/eval-tasks/update-eval-task.mdx b/src/pages/docs/api/eval-tasks/update-eval-task.mdx index 89515b02..954a3930 100644 --- a/src/pages/docs/api/eval-tasks/update-eval-task.mdx +++ b/src/pages/docs/api/eval-tasks/update-eval-task.mdx @@ -1,5 +1,5 @@ --- -title: "Update Eval Task" +title: "Update Eval Task — API" description: "Partially update an eval task's name, evals, sampling rate, run type, or filters. Use edit_type to choose fresh_run or edit_rerun mode." --- diff --git a/src/pages/docs/api/health/healthcheck.mdx b/src/pages/docs/api/health/healthcheck.mdx index 4f1e0f82..2cef3a09 100644 --- a/src/pages/docs/api/health/healthcheck.mdx +++ b/src/pages/docs/api/health/healthcheck.mdx @@ -1,5 +1,5 @@ --- -title: "Health Check" +title: "Health Check — API" description: "Check whether the Future AGI API server is up and reachable. Returns 200 with a status flag and confirmation message when the server is running." --- diff --git a/src/pages/docs/api/personas/createpersona.mdx b/src/pages/docs/api/personas/createpersona.mdx index 670c46d2..bef662ca 100644 --- a/src/pages/docs/api/personas/createpersona.mdx +++ b/src/pages/docs/api/personas/createpersona.mdx @@ -1,5 +1,5 @@ --- -title: "Create Persona" +title: "Create Persona — API" description: "Create a workspace-level persona for voice or text simulation. Set gender, age group, personality, communication style, and tone. Returns the new persona object." --- diff --git a/src/pages/docs/api/personas/deletepersona.mdx b/src/pages/docs/api/personas/deletepersona.mdx index 45ed6194..5c93e719 100644 --- a/src/pages/docs/api/personas/deletepersona.mdx +++ b/src/pages/docs/api/personas/deletepersona.mdx @@ -1,5 +1,5 @@ --- -title: "Delete Persona" +title: "Delete Persona — API" description: "Soft-delete a workspace-level persona by UUID. System (prebuilt) personas cannot be deleted. Returns 204 No Content on success." --- diff --git a/src/pages/docs/api/personas/duplicatepersona.mdx b/src/pages/docs/api/personas/duplicatepersona.mdx index 2dadba6b..24bb5baf 100644 --- a/src/pages/docs/api/personas/duplicatepersona.mdx +++ b/src/pages/docs/api/personas/duplicatepersona.mdx @@ -1,5 +1,5 @@ --- -title: "Duplicate Persona" +title: "Duplicate Persona — API" description: "Copy an existing persona (system or workspace) as a new workspace-level persona. Provide a unique name; all other attributes are inherited from the source." --- diff --git a/src/pages/docs/api/personas/listpersonas.mdx b/src/pages/docs/api/personas/listpersonas.mdx index 20540beb..e7e0f393 100644 --- a/src/pages/docs/api/personas/listpersonas.mdx +++ b/src/pages/docs/api/personas/listpersonas.mdx @@ -1,5 +1,5 @@ --- -title: "List Personas" +title: "List Personas — API" description: "List system and workspace personas with pagination. Filter by type (prebuilt/custom), simulation type (voice/text), or keyword search. Returns persona details and traits." --- diff --git a/src/pages/docs/api/personas/updatepersona.mdx b/src/pages/docs/api/personas/updatepersona.mdx index 40f8c7eb..86c43285 100644 --- a/src/pages/docs/api/personas/updatepersona.mdx +++ b/src/pages/docs/api/personas/updatepersona.mdx @@ -1,5 +1,5 @@ --- -title: "Update Persona" +title: "Update Persona — API" description: "Partially update a workspace-level persona's attributes, including name, personality, tone, and communication style. System personas cannot be modified." --- diff --git a/src/pages/docs/api/run-tests/addevalconfigs.mdx b/src/pages/docs/api/run-tests/addevalconfigs.mdx index b1cf03f4..8915c877 100644 --- a/src/pages/docs/api/run-tests/addevalconfigs.mdx +++ b/src/pages/docs/api/run-tests/addevalconfigs.mdx @@ -1,5 +1,5 @@ --- -title: "Add Eval Configs to Run Test" +title: "Add Eval Configs to Run Test — API" description: "Add one or more evaluation configurations to an existing test run. Specify template, name, config mapping, model, and error localizer per config. Returns created eval config objects." --- @@ -10,25 +10,8 @@ description: "Add one or more evaluation configurations to an existing test run. parameters={[ {"name": "run_test_id", "in": "path", "required": true, "description": "UUID of the test run to add evaluation configurations to.", "type": "string"} ]} - requestBody={{"evaluations_config": [{"template_id": "your-template-id", "name": "My Eval Config", "config": {}, "mapping": {}, "filters": {}, "error_localizer": false, "model": "turing_large"}]}} - responseExample={{ - message: "Successfully added 1 evaluation config(s) to run test", - created_eval_configs: [ - { - id: "ec1b2c3d-e5f6-7890-abcd-ef1234567890", - name: "My Eval Config", - config: {}, - mapping: {}, - filters: {}, - error_localizer: false, - model: "turing_large", - status: null, - eval_group: null, - template_id: "your-template-uuid" - } - ], - run_test_id: "f7a8b9c0-d1e2-3456-789a-bcdef0123456" - }} + requestBody={{"evaluations_config": [{"template_id": "your-template-id", "name": "My Eval Config", "config": {}, "mapping": {}, "error_localizer": false, "model": "turing_large"}]}} + responseExample={{"message": "Evaluation configs added successfully", "created_eval_configs": [{"id": "uuid", "name": "My Eval Config", "config": {}, "mapping": {}, "filters": {}, "error_localizer": false, "model": "turing_large", "status": "pending", "template_id": "your-template-id"}], "run_test_id": "uuid"}} responseStatus={201} responseStatusText="Created" /> @@ -50,59 +33,65 @@ description: "Add one or more evaluation configurations to an existing test run. - Array of evaluation configuration objects. Each object supports the following fields: + Array of evaluation configuration objects. - - **`template_id`** (string, UUID, required) -- UUID of the evaluation template to use. + + + UUID of the evaluation template to use. + - - **`name`** (string, optional) -- Name for this evaluation configuration. Defaults to `Eval-` if omitted. Must be unique within the test run. + + Name for this evaluation configuration. Must be unique within the test run. + - - **`config`** (object, optional) -- Template-specific configuration parameters. + + Template-specific configuration parameters. + - - **`mapping`** (object, optional) -- Maps test execution data fields to the evaluation template's expected inputs. + + Maps test execution data fields to the evaluation template's expected inputs. + - - **`filters`** (object, optional) -- Filter criteria to restrict which test results are evaluated. + + Filter criteria to restrict which test results are evaluated. + - - **`error_localizer`** (boolean, optional) -- Enables granular error localization on evaluation failures. Defaults to `false`. + + Enables granular error localization on evaluation failures. Defaults to `false`. + - - **`model`** (string, optional) -- Model to use for running this evaluation. + + Model to use for running this evaluation. + + - Confirmation message indicating how many evaluation configs were added. + Confirmation of successful addition. + - Array of created evaluation configuration objects. Each object contains: `id`, `name`, `config`, `mapping`, `filters`, `error_localizer`, `model`, `status`, `eval_group`, and `template_id`. + Array of the newly created evaluation config objects. Each object contains: `id`, `name`, `config`, `mapping`, `filters`, `error_localizer`, `model`, `status`, `template_id`. + + + UUID of the test run the configs were added to. + + + Present only when some configs in the request failed to create while others succeeded. Lists per-item error messages. - UUID of the parent test run. - Non-fatal issues encountered while processing individual configs. Only present if partial failures occurred. - Validation error. Common causes: empty `evaluations_config`, duplicate `name` within request, name already exists in test run, non-existent `template_id`. - ```json - { - "evaluations_config": ["Duplicate eval name 'My Eval Config' found in the request. Each evaluation config must have a unique name."] - } - ``` - Or for existing name conflict: - ```json - {"error": "An evaluation config with the name 'My Eval Config' already exists in this run test. Please use a different name."} - ``` + Invalid or missing fields such as a non-existent `template_id`, duplicate `name`, or malformed `config`/`mapping`. Missing or invalid `X-Api-Key` or `X-Secret-Key` headers. No test run found with the specified `run_test_id`. - ```json - {"detail": "No RunTest matches the given query."} - ``` Unexpected server error. - ```json - {"error": "Failed to add evaluation configs: "} - ``` diff --git a/src/pages/docs/api/run-tests/compareevalsummaries.mdx b/src/pages/docs/api/run-tests/compareevalsummaries.mdx index 618e33c6..cc12690f 100644 --- a/src/pages/docs/api/run-tests/compareevalsummaries.mdx +++ b/src/pages/docs/api/run-tests/compareevalsummaries.mdx @@ -1,5 +1,5 @@ --- -title: "Compare Eval Summaries" +title: "Compare Eval Summaries — API" description: "Compare evaluation summaries side-by-side across multiple test executions. Accepts a JSON-encoded array of execution UUIDs; returns per-execution eval metrics keyed by execution ID." --- @@ -11,10 +11,7 @@ description: "Compare evaluation summaries side-by-side across multiple test exe {"name": "run_test_id", "in": "path", "required": true, "description": "UUID of the test run containing the executions to compare.", "type": "string"}, {"name": "execution_ids", "in": "query", "required": false, "description": "JSON-encoded array of test execution UUIDs to compare.", "type": "string"} ]} - responseExample={{ - "execution-uuid-1": [{"name": "Tone Check", "average_score": 0.85, "total_runs": 10, "passed": 8, "failed": 2}], - "execution-uuid-2": [{"name": "Tone Check", "average_score": 0.92, "total_runs": 10, "passed": 9, "failed": 1}] - }} + responseExample={{"execution-uuid-1": {"evaluations": [{"name": "Tone Check", "average_score": 0.85}]}, "execution-uuid-2": {"evaluations": [{"name": "Tone Check", "average_score": 0.92}]}}} responseStatus={200} responseStatusText="OK" /> @@ -36,47 +33,25 @@ description: "Compare evaluation summaries side-by-side across multiple test exe - JSON-encoded array of test execution UUIDs to compare. Must be URL-encoded. Example: `["uuid1","uuid2"]`. + JSON-encoded array of test execution UUIDs to compare. Must be URL-encoded. - - Dictionary keyed by execution UUID. Each value is an array of evaluation summary objects for that execution. - - - Name of the evaluation configuration. - Average score across all evaluated calls. - Total evaluation runs for this config. - Number of passing evaluations. - Number of failing evaluations. - + Dictionary keyed by execution ID, each mapping to its evaluation summary metrics. - Missing, malformed, or empty `execution_ids` parameter. - ```json - {"execution_ids": ["execution_ids must be valid JSON"]} - ``` - Or when empty: - ```json - {"execution_ids": ["execution_ids list is required"]} - ``` + Missing, malformed, or invalid `execution_ids` parameter. Missing or invalid `X-Api-Key` or `X-Secret-Key` headers. No test run found with the specified `run_test_id`. - ```json - {"error": "RunTest not found."} - ``` - Unexpected server error. - ```json - {"error": "Unable to fetch eval summary"} - ``` + Unexpected server error. Retry later or contact support. diff --git a/src/pages/docs/api/run-tests/createruntest.mdx b/src/pages/docs/api/run-tests/createruntest.mdx index 9a3d2dad..6d7054fa 100644 --- a/src/pages/docs/api/run-tests/createruntest.mdx +++ b/src/pages/docs/api/run-tests/createruntest.mdx @@ -1,5 +1,5 @@ --- -title: "Create Run Test" +title: "Create Run Test — API" description: "Create a new test run with scenarios, agent definition, eval configs, and optional tool evaluation. Returns the run test UUID, name, and associated scenario list." --- @@ -7,8 +7,8 @@ description: "Create a new test run with scenarios, agent definition, eval confi method="POST" endpoint="/simulate/run-tests/create/" baseUrl="https://api.futureagi.com" - requestBody={{"name": "your-name", "description": "your-description", "scenario_ids": [], "agent_definition_id": "your-agent-definition-id", "eval_config_ids": [], "evaluations_config": [], "dataset_row_ids": [], "enable_tool_evaluation": true}} - responseExample={{"id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", "name": "your-name", "description": "your-description", "agent_definition": "your-agent-definition-id", "agent_version": null, "agent_definition_detail": null, "source_type": "agent_definition", "source_type_display": "Agent Definition", "scenarios": [], "scenarios_detail": [], "dataset_row_ids": [], "simulator_agent": null, "simulator_agent_detail": null, "simulate_eval_configs": [], "simulate_eval_configs_detail": [], "evals_detail": [], "organization": "org-uuid", "enable_tool_evaluation": true, "created_at": "2026-04-04T12:00:00Z", "updated_at": "2026-04-04T12:00:00Z", "last_run_at": null, "deleted": false, "deleted_at": null}} + requestBody={{"name": "your-name", "description": "your-description", "scenarioIds": [], "agentDefinitionId": "your-agentDefinitionId", "agentVersion": "your-agentVersion", "evalConfigIds": [], "evaluationsConfig": [], "datasetRowIds": [], "enableToolEvaluation": true}} + responseExample={{"id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", "name": "your-name", "description": "your-description", "agent_definition": "your-agentDefinitionId", "scenarios": [], "enable_tool_evaluation": true, "created_at": "2026-04-04T12:00:00Z", "updated_at": "2026-04-04T12:00:00Z"}} responseStatus={201} responseStatusText="Created" /> @@ -29,27 +29,27 @@ description: "Create a new test run with scenarios, agent definition, eval confi Optional free-text description of the test run. - + Array of scenario UUIDs to execute against. Must contain at least one valid scenario ID. - + UUID of the agent definition to evaluate. - + + UUID of a specific agent version to test against. Defaults to the currently active version if omitted. + + Array of existing evaluation configuration UUIDs to associate with this test run. - + Array of inline evaluation configuration objects to create and associate. Each object must include `template_id`, `name`, `config`, and `mapping`. - + Array of dataset row UUIDs to restrict execution to specific data entries. If omitted, all rows are included. - + When `true`, evaluates correctness of tool calls made by the agent. Defaults to `false`. - - Optional UUID of a session to replay. When provided, execution replays the specified session. - @@ -65,45 +65,9 @@ description: "Create a new test run with scenarios, agent definition, eval confi UUID of the associated agent definition. - - UUID of the specific agent version, or `null` if using the active version. - - - Detailed agent definition object, or `null`. - - - Source type identifier (e.g. `"agent_definition"`). - - - Human-readable source type label (e.g. `"Agent Definition"`). - Array of linked scenario UUIDs. - - Array of detailed scenario objects. - - - Array of dataset row UUIDs associated with this test run. - - - UUID of the simulator agent, or `null`. - - - Detailed simulator agent object, or `null`. - - - Array of evaluation configuration UUIDs. - - - Array of detailed evaluation configuration objects. - - - Array of detailed evaluation result objects. - - - UUID of the owning organization. - Whether tool evaluation is enabled. @@ -113,15 +77,6 @@ description: "Create a new test run with scenarios, agent definition, eval confi ISO 8601 last-modified timestamp. - - ISO 8601 timestamp of the most recent execution, or `null`. - - - Whether the test run has been soft-deleted. - - - ISO 8601 timestamp of soft-deletion, or `null`. - diff --git a/src/pages/docs/api/run-tests/deleteevalconfig.mdx b/src/pages/docs/api/run-tests/deleteevalconfig.mdx index c7ad7450..516129ef 100644 --- a/src/pages/docs/api/run-tests/deleteevalconfig.mdx +++ b/src/pages/docs/api/run-tests/deleteevalconfig.mdx @@ -1,5 +1,5 @@ --- -title: "Delete Eval Config" +title: "Delete Eval Config — API" description: "Delete an evaluation configuration from a test run by run test and eval config UUID. Cannot delete the last remaining config in the run. Returns a confirmation message." --- @@ -41,23 +41,14 @@ description: "Delete an evaluation configuration from a test run by run test and Cannot delete the last remaining evaluation configuration in the test run. - ```json - {"error": "Cannot delete the last evaluation config. At least one evaluation config must remain."} - ``` Missing or invalid `X-Api-Key` or `X-Secret-Key` headers. Test run or evaluation configuration not found. - ```json - {"error": "Evaluation config not found"} - ``` - Unexpected server error. - ```json - {"error": "Failed to delete evaluation config: "} - ``` + Unexpected server error. Retry later or contact support. diff --git a/src/pages/docs/api/run-tests/deleteruntest.mdx b/src/pages/docs/api/run-tests/deleteruntest.mdx index c38cfcd9..402245fa 100644 --- a/src/pages/docs/api/run-tests/deleteruntest.mdx +++ b/src/pages/docs/api/run-tests/deleteruntest.mdx @@ -1,5 +1,5 @@ --- -title: "Delete Run Test" +title: "Delete Run Test — API" description: "Soft-delete a test run by UUID. The run must have no currently active executions. Returns a confirmation message on success." --- diff --git a/src/pages/docs/api/run-tests/deletetestexecutions.mdx b/src/pages/docs/api/run-tests/deletetestexecutions.mdx index 895b19a9..3e98bdde 100644 --- a/src/pages/docs/api/run-tests/deletetestexecutions.mdx +++ b/src/pages/docs/api/run-tests/deletetestexecutions.mdx @@ -1,5 +1,5 @@ --- -title: "Delete Test Executions" +title: "Delete Test Executions — API" description: "Bulk-delete test executions from a test run. Specify execution UUIDs or use selectAll. Active executions (running/pending/cancelling) cannot be deleted." --- @@ -11,7 +11,7 @@ description: "Bulk-delete test executions from a test run. Specify execution UUI {"name": "run_test_id", "in": "path", "required": true, "description": "UUID of the test run from which to delete test executions.", "type": "string"} ]} requestBody={{"test_execution_ids": ["execution-uuid-1", "execution-uuid-2"], "select_all": false}} - responseExample={{"message": "Successfully deleted 2 test execution(s).", "runTestId": "run-test-uuid", "deletedCount": 2, "deletedIds": ["execution-uuid-1", "execution-uuid-2"]}} + responseExample={{"message": "Successfully deleted 2 test execution(s).", "run_test_id": "run-test-uuid", "deleted_count": 2, "deleted_ids": ["execution-uuid-1", "execution-uuid-2"]}} responseStatus={200} responseStatusText="OK" /> @@ -49,7 +49,7 @@ description: "Bulk-delete test executions from a test run. Specify execution UUI - Invalid request, empty `test_execution_ids`, or targeted executions are still running/pending/cancelling. + Invalid request, empty `testExecutionIds`, or targeted executions are still running/pending/cancelling. Missing or invalid `X-Api-Key` or `X-Secret-Key` headers. diff --git a/src/pages/docs/api/run-tests/executeruntest.mdx b/src/pages/docs/api/run-tests/executeruntest.mdx index efb49a05..5c6b90ba 100644 --- a/src/pages/docs/api/run-tests/executeruntest.mdx +++ b/src/pages/docs/api/run-tests/executeruntest.mdx @@ -1,5 +1,5 @@ --- -title: "Execute Run Test" +title: "Execute Run Test — API" description: "Trigger a new execution of a test run. Select scenarios by inclusion or exclusion, optionally specify a simulator. Returns execution ID, status, and scenario/call counts." --- @@ -10,7 +10,7 @@ description: "Trigger a new execution of a test run. Select scenarios by inclusi parameters={[ {"name": "run_test_id", "in": "path", "required": true, "description": "UUID of the test run to execute.", "type": "string"} ]} - requestBody={{"select_all": true, "scenario_ids": [], "simulator_id": "your-simulator-id"}} + requestBody={{"selectAll": true, "scenarioIds": [], "simulatorId": "your-simulatorId"}} responseExample={{"message": "Test execution started successfully", "execution_id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", "run_test_id": "b2c3d4e5-f6a7-8901-bcde-f12345678901", "status": "PENDING", "total_scenarios": 5, "total_calls": 25, "scenario_ids": []}} responseStatus={200} responseStatusText="OK" @@ -32,13 +32,13 @@ description: "Trigger a new execution of a test run. Select scenarios by inclusi - - When `true`, all scenarios run except those in `scenario_ids` (exclusion mode). When `false`, only those in `scenario_ids` run (inclusion mode). + + When `true`, all scenarios run except those in `scenarioIds` (exclusion mode). When `false`, only those in `scenarioIds` run (inclusion mode). - - Array of scenario UUIDs to include or exclude based on `select_all`. If empty, all scenarios run. + + Array of scenario UUIDs to include or exclude based on `selectAll`. If empty, all scenarios run. - + UUID of a simulator agent to use. Defaults to the test run or organization default if omitted. diff --git a/src/pages/docs/api/run-tests/getcallexecutions.mdx b/src/pages/docs/api/run-tests/getcallexecutions.mdx index e69de29b..1455ed08 100644 --- a/src/pages/docs/api/run-tests/getcallexecutions.mdx +++ b/src/pages/docs/api/run-tests/getcallexecutions.mdx @@ -0,0 +1,133 @@ +--- +title: "List Call Executions — API" +description: "List paginated call executions for a test run. Filter by status or search string. Returns transcript, eval scores, latency, scenario info, and call summary per call." +--- + + + + + + Your Future AGI API key used to authenticate requests. You can find and manage your API keys in the [Dashboard](https://app.futureagi.com) under Settings. + + + Your Future AGI secret key, used alongside the API key for request authentication. This is generated when you create an API key in the [Dashboard](https://app.futureagi.com). + + + + + + UUID of the test run whose call executions to retrieve. + + + + + + Case-insensitive partial match on phone number or scenario name. + + + Filter by call status. + + + Number of records per page. Defaults to `10`. Must be a positive integer. + + + Page number to retrieve. Defaults to `1`. + + + + + + Total matching call executions across all pages. + + + URL to the next page, or `null` if on the last page. + + + URL to the previous page, or `null` if on the first page. + + + Array of call execution objects for the current page. + + + + UUID of the call execution. + + + Session identifier for external correlation, or `null`. + + + Current call status. + + + Call duration in seconds, or `null` if not completed. + + + ISO 8601 call start timestamp, or `null` if not started. + + + Ordered conversation transcript. Empty array if unavailable. + + + Object with `id` and `name` of the source scenario, or `null` if deleted. + + + Aggregate evaluation score (0-100), or `null` if not yet computed. + + + Raw outputs from each evaluation metric, or `null` if not processed. + + + Computed evaluation metric values by metric name, or `null`. + + + Simulated customer name from scenario data, or `null`. + + + Auto-generated call summary, or `null`. + + + Reason the call ended (e.g., `"completed"`, `"timeout"`, `"error"`), or `null`. + + + Average agent response latency in seconds, or `null`. + + + UUID of the source scenario, or `null`. + + + Whether this is a preserved historical snapshot. + + + Type of rerun, or `null` for original executions. + + + ISO 8601 creation timestamp. + + + + + + + Missing or invalid `X-Api-Key` or `X-Secret-Key` headers. + + + No test run found with the specified `run_test_id`. + + + Unexpected server error. Contact support if it persists. + + diff --git a/src/pages/docs/api/run-tests/getevalsummary.mdx b/src/pages/docs/api/run-tests/getevalsummary.mdx index 13cf9e7f..12f84ede 100644 --- a/src/pages/docs/api/run-tests/getevalsummary.mdx +++ b/src/pages/docs/api/run-tests/getevalsummary.mdx @@ -1,5 +1,5 @@ --- -title: "Get Eval Summary" +title: "Get Eval Summary — API" description: "Retrieve aggregated evaluation summary for a test run. Optionally scope to a specific execution. Returns per-config average scores, pass counts, and fail counts." --- @@ -66,14 +66,8 @@ description: "Retrieve aggregated evaluation summary for a test run. Optionally No test run found with the specified `run_test_id`. - ```json - {"error": "RunTest not found."} - ``` - Unexpected server error. - ```json - {"error": "Unable to fetch eval summary"} - ``` + Unexpected server error. Contact support if it persists. diff --git a/src/pages/docs/api/run-tests/getruntestdetails.mdx b/src/pages/docs/api/run-tests/getruntestdetails.mdx index 10329d9a..0e473e4b 100644 --- a/src/pages/docs/api/run-tests/getruntestdetails.mdx +++ b/src/pages/docs/api/run-tests/getruntestdetails.mdx @@ -1,5 +1,5 @@ --- -title: "Get Run Test Details" +title: "Get Run Test Details — API" description: "Retrieve full details of a test run by UUID, including agent definition, prompt template, scenarios, eval configs, tool evaluation settings, and last run timestamp." --- diff --git a/src/pages/docs/api/run-tests/gettestexecutions.mdx b/src/pages/docs/api/run-tests/gettestexecutions.mdx index 09f451e5..6c68191f 100644 --- a/src/pages/docs/api/run-tests/gettestexecutions.mdx +++ b/src/pages/docs/api/run-tests/gettestexecutions.mdx @@ -1,5 +1,5 @@ --- -title: "List Test Executions" +title: "List Test Executions — API" description: "List paginated test executions for a run test. Filter by status or search string. Returns execution status, call counts, success rate, duration, and scenario IDs." --- diff --git a/src/pages/docs/api/run-tests/gettestscenarios.mdx b/src/pages/docs/api/run-tests/gettestscenarios.mdx index 3d3f2035..aff0d2a6 100644 --- a/src/pages/docs/api/run-tests/gettestscenarios.mdx +++ b/src/pages/docs/api/run-tests/gettestscenarios.mdx @@ -1,5 +1,5 @@ --- -title: "List Run Test Scenarios" +title: "List Run Test Scenarios — API" description: "List paginated scenarios associated with a test run. Supports name search. Returns scenario UUID, display name, and row count per scenario." --- diff --git a/src/pages/docs/api/run-tests/listruntests.mdx b/src/pages/docs/api/run-tests/listruntests.mdx index 7153942d..b311eb24 100644 --- a/src/pages/docs/api/run-tests/listruntests.mdx +++ b/src/pages/docs/api/run-tests/listruntests.mdx @@ -1,5 +1,5 @@ --- -title: "List Run Tests" +title: "List Run Tests — API" description: "List paginated test runs. Filter by name, source type, or prompt template. Returns agent definition, scenarios, tool evaluation flag, and last run timestamp per run." --- diff --git a/src/pages/docs/api/run-tests/reruntestexecutions.mdx b/src/pages/docs/api/run-tests/reruntestexecutions.mdx index 90ab7307..0b3ea726 100644 --- a/src/pages/docs/api/run-tests/reruntestexecutions.mdx +++ b/src/pages/docs/api/run-tests/reruntestexecutions.mdx @@ -1,5 +1,5 @@ --- -title: "Rerun Test Executions" +title: "Rerun Test Executions — API" description: "Rerun test executions within a run test. Choose eval_only to re-evaluate existing data or call_and_eval to re-execute calls from scratch. Supports selectAll mode." --- @@ -10,7 +10,7 @@ description: "Rerun test executions within a run test. Choose eval_only to re-ev parameters={[ {"name": "run_test_id", "in": "path", "required": true, "description": "UUID of the test run containing the executions to rerun.", "type": "string"} ]} - requestBody={{"rerun_type": "eval_only", "test_execution_ids": ["execution-uuid-1"], "select_all": false}} + requestBody={{"rerunType": "eval_only", "testExecutionIds": ["execution-uuid-1"], "selectAll": false}} responseExample={{"message": "Rerun initiated successfully"}} responseStatus={200} responseStatusText="OK" @@ -32,14 +32,14 @@ description: "Rerun test executions within a run test. Choose eval_only to re-ev - + Type of rerun. `eval_only` re-runs evaluations on existing call data. `call_and_eval` re-executes calls and evaluations from scratch. - - Array of test execution UUIDs to rerun. Required when `select_all` is `false`. + + Array of test execution UUIDs to rerun. Required when `selectAll` is `false`. - - When `true`, reruns all executions, ignoring `test_execution_ids`. Defaults to `false`. + + When `true`, reruns all executions, ignoring `testExecutionIds`. Defaults to `false`. @@ -49,7 +49,7 @@ description: "Rerun test executions within a run test. Choose eval_only to re-ev - Invalid or missing `rerun_type`, or no executions specified. + Invalid or missing `rerunType`, or no executions specified. Missing or invalid `X-Api-Key` or `X-Secret-Key` headers. diff --git a/src/pages/docs/api/run-tests/runnewevalsontestexecution.mdx b/src/pages/docs/api/run-tests/runnewevalsontestexecution.mdx index afd5c416..c8dfe116 100644 --- a/src/pages/docs/api/run-tests/runnewevalsontestexecution.mdx +++ b/src/pages/docs/api/run-tests/runnewevalsontestexecution.mdx @@ -1,5 +1,5 @@ --- -title: "Run New Evals on Executions" +title: "Run New Evals on Executions — API" description: "Run new evaluation configs on completed test executions. Specify eval config UUIDs and target executions or use selectAll. Returns call execution count being evaluated." --- @@ -10,7 +10,7 @@ description: "Run new evaluation configs on completed test executions. Specify e parameters={[ {"name": "run_test_id", "in": "path", "required": true, "description": "UUID of the test run containing the executions to evaluate.", "type": "string"} ]} - requestBody={{"test_execution_ids": ["execution-uuid-1"], "select_all": false, "eval_config_ids": ["eval-config-uuid-1"], "enable_tool_evaluation": false}} + requestBody={{"testExecutionIds": ["execution-uuid-1"], "selectAll": false, "evalConfigIds": ["eval-config-uuid-1"], "enableToolEvaluation": false}} responseExample={{"message": "Evaluations started successfully", "run_test_id": "run-test-uuid", "call_execution_count": 5}} responseStatus={200} responseStatusText="OK" @@ -32,16 +32,16 @@ description: "Run new evaluation configs on completed test executions. Specify e - - Array of test execution UUIDs to evaluate. Required when `select_all` is `false`. Only `COMPLETED` executions are eligible. + + Array of test execution UUIDs to evaluate. Required when `selectAll` is `false`. Only `COMPLETED` executions are eligible. - - When `true`, evaluates all completed executions, ignoring `test_execution_ids`. Defaults to `false`. + + When `true`, evaluates all completed executions, ignoring `testExecutionIds`. Defaults to `false`. - + Array of evaluation configuration UUIDs to run on the selected executions. - + When `true`, also evaluates tool usage by the agent. Defaults to `false`. @@ -54,32 +54,15 @@ description: "Run new evaluation configs on completed test executions. Specify e - Validation error. Common causes: missing `eval_config_ids`, neither `select_all` nor `test_execution_ids` provided, no completed executions found. - ```json - {"error": "Either 'select_all' must be True or 'test_execution_ids' must be provided"} - ``` - Or when no completed executions exist: - ```json - {"error": "No test executions found to run evaluations on."} - ``` - Or when executions are not completed: - ```json - {"error": "Only test executions with COMPLETED status can have new evaluations run on them."} - ``` + Missing or empty `evalConfigIds`, no executions specified, or no completed executions found. Missing or invalid `X-Api-Key` or `X-Secret-Key` headers. No test run found with the specified `run_test_id`. - ```json - {"detail": "No RunTest matches the given query."} - ``` - Unexpected server error. - ```json - {"error": "Failed to run evaluations: "} - ``` + Unexpected server error. Retry later or contact support. diff --git a/src/pages/docs/api/run-tests/updateevalconfig.mdx b/src/pages/docs/api/run-tests/updateevalconfig.mdx index 91ab1a34..f04efd28 100644 --- a/src/pages/docs/api/run-tests/updateevalconfig.mdx +++ b/src/pages/docs/api/run-tests/updateevalconfig.mdx @@ -1,29 +1,18 @@ --- -title: "Update Eval Config" +title: "Update Eval Config — API" description: "Update an evaluation configuration for a test run. Modify config, mapping, model, name, or error localizer. Optionally trigger an immediate rerun after saving." --- @@ -56,58 +45,38 @@ description: "Update an evaluation configuration for a test run. Modify config, Model to use for evaluations. - + Enable granular error localization in evaluation results. - - UUID of a knowledge base to use for grounding. Pass `null` to clear. + + UUID of a knowledge base to use for grounding. - Updated name for the evaluation configuration. Cannot be blank. + Updated name for the evaluation configuration. - When `true`, triggers an immediate rerun after updating. Defaults to `false`. Requires `test_execution_id` when set to `true`. + When `true`, triggers an immediate rerun after updating. Defaults to `false`. - + UUID of the test execution to rerun against. Required when `run` is `true`. - Confirmation of successful update. - UUID of the updated evaluation config. - UUID of the parent test run. - UUID of the test execution that was rerun. Only present when `run=true`. - Number of call executions queued for re-evaluation. Only present when `run=true`. - Additional context about parallel task spawning. Only present when `run=true`. + Confirmation message. - Validation error. The response includes a `details` object with per-field errors. - ```json - { - "test_execution_id": ["test_execution_id is required when run is true"] - } - ``` - Or when the test execution has an incompatible status: - ```json - {"error": "Only test executions with COMPLETED, CANCELLED, or FAILED status can have evaluations rerun"} - ``` + Invalid data or missing `testExecutionId` when `run` is `true`. Invalid or missing API credentials. Test run or evaluation configuration not found. - ```json - {"detail": "No RunTest matches the given query."} - ``` Unexpected server error. - ```json - {"error": "Failed to update evaluation config: "} - ``` diff --git a/src/pages/docs/api/run-tests/updatetestcomponents.mdx b/src/pages/docs/api/run-tests/updatetestcomponents.mdx index e69de29b..b5db9ef5 100644 --- a/src/pages/docs/api/run-tests/updatetestcomponents.mdx +++ b/src/pages/docs/api/run-tests/updatetestcomponents.mdx @@ -0,0 +1,69 @@ +--- +title: "Update Run Test Components — API" +description: "Update the agent definition, agent version, simulator agent, scenarios, or tool evaluation settings of an existing test run. Returns the updated test run object." +--- + + + + + + Your Future AGI API key used to authenticate requests. You can find and manage your API keys in the [Dashboard](https://app.futureagi.com) under Settings. + + + Your Future AGI secret key, used alongside the API key for request authentication. This is generated when you create an API key in the [Dashboard](https://app.futureagi.com). + + + + + + The test run ID. + + + + + + UUID of the new agent definition. + + + UUID of a specific agent version. Defaults to the active version if omitted. + + + UUID of the simulator agent. + + + Array of scenario UUIDs. Replaces the entire set. + + + Enable tool call evaluation. Requires `api_key` and `assistant_id` on the agent. Defaults to `false`. + + + + + Updated test run object. + + + + + Invalid data or missing prerequisites for tool evaluation. + + + Invalid or missing API credentials. + + + Test run not found. + + + Unexpected server error. + + diff --git a/src/pages/docs/api/scenarios/addcolumns.mdx b/src/pages/docs/api/scenarios/addcolumns.mdx index ab2267e8..905f3bf6 100644 --- a/src/pages/docs/api/scenarios/addcolumns.mdx +++ b/src/pages/docs/api/scenarios/addcolumns.mdx @@ -1,6 +1,6 @@ --- -title: "Add Columns to Scenario" -description: "Adds new AI-generated columns to a scenario's dataset. Returns 202 Accepted and runs asynchronously." +title: "Add Columns to Scenario — API" +description: "Add up to 10 AI-generated columns to a scenario dataset. Specify name, data type, and description per column. Generation is asynchronous; returns 202 Accepted." --- - The scenario ID. The scenario must have an associated dataset with at least one row. + The scenario ID. The dataset must have at least one row. - Column definitions to add. Min 1, max 10 per request. Column names must be unique within the request and must not already exist in the dataset. - - Each column object: + Column definitions to add (1--10 per request). Names must be unique in the dataset. Each column: `name` (string, max 50 chars), `dataType` (`text`, `boolean`, `integer`, `float`, `json`, `array`, `image`, `images`, `datetime`, `audio`, `document`, `others`, `persona`), `description` (string, max 200 chars, guides AI generation). - - - Column name. Max 50 characters. Cannot be blank or whitespace-only. Must be unique within the request and not already present in the dataset. - - - Column data type. One of: `text`, `boolean`, `integer`, `float`, `json`, `array`, `image`, `images`, `datetime`, `audio`, `document`, `others`, `persona`. - - - Column description. Max 200 characters. Guides the AI when generating values. - - Confirmation that column generation has started. UUID of the scenario. UUID of the underlying dataset. - Names of the columns being generated. + Names of the columns being generated. - Invalid request or dataset state. The response includes an `error` message and (for field errors) a `details` object. - ```json - { - "columns": "Column 'expected_outcome' already exists in the dataset." - } - ``` - Or: - ```json - { - "columns": "Duplicate column name(s): difficulty_level" - } - ``` - Common causes: - - No associated dataset - - Dataset has no rows - - Column name already exists in the dataset - - Duplicate column names within the request - - More than 10 columns submitted - - Invalid `data_type` value + Invalid columns: missing fields, duplicates, exceeds 10-column limit, or dataset has no rows. Invalid or missing API credentials. - Scenario not found or does not belong to your organization. - ```json - {"error": "Scenario not found."} - ``` + Scenario not found. Unexpected server error. - ```json - {"error": "Failed to add columns: "} - ``` diff --git a/src/pages/docs/api/scenarios/addemptyrowstodataset.mdx b/src/pages/docs/api/scenarios/addemptyrowstodataset.mdx index e69de29b..562315c4 100644 --- a/src/pages/docs/api/scenarios/addemptyrowstodataset.mdx +++ b/src/pages/docs/api/scenarios/addemptyrowstodataset.mdx @@ -0,0 +1,61 @@ +--- +title: "Add Empty Rows to Scenario — API" +description: "Add a specified number of empty rows to a scenario's dataset by dataset UUID. Accepts num_rows as a positive integer. Returns success status and confirmation message." +--- + + + + + + Your Future AGI API key used to authenticate requests. You can find and manage your API keys in the [Dashboard](https://app.futureagi.com) under Settings. + + + Your Future AGI secret key, used alongside the API key for request authentication. This is generated when you create an API key in the [Dashboard](https://app.futureagi.com). + + + + + + The dataset ID. + + + + + + Number of empty rows to add. Must be a positive integer. + + + + + `true` on success. + Confirmation message. + + + + + Invalid `num_rows` value or dataset not found. + + + Invalid or missing API credentials. + + + Dataset not found. + + + Unexpected server error. + + diff --git a/src/pages/docs/api/scenarios/addscenariorowswithai.mdx b/src/pages/docs/api/scenarios/addscenariorowswithai.mdx index 9da6351f..f8330d54 100644 --- a/src/pages/docs/api/scenarios/addscenariorowswithai.mdx +++ b/src/pages/docs/api/scenarios/addscenariorowswithai.mdx @@ -1,6 +1,6 @@ --- -title: "Add AI Rows to Scenario" -description: "Generates and adds new rows to a scenario's dataset using AI. Returns 202 Accepted and runs asynchronously." +title: "Add AI Rows to Scenario — API" +description: "Generate and add 10–100 AI-populated rows to a scenario dataset. Provide optional generation guidance; existing rows and columns are used as context. Returns 202 Accepted." --- - The scenario ID. The scenario must have an associated dataset. + The scenario ID. Must have an associated dataset. - - Number of rows to generate. Range: 10–20000. + + Number of rows to generate. Range: 10--100. @@ -51,39 +51,20 @@ description: "Generates and adds new rows to a scenario's dataset using AI. Retu Confirmation that row generation has started. UUID of the scenario. UUID of the underlying dataset. - Number of rows being generated. + Number of rows being generated. - Invalid request or scenario state. The response includes an `error` message. - ```json - {"error": "Scenario does not have an associated dataset."} - ``` - Or for validation failures: - ```json - { - "error": "…", - "details": { - "num_rows": ["Number of rows must be at least 10."] - } - } - ``` - Common causes: no associated dataset, `num_rows` below 10 or above 20000. + No associated dataset or `numRows` outside 10--100. Invalid or missing API credentials. - Scenario not found or does not belong to your organization. - ```json - {"error": "Scenario not found."} - ``` + Scenario not found. Unexpected server error. - ```json - {"error": "Failed to add rows: "} - ``` diff --git a/src/pages/docs/api/scenarios/createscenario.mdx b/src/pages/docs/api/scenarios/createscenario.mdx index d32ead19..565b987a 100644 --- a/src/pages/docs/api/scenarios/createscenario.mdx +++ b/src/pages/docs/api/scenarios/createscenario.mdx @@ -1,5 +1,5 @@ --- -title: "Create Scenario" +title: "Create Scenario — API" description: "Create a simulation scenario from a dataset, script, or conversation graph. Supports AI graph generation, persona assignment, and custom columns. Returns 202 Accepted." --- @@ -10,23 +10,16 @@ description: "Create a simulation scenario from a dataset, script, or conversati requestBody={{ name: "billing-inquiry-scenario", description: "Tests agent handling of billing-related questions", - agent_definition_id: "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + agentDefinitionId: "a1b2c3d4-e5f6-7890-abcd-ef1234567890", kind: "dataset", - dataset_id: "d4e5f6a7-b8c9-0123-def0-123456789abc", - no_of_rows: 20, - add_persona_automatically: false, - source_type: "agent_definition" + datasetId: "d4e5f6a7-b8c9-0123-def0-123456789abc", + noOfRows: 20, + addPersonaAutomatically: false, + sourceType: "agent_definition" }} responseExample={{ - message: "Dataset scenario creation started", - scenario: { - id: "f7a8b9c0-d1e2-3456-789a-bcdef0123456", - name: "billing-inquiry-scenario", - scenario_type: "dataset", - source_type: "agent_definition", - status: "Processing", - created_at: "2026-03-15T10:30:00Z" - }, + message: "Graph scenario creation started", + scenario: { id: "f7a8b9c0-d1e2-3456-789a-bcdef0123456", name: "billing-inquiry-scenario" }, status: "processing" }} responseStatus={202} @@ -44,125 +37,88 @@ description: "Create a simulation scenario from a dataset, script, or conversati - Name for the scenario. Max 255 characters. Cannot be blank or whitespace-only. - - - - Scenario type: `"dataset"` (default), `"script"`, or `"graph"`. - - - - Source for AI-powered generation: `"agent_definition"` (default) or `"prompt"`. - When `"prompt"`, both `prompt_template_id` and `prompt_version_id` are required. + Name for the scenario. Max 255 characters. Optional description of the scenario. - - UUID of the source dataset. **Required** when `kind` is `"dataset"`. - - - - URL of the call script file. **Required** when `kind` is `"script"`. + + Scenario type: `"dataset"` (default), `"script"`, or `"graph"`. - - UUID of the agent definition to test. Required when `generate_graph` is `true` and `source_type` is `"agent_definition"`. + + UUID of the agent definition to test. Required when `generateGraph` is `true` or `sourceType` is `"agent_definition"`. - + UUID of a specific agent version. Defaults to the latest version. - - Auto-generate a conversation graph from the agent definition or prompt template. Default: `false`. + + UUID of the source dataset. Required when `kind` is `"dataset"`. - - Conversation graph data. **Required** when `kind` is `"graph"` and `generate_graph` is `false`. + + URL of the script file. Required when `kind` is `"script"`. - - Number of test case rows to generate. Range: 10–20000. Default: `20`. + + Number of test case rows to generate. Range: 10--100. Default: 20. - + Automatically assign diverse personas to generated test cases. Default: `false`. - - List of persona UUIDs to include in the scenario. + + Conversation graph defining the simulated flow. Required when `kind` is `"graph"` and `generateGraph` is `false`. - - Custom column definitions (max 10). No duplicate names allowed. - Each column must have: - - `name` (string, max 50 chars) - - `data_type` (one of: `text`, `boolean`, `integer`, `float`, `json`, `array`, `image`, `images`, `datetime`, `audio`, `document`, `others`, `persona`) - - `description` (string, max 200 chars) + + Auto-generate a conversation graph from the agent definition. Requires `agentDefinitionId`. Default: `false`. - - UUID of the prompt template. **Required** when `source_type` is `"prompt"`. + + List of persona UUIDs to include in the scenario. - - UUID of the prompt version. **Required** when `source_type` is `"prompt"`. Must belong to `prompt_template_id`. + + Source for AI-powered generation: `"agent_definition"` (default) or `"prompt"`. `"prompt"` requires `promptTemplateId` and `promptVersionId`. - - Additional instruction to steer AI scenario generation. + + UUID of the prompt template. Required when `sourceType` is `"prompt"`. - - Voice provider for simulator agent. Default: `"elevenlabs"`. + + UUID of the prompt version. Required when `sourceType` is `"prompt"`. Must belong to `promptTemplateId`. - - Voice name for simulator agent. Default: `"marissa"`. + + Custom column definitions (max 10). Each column: `name` (string, max 50 chars, unique), `dataType` (`text`, `boolean`, `integer`, `float`, `json`, `array`, `image`, `images`, `datetime`, `audio`, `document`, `others`, `persona`), `description` (string, max 200 chars). - - LLM model for simulator agent. Default: `"gpt-4"`. + + Additional instruction to steer AI scenario generation. - Confirmation that scenario creation has been queued (e.g. `"Dataset scenario creation started"`). - Created scenario object (full `ScenarioSchema` — see [List Scenarios](/docs/api/scenarios/listscenarios) for field reference). - Always `"processing"` on initial response. Poll [Get Scenario](/docs/api/scenarios/getscenario) for the final status. + Confirmation that scenario creation has been queued. + Created scenario with `id` and `name`. + Processing status. Initially `"processing"`, then `"completed"`. - Validation error. The response includes an `error` message and a `details` object with per-field errors. - ```json - { - "error": "Invalid data", - "details": { - "dataset_id": ["dataset_id is required for dataset kind."], - "custom_columns": ["Duplicate column name(s): col_name"] - } - } - ``` - Common causes: - - `name` is blank or whitespace-only - - `dataset_id` missing when `kind="dataset"` - - `script_url` missing when `kind="script"` - - `graph` and `generate_graph` both absent when `kind="graph"` - - `prompt_template_id` or `prompt_version_id` missing when `source_type="prompt"` - - Duplicate column names in `custom_columns` - - Persona column in source dataset has wrong `data_type` + Missing or invalid fields such as blank `name`, missing `datasetId`/`scriptUrl`, or invalid custom columns. Invalid or missing API credentials. Unexpected server error. - ```json - {"error": "Failed to create scenario: "} - ``` diff --git a/src/pages/docs/api/scenarios/deletescenario.mdx b/src/pages/docs/api/scenarios/deletescenario.mdx index 01b3269d..13957f47 100644 --- a/src/pages/docs/api/scenarios/deletescenario.mdx +++ b/src/pages/docs/api/scenarios/deletescenario.mdx @@ -1,6 +1,6 @@ --- -title: "Delete Scenario" -description: "Soft-deletes a scenario by marking it as deleted. The scenario is not removed from the database." +title: "Delete Scenario — API" +description: "Soft-delete a scenario by UUID, marking it as deleted. Returns a confirmation message. Deleted scenarios cannot be recovered through the API." --- - Confirmation of successful deletion: `"Scenario deleted successfully"`. + Confirmation of successful deletion. @@ -39,15 +39,9 @@ description: "Soft-deletes a scenario by marking it as deleted. The scenario is Invalid or missing API credentials. - Scenario not found or does not belong to your organization. - ```json - {"error": "Scenario not found."} - ``` + Scenario not found. Unexpected server error. - ```json - {"error": "Failed to delete scenario: "} - ``` diff --git a/src/pages/docs/api/scenarios/editscenario.mdx b/src/pages/docs/api/scenarios/editscenario.mdx index 585707f6..cde10513 100644 --- a/src/pages/docs/api/scenarios/editscenario.mdx +++ b/src/pages/docs/api/scenarios/editscenario.mdx @@ -1,6 +1,6 @@ --- -title: "Edit Scenario" -description: "Updates a scenario's name, description, graph data, or simulator agent prompt." +title: "Edit Scenario — API" +description: "Update a scenario's name, description, conversation graph, or simulator prompt. Supports persona and situation template variables in the prompt field." --- - Updated scenario name. Max 255 characters. Cannot be blank or whitespace-only. + Updated scenario name. Max 255 characters, cannot be blank. @@ -53,44 +46,30 @@ description: "Updates a scenario's name, description, graph data, or simulator a - Updated conversation graph structure. Replaces the active `ScenarioGraph.graph_config.graph_data`. If no active graph exists, a new one is created. + Updated conversation graph structure. - Updated simulator agent prompt text. Replaces the `simulator_agent.prompt` field. + Updated simulator agent prompt. Supports `{{persona}}` and `{{situation}}` template variables. - Confirmation of successful update: `"Scenario updated successfully"`. - Updated scenario object (full `ScenarioSchema` — see [List Scenarios](/docs/api/scenarios/listscenarios) for field reference). + Confirmation of successful update. + Updated scenario object. - Validation error. The response includes an `error` message and a `details` object with per-field errors. - ```json - { - "error": "…", - "details": { - "name": ["Name cannot be empty or just whitespace."] - } - } - ``` + Invalid request body, such as an empty `name`. Invalid or missing API credentials. - Scenario not found or does not belong to your organization. - ```json - {"error": "Scenario not found."} - ``` + Scenario not found. Unexpected server error. - ```json - {"error": "Failed to update scenario: "} - ``` diff --git a/src/pages/docs/api/scenarios/getscenario.mdx b/src/pages/docs/api/scenarios/getscenario.mdx index d4507f4b..bc0c49d9 100644 --- a/src/pages/docs/api/scenarios/getscenario.mdx +++ b/src/pages/docs/api/scenarios/getscenario.mdx @@ -1,5 +1,5 @@ --- -title: "Get Scenario" +title: "Get Scenario — API" description: "Retrieve a scenario by UUID. Returns scenario type, dataset ID, agent type, status, conversation graph, simulator prompts, and dataset row count." --- @@ -14,16 +14,12 @@ description: "Retrieve a scenario by UUID. Returns scenario type, dataset ID, ag description: "Tests billing questions", source: "dataset", scenario_type: "dataset", - dataset_id: "d4e5f6a7-b8c9-0123-def0-123456789abc", - organization: "org-uuid", - dataset: "d4e5f6a7-b8c9-0123-def0-123456789abc", + dataset_id: "uuid-1", agent_type: "voice", - status: "Completed", + status: "completed", graph: {}, - prompts: [{ role: "system", content: "You are simulating a customer calling about a billing issue." }], + prompts: [{ role: "system", content: "You are simulating a customer..." }], dataset_rows: 20, - deleted: false, - deleted_at: null, created_at: "2026-03-15T10:30:00Z", updated_at: "2026-03-15T10:35:00Z" }} @@ -51,25 +47,17 @@ description: "Retrieve a scenario by UUID. Returns scenario type, dataset ID, ag Scenario name. Scenario description. Data source used to create the scenario. - Type: `"dataset"`, `"script"`, or `"graph"`. - UUID of the underlying dataset. `null` if none. - Organization UUID. - UUID of the underlying dataset (same as `dataset_id`). `null` if none. - - Raw agent type value: `"voice"` or `"text"`. `null` if undetermined. - - **Note:** This endpoint returns the raw `AgentDefinition.agent_type` value (`"voice"` or `"text"`), which differs from the List Scenarios endpoint (which returns `"inbound"`, `"outbound"`, `"chat"`, or `"prompt"`). - - Current status (e.g. `"Processing"`, `"Completed"`, `"Failed"`). + Type: `dataset`, `script`, or `graph`. + UUID of the underlying dataset. `null` if none. + Agent type: `voice` or `text`. + Status: `processing`, `completed`, or `failed`. Conversation graph structure. `{}` if none. Simulator agent prompts. - - Prompt role: `"system"`, `"user"`, or `"assistant"`. - Prompt text content. + + Prompt role (e.g., `"system"`). + Prompt text. - Number of test case rows. `0` if no dataset. - Whether the scenario is soft-deleted. - Deletion timestamp. `null` if not deleted. + Number of test case rows. ISO 8601 creation timestamp. ISO 8601 last-modified timestamp. @@ -79,15 +67,9 @@ description: "Retrieve a scenario by UUID. Returns scenario type, dataset ID, ag Invalid or missing API credentials. - Scenario not found or does not belong to your organization. - ```json - {"error": "Scenario not found."} - ``` + Scenario not found. Unexpected server error. - ```json - {"error": "Failed to retrieve scenario: "} - ``` diff --git a/src/pages/docs/api/scenarios/listscenarios.mdx b/src/pages/docs/api/scenarios/listscenarios.mdx index 48502aa9..0633b7dc 100644 --- a/src/pages/docs/api/scenarios/listscenarios.mdx +++ b/src/pages/docs/api/scenarios/listscenarios.mdx @@ -1,5 +1,5 @@ --- -title: "List Scenarios" +title: "List Scenarios — API" description: "List paginated scenarios with optional search and filtering by agent definition or agent type. Returns scenario type, status, dataset row count, and creation timestamp." --- @@ -8,44 +8,17 @@ description: "List paginated scenarios with optional search and filtering by age endpoint="/simulate/scenarios/" baseUrl="https://api.futureagi.com" parameters={[ - {"name": "search", "in": "query", "required": false, "description": "Filter scenarios by name, source, or type.", "type": "string"}, + {"name": "search", "in": "query", "required": false, "description": "Filter scenarios by name or source.", "type": "string"}, + {"name": "limit", "in": "query", "required": false, "description": "Number of items per page. Default: 10.", "type": "integer"}, + {"name": "page", "in": "query", "required": false, "description": "Page number. Default: 1.", "type": "integer"}, {"name": "agent_definition_id", "in": "query", "required": false, "description": "Filter by agent definition UUID.", "type": "string"}, - {"name": "agent_type", "in": "query", "required": false, "description": "Filter by agent type: \"voice\" or \"text\".", "type": "string"}, - {"name": "page", "in": "query", "required": false, "description": "Page number, starting from 1. Default: 1.", "type": "integer"}, - {"name": "limit", "in": "query", "required": false, "description": "Results per page. Default: 10.", "type": "integer"} + {"name": "agent_type", "in": "query", "required": false, "description": "Filter by agent type.", "type": "string"} ]} responseExample={{ count: 2, next: null, previous: null, - results: [ - { - id: "f7a8b9c0-d1e2-3456-789a-bcdef0123456", - name: "billing-inquiry-scenario", - description: "Tests billing questions", - source: "dataset", - scenario_type: "dataset", - scenario_type_display: "Dataset", - source_type: "agent_definition", - source_type_display: "Agent Definition", - organization: "org-uuid", - dataset: "dataset-uuid", - dataset_rows: 20, - dataset_column_config: {}, - graph: {}, - agent: null, - prompt_template: null, - prompt_template_detail: null, - prompt_version: null, - prompt_version_detail: null, - agent_type: "outbound", - status: "Completed", - deleted: false, - deleted_at: null, - created_at: "2026-03-15T10:30:00Z", - updated_at: "2026-03-15T10:35:00Z" - } - ] + results: [{ id: "uuid-1", name: "billing-inquiry-scenario", description: "Tests billing questions", source: "dataset", scenario_type: "dataset", dataset_rows: 20, status: "completed", created_at: "2026-03-15T10:30:00Z" }] }} responseStatus={200} responseStatusText="OK" @@ -62,7 +35,13 @@ description: "List paginated scenarios with optional search and filtering by age - Case-insensitive filter against scenario name, source, and type. + Case-insensitive search against scenario name and source. + + + Results per page. Default: `10`. + + + Page number, starting from `1`. Default: `1`. Filter by agent definition UUID. @@ -70,44 +49,26 @@ description: "List paginated scenarios with optional search and filtering by age Filter by agent type: `"voice"` or `"text"`. - - Page number, starting from `1`. Default: `1`. - - - Results per page. Default: `10`. - Total matching scenarios. - URL of the next page, or `null`. - URL of the previous page, or `null`. + URL of the next page, or `null`. + URL of the previous page, or `null`. Array of scenario objects. - + UUID of the scenario. Scenario name. Scenario description. - Data source label. - `"dataset"` | `"script"` | `"graph"`. - Human-readable scenario type label. - `"agent_definition"` | `"prompt"`. - Human-readable source type label. - Organization UUID. - UUID of the underlying dataset. `null` if none. - Number of test case rows. `0` if no dataset. - Map of column ID → `{name, type}`. `[]` if no dataset. - Conversation graph data. `{}` if none. - Simulator agent object. `null` if none. - `"inbound"` | `"outbound"` | `"chat"` | `"prompt"` | `null`. - Prompt template UUID. `null` if none. - Prompt template details. `null` if none. - Prompt version UUID. `null` if none. - Prompt version details. `null` if none. - Processing status (e.g. `"Processing"`, `"Completed"`, `"Failed"`). - Whether the scenario is soft-deleted. - Deletion timestamp. `null` if not deleted. + Data source. + Scenario type. + Display label for scenario type. + Source type classification. + Display label for source type. + Number of test case rows. + Agent type: `voice` or `text`. + Current status. ISO 8601 creation timestamp. - ISO 8601 last-modified timestamp. @@ -115,16 +76,7 @@ description: "List paginated scenarios with optional search and filtering by age Invalid or missing API credentials. - - Organization not found for the authenticated user. - ```json - {"error": "Organization not found for the user."} - ``` - Unexpected server error. - ```json - {"error": "Failed to retrieve scenarios: "} - ``` diff --git a/src/pages/docs/api/simulation-analytics/analytics.mdx b/src/pages/docs/api/simulation-analytics/analytics.mdx index c2fc71df..891656a8 100644 --- a/src/pages/docs/api/simulation-analytics/analytics.mdx +++ b/src/pages/docs/api/simulation-analytics/analytics.mdx @@ -1,5 +1,5 @@ --- -title: "Get Simulation Analytics" +title: "Get Simulation Analytics — API" description: "Retrieve aggregated eval scores, per-metric averages, system summary, and Fix My Agent suggestions for a simulation run by run test name or execution ID." --- diff --git a/src/pages/docs/api/simulation-analytics/metrics.mdx b/src/pages/docs/api/simulation-analytics/metrics.mdx index dd4e5686..deb9812e 100644 --- a/src/pages/docs/api/simulation-analytics/metrics.mdx +++ b/src/pages/docs/api/simulation-analytics/metrics.mdx @@ -1,5 +1,5 @@ --- -title: "Get Simulation Metrics" +title: "Get Simulation Metrics — API" description: "Retrieve latency, cost, and conversation metrics for a simulation run. Query by run test name, execution ID, or call execution ID. Supports pagination for run-level results." --- diff --git a/src/pages/docs/api/simulation-analytics/runs.mdx b/src/pages/docs/api/simulation-analytics/runs.mdx index f06b0dc5..2926f7ab 100644 --- a/src/pages/docs/api/simulation-analytics/runs.mdx +++ b/src/pages/docs/api/simulation-analytics/runs.mdx @@ -1,5 +1,5 @@ --- -title: "Get Simulation Runs" +title: "Get Simulation Runs — API" description: "Retrieve simulation run records with eval scores, scenario metadata, and per-call breakdowns. Query by run test name, execution ID, or call execution ID. Supports FMA summary." --- diff --git a/src/pages/docs/api/test-executions/cancelexecution.mdx b/src/pages/docs/api/test-executions/cancelexecution.mdx index 0c4ef7b5..574f98f9 100644 --- a/src/pages/docs/api/test-executions/cancelexecution.mdx +++ b/src/pages/docs/api/test-executions/cancelexecution.mdx @@ -1,5 +1,5 @@ --- -title: "Cancel Test Execution" +title: "Cancel Test Execution — API" description: "Cancel an in-progress test execution by UUID. Execution must be in pending, running, or evaluating state. Returns success flag and cancellation confirmation message." --- diff --git a/src/pages/docs/api/test-executions/getcallexecutiondetails.mdx b/src/pages/docs/api/test-executions/getcallexecutiondetails.mdx index faddbef8..d4025a8d 100644 --- a/src/pages/docs/api/test-executions/getcallexecutiondetails.mdx +++ b/src/pages/docs/api/test-executions/getcallexecutiondetails.mdx @@ -1,5 +1,5 @@ --- -title: "Get Call Execution Details" +title: "Get Call Execution Details — API" description: "Retrieve details of a specific call execution by UUID, including provider call ID, metrics, and full execution data." --- diff --git a/src/pages/docs/api/test-executions/getkpis.mdx b/src/pages/docs/api/test-executions/getkpis.mdx index d6d0f241..297b9a68 100644 --- a/src/pages/docs/api/test-executions/getkpis.mdx +++ b/src/pages/docs/api/test-executions/getkpis.mdx @@ -1,5 +1,5 @@ --- -title: "Get Test Execution KPIs" +title: "Get Test Execution KPIs — API" description: "Retrieve KPI metrics for a test execution. Returns avg score, response time, call connection rate, latency, WPM, interruption rates, and per-eval averages." --- diff --git a/src/pages/docs/api/test-executions/getperformancesummary.mdx b/src/pages/docs/api/test-executions/getperformancesummary.mdx index 40779e55..79596c20 100644 --- a/src/pages/docs/api/test-executions/getperformancesummary.mdx +++ b/src/pages/docs/api/test-executions/getperformancesummary.mdx @@ -1,5 +1,5 @@ --- -title: "Get Performance Summary" +title: "Get Performance Summary — API" description: "Retrieve pass/fail rates and top-performing scenarios for a test execution. Returns overall pass rate, total test runs, latest fail rate, and top 4 scenarios by score." --- diff --git a/src/pages/docs/api/test-executions/gettestexecutiondetails.mdx b/src/pages/docs/api/test-executions/gettestexecutiondetails.mdx index 53913068..7b222f56 100644 --- a/src/pages/docs/api/test-executions/gettestexecutiondetails.mdx +++ b/src/pages/docs/api/test-executions/gettestexecutiondetails.mdx @@ -1,5 +1,5 @@ --- -title: "Get Test Execution Details" +title: "Get Test Execution Details — API" description: "Retrieve a test execution by UUID with paginated call executions. Supports search, filters, and row grouping. Returns status, call counts, eval scores, and transcripts." --- diff --git a/src/pages/docs/api/test-executions/reruncalls.mdx b/src/pages/docs/api/test-executions/reruncalls.mdx index 49b57cbc..55645d0b 100644 --- a/src/pages/docs/api/test-executions/reruncalls.mdx +++ b/src/pages/docs/api/test-executions/reruncalls.mdx @@ -1,5 +1,5 @@ --- -title: "Rerun Call Executions" +title: "Rerun Call Executions — API" description: "Rerun call executions within a test execution. Choose eval_only to re-evaluate existing data or call_and_eval to re-run calls. Returns success and failure counts per call." --- @@ -50,7 +50,7 @@ description: "Rerun call executions within a test execution. Choose eval_only to The type of rerun to perform. Use `eval_only` to re-evaluate existing call data without re-executing the actual calls -- this is useful when you have updated your evaluation configurations and want to see updated scores without the cost of re-running calls. Use `call_and_eval` to fully re-execute the calls and then evaluate the new results -- this produces fresh conversations and is useful when you have modified the agent under test. Note that text agents only support `eval_only` reruns; attempting `call_and_eval` on a text agent will return a 400 error. - An array of call execution UUIDs to rerun. Required when `select_all` is `false` or not provided. Each ID must correspond to a valid call execution within the specified test execution. If a provided ID does not exist or does not belong to the test execution, it will appear in the `failedReruns` array of the response. + An array of call execution UUIDs to rerun. Required when `select_all` is `false` or not provided. Each ID must correspond to a valid call execution within the specified test execution. If a provided ID does not exist or does not belong to the test execution, it will appear in the `failed_reruns` array of the response. When set to `true`, all call executions within the test execution will be rerun, and the `call_execution_ids` field is ignored. Defaults to `false`. You must provide either `select_all: true` or a non-empty `call_execution_ids` array -- the request will fail with a 400 error if neither is specified. diff --git a/src/pages/docs/cookbook/decrease-hallucination.mdx b/src/pages/docs/cookbook/decrease-hallucination.mdx index d4677500..f14ed872 100644 --- a/src/pages/docs/cookbook/decrease-hallucination.mdx +++ b/src/pages/docs/cookbook/decrease-hallucination.mdx @@ -206,7 +206,7 @@ It is the process of adding tracing to your LLM applications. Tracing helps you Where a span represents a single operation within an execution flow, recording input-output data, execution time, and errors, a trace connects multiple spans to represent the full execution flow of a request. -Click [here](/docs/tracing/concepts) to learn more about traces and spans +Click [here](/docs/observe/concepts/observability-model) to learn more about traces and spans This experimentation is done to find the best fit of your application for your use case before deploying in production. @@ -266,7 +266,7 @@ Parameters of `EvalTag` : The trace provider is part of the traceAI ecosystem, which is an OSS package that enables tracing of AI applications and frameworks. It works in conjunction with OpenTelemetry to monitor code executions across different models, frameworks, and vendors. -Click [here](/docs/tracing/concepts/traceai) to learn more about the list of supported frameworks +Click [here](/docs/observe/concepts/traceai) to learn more about the list of supported frameworks To configure a `trace_provider`, we need to pass following parameters to `register` function: diff --git a/src/pages/docs/cookbook/llamaindex-pdf-rag.mdx b/src/pages/docs/cookbook/llamaindex-pdf-rag.mdx index b9a7dff0..fb9192b2 100644 --- a/src/pages/docs/cookbook/llamaindex-pdf-rag.mdx +++ b/src/pages/docs/cookbook/llamaindex-pdf-rag.mdx @@ -94,7 +94,7 @@ LlamaIndexInstrumentor().instrument(tracer_provider=trace_provider) - `LlamaIndexInstrumentor().instrument()` auto-instruments LlamaIndex so you get more AI-aware spans (Embedding, Retriever, LLM, Index build) with rich attributes (model name, token usage, prompt, chunk metadata, latencies, errors). -Click [here](https://docs.futureagi.com/docs/tracing/auto) to learn more about auto-instrumention +Click [here](https://docs.futureagi.com/docs/traceai/auto) to learn more about auto-instrumention This level of detail allows teams to move from “The chatbot failed” to “The chatbot failed because it retrieved irrelevant chunks from document X, page 14, due to an overly generic embedding query.” diff --git a/src/pages/docs/cookbook/mongodb.mdx b/src/pages/docs/cookbook/mongodb.mdx index c7df2cb4..0c1b00a7 100644 --- a/src/pages/docs/cookbook/mongodb.mdx +++ b/src/pages/docs/cookbook/mongodb.mdx @@ -78,7 +78,7 @@ LangChainInstrumentor().instrument(tracer_provider=trace_provider) - `LangChainInstrumentor().instrument()` auto-instruments LangChain so you get more AI-aware spans (Embedding, Retriever, LLM, Index build) with rich attributes (model name, token usage, prompt, chunk metadata, latencies, errors). -Click [here](https://docs.futureagi.com/docs/tracing/auto) to learn more about auto-instrumention +Click [here](https://docs.futureagi.com/docs/traceai/auto) to learn more about auto-instrumention This level of detail allows teams to move from “The chatbot failed” to “The chatbot failed because it retrieved irrelevant chunks from document X, page 14, due to an overly generic embedding query.” diff --git a/src/pages/docs/cookbook/quickstart/distributed-tracing.mdx b/src/pages/docs/cookbook/quickstart/distributed-tracing.mdx index 02038706..8d7c8bff 100644 --- a/src/pages/docs/cookbook/quickstart/distributed-tracing.mdx +++ b/src/pages/docs/cookbook/quickstart/distributed-tracing.mdx @@ -1028,7 +1028,7 @@ Before you ship: Add custom spans to any application without auto-instrumentation. - + Setup guides for 45+ supported frameworks. diff --git a/src/pages/docs/cookbook/quickstart/manual-tracing.mdx b/src/pages/docs/cookbook/quickstart/manual-tracing.mdx index 93132e62..0325c9ff 100644 --- a/src/pages/docs/cookbook/quickstart/manual-tracing.mdx +++ b/src/pages/docs/cookbook/quickstart/manual-tracing.mdx @@ -177,7 +177,7 @@ In Tracing, filter by tags using the **Attribute** filter: select **Attribute** ![Traces filtered by tag in the Tracing dashboard](https://fi-cookbook-assets.s3.ap-south-1.amazonaws.com/quickstart/manual-tracing/step-4-tags.png) -You can combine `using_user`, `using_session`, `using_metadata`, and `using_tags` into a single `using_attributes()` call for convenience. See the [tracing reference](/docs/observe/features/manual-tracing/set-up-tracing) for details. +You can combine `using_user`, `using_session`, `using_metadata`, and `using_tags` into a single `using_attributes()` call for convenience. See the [tracing reference](/docs/traceai/manual-instrumentation/set-up-tracing) for details. @@ -325,7 +325,7 @@ You can now auto-trace LLM calls, add custom spans for non-LLM steps, attach use ## Next steps - + 20+ framework integrations diff --git a/src/pages/docs/cookbook/text-to-sql.mdx b/src/pages/docs/cookbook/text-to-sql.mdx index 17ffe984..476a62b9 100644 --- a/src/pages/docs/cookbook/text-to-sql.mdx +++ b/src/pages/docs/cookbook/text-to-sql.mdx @@ -362,7 +362,7 @@ def setup_database(): - It is the process of adding tracing to your LLM applications. Tracing helps you monitor critical metrics like cost, latency, and evaluation results. - Where a span represents a single operation within an execution flow, recording input-output data, execution time, and errors, a trace connects multiple spans to represent the full execution flow of a request. - > **Click [here](https://docs.futureagi.com/docs/tracing/core-components) to learn more about traces and spans** + > **Click [here](https://docs.futureagi.com/docs/observe/concepts/observability-model) to learn more about traces and spans** > - Tracing using Future AGI requires following steps: @@ -405,7 +405,7 @@ def setup_database(): - The trace provider is part of the traceAI ecosystem, which is an OSS package that enables tracing of AI applications and frameworks. It works in conjunction with OpenTelemetry to monitor code executions across different models, frameworks, and vendors. - > Click [**here**](https://docs.futureagi.com/docs/tracing/traceai) to learn more about the list of supported frameworks + > Click [**here**](https://docs.futureagi.com/docs/traceai/auto) to learn more about the list of supported frameworks > - To configure a **`trace_provider`**, we need to pass following parameters to **`register`** function: - **`project_type`**: Specifies the type of project. In this cookbook, **`ProjectType.EXPERIMENT`** is used since we are experimenting to test agent before deploying in production. **`ProjectType.OBSERVE`** is used to observe your AI application in production and measure the performance in real-time. @@ -417,7 +417,7 @@ def setup_database(): - This is done to integrate with the LangChain framework for the collection of telemetry data. -> **Click [here](https://docs.futureagi.com/docs/tracing/auto) to know about all the supported frameworks by Future AGI** +> **Click [here](https://docs.futureagi.com/docs/traceai/auto) to know about all the supported frameworks by Future AGI** > - The **`instrument`** method is called on the **`LangChainInstrumentor`** instance. This method is responsible for setting up the instrumentation of the LangChain framework using the provided **`tracer_provider`**. - Putting it all together, below is the code that configures **`eval_tags`**, and sets up **`trace_provider`**, which is then passed onto **`LangChainInstrumentor`** . diff --git a/src/pages/docs/integrations/index.mdx b/src/pages/docs/integrations/index.mdx index c61343f7..78157fe9 100644 --- a/src/pages/docs/integrations/index.mdx +++ b/src/pages/docs/integrations/index.mdx @@ -43,7 +43,7 @@ TraceAI provides pre-built auto-instrumentation for the following frameworks and - + diff --git a/src/pages/docs/integrations/traceai/java/anthropic.mdx b/src/pages/docs/integrations/traceai/java/anthropic.mdx index 1012f785..cb53f102 100644 --- a/src/pages/docs/integrations/traceai/java/anthropic.mdx +++ b/src/pages/docs/integrations/traceai/java/anthropic.mdx @@ -12,7 +12,7 @@ description: "Trace Anthropic Messages API calls in Java with TracedAnthropicCli ## Prerequisites -Complete the [Java SDK setup](/docs/tracing/auto/java) first. +Complete the [Java SDK setup](/docs/traceai/auto/java) first. ## Installation diff --git a/src/pages/docs/integrations/traceai/java/bedrock.mdx b/src/pages/docs/integrations/traceai/java/bedrock.mdx index a7bbb147..33bce2b4 100644 --- a/src/pages/docs/integrations/traceai/java/bedrock.mdx +++ b/src/pages/docs/integrations/traceai/java/bedrock.mdx @@ -12,7 +12,7 @@ description: "Trace AWS Bedrock model invocations in Java with TracedBedrockRunt ## Prerequisites -Complete the [Java SDK setup](/docs/tracing/auto/java) first. +Complete the [Java SDK setup](/docs/traceai/auto/java) first. ## Installation diff --git a/src/pages/docs/integrations/traceai/java/cohere.mdx b/src/pages/docs/integrations/traceai/java/cohere.mdx index 05cf379f..22b83095 100644 --- a/src/pages/docs/integrations/traceai/java/cohere.mdx +++ b/src/pages/docs/integrations/traceai/java/cohere.mdx @@ -12,7 +12,7 @@ description: "Trace Cohere chat, embedding, and reranking operations in Java wit ## Prerequisites -Complete the [Java SDK setup](/docs/tracing/auto/java) first. +Complete the [Java SDK setup](/docs/traceai/auto/java) first. ## Installation diff --git a/src/pages/docs/integrations/traceai/java/frameworks.mdx b/src/pages/docs/integrations/traceai/java/frameworks.mdx index 3ce01fcd..bbc24b1e 100644 --- a/src/pages/docs/integrations/traceai/java/frameworks.mdx +++ b/src/pages/docs/integrations/traceai/java/frameworks.mdx @@ -7,12 +7,12 @@ description: "Trace LangChain4j and Semantic Kernel operations in Java. Framewor - LangChain4j: `TracedChatLanguageModel` implements `ChatLanguageModel` as a drop-in replacement - Semantic Kernel: `TracedKernel` wraps `Kernel` and traces function invocations and prompt calls - Both support any underlying LLM provider -- For Spring AI, see the [Spring Boot](/docs/tracing/auto/spring-boot) page +- For Spring AI, see the [Spring Boot](/docs/traceai/auto/spring-boot) page ## Prerequisites -Complete the [Java SDK setup](/docs/tracing/auto/java) first. +Complete the [Java SDK setup](/docs/traceai/auto/java) first. --- diff --git a/src/pages/docs/integrations/traceai/java/index.mdx b/src/pages/docs/integrations/traceai/java/index.mdx index 3c4ee858..2cf65f26 100644 --- a/src/pages/docs/integrations/traceai/java/index.mdx +++ b/src/pages/docs/integrations/traceai/java/index.mdx @@ -275,31 +275,31 @@ This flushes all pending spans (up to 10 second timeout) and resets the tracer. ## Available integrations - + Auto-configuration via `application.yml`. No manual `TraceAI.init()` needed. - + Chat completions, embeddings, streaming. - + Messages API with reflection-based version compatibility. - + InvokeModel (raw JSON) and Converse (typed API). - + Chat, embeddings, and reranking. - + Query, upsert, delete, fetch with namespace support. - + Google GenAI, Vertex AI, Azure OpenAI, Ollama, Watsonx. - + Qdrant, Milvus, ChromaDB, Weaviate, MongoDB, Redis, pgvector, Azure AI Search, Elasticsearch. - + LangChain4j and Semantic Kernel. diff --git a/src/pages/docs/integrations/traceai/java/llm-providers.mdx b/src/pages/docs/integrations/traceai/java/llm-providers.mdx index de42ed46..36817240 100644 --- a/src/pages/docs/integrations/traceai/java/llm-providers.mdx +++ b/src/pages/docs/integrations/traceai/java/llm-providers.mdx @@ -12,7 +12,7 @@ description: "Trace Google GenAI, Vertex AI, Azure OpenAI, Ollama, and Watsonx i ## Prerequisites -Complete the [Java SDK setup](/docs/tracing/auto/java) first. All providers below need `traceai-java-core` and `TraceAI.init()` called before use. +Complete the [Java SDK setup](/docs/traceai/auto/java) first. All providers below need `traceai-java-core` and `TraceAI.init()` called before use. --- diff --git a/src/pages/docs/integrations/traceai/java/openai.mdx b/src/pages/docs/integrations/traceai/java/openai.mdx index 265e789e..8a6ad9b3 100644 --- a/src/pages/docs/integrations/traceai/java/openai.mdx +++ b/src/pages/docs/integrations/traceai/java/openai.mdx @@ -12,7 +12,7 @@ description: "Trace OpenAI chat completions, embeddings, and streaming responses ## Prerequisites -Complete the [Java SDK setup](/docs/tracing/auto/java) first. You need `TraceAI.init()` called before using this wrapper. +Complete the [Java SDK setup](/docs/traceai/auto/java) first. You need `TraceAI.init()` called before using this wrapper. ## Installation diff --git a/src/pages/docs/integrations/traceai/java/pinecone.mdx b/src/pages/docs/integrations/traceai/java/pinecone.mdx index f17fad88..b8420ddc 100644 --- a/src/pages/docs/integrations/traceai/java/pinecone.mdx +++ b/src/pages/docs/integrations/traceai/java/pinecone.mdx @@ -12,7 +12,7 @@ description: "Trace Pinecone vector operations in Java with TracedPineconeIndex. ## Prerequisites -Complete the [Java SDK setup](/docs/tracing/auto/java) first. +Complete the [Java SDK setup](/docs/traceai/auto/java) first. ## Installation diff --git a/src/pages/docs/integrations/traceai/java/vector-databases.mdx b/src/pages/docs/integrations/traceai/java/vector-databases.mdx index 206248c6..7018d47e 100644 --- a/src/pages/docs/integrations/traceai/java/vector-databases.mdx +++ b/src/pages/docs/integrations/traceai/java/vector-databases.mdx @@ -12,7 +12,7 @@ description: "Trace vector database operations in Java. Qdrant, Milvus, ChromaDB ## Prerequisites -Complete the [Java SDK setup](/docs/tracing/auto/java) first. For Pinecone, see the [dedicated Pinecone page](/docs/tracing/auto/java/pinecone). +Complete the [Java SDK setup](/docs/traceai/auto/java) first. For Pinecone, see the [dedicated Pinecone page](/docs/traceai/auto/java/pinecone). --- diff --git a/src/pages/docs/integrations/traceai/langgraph.mdx b/src/pages/docs/integrations/traceai/langgraph.mdx index 78d5dc6e..0bebdc62 100644 --- a/src/pages/docs/integrations/traceai/langgraph.mdx +++ b/src/pages/docs/integrations/traceai/langgraph.mdx @@ -3,7 +3,7 @@ title: "LangGraph Integration with Future AGI for Agent Tracing" description: "Integrate LangGraph with Future AGI observability. Trace agent graph execution, tool usage, and state transitions using the LangChain instrumentor." --- -Our [LangChainInstrumentor](/docs/tracing/auto/langchain) automatically captures traces for both LangGraph and LangChain. If you've already enabled that instrumentor, you do not need to complete the steps below. +Our [LangChainInstrumentor](/docs/traceai/auto/langchain) automatically captures traces for both LangGraph and LangChain. If you've already enabled that instrumentor, you do not need to complete the steps below. ## 1. Installation First install the traceAI package and necessary LangChain packages. @@ -46,7 +46,7 @@ trace_provider = register( --- ## 4. Instrument your Project -Initialize the LangChain Instrumentor to enable automatic tracing. Our [LangChainInstrumentor](/docs/tracing/auto/langchain) automatically captures traces for both LangGraph and LangChain. +Initialize the LangChain Instrumentor to enable automatic tracing. Our [LangChainInstrumentor](/docs/traceai/auto/langchain) automatically captures traces for both LangGraph and LangChain. ```python from traceai_langchain import LangChainInstrumentor diff --git a/src/pages/docs/integrations/traceai/llamaindex-workflows.mdx b/src/pages/docs/integrations/traceai/llamaindex-workflows.mdx index ae99e67b..e99214ce 100644 --- a/src/pages/docs/integrations/traceai/llamaindex-workflows.mdx +++ b/src/pages/docs/integrations/traceai/llamaindex-workflows.mdx @@ -5,7 +5,7 @@ description: "Integrate LlamaIndex Workflows with Future AGI. Trace workflow-bas [LlamaIndex Workflows](https://www.llamaindex.ai/blog/introducing-workflows-beta-a-new-way-to-create-complex-ai-applications-with-llamaindex) are a subset of the LlamaIndex package specifically designed to support agent development. -Our [LlamaIndexInstrumentor](/docs/tracing/auto/llamaindex) automatically captures traces for LlamaIndex Workflows agents. If you've already enabled that instrumentor, you do not need to complete the steps below. +Our [LlamaIndexInstrumentor](/docs/traceai/auto/llamaindex) automatically captures traces for LlamaIndex Workflows agents. If you've already enabled that instrumentor, you do not need to complete the steps below. ## 1. Installation First install the traceAI and necessary llama-index packages. diff --git a/src/pages/docs/integrations/traceai/mastra.mdx b/src/pages/docs/integrations/traceai/mastra.mdx index 162245b2..68a4cd85 100644 --- a/src/pages/docs/integrations/traceai/mastra.mdx +++ b/src/pages/docs/integrations/traceai/mastra.mdx @@ -3,117 +3,56 @@ title: "Mastra Integration with Future AGI TypeScript Agent Tracing" description: "Integrate Mastra with Future AGI for TypeScript agent observability. Configure trace export using the @traceai/mastra package for LLM monitoring." --- -Integrate Future AGI observability into your [Mastra](https://mastra.ai) agents and -workflows. Every agent run, tool call, and LLM interaction is exported to Future AGI -for monitoring, evaluation, and debugging. - - -This guide targets **Mastra v1** (`@mastra/core` ≥ 1.16). Mastra v1 removed the old -`telemetry:` config key, so the previous `FITraceExporter` setup no longer exports any -spans. Use `createFIObservability` from `@traceai/mastra` as shown below. Still on -Mastra v0.x? See [Legacy (Mastra v0.x)](#legacy-mastra-v0x) at the bottom. - - ## 1. Installation -Install Mastra's observability packages, the OTLP/protobuf exporter, and `@traceai/mastra`. +First install the Mastra and traceAI packages. ```bash JS/TS -npm install @mastra/core @mastra/observability @mastra/otel-exporter \ - @opentelemetry/exporter-trace-otlp-proto @traceai/mastra +npm install @mastra/core @traceai/mastra @traceai/fi-core ``` --- ## 2. Set Environment Variables -Add your Future AGI credentials to your `.env`. `@traceai/mastra` reads them automatically. +Configure your Future AGI credentials. -```bash .env -FI_API_KEY=your-futureagi-api-key -FI_SECRET_KEY=your-futureagi-secret-key +```typescript JS/TS +process.env.FI_API_KEY = "your-futureagi-api-key"; +process.env.FI_SECRET_KEY = "your-futureagi-secret-key"; ``` --- -## 3. Configure Observability -Wire Future AGI into your Mastra instance with `createFIObservability`. It points the -exporter at Future AGI's collector, authenticates with your keys, and exports traces only -(Future AGI's collector does not accept the OTLP logs signal). +## 3. Configure Mastra Telemetry Export +Use the custom exporter from `@traceai/mastra` to send traces to Future AGI. You can optionally filter out non-LLM spans using `isFISpan`. ```typescript JS/TS import { Mastra } from "@mastra/core"; -import { createFIObservability } from "@traceai/mastra"; +import { FITraceExporter, isFISpan } from "@traceai/mastra"; export const mastra = new Mastra({ - // ... your agents, workflows, etc. - observability: createFIObservability({ - serviceName: "traceai-mastra-agent", // appears in the Future AGI trace list - }), + // ... other config + telemetry: { + serviceName: "traceai-mastra-agent", // customize the service name + enabled: true, + export: { + type: "custom", + exporter: new FITraceExporter({ + url: "https://app.futureagi.com/tracer/v1/traces", + headers: { + "x-api-key": process.env.FI_API_KEY as string, + "x-secret-key": process.env.FI_SECRET_KEY as string, + }, + // Optional: filter out non-LLM/node spans from being sent to Future AGI + spanFilter: isFISpan, + }), + }, + }, }); ``` -No changes are needed to your agent code. Every span is mapped to OpenTelemetry -`gen_ai.*` conventions, given the right span kind (LLM / agent / tool / chain), and -its input/output is captured — so the trace renders fully in Future AGI. - --- ## 4. Run your Agent -Run your Mastra agent as usual. Traces appear in your Future AGI project under -**Observability** (service `traceai-mastra-agent`). - -```typescript JS/TS -const agent = mastra.getAgent("yourAgent"); -const result = await agent.generate("What's the weather in Bangalore?"); -``` - - -**Short-lived scripts & serverless.** Spans are batched, so a process that exits -immediately may drop them. Keep a reference to the observability instance and flush -before exit: - -```typescript JS/TS -export const observability = createFIObservability({ serviceName: "traceai-mastra-agent" }); -export const mastra = new Mastra({ observability /* , agents, ... */ }); - -// at the end of your script / request handler: -await observability.shutdown(); // flushes buffered spans -``` - - ---- - -## Configuration Options - -`createFIObservability(options)` accepts: - -| Option | Default | Description | -| --- | --- | --- | -| `serviceName` | `"mastra-app"` | Service name; also the default Future AGI **project** name. | -| `projectName` | `serviceName` (or `FI_PROJECT_NAME`) | Future AGI project the traces are filed under. | -| `projectType` | `"observe"` | `"observe"` for tracing, `"experiment"` for eval runs. | -| `apiKey` | `process.env.FI_API_KEY` | Future AGI API key. | -| `secretKey` | `process.env.FI_SECRET_KEY` | Future AGI secret key. | -| `baseUrl` | `https://api.futureagi.com` | Collector base URL (`/tracer/v1/traces` is appended). | -| `endpoint` | — | Full traces endpoint URL; overrides `baseUrl`. | -| `headers` | — | Extra headers merged into the export request. | -| `excludeSpanTypes` | `[MODEL_CHUNK]` | Mastra span types to drop before export (chunk spans are noise). | -| `timeout` | `30000` | Export request timeout (ms). | -| `batchSize` | — | Spans per batch. | - -If you need to compose the exporter into your own `Observability` config, use -`createFIMastraExporter(options)` instead — it returns a pre-configured exporter you -can drop into `new Observability({ configs: { otel: { exporters: [...] } } })`. - ---- - -## Legacy (Mastra v0.x) - -The old `telemetry:` + `FITraceExporter` integration is deprecated and does **not** work -on Mastra v1. If you are still on Mastra v0.x, import it from the `/legacy` subpath: - -```typescript JS/TS -import { FITraceExporter, isFISpan } from "@traceai/mastra/legacy"; -``` +Once configured, run your Mastra agent as usual. The exporter will automatically send trace data to your Future AGI project. -We recommend upgrading to Mastra v1 and the `createFIObservability` setup above. diff --git a/src/pages/docs/observe/concepts/observability-model.mdx b/src/pages/docs/observe/concepts/observability-model.mdx new file mode 100644 index 00000000..271a7fa5 --- /dev/null +++ b/src/pages/docs/observe/concepts/observability-model.mdx @@ -0,0 +1,148 @@ +--- +title: "Observability model" +description: "How the pieces of Observe fit together: spans nest into traces, traces group into sessions and users, and eval scores attach on top — all collected as OpenTelemetry data through the traceAI SDK." +slug: "observability-model" +page_type: "concept" +diataxis: "explanation" +products: ["Observe"] +concept_family: "observability" +concept_level: "foundational" +audience: ["engineer"] +difficulty: "beginner" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-18" +last_diagram_reviewed: "2026-06-17" +schema_type: "TechArticle" +primary_question: "How do traces, spans, sessions, users, and evals relate in FutureAGI Observe?" +direct_answer: "Observe is built on a hierarchy: a span is one operation, spans sharing a trace ID form a trace (one request), traces sharing a session ID form a session, and sessions belong to a user. Eval scores attach to spans or traces. All of it is captured as OpenTelemetry data via the traceAI SDK." +seo: + title: "The FutureAGI Observe observability model" + description: "The entity hierarchy behind Observe — spans, traces, sessions, users, and eval scores — and how traceAI and OpenTelemetry collect it." + primary_keyword: "llm observability data model" + direct_answer: true +geo: + answer_target: "How do traces, spans, sessions, and users relate in FutureAGI Observe?" + llm_summary: "Spans nest into traces by trace ID, traces group into sessions by session ID, sessions belong to users, and eval scores attach to spans or traces — all collected via traceAI over OpenTelemetry." +canonical: "/docs/observe/concepts/observability-model" +related: + - "/docs/observe/concepts/traces" + - "/docs/observe/concepts/spans" + - "/docs/observe/concepts/sessions-and-users" + - "/docs/observe/concepts/otel" +--- + +## About + +Observe doesn't have one data type — it has a small set of objects that nest. Understanding how they relate is the difference between knowing *where* to look and guessing. A **span** is a single operation. Spans that share a trace ID make up a **trace** — one full request. Traces that share a session ID make up a **session** — one multi-turn conversation. Sessions belong to a **user**. And **eval scores** attach on top, to a span or a whole trace. Every view in Observe — the trace list, the span detail drawer, sessions, dashboards, alerts — is a different lens on this one hierarchy. + +All of it is collected the same way: your app emits spans through the [traceAI](/docs/observe/concepts/traceai) SDK, which is built on [OpenTelemetry](/docs/observe/concepts/otel), and Observe reads them. + +--- + +## Mental model + +The objects form a strict containment hierarchy. The ID on each span is what reconstructs it — you don't assemble traces or sessions by hand; shared IDs do that automatically. + + S["Session · session.id"] + S --> T1["Trace · one request"] + S --> T2["Trace · one request"] + T1 --> SP1["Span · llm call"] + T1 --> SP2["Span · tool call"] + SP1 --> EV["Eval score"] + T1 --> EV2["Eval score"]`} /> + +Read it bottom-up when debugging (a bad span → which trace → which session → which user) and top-down when analyzing (a user's sessions → their traces → the spans inside). + +--- + +## Key terms + +| Object | What it is | Identified by | Learn more | +|---|---|---|---| +| **Span** | One operation — an LLM call, tool call, retrieval, or agent step — with input, output, timing, and cost. | Span ID (+ parent span ID) | [Spans](/docs/observe/concepts/spans) | +| **Trace** | One complete request, made of all the spans that share its trace ID. | Trace ID | [Traces](/docs/observe/concepts/traces) | +| **Session** | A multi-turn conversation — the traces that share a session ID. | `session.id` | [Sessions and users](/docs/observe/concepts/sessions-and-users) | +| **User** | One end user, across all their sessions and traces. | `user.id` | [Sessions and users](/docs/observe/concepts/sessions-and-users) | +| **Eval score** | A quality score attached to a span or trace. | Attached to span/trace | [Trace evals](/docs/observe/features/evals) | +| **OpenTelemetry** | The open standard the spans are emitted in. | — | [OpenTelemetry](/docs/observe/concepts/otel) | +| **traceAI** | The SDK that produces the spans. | — | [traceAI SDK](/docs/observe/concepts/traceai) | + +--- + +## How it works in FutureAGI + +Your app emits **spans** through traceAI (or OpenTelemetry directly). Each span carries a trace ID, so all spans from one request form a single **trace**. The backend receives them over OTLP (HTTP or gRPC) and stores them by project — and from there every Observe view runs on the same data. + +|traceAI / OTel SDK| B["OTLP: HTTP or gRPC"] + B --> C["FutureAGI backend"] + C --> D["Observe dashboard"]`} /> + +To enrich the hierarchy — set a `session.id`, a `user.id`, or [tags](/docs/observe/features/tags) — you attach those attributes in code. Wrap the traced work in the context managers and every span inside picks them up: + + +```python +from fi_instrumentation import using_session, using_user + +with using_session("session_123"), using_user("user_456"): + response = run_agent(prompt) +``` + + +For the full set of attributes, see [add attributes and metadata](/docs/traceai/manual-instrumentation/add-attributes-metadata-tags). + +--- + +## Walking the hierarchy when debugging + +The hierarchy is most useful read bottom-up. A typical investigation starts at the smallest object and climbs: + +1. **Start at the span.** A user complained the assistant gave a wrong answer. You open the span that produced it and read its real input and output — the exact prompt and the exact completion, not a paraphrase. +2. **Climb to the trace.** The span alone rarely explains the failure. You move up to its trace and read the other spans in order — the retrieval that fed bad context, the tool call that returned stale data, the agent step that chose the wrong path. The trace is where the *request* becomes legible. +3. **Climb to the session.** If the request looked fine in isolation but the conversation still went wrong, you open its session and read the earlier turns. Multi-turn problems — the assistant losing track, contradicting itself — only show up here. +4. **Climb to the user.** Finally, if the pattern repeats, you pivot to the user to see whether it's one customer's data or a systemic issue across everyone. + +Because the IDs link each level to the next, every climb is one click — you never reassemble context by hand. The same path runs top-down for analysis: start at a user, expand their sessions, then traces, then the spans inside. + +--- + +## When to use this model + +- **Debugging a bad answer** — start at the span that produced it, then read the rest of the trace for context. +- **Analyzing a conversation** — open the session to see every turn in order. +- **Investigating a user** — pivot from a user to all their sessions and traces. +- **Measuring quality** — read eval scores attached at the span or trace level. + +--- + +## Common mistakes + +- **Confusing a span with a trace** — a slow *trace* tells you a request was slow; the *span* tells you which step was slow. They are different levels. See [Spans](/docs/observe/concepts/spans). +- **Expecting sessions without a session ID** — traces only group into a session if they share a `session.id`. Set it in code. +- **Looking for users you never tagged** — per-user views need `user.id` on the spans. + +--- + +## Next steps + + + + The top-level unit: one request, one trace. + + + The operations that make up every trace. + + + Grouping traces by conversation and end user. + + + Send a trace and watch the model fill in. + + diff --git a/src/pages/docs/observe/concepts/otel.mdx b/src/pages/docs/observe/concepts/otel.mdx new file mode 100644 index 00000000..bbcef2b2 --- /dev/null +++ b/src/pages/docs/observe/concepts/otel.mdx @@ -0,0 +1,160 @@ +--- +title: "OpenTelemetry" +description: "OpenTelemetry is the open, vendor-neutral standard Observe uses to collect and export traces — your app emits OTel spans that the traceAI SDK sends to Observe." +slug: "otel" +page_type: "concept" +diataxis: "explanation" +products: ["Observe"] +concept_family: "tracing" +concept_level: "foundational" +primary_question: "What is OpenTelemetry and how does FutureAGI use it?" +direct_answer: "OpenTelemetry (OTel) is the open standard for collecting traces, metrics, and logs. FutureAGI builds on it: your app emits OTel spans that the traceAI SDK exports to Observe." +audience: ["engineer"] +difficulty: "beginner" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-18" +last_diagram_reviewed: "2026-05-25" +schema_type: "TechArticle" +seo: + title: "OpenTelemetry in Observe" + description: "How Observe builds on OpenTelemetry — your app emits OTel spans, the traceAI SDK exports them over OTLP, and Observe reads them. Vendor-neutral instrumentation." + primary_keyword: "what is opentelemetry llm tracing" + direct_answer: true +geo: + answer_target: "What is OpenTelemetry and how does FutureAGI Observe use it?" + llm_summary: "OpenTelemetry (OTel) is the open, vendor-neutral standard for collecting traces, metrics, and logs. Observe builds on it: your app emits OTel spans and the traceAI SDK exports them over OTLP to Observe, so the same instrumentation works across languages and backends." +canonical: "/docs/observe/concepts/otel" +related: + - "/docs/observe/concepts/traceai" + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/observe/features/llm-tracing" +--- + +## About + +[OpenTelemetry](https://opentelemetry.io/) (OTel) is the open, vendor-neutral standard for collecting traces, metrics, and logs from software. It defines how a [span](/docs/observe/concepts/spans) is structured and how spans are exported, so any tool that speaks OTel can read them. FutureAGI is built on it: your app emits OTel spans, and [traceAI](/docs/observe/concepts/traceai) exports them over OTLP to Observe. Because it's a standard, the same instrumentation works across languages, frameworks, and backends. + +--- + +## Why it matters + +Building on a standard is what keeps you un-locked-in. Your instrumentation isn't proprietary to FutureAGI — the same OTel spans can go to any OTel-compatible backend, and FutureAGI can ingest spans from anything that emits them. OTel is also built for scale: batch export handles high-volume tracing without blocking your app. So the choice to standardize on OTel is why "set up tracing once" works regardless of your stack. + +--- + +## Mental model + +OTel sits between your app and FutureAGI: your code (or an instrumentor) creates spans, a processor batches them, and an exporter ships them over OTLP. + + B[OTel SDK: create spans] + B --> C[Span processor: batch] + C -->|OTLP HTTP/gRPC| D[FutureAGI backend] + D --> E[Observe]`} /> + +traceAI plugs into this pipeline — it's the layer that creates *LLM-shaped* spans and configures the exporter to point at FutureAGI. + +--- + +## What OTel gives traceAI + +traceAI doesn't reinvent any of the tracing machinery — it inherits three things from OpenTelemetry and adds LLM meaning on top: + +- **The span model.** OTel defines what a [span](/docs/observe/concepts/spans) is: a named, timed unit of work with a start, an end, attributes, status, and a parent. traceAI uses that exact shape and fills the attributes with LLM-specific keys (prompt, completion, token counts, span kind). Because the shape is standard, every span traceAI produces is a valid OTel span. +- **Context propagation.** OTel carries a trace ID and the current parent through an in-process context, so a span created deeper in your call stack automatically nests under the one above it. This is what lets spans from a single request reconstruct into one trace, and what lets `session.id` and `user.id` flow down to child spans without being passed manually. +- **OTLP export.** Finished spans are batched and shipped over OTLP — the OpenTelemetry wire protocol — to a backend. traceAI's job here is just to point that exporter at FutureAGI; the transport itself is plain OTel. + +The split is deliberate: OTel owns *how spans are structured, linked, and shipped*, and traceAI owns *what an LLM span should say*. + +--- + +## Key terms + +| Term | What it is | +|---|---| +| **OTLP** | The OpenTelemetry Protocol — the wire format spans are exported in, over HTTP or gRPC. | +| **Span processor** | The stage that collects finished spans and hands them to the exporter; batched by default. | +| **Exporter** | The component that ships spans over OTLP to a backend such as FutureAGI. | +| **TracerProvider** | The root object that holds the processor and exporter and produces tracers that create spans. | + +--- + +## When to use + +- You want instrumentation that isn't locked to one vendor. +- You already emit OTel spans and want them in Observe. +- You're tracing across multiple languages or services and need one standard. +- You need high-volume tracing with batched export. + +--- + +## When not to use raw OpenTelemetry directly + +You build on OTel either way — Observe only reads OTel spans. The question is whether to write OTel calls yourself or let [traceAI](/docs/observe/concepts/traceai) do it for you. + +- **For LLM and agent calls, don't hand-write OTel spans.** Raw OTel has no notion of a prompt, a completion, or token cost, so you'd be re-deriving traceAI's LLM conventions by hand and risking inconsistent keys. Use a traceAI instrumentor instead — it produces the standard LLM-shaped spans Observe's filters, evals, and dashboards expect. +- **Don't reach for OTel when you just need one custom step traced.** Wrapping your own function is what [tool spans](/docs/traceai/manual-instrumentation/create-tool-spans) are for; they ride the same provider traceAI already configured. +- **Don't treat OTel as a destination.** It collects and ships spans; it does not store, index, or display them. If you need to read traces, that's Observe's job, not OTel's. + +Raw OTel is the right tool only for non-LLM work — a database call, an HTTP handler — that you want in the same trace tree. + +--- + +## What it isn't + +- **OTel is not a backend.** It collects and exports spans; storing and displaying them is FutureAGI's job. +- **OTel is not traceAI.** traceAI is the FutureAGI layer *on top of* OTel that adds LLM-specific spans and conventions. See [traceAI](/docs/observe/concepts/traceai). +- **OTel is not logging.** It's structured, timed traces — not flat log lines. + +--- + +## How FutureAGI uses OpenTelemetry + +`register()` configures an OTel `TracerProvider` with an OTLP exporter pointed at FutureAGI, and a span processor (batched by default). Auto-instrumentors and manual spans both feed that provider. The `transport` argument selects the OTLP transport — `Transport.HTTP` for OTLP/HTTP or `Transport.GRPC` for OTLP/gRPC: + + +```python +from fi_instrumentation import register, Transport +from fi_instrumentation.fi_types import ProjectType + +# OTLP/HTTP +tp = register(project_type=ProjectType.OBSERVE, project_name="my-app", transport=Transport.HTTP) + +# OTLP/gRPC +tp = register(project_type=ProjectType.OBSERVE, project_name="my-app", transport=Transport.GRPC) +``` + + +Self-hosted deployments point the exporter at their own collector URL. See [Set up tracing](/docs/traceai/manual-instrumentation/set-up-tracing). + +--- + +## Common mistakes + +- **Picking a transport your network blocks.** OTLP/gRPC and OTLP/HTTP use different ports and protocols. If gRPC is blocked by a proxy or firewall, switch to `Transport.HTTP`. +- **Expecting spans without calling `register()`.** Nothing reaches Observe until a `TracerProvider` with the FutureAGI exporter exists. `register()` builds it. +- **Treating OTel as the backend.** OTel only collects and exports spans; storing, indexing, and displaying them is Observe's job. + +--- + +## Next steps + + + + The FutureAGI layer built on OpenTelemetry. + + + The unit OpenTelemetry collects. + + + Configure the OTel exporter to FutureAGI. + + + Read the spans once they arrive. + + diff --git a/src/pages/docs/observe/concepts/sessions-and-users.mdx b/src/pages/docs/observe/concepts/sessions-and-users.mdx new file mode 100644 index 00000000..3e631605 --- /dev/null +++ b/src/pages/docs/observe/concepts/sessions-and-users.mdx @@ -0,0 +1,154 @@ +--- +title: "Sessions and users" +description: "Sessions and users are how Observe groups traces — a session is one multi-turn conversation, a user is one end user across all their sessions and traces." +slug: "sessions-and-users" +page_type: "concept" +diataxis: "explanation" +products: ["Observe"] +concept_family: "tracing" +concept_level: "foundational" +primary_question: "What are sessions and users in FutureAGI?" +direct_answer: "A session groups the traces of one multi-turn conversation; a user groups every session and trace from one end user. Both are set with attributes on your spans." +audience: ["engineer"] +difficulty: "beginner" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-18" +last_diagram_reviewed: "2026-05-25" +schema_type: "TechArticle" +seo: + title: "Sessions and users in Observe" + description: "How Observe groups traces by conversation and by end user — set session.id and user.id on your spans to roll requests up into sessions and users." + primary_keyword: "group llm traces by session and user" + direct_answer: true +geo: + answer_target: "What are sessions and users in FutureAGI Observe?" + llm_summary: "A session groups the traces of one multi-turn conversation; a user groups every session and trace from one end user. You create both by setting session.id and user.id attributes on your spans, and Observe rolls up per-session and per-user metrics." +canonical: "/docs/observe/concepts/sessions-and-users" +related: + - "/docs/observe/concepts/traces" + - "/docs/traceai/manual-instrumentation/set-session-user-id" + - "/docs/observe/features/session" + - "/docs/observe/features/users" +--- + +## About + +Sessions and users are the two ways Observe groups individual traces into something larger. + +A single [trace](/docs/observe/concepts/traces) is one request. But a chatbot conversation is many requests, and one customer has many conversations. A **session** ties together the traces of one multi-turn interaction — a whole support chat, start to finish. A **user** ties together every session and trace belonging to one end user. You create both by setting an attribute on your spans (`session.id` and `user.id`); Observe does the grouping. Use them when a single trace is too small a unit — when you need a customer's whole history, not one message. + +--- + +## Why it matters + +Most real problems span more than one request. A bug report says "the assistant kept losing track" — that is a *session* problem, visible only when you read the conversation in order. A cost spike traces back to *one user* hammering an expensive model — visible only when you roll traces up per user. Without these groupings you are stuck reading isolated requests and reconstructing context by hand. With them, Observe shows per-session and per-user metrics — trace count, latency, cost, eval pass-rate — so behavior over time becomes legible. + +--- + +## Mental model + +Users contain sessions; sessions contain traces; traces contain spans. It is one hierarchy, built from two attributes you set. + + S1["Session (session.id)"] + U --> S2["Session (session.id)"] + S1 --> T1["Trace"] + S1 --> T2["Trace"] + S2 --> T3["Trace"]`} /> + +A trace with a `session.id` joins (or starts) that session; a trace with a `user.id` is attributed to that user. A trace can carry both, one, or neither. + +--- + +## When to use + +- A user reports an issue that played out over several turns, and you need the whole conversation. +- You're analyzing drop-off or escalation across a multi-step chatbot flow. +- A cost or latency spike looks tied to specific end users and you want a per-user breakdown. +- You're checking whether quality (eval pass-rate) differs across users or cohorts. +- You want to filter or group the trace list by conversation or by person. + +--- + +## What it isn't + +- **A session is not a trace.** A trace is one request; a session is the set of traces in one conversation. See [Traces](/docs/observe/concepts/traces). +- **A user is not a session.** A user spans many sessions over time; a session is one sitting. +- **A user ID is not an auth identity.** It is whatever stable identifier you choose to set (often a hashed customer ID) — FutureAGI does not authenticate it, it only groups by it. Avoid putting raw PII in it. + +--- + +## How FutureAGI represents sessions and users + +Both are span attributes you set in your instrumentation: + +| Attribute | What it does | +|---|---| +| `session.id` | Groups every trace carrying the same value into one session. | +| `user.id` | Attributes every trace carrying the same value to one user. | + +A session is created when the first trace with a given `session.id` arrives; a user is tracked the same way for `user.id`. You can see the groupings in the [Sessions](/docs/observe/features/session) and [Users](/docs/observe/features/users) views, each with its own rolled-up metrics. To set the attributes, see [Set session and user IDs](/docs/traceai/manual-instrumentation/set-session-user-id). + +Set both in code by wrapping the traced work in the context managers — every span created inside picks up the IDs: + + +```python +from fi_instrumentation import using_session, using_user + +with using_session("session_123"), using_user("user_456"): + # traced work here; spans get session.id and user.id + response = run_agent(prompt) +``` + + +--- + +## How the IDs propagate + +You don't set `session.id` and `user.id` on every span by hand. You set them once, around a block of work, and they flow down to every span created inside that block. This is the same context-propagation mechanism [OpenTelemetry](/docs/observe/concepts/otel) uses to attach a trace ID to its child spans — the context managers put the IDs into the active context, and each new span reads them from there. + +That has two consequences worth understanding: + +- **Scope follows the `with` block.** Spans created inside the block carry the IDs; spans created after it exits do not. So wrap the unit you mean to group — the whole turn's handler, not just the model call — and the retrieval, tool, and LLM spans of that turn all land in the same session and user. +- **The same value across turns is what builds the grouping.** A session is just "the set of traces that share one `session.id`." To extend a conversation, reuse its `session.id` on the next turn's block; to attribute every turn to one person, reuse that person's `user.id`. Observe does the rollup the moment a second trace arrives with a value it has already seen. + +Because the grouping is built purely from these attribute values, getting the IDs right at instrumentation time is the whole job — there's no separate "create session" or "create user" call. + +--- + +## When not to use sessions and users + +- **A one-shot request with no follow-up.** A single classify-or-extract call that never continues a conversation gains nothing from a session — the trace alone is the right unit. Reach for sessions when work spans more than one request. +- **Per-request technical IDs.** Don't put a request ID, a trace ID, or a fresh UUID in `session.id`. That produces a session of exactly one trace and defeats the grouping. Use a value that's stable across the turns you want joined. +- **As an access or auth identity.** `user.id` only groups traces; FutureAGI doesn't authenticate it. Don't rely on it for authorization, and don't store raw PII in it. + +--- + +## Common mistakes + +- **Expecting a session without setting `session.id`.** Traces only roll up into a session when they share a `session.id`. Without it, every turn stays an isolated trace. +- **Generating a new session ID per request.** A session is one conversation; reuse the same `session.id` across its turns. A fresh ID each turn produces a session of one trace. +- **Putting raw PII in `user.id`.** The user ID only groups traces — FutureAGI doesn't authenticate it. Use a stable, non-sensitive identifier (often a hashed customer ID), not an email or name. + +--- + +## Next steps + + + + The single request that sessions and users group together. + + + Add the attributes that drive the grouping. + + + Read multi-turn conversations and per-session metrics. + + + Read per-user activity, cost, and quality. + + diff --git a/src/pages/docs/observe/concepts/spans.mdx b/src/pages/docs/observe/concepts/spans.mdx new file mode 100644 index 00000000..fbfc5780 --- /dev/null +++ b/src/pages/docs/observe/concepts/spans.mdx @@ -0,0 +1,158 @@ +--- +title: "Spans" +description: "A span is one operation inside a trace in FutureAGI Observe — a model call, tool call, retrieval, agent step, or evaluator run — with its inputs, outputs, timing, and cost." +slug: "spans" +page_type: "concept" +diataxis: "explanation" +products: ["Observe"] +concept_family: "tracing" +concept_level: "foundational" +audience: ["engineer"] +difficulty: "beginner" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-18" +last_diagram_reviewed: "2026-06-17" +schema_type: "TechArticle" +primary_question: "What is a span in FutureAGI?" +direct_answer: "A span is one timed operation inside a trace — a model call, tool call, retrieval, agent step, or evaluator run — recording its inputs, outputs, duration, and cost." +seo: + title: "What is a span in FutureAGI Observe?" + description: "A span is one timed operation inside a trace — a model call, tool call, retrieval, agent step, or evaluator run — with inputs, outputs, duration, cost, and a span kind." + primary_keyword: "what is a span llm tracing" + direct_answer: true +geo: + answer_target: "What is a span in LLM tracing?" + llm_summary: "A span is one timed operation inside a trace in FutureAGI Observe — a model call, tool call, retrieval, agent step, guardrail, or evaluator run — recording its inputs, outputs, duration, cost, and a span kind, nested under the step that triggered it." +canonical: "/docs/observe/concepts/spans" +related: + - "/docs/observe/concepts/traces" + - "/docs/observe/concepts/observability-model" + - "/docs/observe/features/llm-tracing" + - "/docs/traceai/manual-instrumentation/create-tool-spans" + - "/docs/traceai/manual-instrumentation/semantic-conventions" +--- + +## About + +A span is one operation inside a trace. + +In FutureAGI, a span records a single step in an AI request — a model call, tool call, retrieval, agent decision, guardrail check, evaluator run, or custom function. A trace is the whole request; spans are the steps inside it. Each span carries its own input and output, start and end time, duration, and (for model calls) token counts and cost. Spans matter because they show *where* latency, cost, errors, or bad outputs entered the system — the trace tells you a request was slow, the span tells you which step was slow. + +--- + +## Why it matters + +A response is the sum of its steps, and failures usually live in one step, not the whole request. A span isolates that step: it shows the exact prompt sent to the model, the arguments passed to a tool, the chunks a retriever returned, or the score an evaluator assigned. traceAI captures spans automatically for supported frameworks, and lets you [add custom spans](/docs/traceai/manual-instrumentation/create-tool-spans) where automatic instrumentation does not reach — so the part of your pipeline you most need to debug is never a black box. + +--- + +## Mental model + +Spans nest. A parent span (say, an agent) contains the child spans it triggered, and each child can have children of its own. The nesting is how FutureAGI reconstructs which step caused which. + + B["chain span"] + A --> C["tool span"] + B --> D["retriever span"] + B --> E["llm span"]`} /> + +Each box is a span with its own timing and attributes. The edges are parent-child links recorded on the spans, which is what lets the [trace explorer](/docs/observe/features/llm-tracing) draw the tree. + +--- + +## When to use + +Spans aren't optional or an alternative to traces — every trace is made of spans, captured automatically. You'll look at an individual span when: + +- A request is slow and you need to find the single step responsible. +- An answer is wrong and you want the exact prompt and output of the step that produced it. +- A tool or retrieval misbehaved and you need its real arguments and results. +- You're adding instrumentation and need to decide what each step should capture. +- You want an eval or annotation attached to a specific step, not the whole request. + +--- + +## When a span isn't the right level + +A span is the right unit for *one step*. When your question is about something bigger or smaller, look elsewhere: + +- **For the whole request, read the [trace](/docs/observe/concepts/traces).** "Was this request slow?" or "what was the full sequence of steps?" is a trace question — the span only tells you about its own step, not how the steps fit together. +- **For behavior across many requests, use [dashboards](/docs/observe/features/dashboard).** Trends like p95 latency, cost over time, or eval pass-rate are aggregates over many traces. A single span can't answer them; chart the metric instead. + +Reach for the span when you've already narrowed the problem to a step and need its exact inputs, outputs, and timing. + +--- + +## What it isn't + +- **A span is not a trace.** A trace is the full request; a span is one operation in it. See [Traces](/docs/observe/concepts/traces). +- **A span is not a log line.** A log is a flat text event; a span is a timed unit with structured input, output, status, and attributes, linked to a parent. +- **A span is not an event.** Events are point-in-time markers *inside* a span (e.g. an exception); the span is the operation that contains them. + +--- + +## How FutureAGI represents a span + +Every span has identification (its own ID, the trace ID, and an optional parent span ID), execution details (name, kind, input, output, error metadata), timing (start, end, latency), and attributes (key-value metadata, including token counts and cost for model calls). The **kind** classifies the operation, set via `fi.span.kind`: + +| Span kind | Captures | +|---|---| +| `LLM` | A model call — prompt, completion, tokens, cost, model name. | +| `TOOL` | A function or external tool call — name, arguments, output. | +| `CHAIN` | A step in a sequential pipeline. | +| `RETRIEVER` | A search or document fetch — query and results. | +| `EMBEDDING` | A text-to-vector operation. | +| `RERANKER` | A reordering of retrieved results. | +| `AGENT` | An autonomous decision step. | +| `GUARDRAIL` | A safety or policy check. | +| `EVALUATOR` | A quality assessment attached to the span. | +| `UNKNOWN` | Fallback for an unclassified operation. | + +Attribute keys must be non-null strings; values must be a string, boolean, integer, float, or an array of those. Use the standard keys so spans stay queryable — see [semantic conventions](/docs/traceai/manual-instrumentation/semantic-conventions). + +To enrich the current span, set attributes on it in code: + + +```python +from opentelemetry import trace + +span = trace.get_current_span() +span.set_attribute("fi.span.kind", "LLM") +span.set_attribute("input.value", prompt) +span.set_attribute("output.value", completion) +``` + + +--- + +## Common mistakes + +- **Putting everything in one span.** A single span that wraps the whole request hides where latency, cost, or a bad output came from. Give each operation its own span so the trace tree is readable. +- **Leaving the span kind unset.** Without `fi.span.kind`, a span falls back to `UNKNOWN` and drops out of kind-based filters and dashboards. Set it to `LLM`, `TOOL`, `RETRIEVER`, and so on. +- **Storing non-primitive attribute values.** Attribute values must be a string, boolean, integer, float, or an array of those. Serialize objects or dicts to a string first, or the attribute is dropped. + +--- + +## Next steps + + + + The full request that spans are grouped into. + + + How spans, traces, sessions, and users fit together. + + + Add custom spans where auto-instrumentation stops. + + + The standard attribute keys for spans. + + + Inspect spans inside a trace. + + diff --git a/src/pages/docs/observe/concepts/traceai.mdx b/src/pages/docs/observe/concepts/traceai.mdx new file mode 100644 index 00000000..de7252b1 --- /dev/null +++ b/src/pages/docs/observe/concepts/traceai.mdx @@ -0,0 +1,158 @@ +--- +title: "traceAI SDK" +description: "traceAI is FutureAGI's open-source instrumentation SDK on OpenTelemetry — it captures LLM, tool, and retrieval calls as standardized spans that Observe reads as traces." +slug: "traceai" +page_type: "concept" +diataxis: "explanation" +products: ["Observe"] +concept_family: "tracing" +concept_level: "foundational" +primary_question: "What is the traceAI SDK?" +direct_answer: "traceAI is FutureAGI's open-source instrumentation SDK on OpenTelemetry. It captures model, tool, and retrieval calls as standardized spans, and Observe is the product that reads them as traces." +audience: ["engineer"] +difficulty: "beginner" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-18" +last_diagram_reviewed: "2026-05-25" +schema_type: "TechArticle" +seo: + title: "The traceAI SDK" + description: "traceAI is FutureAGI's open-source instrumentation SDK on OpenTelemetry. It captures LLM, tool, and retrieval calls as standardized spans that Observe reads as traces." + primary_keyword: "what is traceai instrumentation" + direct_answer: true +geo: + answer_target: "What is the traceAI SDK and how does it relate to Observe?" + llm_summary: "traceAI is FutureAGI's open-source instrumentation SDK, built on OpenTelemetry. It captures model, tool, and retrieval calls as standardized spans across frameworks; Observe is the product that reads those spans as traces." +canonical: "/docs/observe/concepts/traceai" +related: + - "/docs/observe/concepts/otel" + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/auto" +--- + +## About + +traceAI is the instrumentation SDK; Observe is the product that reads its traces. traceAI is FutureAGI's open-source instrumentation SDK, built on [OpenTelemetry](/docs/observe/concepts/otel). It's a set of conventions and per-framework instrumentors that capture what your AI app does — model calls, tool calls, retrievals, agent steps — and map them to standardized [span](/docs/observe/concepts/spans) attributes. Add the instrumentor for your framework, and those calls become traces in Observe without hand-writing spans. traceAI is natively supported by FutureAGI but emits standard OTel, so it works with any OTel-compatible backend too. + +--- + +## Why it matters + +Raw OpenTelemetry knows nothing about LLMs — it has no concept of a prompt, a completion, token cost, or a tool call. traceAI fills that gap: it turns framework calls into *LLM-shaped* spans with consistent keys, so a LangChain trace and an OpenAI trace look the same in Observe and are queryable the same way. That standardization is what makes filtering, evals, and dashboards work across different stacks. + +--- + +## Mental model + +traceAI is the adapter between your framework and OpenTelemetry: the instrumentor wraps the framework, produces standardized spans, and hands them to the OTel pipeline that exports to FutureAGI. + + B[traceAI instrumentor] + B --> C[Standardized OTel spans] + C --> D[FutureAGI Observe]`} /> + +You pick the instrumentor that matches your framework; the rest of the pipeline is the same OTel flow for everyone. + +--- + +## SDK vs product, auto vs manual + +Two distinctions explain most of how traceAI is used. + +**traceAI is the SDK; Observe is the product.** traceAI runs inside your application and produces spans. [Observe](/docs/observe) runs in FutureAGI and reads them — searching, replaying, scoring, and alerting on the traces those spans form. One emits the data; the other consumes it. They meet only at the span: a standard [OpenTelemetry](/docs/observe/concepts/otel) span on the wire. + +**Auto and manual instrumentation are two ways to produce those spans, and they coexist.** Auto-instrumentation is a per-framework instrumentor that wraps a library — install `traceAI-openai`, call `.instrument()`, and every OpenAI call becomes a span with no span code in your app. Manual instrumentation is for the parts no instrumentor reaches: your own business functions, custom retrieval, glue logic. You wrap those as [tool spans](/docs/traceai/manual-instrumentation/create-tool-spans) yourself. Most real apps use both — auto for the framework calls, manual for the code between them — and both feed the same provider, so they nest into one trace. + +That shared provider is what `register()` sets up. Conceptually it does one thing: it builds the OpenTelemetry `TracerProvider` — the exporter pointed at FutureAGI plus a batched span processor — and makes it the active provider. After that call, both the auto-instrumentors and any manual spans attach to it automatically. Nothing reaches Observe until `register()` has run, because before it there is no exporter to ship spans to. + +--- + +## When to use + +- You want LLM/agent calls traced without writing spans by hand. +- You use a supported framework (OpenAI, LangChain, LlamaIndex, CrewAI, …). +- You want consistent, queryable span attributes across different SDKs. +- You want instrumentation that stays portable across OTel backends. + +--- + +## When not to use traceAI + +- **For non-LLM work that has no LLM meaning to capture.** A plain database query or HTTP handler doesn't benefit from traceAI's LLM conventions. Trace it with raw [OpenTelemetry](/docs/observe/concepts/otel) instead; it still lands in the same trace tree. +- **As a stand-in for the backend.** traceAI only produces spans — it doesn't store, search, or display them. If you need to read traces, that's [Observe](/docs/observe), not the SDK. +- **For a framework with no instrumentor, expecting auto-capture.** Each instrumentor wraps one specific framework. If yours isn't in the [catalog](/docs/traceai/auto), auto-instrumentation won't see it — reach for [manual spans](/docs/traceai/manual-instrumentation/create-tool-spans) rather than the wrong instrumentor. + +--- + +## What it isn't + +- **traceAI is not a backend.** It produces spans; FutureAGI stores and displays them. +- **traceAI is not a replacement for OpenTelemetry.** It's complementary — conventions and instrumentors *on top of* OTel. See [OpenTelemetry](/docs/observe/concepts/otel). +- **traceAI is not only for FutureAGI.** It emits standard OTel and works with any compatible backend. + +--- + +## How FutureAGI represents traceAI + +traceAI ships as the core `fi-instrumentation-otel` package plus a per-framework instrumentor you install alongside it. A sample of the Python instrumentors: + +| Package | Instruments | +|---|---| +| `traceAI-openai` | OpenAI | +| `traceAI-anthropic` | Anthropic | +| `traceAI-langchain` | LangChain | +| `traceAI-llamaindex` | LlamaIndex | +| `traceAI-crewai` | CrewAI | +| `traceAI-bedrock` | AWS Bedrock | +| `traceAI-litellm` | LiteLLM | +| `traceAI-google-adk` | Google ADK | +| `traceAI-dspy` | DSPy | +| `traceAI-haystack` | Haystack | + +See the full, current list in the [Auto Instrumentation catalog](/docs/traceai/auto). To wire one up, see [Set up tracing](/docs/traceai/manual-instrumentation/set-up-tracing). + +Install the core package and an instrumentor, register a tracer provider, then instrument the framework with it: + + +```python +# pip install fi-instrumentation-otel traceAI-openai +from fi_instrumentation import register, Transport +from fi_instrumentation.fi_types import ProjectType +from traceai_openai import OpenAIInstrumentor + +tp = register(project_type=ProjectType.OBSERVE, project_name="my-app", transport=Transport.GRPC) +OpenAIInstrumentor().instrument(tracer_provider=tp) +``` + + +--- + +## Common mistakes + +- **Calling `.instrument()` after the client is created → no spans.** Run `OpenAIInstrumentor().instrument(...)` before you construct the framework client, or its calls aren't wrapped and nothing is traced. +- **Registering with the wrong `project_type`.** Use `ProjectType.OBSERVE` for production tracing; a mismatched project type sends spans somewhere you won't find them in Observe. +- **Installing the wrong instrumentor for your framework.** Each instrumentor wraps one framework. A LangChain app needs `traceAI-langchain`; `traceAI-openai` won't capture it. + +--- + +## Next steps + + + + The standard traceAI is built on. + + + What traceAI produces. + + + Install an instrumentor and start capturing. + + + Every supported framework. + + diff --git a/src/pages/docs/observe/concepts/traces.mdx b/src/pages/docs/observe/concepts/traces.mdx new file mode 100644 index 00000000..bc772bf2 --- /dev/null +++ b/src/pages/docs/observe/concepts/traces.mdx @@ -0,0 +1,138 @@ +--- +title: "Traces" +description: "A trace is the complete record of one AI request in FutureAGI Observe — every model call, tool call, retrieval, output, latency, and cost for a single response, linked into a tree." +slug: "traces" +page_type: "concept" +diataxis: "explanation" +products: ["Observe"] +concept_family: "tracing" +concept_level: "foundational" +audience: ["engineer"] +difficulty: "beginner" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_diagram_reviewed: "2026-06-17" +schema_type: "TechArticle" +primary_question: "What is a trace in FutureAGI?" +direct_answer: "A trace is the complete record of one AI request: the ordered spans — model calls, tool calls, retrievals, and evaluator runs — that produced a single response." +seo: + title: "What is a trace in FutureAGI Observe?" + description: "A trace is the complete record of one AI request — the tree of spans (model, tool, retrieval, evaluator) that produced a single response, with latency, cost, and quality." + primary_keyword: "what is a trace llm observability" + direct_answer: true +geo: + answer_target: "What is a trace in LLM observability?" + llm_summary: "A trace is the complete record of one AI request in FutureAGI Observe: the ordered, nested spans — model calls, tool calls, retrievals, and evaluator runs — that produced a single response, linked into a tree by a shared trace ID." +canonical: "/docs/observe/concepts/traces" +related: + - "/docs/observe/concepts/spans" + - "/docs/observe/concepts/observability-model" + - "/docs/observe/features/llm-tracing" + - "/docs/traceai/manual-instrumentation/set-up-tracing" +--- + +## About + +A trace is the complete record of one AI request. + +For example, when a support agent answers a refund question, the reply is only the last artifact. The trace shows everything that produced it: the user input, the model's intent classification, the order-lookup tool call, the retrieved policy text, the final generation, and the latency, token cost, and evaluator scores at each step. A trace is made of **spans** — one per operation — linked into a tree by a shared trace ID. Reach for a trace when you need to debug why a request failed, reproduce a production issue, or turn a bad response into a regression test. + +--- + +## Why it matters + +Without traces, a wrong or slow AI response is a dead end — you see the output but not the five steps that caused it, so debugging becomes guesswork over logs. A trace turns that into a readable execution path: you can see that the retriever returned the wrong policy chunk, or that one tool call took four seconds, or that an eval flagged the answer as unsupported. In FutureAGI, traceAI captures this automatically for supported frameworks and sends it to Observe, where latency, cost, errors, and quality are all attached to the same request. + +--- + +## Mental model + +A trace is a **tree** — a branching hierarchy where each step nests under the step that triggered it. The whole request *is* the trace; its **root span** is the operation that started the request, and every other span hangs beneath the step that called it. For example, a single support-agent request looks like this: + + S1["llm.intent_classification"] + T --> S2["tool.check_order_status"] + T --> S3["chain.generate_reply"] + S3 --> S4["retriever.knowledge_base"] + S3 --> S5["llm.response_generation"]`} /> + +Every span in that tree shares one trace ID, so FutureAGI can reassemble the steps — even when they run across async tasks or services — into a single request you can read top to bottom. + +--- + +## Key terms + +| Term | What it is | +|---|---| +| **Trace** | The complete record of one AI request — the whole tree of spans that produced a single response. | +| **Root span** | The top-level operation that started the request; every other span nests beneath it. | +| **Span** | One operation inside the trace — a model call, tool call, retrieval, or evaluator run. | +| **Trace ID** | The shared identifier that links every span into one trace. | + +--- + +## When to use + +- A user reports a bad answer and you need the exact request to see what the AI actually did. +- A request was slow and you want to find which step — model, tool, or retrieval — spent the time. +- You're reproducing a production failure and need the real inputs, not a paraphrase. +- You want to promote a real production request into a dataset for evals or fine-tuning. +- You're reviewing how an agent reasoned, step by step, before it answered. + +--- + +## What it isn't + +- **A trace is not a span.** A span is one operation; a trace is the whole request made of many spans. See [Spans](/docs/observe/concepts/spans). +- **A trace is not a session.** A session groups many traces from one conversation or user; a trace is a single request inside it. See [Sessions](/docs/observe/features/session). +- **A trace is not a log line.** Logs are flat text events; a trace is a timed, structured tree with inputs, outputs, cost, and [eval scores](/docs/observe/features/evals) attached to each step. + +--- + +## How FutureAGI represents a trace + +traceAI emits OpenTelemetry spans; FutureAGI groups them by trace ID and stores the tree. In the [trace explorer](/docs/observe/features/llm-tracing), one row is one trace, and opening it shows the span tree plus per-step input, output, latency, token counts, cost, and any eval or annotation attached. A trace carries: + +| Field | What it holds | +|---|---| +| **Trace ID** | The shared identifier that links every span in the request. | +| **Trace name** | The top-level task, e.g. `support_agent.run`. | +| **Spans** | The ordered operations — model, tool, retriever, evaluator — that ran. | +| **Status** | OK, or ERROR if any span failed. | +| **Latency / tokens / cost** | Totals rolled up from the spans. | +| **Session & user IDs** | Optional links that group the trace by conversation and end user. | + +To get traces flowing, see [Set up tracing](/docs/traceai/manual-instrumentation/set-up-tracing). To name the operations inside a trace consistently, see [semantic conventions](/docs/traceai/manual-instrumentation/semantic-conventions). + +--- + +## Common mistakes + +- **Treating every model call as its own trace.** One user request is one trace; the model, tool, and retrieval calls inside it are spans, not separate traces. Splitting them breaks the request into fragments you can't read end to end. +- **Logging only the final answer instead of the whole request.** The reply is the last artifact, not the trace. If you only record the output, you lose the inputs, intermediate steps, and per-step cost — exactly what you need to debug. +- **Reusing one trace ID across unrelated requests.** A trace ID links spans into a single request; sharing it across requests collapses them into one unreadable tree. Group related requests with a `session.id` instead. + +--- + +## Next steps + + + + The individual operations a trace is built from. + + + How spans, traces, sessions, and users fit together. + + + Read and debug traces in Observe. + + + Instrument your app so it emits traces. + + + Group multiple traces into one conversation. + + diff --git a/src/pages/docs/observe/features/alerts.mdx b/src/pages/docs/observe/features/alerts.mdx index b86f7f2c..66ab7ca1 100644 --- a/src/pages/docs/observe/features/alerts.mdx +++ b/src/pages/docs/observe/features/alerts.mdx @@ -1,84 +1,162 @@ --- -title: "Alerts and Monitors: Observe Metric Threshold Notifications" -description: "Define monitors on Observe project metrics (system or evaluation) and get notified by email or Slack when values cross a threshold." +title: "Alerts and monitors" +description: "Define monitors on Observe metrics — error rate, latency, cost, or eval scores — and get notified by email or Slack when a value crosses a threshold." +slug: "alerts" +page_type: "feature" +products: ["Observe"] +feature: "Alerts and monitors" +feature_status: "stable" +ui_surfaces: ["Observe > Alerts"] +audience: ["engineer"] +difficulty: "beginner" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-05-25" +last_screenshotted: "2026-05-25" +schema_type: "TechArticle" +seo: + title: "Alerts and monitors in Observe" + description: "Define monitors on Observe metrics — error rate, latency, cost, or eval scores — and get notified by email or Slack when a value crosses a threshold." + primary_keyword: "llm observability alerts and monitors" + direct_answer: true +geo: + answer_target: "How do I get alerted when an Observe metric crosses a threshold?" + llm_summary: "A monitor watches one Observe metric on a schedule — error rate, latency, cost, or an eval score — and notifies you by email or Slack when a value crosses a threshold. You set the metric, threshold, direction, and frequency, then review alert history and mute or resolve as needed." +canonical: "/docs/observe/features/alerts" +related: + - "/docs/observe/features/dashboard" + - "/docs/observe/features/evals" + - "/docs/observe/reference/dashboard-metric-definitions" + - "/docs/observe/troubleshooting/alerts-did-not-fire" --- ## About -**Alerts and monitors** notify you when a metric goes above or below a value you set. Pick a metric (error rate, latency, cost, or an eval score), define a threshold, and choose where to get notified: email, Slack, or both. Monitors check the metric on a schedule. If the threshold is breached, you get an alert. You can review past alerts, mark them resolved, or mute a monitor without deleting it. +A monitor watches one Observe metric on a schedule and notifies you when it crosses a threshold. Pick a metric — error rate, latency, cost, or an eval score — set a threshold and direction, and choose where alerts go: email, Slack, or both. When the threshold is breached the monitor creates an alert log and sends the notification. You can review past alerts, mark them resolved, or mute a monitor without deleting it. Monitors are how Observe tells *you* something broke instead of you watching a dashboard. --- ## When to use -- **Catch errors early**: Get notified when error rate or API failure rate spikes after a deployment. -- **Stay within latency limits**: Alert when response time goes above your target. -- **Control costs**: Track token usage and get a warning before you hit your budget. -- **Monitor eval quality**: Know when a pass/fail eval like toxicity starts failing more often. -- **Stay informed without watching dashboards**: Send alerts to email, Slack, or both. +- **Catch errors early** — alert when error rate or LLM API failure rate spikes after a deploy. +- **Hold latency limits** — alert when response time goes above your target. +- **Control cost** — warn before token usage hits your budget. +- **Guard quality** — alert when a pass/fail eval (e.g. toxicity) starts failing more often. --- -## How to +## When not to use + +- **Exploring trends** — a monitor is a tripwire, not a chart; use [Dashboards](/docs/observe/features/dashboard). +- **Debugging one request** — use the [trace explorer](/docs/observe/features/llm-tracing). +- **A non-`observe` project** — monitors run on `observe` projects only; an experiment project has nothing for a monitor to watch. + +--- + +## How it works + +A monitor pairs one metric with one threshold rule and runs on a fixed cadence. Each run reads the metric over its window and compares the value to the threshold; when the comparison is true the monitor writes a `UserAlertMonitorLog` record and sends the notification to its configured channels. + +There are two threshold shapes: + +- **Static** — the value is compared against a fixed number. You can set a `critical_threshold_value` and an optional `warning_threshold_value`, so one monitor can raise a softer warning before a hard breach. +- **Percentage change** — the value is compared against a baseline computed over `auto_threshold_time_window` (default one week), so the monitor fires on a *relative* swing rather than an absolute number. This needs enough history in the baseline window before it can evaluate. + +The `threshold_operator` (`Greater than` / `Less than`) sets the direction of the comparison, so the same metric can alert on a spike or on a drop. Alerts carry their severity, the message, the time window, and a link back into Observe, which is what you act on in the alert history. + +--- + +## Set up a monitor - Create a monitor for an Observe project and select the **metric type**: - ![Choose the metric](/screenshot/product/observe/1.png) + Create a monitor for an Observe project and pick the metric type. - - **System metrics**: count of errors, error-free session rates, LLM API failure rates, span response time, LLM response time, token usage, daily/monthly tokens spent. - - **Evaluation metrics**: attach an eval config for that project. For pass/fail or choice evals you can set **threshold_metric_value** to the specific value to monitor (e.g. fail rate or a choice label). + Create-monitor form selecting a metric, threshold, and notification channels + *Building a monitor: metric → threshold → notifications, in one form.* - The monitor is scoped to one project (Observe projects only). + - **System metrics:** error count, error-free session rate, LLM API failure rate, span response time, LLM response time, token usage, daily/monthly tokens spent. + - **Evaluation metrics:** attach an eval config for the project. For pass/fail or choice evals, set `threshold_metric_value` to the value to watch (e.g. a fail rate or a choice label). - Set how the alert is triggered: - ![Define the threshold](/screenshot/product/observe/2.png) - - - **threshold_operator**: **Greater than** or **Less than** (the current metric value is compared to the threshold). - - **threshold_type**: how the threshold is determined: - - **Static**: you set fixed **critical_threshold_value** and optionally **warning_threshold_value**. Alert fires when the metric is greater than (or less than) these values. - - **Percentage change**: threshold is based on percentage change from a baseline (e.g. historical mean over a time window). You set **critical_threshold_value** and optionally **warning_threshold_value** as percentage values. **auto_threshold_time_window** (default one week, in minutes) defines the window used to compute the baseline. - - When the condition is met, the system creates an alert log (critical or warning) and triggers notifications. + - **`threshold_operator`** — `Greater than` or `Less than`. + - **`threshold_type`** — `Static` (fixed `critical_threshold_value`, optional `warning_threshold_value`) or `Percentage change` (compared to a baseline; `auto_threshold_time_window` sets the baseline window, default one week). - - **alert_frequency** is how often the monitor is evaluated, in minutes (minimum 5, default 60). The monitor runs on this schedule and checks the metric over the relevant time window. If the threshold is breached, an alert is created and notifications are sent. + + `alert_frequency` is how often the monitor runs, in minutes — **minimum 5, default 60**. Each run checks the metric over its window and fires an alert if the threshold is breached. - - **Email**: add up to five addresses in **notification_emails**. They receive an email when an alert is triggered (subject and body include alert name, message, and type). - - **Slack**: set **slack_webhook_url** to your Slack incoming webhook. Optional **slack_notes** are included in the message. - ![Configure notifications](/screenshot/product/observe/3.png) - You can use email only, Slack only, or both. Mute a monitor with **is_mute** to stop notifications without deleting it. + - **Email** — up to **5** addresses in `notification_emails`. + - **Slack** — set `slack_webhook_url` (an incoming webhook); optional `slack_notes` are included. + + Use email only, Slack only, or both. Mute with `is_mute` to pause notifications without deleting the monitor. - Alert history is stored as **UserAlertMonitorLog** records (critical/warning, message, time window, link). You can list logs for a monitor, see when each alert fired, and mark them resolved. Use the monitor detail view in the UI to see trend data and unresolved count. + Alert history is stored as `UserAlertMonitorLog` records (critical or warning, with message, time window, and a link). List them per monitor, see when each fired, and mark them resolved. + + Alerts list showing past alerts with severity, message, and resolved status + *Alert history. The unresolved count is your live to-do list.* - - Monitors are only available for projects with **trace_type** `observe`. Optional **filters** (same structure as eval-task filters) can narrow which spans are included when computing the metric. - +--- + +## Inputs and parameters + +| Parameter | Detail | +|---|---| +| `metric type` | System metric or an attached evaluation metric. | +| `threshold_operator` | `Greater than` / `Less than`. | +| `threshold_type` | `Static` or `Percentage change`. | +| `critical_threshold_value` / `warning_threshold_value` | The trigger values (warning optional). | +| `auto_threshold_time_window` | Baseline window for percentage-change, default one week (minutes). | +| `alert_frequency` | Evaluation cadence, min 5 / default 60 minutes. | +| `notification_emails` | Up to 5 recipients. | +| `slack_webhook_url`, `slack_notes` | Slack channel + optional message. | +| `is_mute` | Pause notifications without deleting. | +| `filters` | Optional; same structure as eval-task filters, to narrow the spans. | + +--- + +## Edge cases and limits + +- Monitors are available **only for `observe` projects**. +- A `Percentage change` monitor needs enough history in its baseline window to compute against — a brand-new project may not alert until data accumulates. +- Muting (`is_mute`) stops notifications but the monitor keeps evaluating and logging. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| Alert never fired | The frequency window is longer than the spike, so the breach passed between runs. | Lower `alert_frequency` so a run lands inside the spike. | +| Slack notification is silent | The webhook was revoked and now returns 401. | Generate a new incoming webhook and update `slack_webhook_url`. | +| Percentage-change alert never triggers | The baseline window is empty on a new project. | Wait for the baseline window to accumulate data, or switch to a `Static` threshold. | +| Email not received | The address isn't in the recipient list. | Add the address to `notification_emails` (up to 5). | +| Only critical alerts, never a heads-up | No `warning_threshold_value` was set, so the monitor jumps straight to critical. | Set a `warning_threshold_value` ahead of the critical one to get an earlier, softer alert. | +| Notifications stopped but the monitor still logs | The monitor is muted (`is_mute`), which pauses notifications while it keeps evaluating. | Clear `is_mute` to resume notifications. | --- -## Next Steps +## Related features - - Connect the SDK and start capturing traces. + + Chart the same metrics you alert on. - - Run evaluations on your traced spans to score quality. + + Produce the eval scores a monitor can watch. - - Group traces into sessions for multi-turn analysis. + + Investigate the requests behind a breached threshold. - - View activity and metrics per end user. + + Check whether a spike is concentrated in specific users. diff --git a/src/pages/docs/observe/features/dashboard.mdx b/src/pages/docs/observe/features/dashboard.mdx index 2ef91ad5..37e42f88 100644 --- a/src/pages/docs/observe/features/dashboard.mdx +++ b/src/pages/docs/observe/features/dashboard.mdx @@ -1,85 +1,148 @@ --- -title: "Dashboards: Custom Metric Visualization in Observe" -description: "Build custom dashboards with widgets to visualize your Observe project metrics, traces, and performance data in one place." +title: "Dashboards" +description: "Build dashboards in FutureAGI Observe to chart latency, cost, tokens, error rate, and eval scores over time — create one, add widgets, and share it." +slug: "dashboard" +page_type: "feature" +products: ["Observe"] +feature: "Dashboards" +feature_status: "stable" +ui_surfaces: ["Observe > Dashboards"] +frameworks: [] +audience: ["engineer"] +difficulty: "beginner" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-05-25" +last_screenshotted: "2026-06-18" +schema_type: "TechArticle" +seo: + title: "Dashboards in FutureAGI Observe — chart LLM trace metrics" + description: "Build Observe dashboards from production traces: create a dashboard, add widgets, pick a metric and chart type, and share the view. Setup, widget options, and limits." + primary_keyword: "llm observability dashboards" + direct_answer: true +geo: + answer_target: "How do I build a dashboard over LLM trace metrics in FutureAGI Observe?" + llm_summary: "In FutureAGI Observe, create a dashboard, add a widget, choose its metric and chart type with optional aggregation, grouping, and filters, then share the dashboard with your project. Widgets chart latency, cost, tokens, error rate, and eval scores over time." +canonical: "/docs/observe/features/dashboard" +related: + - "/docs/observe/features/alerts" + - "/docs/observe/reference/dashboard-metric-definitions" + - "/docs/observe/features/llm-tracing" + - "/docs/observe/features/evals" --- ## About -**Dashboards** let you create custom views of your project data. Each dashboard is a collection of widgets, and each widget runs a query against your data and displays the result as a chart or table. You can track error rates, latency, token usage, eval scores, or any metric from your spans and experiments. Dashboards work across project types and are shareable across your team. +A dashboard is a set of widgets that chart your Observe metrics over time. Where the [trace explorer](/docs/observe/features/llm-tracing) shows individual requests, a dashboard shows the roll-up — latency trends, daily cost, error rate, token usage, eval pass-rate — so you can watch production health at a glance and compare periods. + +Each widget runs one query and renders it as a chart or table. You choose its metric, chart type, aggregation, grouping, filters, and time range. Dashboards are shared across the project, so the whole team reads the same view. --- ## When to use -- **You want a single view of key metrics**: Combine error rate, latency, cost, and eval scores into one dashboard instead of switching between pages. -- **You need to monitor a deployment**: Create a dashboard with widgets that show the metrics you care about, then filter by time range to see how things changed after a release. -- **Your team needs a shared overview**: Build a dashboard that everyone on the team can open to see the current state of the project. -- **You want to compare metrics side by side**: Place multiple widgets on the same dashboard to spot correlations between latency spikes and error rate increases. -- **You need to export or present data**: Use table widgets to view raw data and export it as CSV. +- **Watch production health** — keep latency, error rate, and cost on one screen. +- **Monitor a deployment** — chart the metrics you care about, then filter by time to see how a release moved them. +- **Share an overview** — give the team one page instead of asking everyone to run their own queries. +- **Spot correlations** — place widgets side by side to see a latency spike line up with an error-rate jump. +- **Export** — use a table widget to view and export raw data as CSV. + +Reach for something else when you need to debug one request (use the [trace explorer](/docs/observe/features/llm-tracing)), get notified on a threshold (use [Alerts](/docs/observe/features/alerts)), or break a metric down per end user (use [Users](/docs/observe/features/users)). + +--- + +## How it works + +Observe records every request as a [trace](/docs/observe/concepts/traces) made of spans. A widget queries those spans, applies an aggregation over a time bucket, and draws the result. Nothing is precomputed — change the time range, granularity, or filters and every widget re-queries the same underlying traces. + +That means a dashboard is only ever as good as the metric definitions behind it. The exact formula for each metric — how latency, cost, tokens, and error rate are derived from spans — is in the [dashboard metric definitions](/docs/observe/reference/dashboard-metric-definitions) reference. + +Build a dashboard in four steps. + +--- + +## Step 1: Create a dashboard + +Open **Dashboards** in Observe and create a new one. Give it a name and an optional description — this is the container that will hold every widget. + +Observe Dashboards view listing existing dashboards with the create action +*The Dashboards view. One dashboard holds many widgets, and the whole project shares it.* + +--- + +## Step 2: Add a widget + +Inside the dashboard, click **Add widget**. This opens the widget editor, where each widget becomes one chart or table backed by a single query. + +Widget editor with chart type, metric, aggregation, group-by, and granularity controls +*The widget editor. The same data reads very differently as a line over time versus a single metric tile — pick the shape that answers your question.* + +--- + +## Step 3: Configure the metric + +In the widget editor, choose the **metric** (latency, cost, tokens, error count, eval score), then set the **chart type**, **aggregation**, optional **filters** and **group-by**, and the **granularity**. Preview updates as you change controls, so you can confirm the widget answers the question before you save. + +Widget editor cropped to the metric configuration: Latency selected with Average aggregation on the right, a Line chart over a 30-day range, and the live preview and data table +*Configure the widget once: pick the metric and aggregation on the right, set chart type and granularity up top, and read the live preview before saving.* + +Pick aggregation and granularity together: *average latency per hour* and *sum of cost per day* ask very different questions of the same traces. --- -## How to +## Step 4: Share the dashboard - - - Open the **Dashboards** section and click **Create Dashboard**. Give it a name and optional description. - ![Create dashboard](/screenshot/product/observe/dashboard/1.png) - +Set a global time range — it applies to every widget at once — then drag and resize widgets into a layout, with the views you check most often at the top. Because dashboards live at the project level, anyone on the project opens the same saved view automatically. - - Click **Add Widget** and configure the query: - ![Add widget](/screenshot/product/observe/dashboard/2.png) +A populated dashboard with widgets for latency, cost, and request volume +*A populated dashboard, shared with the project and ready to read at a glance.* - - **Chart type**: line, stacked line, column, stacked column, bar, stacked bar, pie, table, or metric (single number). - - **Metric**: select from available metrics (e.g. span count, error count, latency, token usage, eval scores). - - **Aggregation**: sum, average, median, count, distinct count, min, or max. - - **Granularity**: minute, hour, day, week, or month (options adjust based on the time range). - - **Filters**: narrow the query to specific spans. - - **Group by**: break down the metric by a span attribute (e.g. model, user, status). +--- - Preview the result before saving. - +## Widget and metric options - - Choose a global time range that applies to all widgets on the dashboard: - ![Time range](/screenshot/product/observe/dashboard/3.png) +| Widget type | Metric | What it shows | +|---|---|---| +| Line / stacked line | Latency, cost, tokens, error rate | A metric over time — the default for spotting trends and spikes. | +| Column / bar (and stacked) | Span count, error count, cost | Discrete totals per bucket or per group — good with a group-by. | +| Pie | Span count, cost, tokens | Share of a total across one dimension (model, status, user). | +| Metric (single value) | Any aggregate | One headline number for the selected range — latency, total cost, error rate. | +| Table | Raw or grouped rows | Tabular detail you can read and export as CSV. | + +Supporting controls apply to every widget type: **aggregation** (sum, average, median, count, distinct count, min, max), **granularity** (minute, hour, day, week, month — adjusts to the range), **group-by** (break a metric down by a span attribute like model, user, or status), and **time range** (30 minutes through 12 months, or custom). Full formulas are in the [dashboard metric definitions](/docs/observe/reference/dashboard-metric-definitions) reference. + +--- - - **Presets**: 30 mins, 6 hrs, Today, Yesterday, 7D, 30D, 3M, 6M, 12M. - - **Custom**: pick a specific start and end date. - +## Limits and edge cases - - Resize and reorder widgets to build your layout: - ![Arrange widgets](/screenshot/product/observe/dashboard/4.png) - Drag and drop to reorder. - +| Symptom | Cause | Fix | +|---|---|---| +| Widget shows no data | Granularity is finer than your data retention. | Widen the granularity (or shorten the range) so each bucket falls inside the retained window. | +| Eval-score widget is empty | No eval task has run for that metric yet. | Run an eval that produces the score, then reload the widget. | +| Export is truncated | Very large ranges cap the number of rows returned. | Narrow the time range or filters so the result fits under the row cap. | +| Numbers shift on reload | The range is relative to "now", so the window moves with each reload. | Set an absolute custom range to freeze the numbers. | - - Use the menu on each widget to **edit**, **duplicate**, **resize**, or **delete** it. - ![Widget menu](/screenshot/product/observe/dashboard/5.png) - - +Two general limits to keep in mind: - - Dashboards are scoped to your organization and project. All team members with access to the project can view and edit dashboards. - +- Dashboards visualize **aggregates** — they don't drill to a single span. Click through to the [trace explorer](/docs/observe/features/llm-tracing) for that. +- Dashboards are **passive** — they don't notify you. Pair a metric with [Alerts](/docs/observe/features/alerts) to get told when it crosses a threshold. --- -## Next Steps +## Next steps - - Connect the SDK and start capturing traces. + + Turn a dashboard metric into a threshold notification. - - Run evaluations on your traced spans to score quality. + + The exact formula behind every metric you can chart. - - Group traces into sessions for multi-turn analysis. + + Drill from a trend down to the request behind it. - - Get notified when metrics cross a threshold. + + Produce the eval scores you can chart here. diff --git a/src/pages/docs/observe/features/evals.mdx b/src/pages/docs/observe/features/evals.mdx index 24225eca..bc74263b 100644 --- a/src/pages/docs/observe/features/evals.mdx +++ b/src/pages/docs/observe/features/evals.mdx @@ -1,95 +1,161 @@ --- -title: "Run Evals on Traces in Future AGI Observe" -description: "Run automated quality checks on traced spans in Observe: filter spans, choose historic or continuous runs, set sampling, and attach preset or custom evals." +title: "Trace evals" +description: "Score traced spans for hallucination, tone, bias, and more — filter which spans, run historically or continuously, sample to control cost, and read results per span." +slug: "evals" +page_type: "feature" +products: ["Observe"] +feature: "Trace evals" +feature_status: "stable" +ui_surfaces: ["Observe > Evals"] +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-05-25" +last_screenshotted: "2026-06-18" +schema_type: "TechArticle" +seo: + title: "Trace evals in Observe" + description: "Score traced spans for hallucination, tone, bias, and more — filter which spans, run historically or continuously, sample to control cost, and read results per span." + primary_keyword: "run evals on production llm traces" + direct_answer: true +geo: + answer_target: "How do I run evals on production traces in Observe?" + llm_summary: "Trace evals run automated quality checks on production traces in Observe — scoring spans for hallucination, tone, bias, toxicity, and more. You filter which spans are scored, choose historical or continuous runs, sample to control cost, and read results per span; scores can feed alerts." +canonical: "/docs/observe/features/evals" +related: + - "/docs/observe/concepts/traces" + - "/docs/evaluation/builtin" + - "/docs/observe/features/alerts" + - "/docs/traceai/manual-instrumentation/in-line-evals" --- ## About -Evals run automated quality checks on your production traces, scoring every LLM response for hallucination, tone, bias, toxicity, and more. You configure which checks to run, filter which spans they apply to, and choose whether to evaluate historical data or new spans as they arrive. Results appear per span in the Observe dashboard and can trigger alerts when quality drops. - -{/* ARCADE EMBED START */} - -
-{/* ARCADE EMBED END */} +Evals run automated quality checks on your production traces, scoring each response for hallucination, tone, bias, toxicity, and more. You pick which checks to run, filter which spans they apply to, and choose whether to score existing traffic (historical) or new spans as they arrive (continuous). Scores attach to the span in the [trace explorer](/docs/observe/features/llm-tracing) and can feed [alerts](/docs/observe/features/alerts) when quality drops. --- ## When to use -- **Scoring production output quality**: Run historic evals after a release to check for hallucinations, bias, or unsafe content across real traffic. -- **Catching regressions in production**: Set up a continuous eval task so new spans are scored automatically and you see quality drops before users report them. -- **Spot-checking a specific time window**: Filter by date range or session to evaluate only the spans from an incident or a specific user flow. -- **Controlling eval cost**: Use sampling rate and span limits to evaluate a representative subset instead of every span. -- **Running multiple quality checks at once**: Attach several evals to one task so each span gets scored for tone, safety, and accuracy in a single run. +- **Score production quality** — after a release, run a historical eval for hallucinations, bias, or unsafe content across real traffic. +- **Catch regressions automatically** — a continuous task scores new spans so you see quality drops before users report them. +- **Investigate an incident** — filter by date range or session to evaluate only the spans from one window or flow. +- **Run several checks at once** — attach multiple evals so each span is scored for tone, safety, and accuracy in one run. + +--- + +## When not to use + +- **A one-off check while developing** — run an eval inline at the span; see [in-line evaluations](/docs/traceai/manual-instrumentation/in-line-evals). +- **Watching a score trend over time** — chart it on a [Dashboard](/docs/observe/features/dashboard). +- **Defining a new evaluator** — that lives in the evaluation section; this page only *runs* evals against traces. + +--- + +## What you can evaluate: span, trace, and session + +An eval can run at three levels, depending on what you want to judge: + +- **Span level** — scores a single step on its own: one LLM call, one tool call, one retrieval. Use it to check the quality of an individual operation, such as whether a generation was faithful or whether a tool received the right arguments. +- **Trace level** — scores one whole request end to end. The eval looks at the agent's internal steps and how they produced the final output, so you can catch issues that only appear across steps, like a hallucination introduced mid-chain or a tool-calling mistake. At this level the eval can read any span's data through `spans.0.*`, `spans.1.*`, … paths (the same attribute keys you see in a span's attribute section), so you can point it at the exact field to judge. +- **Session level** — scores a full multi-turn conversation between a user and your agent. Instead of one request, the eval looks at the whole exchange: how the agent handled several inputs in one conversation, and whether it stayed consistent — in both its answers and its actions — across turns. Use it for conversation-level behavior that no single trace captures. + +You pick the level by where you launch the eval from: the **Traces** view for span- and trace-level evals, the **Sessions** view for session-level. + +### Adding context to an eval + +When you configure an eval, you can attach **context** — extra trace or span data the evaluator sees beyond the field being scored. For example, you can hand a faithfulness eval the documents a retriever span returned, so it can judge the answer against the right source. Context lets an eval reason about one step in light of the rest of the request. --- -## How to +## How to run evals - Define filters so the task runs only on the spans you care about. + Filter so the task runs only on the spans you care about. - ![Set filters](/images/docs/observe/1.png) + Eval task configuration showing span filters, run type, and sampling + *Scope the task before you run it — filtering down keeps cost and runtime predictable.* | Filter | Description | - |--------|-------------| - | `observation_type` | Node/span type (e.g. `llm`, `chain`, `agent`). | - | `date_range` | Time range: `[start_date, end_date]` applied to `created_at`. | - | `created_at` | Minimum creation time (spans at or after this value). | + |---|---| + | `observation_type` | Span type (`llm`, `chain`, `agent`, …). | + | `date_range` | `[start_date, end_date]` applied to `created_at`. | + | `created_at` | Minimum creation time. | | `project_id` | Restrict to a specific Observe project. | - | `session_id` | Restrict to traces in a given session. | - | `span_attributes_filters` | List of span-attribute conditions. | + | `session_id` | Restrict to one session. | + | `span_attributes_filters` | A list of span-attribute conditions. | + - Filters are stored in the task's `filters` field and applied when the task runs. + + - **Historical** — runs on existing rows matching the filters, up to the sampling rate and row limit, then completes. + - **Continuous** — runs on new spans as they arrive; each run processes only spans created since the last, and the task stays active. - - Set the **run type**: + + - **`sampling_rate`** — the percentage of matching rows to evaluate (0–100). + - **`spans_limit`** — the maximum number of matching rows the task processes (**default 1000**), shown in the UI as **Row limit**. The run stops at whichever cap is hit first. + - ![Choose run type](/images/docs/observe/2.png) + + Attach one or more eval configs. For evals that need an input (e.g. Bias Detection), set the **input key** to a span-attribute path like `gen_ai.output.messages.0.message.content` so the eval reads the right field. Then run the task. - - **Historical**: Run on existing spans matching the filters, up to the sampling cap and span limit. The task completes after processing. - - **Continuous**: Run on new spans as they arrive. Each run only processes spans created after the last run; the task stays active for ongoing evaluation. + Selecting eval templates to attach to a task and running it + *Attach the checks, map their inputs, run. See [built-in evals](/docs/evaluation/builtin) for what each one needs.* + - - ![Set sampling rate and span limit](/images/docs/observe/3.png) +--- - - **sampling_rate**: Percentage of matching spans to evaluate (0-100). For example, `50` evaluates 50% of filtered spans per run. - - **spans_limit**: Maximum number of spans to process per run (default 1000). The task stops when either the sampled count or this limit is reached. - +## Outputs and interpretation - - Attach one or more eval configs to the task. The task runs each selected eval on every span it processes. For evals that need an input (e.g. Bias Detection), set the **input key** to a span attribute path (e.g. `gen_ai.output.messages.0.message.content`) so the eval reads the right field from each span. See [built-in evals](/docs/evaluation/builtin) for supported evaluations and their required inputs. - +Scores attach to each evaluated span and appear under the span's **Evals** tab in the trace explorer; you can also alert on them. - - ![run](/images/docs/observe/4.png) +The Evals tab on a trace, showing a pass/fail summary and a matrix of each eval's score against each span — is_good_summary, is_harmful_advice, and word_count_in_range across the agent, classification, tool, and response spans +*The Evals tab on a span: a pass/fail rollup plus the per-eval, per-span result matrix. Each row is one eval scored against one span, so you see exactly which step failed.* - Create or update the eval task via the API or UI, then run it. You can test the configuration before saving. Task status values: `pending`, `running`, `completed`, `failed`, `paused`, `deleted`. Results appear on the spans in the Observe dashboard and can be used for alerts. - - +A task moves through these statuses: + +| Status | Meaning | +|---|---| +| `pending` | Created, not yet started. | +| `running` | Processing spans. | +| `completed` | Historical run finished. | +| `paused` | Temporarily stopped. | +| `failed` | The run errored. | +| `deleted` | Removed. | + +--- + +## Edge cases and limits + +- Eval tasks are **asynchronous** — status and results update as runs complete. +- **Continuous** tasks only pick up spans created after their last run; historical spans need a separate historical task. +- Model-based evals call a judge model, so the sampling rate and row limit directly control how much you spend — start small on real traffic. - - Eval tasks are processed asynchronously. Status and results update as runs complete. For continuous tasks, new spans are picked up on subsequent runs. - +| Symptom | Cause | Fix | +|---|---|---| +| Eval shows no scores | The input key path is wrong (e.g. a mistyped `gen_ai.output.messages.0.message.content`). | Set the **input key** to the exact span-attribute path the field lives at, then rerun. | +| Continuous task not scoring new spans | The task filters exclude those spans. | Loosen the filters so the new spans match, or confirm `observation_type` and `span_attributes_filters`. | +| Task status `failed` | Judge-model auth or quota error. | Re-check the judge model's API key/quota and rerun the task. | --- -## Next Steps +## Related features - - Connect the SDK and start capturing traces. + + Alert when an eval score crosses a threshold. - - Group traces into sessions for multi-turn analysis. + + Score a response inside a span as it runs. - - View activity and metrics per end user. + + The available checks and their required inputs. - - Get notified when metrics cross a threshold. + + Read eval scores attached to a span. diff --git a/src/pages/docs/observe/features/llm-tracing.mdx b/src/pages/docs/observe/features/llm-tracing.mdx new file mode 100644 index 00000000..0815cecb --- /dev/null +++ b/src/pages/docs/observe/features/llm-tracing.mdx @@ -0,0 +1,174 @@ +--- +title: "Trace explorer" +description: "Inspect one production AI request in Observe — read its span tree, check latency, cost, tokens and errors, and open any step's exact input and output." +page_type: "feature" +slug: "llm-tracing" +products: ["Observe"] +feature: "Trace explorer" +feature_status: "stable" +ui_surfaces: ["Observe > Tracing"] +audience: ["engineer"] +difficulty: "beginner" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-05-25" +last_screenshotted: "2026-06-18" +schema_type: "TechArticle" +seo: + title: "Trace explorer" + description: "Inspect one production AI request in Observe — read its span tree, check latency, cost, tokens and errors, and open any step's exact input and output." + primary_keyword: "view llm traces in production" + direct_answer: true +geo: + answer_target: "How do I inspect a single production AI request in Observe?" + llm_summary: "The Observe trace explorer lists production AI requests newest-first, one row per trace. Open a row to read the span tree — every model, tool, and retrieval step with its input, output, latency, tokens, cost, and pass/fail state — and use the Filter panel to narrow by status, model, user, and more." +canonical: "/docs/observe/features/llm-tracing" +related: + - "/docs/observe/concepts/traces" + - "/docs/observe/concepts/spans" + - "/docs/observe/features/session" + - "/docs/observe/features/evals" +--- + +## About + +The trace explorer is the page in Observe where you debug a single production AI request. When one user hit an error, one answer came back wrong, or one request ran slow, this is where you go to see exactly what your app did. + +The unit you work with is a *trace row*. Each row is one complete request — the user input, every model and tool call made along the way, the final output, the latency and token cost of each step, and whether anything failed. + +The explorer lists those rows newest-first. Click any row to open its span tree and read the request step by step. + +Observe trace explorer listing production traces with name, input, output, status, latency, and token columns +*The trace list, newest request first. Each row is one full request; the Status and Latency columns are the fastest way to spot what broke or ran slow.* + +--- + +## When to use + +Reach for the trace explorer when you need to look at *individual* requests: + +- **One user hit a problem** — find their exact request and read what the AI did, step by step. +- **The app feels slow** — sort by Latency to find the slowest requests, then open one to see which step inside it is the bottleneck. +- **You're chasing an error pattern** — filter to failed requests and look for what they share: a model, a tool, an input shape. +- **You're reviewing an agent's decisions** — open the span tree to follow the full path the agent took. +- **You need real production data for testing** — select rows in bulk and move them into a dataset. + +For aggregate trends (error rate over time, p95 latency, daily cost) you want roll-ups, not rows — build a [dashboard](/docs/observe/features/dashboard) instead. To group by conversation or end user, use [Sessions](/docs/observe/features/session) or [Users](/docs/observe/features/users). + +--- + +## How it works + +Your app sends [spans](/docs/observe/concepts/spans) to Observe through the [traceAI](/docs/traceai/manual-instrumentation/set-up-tracing) SDK. A span is one operation — a model call, a tool call, a retrieval step. Spans that belong to the same request share a trace ID, and Observe reassembles them into one [trace](/docs/observe/concepts/traces): a tree with the top-level request at the root and each step nested underneath. The explorer reads that tree. + + + No traces yet? Spans arrive via the traceAI SDK — instrument your app first. See the [Quick start](/docs/observe/quickstart) and [Set up tracing](/docs/traceai/manual-instrumentation/set-up-tracing). + + +|"traceAI spans"| B["Observe ingest"] + B -->|"group by trace ID"| C["Span tree per request"] + C --> D["Trace list rows"] + D -->|"open a row"| E["Detail drawer: span tree + step details"]`} /> + +Traces appear within seconds of a request finishing. A short-lived script must flush its spans before it exits, or the last trace may never arrive — see [Set up tracing](/docs/traceai/manual-instrumentation/set-up-tracing) for the flush step. + + + Auto-refresh (in the header) re-checks for new traces every 10 seconds, and the date picker defaults to the past 7 days. If a request you recently sent is missing, widen the time window and confirm auto-refresh is on before assuming the trace was dropped. + + +--- + +## Trace list columns + +The **Trace** tab is the default view — one row per request. The columns are how you scan for trouble without opening anything. + +| Column | What it means | +|---|---| +| Status | `OK` (green) when every step succeeded, `ERROR` (red) when a step failed. The fastest way to spot a broken request. | +| Trace name | The name of the top-level task, e.g. `support_agent.run`. | +| Model | The model used on the request's LLM step(s), e.g. `gpt-4o`. | +| Latency | Total wall-clock time from request to response. Sort by this to find slow requests. | +| Tokens | Total tokens used across the whole request. | +| Cost | Estimated cost of the request, derived from token counts and the model's price. | +| Time | When the request happened (the trace timestamp). | + +The list also previews the request **Input** and the model **Output** inline, so you can read what the user asked and what came back without opening the row. + + + A **voice project** shows different columns — Call Details, Status, Duration, Avg Latency, Turn Count, and Tokens — because calls are measured per turn, not per text exchange. See [Voice observability](/docs/observe/features/voice). + + +--- + +## Inspect a trace + +Click any row and a detail drawer slides in, split into two sides. Use the ↑ ↓ buttons to step between traces without closing it. + +Trace detail drawer cropped to the span tree on the left and the selected step's type, status, tokens, cost, input, and output on the right +*Left: the ordered steps the AI took. Right: everything about the step you clicked. Start at the top of the tree and walk down until a step's timing or output looks wrong.* + +**Left — the span tree.** Every [span](/docs/observe/concepts/spans) in the request, in order, each with its name, duration, and pass/fail state. A support agent might show `llm.intent_classification` → `tool.check_order_status` → `llm.response_generation`. Click any step to inspect it. + +**Right — the step details.** For the selected span: + +- **Header** — type, status, start time, duration, prompt / completion / total tokens, and cost. LLM steps also show the model, e.g. `gpt-4o`. +- **Preview** — the exact input and output text, plus the full attribute list (model, provider, token counts). +- **Log view** — raw logs for the step. +- **Evals** — any quality scores attached to the step. See [Evals](/docs/observe/features/evals). +- **Annotations** — notes a human reviewer added. +- **Events** — anything that fired during the step. + +--- + +## Filter and search + +Click **Filter** to narrow the trace list. There are three modes — pick the one that matches how precisely you already know what you're looking for. + +Filter panel cropped to the AI search bar, the Basic and Query tabs, and the property/condition/value builder with Apply +*The filter panel. Stack several Basic conditions and they apply together (AND).* + +- **AI search** — describe what you want in plain English (*"errors on gpt-4o today"*) and the filter is built for you. +- **Basic** — pick a property, a condition, and a value; add as many conditions as you need. +- **Query** — write a filter expression directly, for conditions the Basic builder can't express. + +You can filter on trace name, status, model, span kind, user ID, provider, eval scores, and more. For every property, attribute key, operator, and a set of ready-to-paste expressions, see [Filter syntax](/docs/observe/reference/trace-filter-syntax). + +--- + +## Limits and edge cases + +| Behaviour | Detail | +|---|---| +| Default time window | The date picker opens on the **past 7 days**. Traces outside that window are hidden until you widen it (selectable up to the past 12 months, or a custom range). | +| Ingestion lag | Traces appear within seconds of a request finishing, not instantly. Auto-refresh polls every 10 seconds; a just-sent trace may take a refresh cycle to show. | +| Unflushed spans | A short-lived script that exits before flushing may never send its last trace — that is a missing flush, not a dropped trace. | +| Redacted fields | If you mask span input/output at the SDK, those fields show as hidden here. That is expected, not a missing trace — see [Mask span attributes](/docs/traceai/manual-instrumentation/mask-span-attributes). | +| What's not shown | The explorer shows individual requests, not roll-ups. Error rate over time, p95 latency, and daily cost live on a [dashboard](/docs/observe/features/dashboard). | +| Retention | How far back traces are kept depends on your plan. | + + + Span input and output can contain customer data. Redact it at the SDK with `TraceConfig` or the `FI_HIDE_*` environment variables before traces leave your app — see [Mask span attributes](/docs/traceai/manual-instrumentation/mask-span-attributes). + + +--- + +## Next steps + + + + The full record of one request, and how spans roll up into it. + + + The single operation each row in the span tree represents. + + + Group traces into multi-turn conversations and read per-session metrics. + + + Attach automated quality scores to production traces. + + diff --git a/src/pages/docs/observe/features/quickstart.mdx b/src/pages/docs/observe/features/quickstart.mdx deleted file mode 100644 index f810737d..00000000 --- a/src/pages/docs/observe/features/quickstart.mdx +++ /dev/null @@ -1,155 +0,0 @@ ---- -title: "Set Up Observability with Future AGI Observe" -description: "Instrument your application and send traces to an Observe project so you can monitor LLM calls, latency, and cost in one place." ---- - -## About - -This is how you connect your application to Future AGI so LLM calls are captured in the Observe dashboard. Register a project, instrument your app, and every request appears automatically with its inputs, outputs, cost, latency, and token usage. - ---- - -## When to use - -- **First-time setup**: Get traces flowing into the Observe dashboard so you can start monitoring production LLM calls. -- **Production monitoring**: See latency, cost, and token usage for every LLM call in one place instead of scraping logs. -- **Debugging**: Tie a user report or failure to a specific trace and span so you can reproduce and fix issues. -- **Baseline for other Observe features**: Sessions, evals, user tracking, and alerts all require traces to be set up first. - ---- - -## How to - - - - Install the core instrumentation package and the framework instrumentor for your LLM provider. - - - ```bash Python - pip install fi-instrumentation-otel traceAI-openai - ``` - ```bash JS/TS - npm install @traceai/fi-core @traceai/openai - ``` - - - - - Set environment variables so the SDK can connect to Future AGI. Get your API keys from the [dashboard](https://app.futureagi.com/dashboard/keys). - - - ```python Python - import os - os.environ["FI_API_KEY"] = "YOUR_API_KEY" - os.environ["FI_SECRET_KEY"] = "YOUR_SECRET_KEY" - ``` - ```typescript - process.env.FI_API_KEY = FI_API_KEY; - process.env.FI_SECRET_KEY = FI_SECRET_KEY; - ``` - - - - - Call `register` with `project_type` set to Observe and a `project_name`. Optionally set `transport` (e.g. GRPC or HTTP). - - - ```python - from fi_instrumentation import register, Transport - from fi_instrumentation.fi_types import ProjectType - - trace_provider = register( - project_type=ProjectType.OBSERVE, - project_name="FUTURE_AGI", - transport=Transport.GRPC, - ) - ``` - ```typescript - import { register, ProjectType } from "@traceai/fi-core"; - - const traceProvider = register({ - project_type: ProjectType.OBSERVE, - project_name: "FUTURE_AGI" - }); - ``` - - - - - Use one of two options: - - - **Auto Instrumentor**: For supported frameworks (e.g. OpenAI). Use Future AGI's [Auto Instrumentation](/docs/tracing/auto); recommended for most apps. - - **Manual tracing**: For custom spans, use [OpenTelemetry](/docs/tracing/concepts/otel). [Learn more →](/docs/observe/features/manual-tracing/set-up-tracing) - - Example with the OpenAI instrumentor: install the package, instrument with your trace provider, then use the OpenAI client as usual. Traces appear in your [Observe dashboard](https://app.futureagi.com/dashboard/projects/observe). - - - ```python - pip install traceAI-openai - ``` - ```typescript - npm install @traceai/openai - ``` - - - - ```python - from traceai_openai import OpenAIInstrumentor - - OpenAIInstrumentor().instrument(tracer_provider=trace_provider) - ``` - ```typescript - import { OpenAIInstrumentation } from "@traceai/openai"; - - const openaiInstrumentation = new OpenAIInstrumentation({}); - ``` - - - - ```python - from openai import OpenAI - - os.environ["OPENAI_API_KEY"] = "your-openai-api-key" - client = OpenAI() - - completion = client.chat.completions.create( - model="gpt-4o", - messages=[{"role": "user", "content": "Write a one-sentence bedtime story about a unicorn."}] - ) - print(completion.choices[0].message.content) - ``` - ```typescript - import { OpenAI } from "openai"; - - const client = new OpenAI({ apiKey: process.env.OPENAI_API_KEY }); - const completion = await client.chat.completions.create({ - model: "gpt-4o", - messages: [{ role: "user", content: "Write a one-sentence bedtime story about a unicorn." }], - }); - console.log(completion.choices[0].message.content); - ``` - - - - -For supported frameworks and more options, see the [Auto Instrumentation](/docs/tracing/auto) page. - ---- - -## Next Steps - - - - Run evaluations on your traced spans to score quality. - - - Group traces into sessions for multi-turn analysis. - - - View activity and metrics per end user. - - - Get notified when metrics cross a threshold. - - - diff --git a/src/pages/docs/observe/features/session.mdx b/src/pages/docs/observe/features/session.mdx index 7644ef9a..719afe9c 100644 --- a/src/pages/docs/observe/features/session.mdx +++ b/src/pages/docs/observe/features/session.mdx @@ -1,123 +1,219 @@ --- -title: "Group Traces by Session: Multi-turn Conversation Analysis" -description: "Group traces into sessions so you can view and analyze multi-turn conversations, chatbot flows, and per-session metrics in Observe." +title: "Sessions" +description: "Group LLM traces into sessions so you can read a full multi-turn conversation, see per-session duration, cost, and tokens, and debug the exact turn where something went wrong." +slug: "session" +page_type: "feature" +feature: "Sessions" +feature_status: "stable" +ui_surfaces: ["Observe > Sessions"] +products: ["Observe"] +audience: ["engineer"] +difficulty: "beginner" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-05-25" +last_screenshotted: "2026-05-26" +schema_type: "TechArticle" +seo: + title: "Sessions in Observe — group traces into conversations" + description: "Set session.id on your spans and Observe groups every trace into one conversation with total duration, cost, and tokens. Read, filter, and debug whole sessions." + primary_keyword: "group llm traces into sessions" + direct_answer: true +geo: + answer_target: "How do I group LLM traces into a single conversation in Observe?" + llm_summary: "A session is one multi-turn conversation reassembled from its traces. Set the session.id attribute on your spans — in traceAI use `with using_session(\"session_123\"):` (Python) or the session.id baggage key (JS/TS) — and Observe groups every trace sharing that value into one session with total duration, cost, and tokens. View and filter sessions in the Observe Sessions tab." +canonical: "/docs/observe/features/session" +related: + - "/docs/observe/features/users" + - "/docs/observe/concepts/sessions-and-users" + - "/docs/observe/concepts/observability-model" + - "/docs/traceai/manual-instrumentation/set-session-user-id" --- ## About -Sessions group related traces together under a single identifier. A chatbot conversation, a multi-step user journey, or any sequence of LLM calls that belong to the same flow can be tracked as one session. The Observe dashboard shows sessions with their duration, cost, and token usage so you can review the full flow, drill into individual traces, and spot where things went wrong. +A session is one multi-turn conversation, reassembled from its traces. + +When a chatbot answers five messages, that is five separate [traces](/docs/observe/concepts/traces). A session ties them back together under one ID so you can read the whole exchange in order, see its total duration, cost, and tokens, and jump straight to the turn that failed. + +You create a session by setting the `session.id` attribute on your spans. Observe groups every trace that shares a value into one session — there is nothing to pre-register, and the session appears the moment the first trace carrying that ID arrives. + +`session.id` is the per-conversation grouping key. For where it sits in the wider span model, see the [observability model](/docs/observe/concepts/observability-model), and for how sessions relate to users, see [sessions and users](/docs/observe/concepts/sessions-and-users). --- ## When to use -- **Chatbot and multi-turn flows**: Group all traces for a single conversation so you can review the full exchange and debug a specific turn. -- **User journey analysis**: Treat one user's sequence of requests as a session to understand behavior and find drop-off points. -- **Session-level metrics**: See total duration, cost, and tokens for an entire session instead of checking each trace individually. -- **Filtering and drill-down**: Filter sessions by time range, open a session to see its traces, then open a trace to see spans and eval results. + + + Read a full conversation instead of disconnected requests, and debug one turn in the context of the turns around it. + + + Treat a sequence of requests as one flow to find where users drop off, repeat themselves, or escalate. + + + See total duration, cost, and token usage for a whole conversation at once, not per request. + + + Filter sessions by time, open one to see its traces in order, then open a trace for span-level detail and [eval results](/docs/observe/features/evals). + + + +Reach for a different surface when the granularity is wrong: + +- Debugging a single request — use the [trace explorer](/docs/observe/features/llm-tracing); a session is too coarse. +- Rolling up by person, not conversation — use [Users](/docs/observe/features/users), which spans all of a person's sessions. +- Aggregate trends across many sessions — build a [Dashboard](/docs/observe/features/dashboard). --- -## How to +## How it works + +A trace joins a session when its spans carry the `session.id` attribute. You set the value once, and Observe does the grouping. - - - For a trace to appear in a session, the span must carry a **session identifier** via the `session.id` attribute. All traces with the same session name in the same project form one session. The backend creates the session automatically when the first trace with that identifier arrives. - +session.id = abc] --> S[(Session abc)] + B[Request 2
session.id = abc] --> S + C[Request 3
session.id = abc] --> S + S --> V[Sessions view
one row, ordered turns,
total duration / cost / tokens] +`} /> - - When creating a span manually, set the attribute so the trace is attached to the session: +1. Your app sets `session.id` on the spans of each request in a conversation, reusing the same string for every turn. +2. traceAI auto-instrumentors read the value from the OpenTelemetry context and write it onto every span in the block. +3. Observe ingests the traces and groups all of them sharing a `session.id` into one session, computing duration, cost, and token totals across the turns. - - ```python Python - from fi_instrumentation import register, FITracer +The ID is a plain string and the grouping is automatic, so the only thing you own in code is setting `session.id` early enough — see the next section. - trace_provider = register( - project_type=ProjectType.OBSERVE, - project_name="PROJECT_NAME", +--- + +## Set a session ID in code + +In traceAI, set `session.id` for a block with a context helper, and every span created inside inherits it. Use the same string for every turn of one conversation. + + +```python Python +from fi_instrumentation import using_session + +# Every span created in this block gets session.id = "session_123" +with using_session("session_123"): + response = client.chat.completions.create( + model="gpt-4o", + messages=[{"role": "user", "content": "Book me a flight to Tokyo."}], ) +``` +```javascript JS/TS +import { context, propagation } from "@opentelemetry/api"; + +// Set the session.id baggage key on the active context. +const baggage = propagation.createBaggage({ + "session.id": { value: "session_123" }, +}); +const ctx = propagation.setBaggage(context.active(), baggage); + +context.with(ctx, () => { + // Every span created here gets session.id = "session_123" + // e.g. await client.chat.completions.create({ ... }); +}); +``` + + +The decorator form works too, when a whole function handles one conversation: + +```python Python +from fi_instrumentation import using_session + +@using_session("session_123") +def handle_turn(message): + ... +``` + +For every variant — `using_attributes`, setting a user ID at the same time, and the full JS/TS baggage pattern — see [Set session and user IDs](/docs/traceai/manual-instrumentation/set-session-user-id). + +--- + +## View and filter sessions + +Open the project and switch to the **Sessions** tab. Each row is one conversation. + +Observe Sessions tab listing conversations with first and last message, duration, total cost, and trace count +*One row per conversation. Sort by total cost or total traces to find the longest or most expensive sessions.* + +The list columns roll the conversation up at a glance: + +| Column | What it shows | +|---|---| +| **Session Id** | The shared identifier for the conversation. | +| **First Message** | The opening message. | +| **Last Message** | The most recent message. | +| **Duration** | How long the conversation lasted. | +| **Total Cost** | Combined cost of all calls in the session. | +| **Total Traces** | How many requests were part of it. | + +Open a session for its detail view — the traces in order, each with its eval scores and annotations. From there you open any trace for the full span tree. + +Session detail view showing the traces of one conversation in order with per-turn timing and cost +*A session opened: its traces in order, with per-turn timing and cost. Click a trace to drop into the span tree.* + +### Filter and date range + +Use the filter bar to narrow the list — by `session.id`, by metadata, or by any span attribute. The filter grammar is shared across Observe; see the [trace filter syntax reference](/docs/observe/reference/trace-filter-syntax) for operators and field names. + +Sessions filter bar with conditions applied to narrow the conversation list +*Filter sessions down to the conversations you care about — by ID, metadata, or any span attribute.* + +Scope the list to a time window with the date-range picker; metrics in the columns recompute for the selected window. You can also choose which columns to show and run bulk actions on selected sessions. + +### Session replay + +For voice and other replayable sessions, configure session replay so you can step back through a conversation as it happened. + +Session replay configuration panel for stepping through a conversation as it happened +*Replay configuration: step back through a session turn by turn.* + +--- + +## Limits and edge cases + +- A trace **without** a `session.id` does not belong to any session. It still appears in the [trace explorer](/docs/observe/features/llm-tracing) — it just will not group. +- The ID must be set **before or at the start of the request**, so the spans of that turn carry it. Setting it after the model call has already run is too late for those spans. +- `session.id` values are **strings**. Use a stable conversation ID; an empty string does not group. +- Sessions are scoped to a project. The same `session.id` in two projects is two different sessions. +- A session is created automatically on the first trace carrying its ID — there is nothing to pre-register. +- Do not put raw PII in the `session.id`. To redact sensitive span data, see [Mask span attributes](/docs/traceai/manual-instrumentation/mask-span-attributes). + + + The per-conversation grouping key. Any non-empty string; reuse the same value for every turn of one conversation. + + +--- + +## Troubleshooting - tracer = FITracer(trace_provider.get_tracer(__name__)) - - with tracer.start_as_current_span( - f"SPAN_NAME", - ) as span: - span.set_status(Status(StatusCode.OK)) - span.set_attribute("session.id", "session123") - span.set_attribute("input.value", "input") - span.set_attribute("output.value", "output") - ``` - ```javascript JS/TS - const { register, ProjectType } = require("@traceai/fi-core"); - - const traceProvider = register({ - projectType: ProjectType.OBSERVE, - projectName: "FUTURE_AGI" - }); - - const tracer = traceProvider.getTracer("manual-instrumentation-example"); - - tracer.startActiveSpan("HandleFunctionCall", {}, (span) => { - span.setAttribute("session.id", "my-session-id"); - span.end(); - }); - ``` - - - - - To tag all spans in a block with the same session, use context so every span gets `session.id` automatically: - - - ```python Python - from fi_instrumentation import using_session - - with using_session(session_id="my-session-id"): - # All spans created within this block get session.id = "my-session-id" - ... - ``` - ```javascript JS/TS - import { context, propagation } from "@opentelemetry/api"; - - const sessionId = "my-js-session-id"; - - const activeContext = context.active(); - const baggageWithSession = propagation.createBaggage({ - "session.id": { value: sessionId } - }); - const newContext = propagation.setBaggage(activeContext, baggageWithSession); - - context.with(newContext, () => { - // All spans created within this block get session.id = "my-js-session-id" - }); - ``` - - - - - In the Observe UI, open the project and go to the Sessions view. You can filter by time range, see a list of sessions with duration and metrics, open a session to see its traces, and click **View Trace** for span-level detail and [eval](/docs/observe/features/evals) results. - - - - - For more on setting `session.id` with Trace AI helpers and context, see the [manual tracing guide](/docs/observe/features/manual-tracing/set-session-user-id). - +| Symptom | Cause | Fix | +|---|---|---| +| Traces not grouping into a session | The model call ran outside the `using_session` block, so spans never got `session.id`. | Make the call **inside** the block (or function decorated with `@using_session`). | +| Conversation split across several sessions | A different `session.id` was used on some turns. | Reuse one stable string for the whole conversation. | +| JS/TS: `session.id` not appearing | Baggage was not set on the active context. | Use `propagation.createBaggage()` + `context.with()` as shown above. | +| Session exists but has no metrics | Spans carried the ID but no cost/token attributes. | Confirm the LLM spans are auto-instrumented so cost and tokens are captured. | +| Empty grouping | An empty-string ID was passed. | IDs must be non-empty strings. | --- -## Next Steps +## Next steps - - Connect the SDK and start capturing traces. + + Roll traces and sessions up per end user. - - Run evaluations on your traced spans to score quality. + + How grouping by conversation and user works. - - View activity and metrics per end user. + + Every way to attach a session ID in traceAI. - - Get notified when metrics cross a threshold. + + Where session.id sits in the span model. diff --git a/src/pages/docs/observe/features/tags.mdx b/src/pages/docs/observe/features/tags.mdx new file mode 100644 index 00000000..9b2a7754 --- /dev/null +++ b/src/pages/docs/observe/features/tags.mdx @@ -0,0 +1,160 @@ +--- +title: "Tags" +description: "Tags are categorical labels you attach to spans in code so you can slice production traffic in Observe — by environment, release, experiment, or any dimension you choose — then filter the trace explorer down to just those traces." +slug: "tags" +page_type: "feature" +products: ["Observe"] +frameworks: [] +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-05-25" +last_screenshotted: "2026-06-18" +schema_type: "TechArticle" +seo: + title: "Tags in FutureAGI Observe — label and filter traces" + description: "Attach categorical tags to spans with traceAI and filter the Observe trace explorer by them. SDK setup, UI filtering, limits, and troubleshooting." + primary_keyword: "tag llm traces observability" + direct_answer: true +geo: + answer_target: "How do I tag traces and filter by tags in FutureAGI Observe?" + llm_summary: "Set tags in code with traceAI's using_tags context manager; they attach to every span in the block as the tag.tags attribute, and you filter the Observe trace explorer by them." +canonical: "/docs/observe/features/tags" +related: + - "/docs/observe/features/llm-tracing" + - "/docs/observe/reference/trace-filter-syntax" + - "/docs/traceai/manual-instrumentation/add-attributes-metadata-tags" + - "/docs/observe/concepts/spans" +--- + + +## About + +A tag is a categorical label you attach to a span — `production`, `release-2.1`, `experiment-a`, `eu-region`. Tags don't change what your app does; they give you an axis to slice production traffic on later. When you need to ask "how did the new prompt do versus the old one?" or "are EU users hitting more errors?", tags are how you carved that question into the data ahead of time. You set them once in code with [traceAI](/docs/observe/concepts/traceai), they ride along on every [span](/docs/observe/concepts/spans) in the block, and then you filter the [trace explorer](/docs/observe/features/llm-tracing) down to exactly the slice you care about. + +Tags differ from [metadata](/docs/traceai/manual-instrumentation/add-attributes-metadata-tags): metadata is structured key/value context (`{"experiment": "A/B", "version": "2.1"}`), while a tag is a flat label from a controlled vocabulary you define. Use tags for the handful of dimensions you slice on constantly; use metadata for richer per-request context. + +--- + +## When to use + +- **Separate environments** — tag spans `production`, `staging`, or `dev` so production dashboards and alerts never mix in test traffic. +- **Compare releases** — tag each deploy (`release-2.1`) and compare error rate, latency, and eval scores across versions. +- **Track experiments** — tag the variant (`prompt-v2`, `control`) and read the difference in the trace list. +- **Segment traffic** — tag by tier, region, or surface (`free`, `eu-region`, `mobile`) to find where problems concentrate. + +--- + +## How it works + +Tags are attached at the OpenTelemetry **context** level, not to one span by hand. You open a `using_tags` block (or decorate a function), and every span traceAI creates inside that block carries the tags automatically — including spans from auto-instrumented frameworks. They're stored on each span as a single attribute, `tag.tags`, holding a JSON list of strings. Because they propagate to child spans, tagging the top of a request tags the whole trace. + +Because the tags live in the active context, they follow execution wherever it goes for the life of the block — into helper functions, nested framework calls, and the spans those create — without you threading a value through your code. Once the block exits, the context is restored and later spans are untagged, so the boundary of the block is exactly the set of spans that get the label. On the storage side `tag.tags` is a first-class span attribute, which is why Observe can index it and offer it as a filter property rather than making you dig through raw metadata. + + root["agent span · tag.tags set"] + root --> llm["llm span · tag.tags inherited"] + root --> tool["tool span · tag.tags inherited"] + root --> obs["Observe · filter by tag.tags"]`} /> + +--- + +## Set tags in code + +Wrap the work you want tagged. Tags must be a **list of strings**. + + +```python Python (context manager) +from fi_instrumentation import using_tags + +with using_tags(["production", "release-2.1"]): + # every span created here gets tag.tags = ["production","release-2.1"] + response = run_agent(user_input) +``` +```python Python (decorator) +from fi_instrumentation import using_tags + +@using_tags(["production", "release-2.1"]) +def handle_request(user_input): + # every span this function creates is tagged + return run_agent(user_input) +``` +```typescript JS/TS +import { context, propagation } from "@opentelemetry/api"; + +const tags = ["production", "release-2.1"]; +const baggage = propagation.createBaggage({ + "tag.tags": { value: JSON.stringify(tags) }, +}); + +context.with(propagation.setBaggage(context.active(), baggage), () => { + // spans created here pick up tag.tags from baggage + runAgent(userInput); +}); +``` + + +To tag a single span directly instead of a block, set the attribute yourself: `span.set_attribute("tag.tags", json.dumps(["production"]))`. + +### Tag attribute + +| Attribute key | Type | Example | Where it's set | Propagation | +|---|---|---|---|---| +| `tag.tags` | `list[string]` | `["production","release-2.1"]` | `using_tags([...])` block | Inherited by all child spans in the block. | + +--- + +## Filter by tags + +Once tags are flowing, open the **Filter** panel in the [trace explorer](/docs/observe/features/llm-tracing) and narrow to the slice you want. + +Filter panel in the Observe trace explorer cropped to the Basic and Query tabs and the property, condition, and value builder used to filter by a tag +*Filter the trace list to one tag, then save it as a view you can return to.* + +- In **Basic** mode, pick the tag property, choose `contains`, and enter the tag value. +- In **Query** mode, write the expression directly against `tag.tags`. +- In **AI search**, describe it in plain English ("production traces with errors today"). + +Tags also show up in each span's attribute list, so you can confirm them on an individual trace. See [Filter syntax](/docs/observe/reference/trace-filter-syntax) for every mode and operator. + +--- + +## Limits and edge cases + +- **Type** — tags must be a list of strings. Numbers or objects are dropped; use [metadata](/docs/traceai/manual-instrumentation/add-attributes-metadata-tags) for non-string context. +- **Propagation** — tags apply to spans created *inside* the block. Spans created before you enter the block, or after you leave it, are not tagged. +- **No secrets in tags** — tags are labels and are indexed for filtering; never put API keys, emails, or customer data in a tag value. Redact payloads with [mask span attributes](/docs/traceai/manual-instrumentation/mask-span-attributes), not by hiding them in tags. +- **Keep the vocabulary small** — a controlled set (`production`/`staging`, `v1`/`v2`) keeps filters meaningful. Free-form per-request strings belong in metadata. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| Tag not on the span | The span was created outside the `using_tags` block. | Move the work inside the block, or set `tag.tags` on the span directly. | +| Can't filter by a tag | Tag value was not a string, so it was dropped. | Pass tags as a list of strings only. | +| Only some spans tagged | Tagging started mid-request. | Open the block at the top of the request so child spans inherit it. | + +--- + +## Next steps + + + + Every filter mode and operator for tags and other properties. + + + Set tags, metadata, session, and user IDs in code. + + + Where you filter and save tagged views. + + + The unit that carries the tag.tags attribute. + + diff --git a/src/pages/docs/observe/features/users.mdx b/src/pages/docs/observe/features/users.mdx index 5f689619..91824c5e 100644 --- a/src/pages/docs/observe/features/users.mdx +++ b/src/pages/docs/observe/features/users.mdx @@ -1,118 +1,221 @@ --- -title: "User Dashboard: Per-User Trace and Session Analytics" -description: "View all traces, sessions, and metrics per end user in one place so you can debug, analyze behavior, and optimize at the user level." +title: "Users" +description: "Group every LLM trace and session by end user so you can debug one customer's full history, find who is driving cost, and spot quality drops per user in Observe." +slug: "users" +page_type: "feature" +feature: "Users" +feature_status: "stable" +ui_surfaces: ["Observe > Users"] +products: ["Observe"] +audience: ["engineer"] +difficulty: "beginner" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-05-25" +last_screenshotted: "2026-05-26" +schema_type: "TechArticle" +seo: + title: "Users in Observe — per-user trace and session analytics" + description: "Set user.id on your spans and Observe rolls every trace and session up per end user, with per-user cost, evals, and history. View, filter, and debug by user." + primary_keyword: "per user llm analytics observability" + direct_answer: true +geo: + answer_target: "How do I see all of one end user's LLM traces and sessions in Observe?" + llm_summary: "The Users view groups every trace and session by end user. Set the user.id attribute on your spans — in traceAI use `with using_user(\"user_456\"):` (Python) or the user.id baggage key (JS/TS) — and Observe creates the user on first sight and links every matching span, with per-user trace and session counts, cost, and evals. View and filter users in the Observe Users view." +canonical: "/docs/observe/features/users" +related: + - "/docs/observe/features/session" + - "/docs/observe/concepts/sessions-and-users" + - "/docs/observe/concepts/observability-model" + - "/docs/traceai/manual-instrumentation/set-session-user-id" --- ## About -The **user dashboard** in Observe groups all traces and sessions by end user. Each user row shows aggregated metrics like cost, tokens, latency, error count, eval pass rate, and guardrail triggers. You identify users by setting a `user.id` attribute on your spans. Once the backend sees that attribute, it creates a user entry and links all matching spans to it. Open any user to see their full activity: traces, sessions, and metrics in one view. +The Users view groups every [trace](/docs/observe/concepts/traces) and [session](/docs/observe/features/session) by end user. + +Each row shows a user's activity at a glance — trace count, session count, and when they were first and last active. Open a user and you get their full history: every session and trace, with per-user cost, evals, and guardrail results. + +So you can answer "what happened to this customer?" without writing a query. + +You identify a user by setting the `user.id` attribute on your spans. Observe creates the user on first sight and links every matching span to it — nothing to pre-register. + +`user.id` is the per-person grouping key. For where it sits in the wider span model, see the [observability model](/docs/observe/concepts/observability-model), and for how users relate to conversations, see [sessions and users](/docs/observe/concepts/sessions-and-users). --- ## When to use -- **A user reports a bug**: Open their row in the dashboard, see every trace and session they triggered, and pinpoint which request failed and why. -- **Costs spike unexpectedly**: Sort users by cost or token usage to find who is driving the increase and whether it is normal usage or a runaway loop. -- **You need to measure engagement**: Check activation date, last active, active days, and session counts per user to see who is adopting the product and who dropped off. -- **Eval scores drop for a segment**: Filter users by eval pass rate to find accounts with low quality scores, then drill into their traces to understand the pattern. -- **Support asks "what happened to this user?"**: Search by user ID, open their detail view, and walk through their traces and sessions without writing a single query. + + + Open their row to see every trace and session they triggered, and find the failed request. + + + Open a heavy user to see cost and token usage per session and spot a runaway loop. + + + Read activation date, last-active, and session counts to see who adopted and who dropped off. + + + Open users and check their [eval results](/docs/observe/features/evals) to find low-quality accounts, then drill into their traces. + + + +Reach for a different surface when the granularity is wrong: + +- One conversation, not a person — use [Sessions](/docs/observe/features/session). +- A single request — use the [trace explorer](/docs/observe/features/llm-tracing). +- Authentication or identity — `user.id` is a grouping key, not an auth identity. Observe never verifies it. --- -## How to +## How it works - - - For a span to count under a user in the dashboard, it must carry a **user identifier**. In the OTLP path this comes from the span attribute **`user.id`**. When a span is ingested with this attribute (for an Observe project), the backend gets or creates an `EndUser` for that project and organization with that `user_id` (and optional `user_id_type`) and links the observation span to that end user. All spans with the same `user.id` in the same project contribute to that user's metrics and appear in their detail view. - +A span counts under a user when it carries the `user.id` attribute. You set the value once, and Observe does the grouping. - - Set **`user.id`** (required). You can also set **`user.id.type`** (email, phone, uuid, custom), **`user.id.hash`**, and **`user.metadata`** (JSON) for display or filtering. +user.id = user_456] --> U[(User user_456)] + B[Session
user.id = user_456] --> U + C[Trace
user.id = user_456] --> U + U --> V[Users view
one row, trace + session counts,
per-user cost / evals / history] +`} /> - - ```python Python - from fi_instrumentation import register, FITracer - from fi_instrumentation.fi_types import ProjectType - from opentelemetry.trace import Status, StatusCode +1. Your app sets `user.id` on the spans of each request a given person triggers, reusing the same string for that person. +2. traceAI auto-instrumentors read the value from the OpenTelemetry context and write it onto every span in the block. +3. Observe ingests the traces, creates the user on first sight, and rolls all of that user's traces and sessions up under one row. - trace_provider = register( - project_type=ProjectType.OBSERVE, - project_name="PROJECT_NAME", +The ID is a plain string and the grouping is automatic, so the only thing you own in code is setting `user.id` early enough — see the next section. + +--- + +## Set a user ID in code + +In traceAI, set `user.id` for a block with a context helper, and every span created inside inherits it. Use the same string for every request from one person. + + +```python Python +from fi_instrumentation import using_user + +# Every span created in this block gets user.id = "user_456" +with using_user("user_456"): + response = client.chat.completions.create( + model="gpt-4o", + messages=[{"role": "user", "content": "Write a haiku."}], ) - tracer = FITracer(trace_provider.get_tracer(__name__)) - - with tracer.start_as_current_span("SPAN_NAME") as span: - span.set_status(Status(StatusCode.OK)) - span.set_attribute("user.id", "vivek.gupta") - span.set_attribute("user.id.type", "email") # email | phone | uuid | custom - span.set_attribute("user.id.hash", "") # optional - span.set_attribute("user.metadata", {}) # optional - span.set_attribute("input.value", "input") - span.set_attribute("output.value", "output") - ``` - ```javascript JS/TS - const { register, ProjectType } = require("@traceai/fi-core"); - - const traceProvider = register({ - projectType: ProjectType.OBSERVE, - projectName: "FUTURE_AGI" - }); - const tracer = traceProvider.getTracer("manual-instrumentation-example"); - - tracer.startActiveSpan("SPAN_NAME", {}, (span) => { - span.setAttribute("user.id", "vivek.gupta"); - span.setAttribute("user.id.type", "email"); - span.end(); - }); - ``` - - - - - To tag all spans in a block with the same user, use a context that sets `user.id` (and optional type/metadata) so every span in that block is linked to that end user. With the Python SDK you can use **`using_attributes`** and pass `user_id` (and optionally `session_id`). - - - ```python Python - from fi_instrumentation import using_attributes - - with using_attributes(user_id="newuser@example.com", session_id="new-session"): - response = client.chat.completions.create( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "Write a haiku."}], - max_tokens=20, - ) - ``` - - - - - - Open the project and go to the **Users** (user dashboard) view. - - Table columns: user_id, activation date, last active, trace count, error count, session count, avg latency, LLM calls, eval pass rate, guardrail triggers, tokens, cost. - - Search by user ID; apply filters as needed. - - Click a user for detail: **Summary** metrics, **Traces** tab (trace ID, session, latency, input/output, evals, cost, annotations), **Sessions** tab (session ID, time range, trace count, evals, cost). - ![Dashboard](/images/docs/observe/5.png) - - - - - End users are unique per project and organization by `(user_id, user_id_type)`. Sending the same `user.id` (and type) on spans in the same Observe project ties those spans to one end user in the dashboard. - +``` +```javascript JS/TS +import { context, propagation } from "@opentelemetry/api"; + +// Set the user.id baggage key on the active context. +const baggage = propagation.createBaggage({ + "user.id": { value: "user_456" }, +}); +const ctx = propagation.setBaggage(context.active(), baggage); + +context.with(ctx, () => { + // Every span created here gets user.id = "user_456" + // e.g. await client.chat.completions.create({ ... }); +}); +``` + + +The decorator form works too, when a whole function handles one user's request: + +```python Python +from fi_instrumentation import using_user + +@using_user("user_456") +def handle_request(message): + ... +``` + +To set a session ID at the same time, use `using_attributes(session_id=..., user_id=...)`. For every variant and the full JS/TS baggage pattern, see [Set session and user IDs](/docs/traceai/manual-instrumentation/set-session-user-id). + + + `user.id` is exported in span data. Use a stable but non-sensitive identifier — a hashed customer ID, not a raw email or phone number. To redact sensitive span data, see [Mask span attributes](/docs/traceai/manual-instrumentation/mask-span-attributes). + + +--- + +## View and filter users + +Open the project and go to the **Users** view. Each row is one end user. + +Observe Users view listing end users with User ID, First Active, Last Active, number of traces, and number of sessions columns +*One row per user, with trace and session counts rolled up. Sort by trace or session count to find your most active users, then open one for the full breakdown.* + +The list columns roll each user up at a glance: + +| Column | What it shows | +|---|---| +| **User ID** | The `user.id` value you set in code. | +| **First Active** | When the user's first trace arrived. | +| **Last Active** | When the user's most recent trace arrived. | +| **No. of Traces** | How many traces are attributed to the user. | +| **No. of Sessions** | How many conversations the user had. | +| **Actions** | Open the user's detail view. | + +Click a user for their detail view, where cost, evals, and guardrail results are broken down per session and trace. + +User detail view with summary metrics, a Traces tab, and a Sessions tab for one end user +*User detail: a Traces tab and a Sessions tab. The Sessions tab lists each session with its first and last message, duration, total traces, and total cost.* + +### Filter and date range + +Use the filter bar to narrow the list — by `user.id`, by metadata, or by any span attribute. The filter grammar is shared across Observe; see the [trace filter syntax reference](/docs/observe/reference/trace-filter-syntax) for operators and field names. + +Users filter bar with conditions applied to narrow the end-user list +*Filter users down to the segment you care about — by ID, metadata, or any span attribute.* + +Scope the list to a time window with the date-range picker; trace and session counts recompute for the selected window. + +Users date-range picker scoping the end-user list to a time window +*Scope to a window — last hour, last day, or a custom range. You can also choose which columns to show.* + +--- + +## Limits and edge cases + +- A span **without** a `user.id` is unattributed — it belongs to no user. It still appears in the [trace explorer](/docs/observe/features/llm-tracing); it just will not roll up. +- The ID must be set **before or at the start of the request**, so the spans of that request carry it. Setting it after the model call has already run is too late for those spans. +- `user.id` values are **strings**. Use a stable identifier; an empty string does not group. +- A user is created automatically on the first span carrying its ID — there is nothing to pre-register. +- `user.id` is a grouping key, not an auth identity. Observe never verifies it. +- Do not put raw PII in the `user.id`. To redact sensitive span data, see [Mask span attributes](/docs/traceai/manual-instrumentation/mask-span-attributes). + + + The per-user grouping key. Any non-empty string; reuse the same value for every request from one person. Prefer a hashed customer ID over raw PII. + + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| Traces not attributed to a user | The model call ran outside the `using_user` block, so spans never got `user.id`. | Make the call **inside** the block (or function decorated with `@using_user`). | +| One person split across several users | A different `user.id` was used on some requests. | Reuse one stable string for that person. | +| JS/TS: `user.id` not appearing | Baggage was not set on the active context. | Use `propagation.createBaggage()` + `context.with()` as shown above. | +| User shows no cost or evals | Spans carried the ID but no cost/eval attributes. | Confirm the LLM spans are auto-instrumented and evals are configured. | +| Empty grouping | An empty-string ID was passed. | IDs must be non-empty strings. | --- -## Next Steps +## Next steps - - Connect the SDK and start capturing traces. + + Group a user's traces into conversations. - - Run evaluations on your traced spans to score quality. + + How grouping by user and conversation works. - - Group traces into sessions for multi-turn analysis. + + Every way to attach a user ID in traceAI. - - Get notified when metrics cross a threshold. + + Where user.id sits in the span model. diff --git a/src/pages/docs/observe/features/voice.mdx b/src/pages/docs/observe/features/voice.mdx index e526ce5c..9a2adb8d 100644 --- a/src/pages/docs/observe/features/voice.mdx +++ b/src/pages/docs/observe/features/voice.mdx @@ -1,106 +1,177 @@ --- -title: "Voice Observability: Call Logs as Traces in Observe" -description: "Connect a voice provider like Vapi or Retell and get call logs as traces in Observe without any SDK instrumentation or code changes." +title: "Voice observability" +description: "Connect a voice provider like Vapi or Retell and get every call as a trace in Observe — transcript, recording, cost, and duration — with no SDK or code changes." +slug: "voice" +page_type: "feature" +products: ["Observe"] +feature: "Voice observability" +feature_status: "stable" +ui_surfaces: ["Observe > Agent definitions", "Observe > Projects"] +audience: ["engineer"] +difficulty: "beginner" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-05-25" +last_screenshotted: "2026-05-25" +schema_type: "TechArticle" +seo: + title: "Voice observability in Observe" + description: "Connect a voice provider like Vapi or Retell and get every call as a trace in Observe — transcript, recording, cost, and duration — with no SDK or code changes." + primary_keyword: "voice agent observability vapi retell" + direct_answer: true +geo: + answer_target: "How do I observe voice agent calls in Observe?" + llm_summary: "Voice observability pulls call logs from a voice provider (Vapi, Retell, or Eleven Labs) into Observe automatically, with no SDK or code changes. Connect the provider with its API key and assistant ID, and every call appears as a trace with transcript, recording URLs, cost, and duration — ready for evals, alerts, and export." +canonical: "/docs/observe/features/voice" +related: + - "/docs/observe/features/llm-tracing" + - "/docs/observe/features/evals" + - "/docs/observe/features/alerts" + - "/docs/observe/concepts/traces" --- - ## About -Voice agents are hard to debug. Conversations happen in real time, across multiple turns, and when something goes wrong you usually find out from a user complaint, not a log. **Voice observability** fixes this by pulling call logs from your voice provider into Observe automatically. No SDK or code changes needed. Connect a provider (Vapi, or Retell) using its API key and assistant ID, and every call shows up as a trace with its transcript, recording URLs, cost, and duration. From there you can run [evaluations](/docs/observe/features/evals), set [alerts](/docs/observe/features/alerts), search, filter, and export, the same way you would with any other trace. +Voice agents are hard to debug: conversations happen in real time, across turns, and you usually hear about failures from a user, not a log. Voice observability pulls call logs from your voice provider into Observe automatically — **no SDK or code changes**. Connect a provider (Vapi, Retell, or Eleven Labs) with its API key and assistant ID, and every call shows up as a [trace](/docs/observe/concepts/traces) with its transcript, recording URLs, cost, and duration. From there you can run [evals](/docs/observe/features/evals), set [alerts](/docs/observe/features/alerts), and filter and export, exactly like any other trace. + --- ## When to use -- **Visibility into voice agent calls**: See all conversations for a voice agent in one project without adding SDK instrumentation. -- **Evaluate voice conversations**: Run evals (quality, bias, adherence) on conversation spans from voice calls. -- **Alerts on voice metrics**: Set monitors on voice project metrics and get notified when something degrades. -- **Transcripts and recordings for debugging**: Access transcript and recording URLs from the trace view. -- **Multiple voice providers**: Support for Vapi, Retell so you can monitor agents regardless of provider. +- **See every voice call** for an agent in one project, without instrumenting code. +- **Evaluate conversations** — run quality, bias, or adherence evals on voice call spans. +- **Alert on voice metrics** — get notified when a voice project degrades. +- **Debug with transcripts and recordings** — open a call to read the transcript and play the recording. --- -## How to +## When not to use + +- **A text/SDK app** — that uses normal instrumentation; start at the [quickstart](/docs/observe/quickstart). +- **An unsupported provider** — only Vapi, Retell, and Eleven Labs are supported today (see below). + +--- + +## Connect a voice provider - From your voice provider's dashboard, obtain: - - **API key** - - **Assistant ID** (or agent ID) - - These are required when observability is enabled. Supported providers: [Vapi](https://dashboard.vapi.ai), [Retell](https://www.retellai.com/). + From your provider's dashboard, get the **API key** and **Assistant ID** (a.k.a. agent ID). Supported providers: [Vapi](https://dashboard.vapi.ai), [Retell](https://www.retellai.com/), [Eleven Labs](https://elevenlabs.io). The API key and Assistant ID are both required when observability is enabled. - Go to the **Agent definition** section and click **Create agent definition**. - ![Agent definition list](/screenshot/product/observe/voice/agent_definition_list.png) + Open **Agent definitions** and create one. Fill in the agent name and provider; the API key and Assistant ID are masked. Check **Enable Observability**, then **Create** — you're returned to the list with the new agent. - Fill in agent name, provider, and other required fields. The API key and Assistant ID are masked for security. - ![Create agent definition form](/screenshot/product/observe/voice/agent_definition_filled.png) + Create agent definition form with provider, masked API key, Assistant ID, and Enable Observability toggle + *The API key and Assistant ID are only required when Observability is on.* - Check **Enable Observability**. The API key and Assistant ID are required only if observability is enabled. - ![Agent definition details](/screenshot/product/observe/voice/agent_definition_details.jpeg) - - Click **Create**. You are redirected to the agent list where the new agent is now visible. - ![Agent definition list with new agent](/screenshot/product/observe/voice/agent_definition_list_with_new.jpeg) + Agent definitions list showing the newly created voice agent + *Your agent definitions. Each one maps to a project that collects its calls.* - Open the **Projects** tab. A project with the same name as your agent lists all call logs. - ![Projects list](/screenshot/product/observe/voice/project_list.png) + Open **Projects** — a project named after your agent collects its calls. Open it to see the voice table (calls with status, duration, cost), then click a call for the detail drawer. - Open the project to see the voice observability table (calls with status, duration, cost). - ![Voice observability table](/screenshot/product/observe/voice/voice_observability_table.png) + Voice project table listing calls with status, duration, and cost + *Each row is a call, captured as a trace.* - Click a call to open the detail drawer (transcript, recording URLs, call data). - ![Call log detail drawer](/screenshot/product/observe/voice/call_log_detail_drawer_marked.jpeg) + Call detail drawer with transcript, recording URLs, and call metadata + *Inside a call: transcript, recording URLs, and call data — the context a user complaint never gives you.* - Click an agent definition to open the edit form. You can edit any field. - - - If you **disable** observability, the API key and Assistant ID become optional. - - If you **enable** observability (or keep it on), API key and Assistant ID are required. - - - - ![Agent with observability disabled](/screenshot/product/observe/voice/agent_update_observability_disabled.png) - - - ![Agent with observability enabled](/screenshot/product/observe/voice/agent_update_observability_enabled.png) - - + Open an agent definition to edit it. Disabling observability makes the API key and Assistant ID optional; enabling it makes them required again. +--- + +## How it works + +You connect a provider once through an agent definition, and Observe does the rest: + +1. You create an [agent definition](/docs/observe/features/voice) with a provider, its API key, and the Assistant ID, and turn **Enable Observability** on. +2. Each agent definition maps to a project named after the agent. That project is where the agent's calls land. +3. Observe pulls the provider's call logs and writes each completed call in as a [trace](/docs/observe/concepts/traces) — no SDK, no code on your side. + +Because the data arrives from the provider's logs, calls appear after the provider delivers them, and a call shows up once its events are complete. + +--- + +## What a voice project shows + +A voice project is read at two levels. The project table is the list of calls — one row per call, with status, duration, and cost so you can scan for failed or expensive calls at a glance. Open a row and the **detail drawer** gives you the inside of that one call: the full transcript, the recording URLs, and the call metadata. + +The list also rolls up the per-turn shape of each call. **Turn count** tells you how many back-and-forth turns the conversation had, and **avg latency** is the average response latency across those turns — so a call that completed but felt slow stands out from one that was quick, before you ever open it. Tokens and cost are totalled across the whole call. Once a call is in as a trace it behaves like any other trace: run [evals](/docs/observe/features/evals) on its spans, set [alerts](/docs/observe/features/alerts) on the project, and filter or export the calls. + +--- + +## Voice call columns + +Voice projects use **call-shaped columns** rather than the text trace columns. + +| Column | What it shows | +|---|---| +| Call details | The call identifier and a link into the detail drawer (transcript, recording). | +| Status | Whether the call completed or errored. | +| Duration | Total wall-clock length of the call. | +| Avg latency | Average response latency across the call's turns. | +| Turn count | Number of conversational turns in the call. | +| Tokens | Total tokens used across the call. | +| Cost | Estimated cost of the call. | + +--- + +## Edge cases and limits + +- The **API key and Assistant ID are mandatory** whenever observability is enabled; the agent definition won't capture calls without them. +- Transcripts and recordings can contain sensitive customer audio — treat the recording URLs as sensitive data. + +--- + ## Supported providers -- [Vapi](https://dashboard.vapi.ai) -- [Retell](https://www.retellai.com/) +| Provider | What's captured | Credentials needed | +|---|---|---| +| [Vapi](https://dashboard.vapi.ai) | Call logs as traces — transcript, recording URLs, cost, duration. | API key + Assistant ID | +| [Retell](https://www.retellai.com/) | Call logs as traces — transcript, recording URLs, cost, duration. | API key + Assistant ID | +| [Eleven Labs](https://elevenlabs.io) | Call logs as traces — transcript, recording URLs, cost, duration. | API key + Assistant ID | + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No calls appear | Wrong Assistant ID, or observability is disabled on the agent definition. | Re-check the Assistant ID and confirm **Enable Observability** is on. | +| Calls partial | Provider webhook lag. | Wait for the provider to deliver the remaining call events. | +| Recording URL returns 403 | The signed URL has expired. | Reopen the call to fetch a fresh signed recording URL. | --- -## Next Steps +## Related features - - Connect the SDK and start capturing traces. + + Inspect voice calls alongside your other traces. - - Run evaluations on your traced spans to score quality. + + Score voice conversations for quality and safety. - - Get notified when metrics cross a threshold. + + Get notified when a voice project degrades. - - View activity and metrics per end user. + + Instrument a text/SDK app instead. diff --git a/src/pages/docs/observe/index.mdx b/src/pages/docs/observe/index.mdx index 478e3c5f..42ae0fc5 100644 --- a/src/pages/docs/observe/index.mdx +++ b/src/pages/docs/observe/index.mdx @@ -1,45 +1,67 @@ --- -title: "Future AGI Observe: Monitor LLM Apps in Production" -description: "Monitor and evaluate LLM applications in production with real-time tracing, session analysis, cost tracking, and alerting." +title: "Get started with Observe" +description: "Observe shows you what your AI app did in production. Send your first trace, then search, replay, score, and alert on every request. Start here." +slug: "observe" +page_type: "product-overview" +products: ["Observe"] +frameworks: [] +audience: ["engineer"] +difficulty: "beginner" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_screenshotted: "2026-05-25" +schema_type: "SoftwareApplication" +seo: + title: "Get started with FutureAGI Observe" + description: "Observe records every LLM request as a trace so you can replay it, score it, and get alerted when it degrades. Send your first trace and start here." + primary_keyword: "llm production observability monitoring" + direct_answer: true +geo: + answer_target: "What is FutureAGI Observe and how do I get started?" + llm_summary: "FutureAGI Observe records each LLM request as a trace through the traceAI SDK, then lets you replay requests, score them with evals, group them by session or user, and get alerted when a metric degrades." +canonical: "/docs/observe" +related: + - "/docs/observe/quickstart" + - "/docs/observe/concepts/traces" + - "/docs/observe/features/llm-tracing" + - "/docs/observe/reference/trace-filter-syntax" --- -## About +## What is Observe? -Observability is how you monitor your AI application after it goes live. Once your app is in production, things change: user inputs vary, model behavior shifts, and issues come up that testing never caught. Observability gives you a continuous view of how your application is performing so you can stay on top of it. +Observe shows you what your AI app did in production. Every request is recorded as a [trace](/docs/observe/concepts/traces), and each step inside it — a model call, a tool call, a retrieval — is a [span](/docs/observe/concepts/spans). When an answer is wrong, a request is slow, or cost jumps, you open the trace and see what actually happened instead of guessing. -It tracks every response your application generates, groups them by session and user, scores them for quality, and alerts you when something goes wrong. Instead of finding out about problems from users, you see them in the dashboard first. +Send one trace and the rest opens up: search and replay requests, group them by [session](/docs/observe/features/session) or [user](/docs/observe/features/users), score them with [evals](/docs/observe/features/evals), chart them on [dashboards](/docs/observe/features/dashboard), and get [alerted](/docs/observe/features/alerts) when something slips. -Sessions Overview +Observe trace explorer listing production traces with status, latency, and token columns +*Every production request, captured as a trace and ready to inspect.* --- -## How Observability Connects to Other Features - -- **Prototype**: After you promote a winning version in Prototype, its traces continue flowing into Observe so you can monitor production performance against the same quality criteria. [Learn more](/docs/prototype) -- **Evaluation**: Observability uses the same built-in eval templates to score production traces automatically. Any eval you configured in Prototype or Datasets runs the same way here. [Learn more](/docs/evaluation) -- **Alerts**: Observability feeds into the alerting system so you are notified when quality, cost, or latency crosses a threshold in production. [Learn more](/docs/observe/features/alerts) - ---- - -## Getting Started with Observability +## Start here - - Connect the SDK and start capturing traces in minutes. - - - Run evaluations on observed traces and sessions. - - - Group and analyze multi-turn interactions. + + Instrument your app and see a request in Observe in about five minutes. - - Track and analyze activity by user. + + One line to trace OpenAI, LangChain, LlamaIndex, and 30+ others. - - Configure alerts for real-time issue detection. + + The mental model the rest of Observe is built on. - - Monitor voice agent interactions and call quality. + + Open a request and read it step by step. + +--- + +## Look something up + +Need exact fields, operators, or limits? Go straight to the [reference](/docs/observe/reference/trace-filter-syntax): + +- [Filter syntax](/docs/observe/reference/trace-filter-syntax) — every filter property and operator, with ready-to-paste examples. +- [Dashboard metrics](/docs/observe/reference/dashboard-metric-definitions) — what each metric measures and how it's derived. +- [Export and endpoints](/docs/observe/reference/export-formats) — getting data out, and the OTLP endpoints traces are sent to. diff --git a/src/pages/docs/observe/quickstart.mdx b/src/pages/docs/observe/quickstart.mdx new file mode 100644 index 00000000..3423f7e4 --- /dev/null +++ b/src/pages/docs/observe/quickstart.mdx @@ -0,0 +1,210 @@ +--- +title: "Quick start: send your first trace to Observe" +description: "Install traceAI, register an Observe project, run one OpenAI request, and confirm the trace appears in FutureAGI with model, latency, and token cost — in about five minutes." +slug: "quickstart" +page_type: "quickstart" +products: ["Observe", "traceAI"] +audience: ["engineer"] +difficulty: "beginner" +time_to_complete: "5 minutes" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-05-25" +last_screenshotted: "2026-05-26" +schema_type: "HowTo" +success_artifact: "trace" +api_keys_needed: ["FI_API_KEY", "FI_SECRET_KEY", "OPENAI_API_KEY"] +dashboard_verification_path: "Observe > your project > Tracing" +seo: + title: "Observe quick start: your first LLM trace" + description: "Send your first traced LLM request to FutureAGI Observe in five minutes — install, register a project, run one call, and confirm the trace." + primary_keyword: "send first llm trace to futureagi" + direct_answer: true +geo: + answer_target: "How do I send my first trace to FutureAGI Observe?" + llm_summary: "Install fi-instrumentation-otel and a traceAI instrumentor, call register() with project_type OBSERVE, run one instrumented request, and the trace appears in Observe within seconds." +canonical: "/docs/observe/quickstart" +related: + - "/docs/observe/concepts/traces" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/observe/features/llm-tracing" +--- + +## About + +Send your first traced request to FutureAGI in about five minutes. You'll install the traceAI instrumentor, register an Observe project, run one normal OpenAI chat completion, and confirm the request shows up in the [trace explorer](/docs/observe/features/llm-tracing) with its model, latency, and token cost. Everything after this — sessions, evals, users, alerts — builds on traces, so this is the page to start from. + +This example traces OpenAI because it's the shortest path to a visible result, but the flow is the same for every [supported framework](/docs/traceai/auto): install the instrumentor, register a project, attach the instrumentor, run your code. Nothing here changes your application logic — auto-instrumentation captures the call you already make. Five minutes from now you'll have one real trace to read, and the pattern to instrument the rest of your app. + +--- + +## Prerequisites + +- **Python 3.9+** (or Node 18+ for the TypeScript path). +- A **FutureAGI account** and your **`FI_API_KEY`** + **`FI_SECRET_KEY`** from the [dashboard keys page](https://app.futureagi.com/dashboard/keys). +- An **`OPENAI_API_KEY`** (this example traces an OpenAI call). + + + Pin packages to the version you test against. The commands below were last verified on **2026-05-25**; add a version (e.g. `traceAI-openai==`) before shipping to CI so a future release can't silently change behavior. + + +--- + +## Install + +Install the core instrumentation package and the instrumentor for your provider. + + +```bash Python +pip install fi-instrumentation-otel traceAI-openai +``` +```bash JS/TS +npm install @traceai/fi-core @traceai/openai +``` + + +--- + +## Configure + +Read keys from the environment — never hardcode them in source. + + +```python Python +import os +os.environ["FI_API_KEY"] = "YOUR_FI_API_KEY" +os.environ["FI_SECRET_KEY"] = "YOUR_FI_SECRET_KEY" +``` +```typescript JS/TS +process.env.FI_API_KEY = "YOUR_FI_API_KEY"; +process.env.FI_SECRET_KEY = "YOUR_FI_SECRET_KEY"; +``` + + +--- + +## Run your first request + + + + `register` returns a tracer provider. Set `project_type` to `OBSERVE` and give the project a name. + + + ```python Python + from fi_instrumentation import register, Transport + from fi_instrumentation.fi_types import ProjectType + + trace_provider = register( + project_type=ProjectType.OBSERVE, + project_name="my-first-project", + transport=Transport.GRPC, + ) + ``` + ```typescript JS/TS + import { register, ProjectType } from "@traceai/fi-core"; + + const traceProvider = register({ + project_type: ProjectType.OBSERVE, + project_name: "my-first-project", + }); + ``` + + + + + Attach the OpenAI instrumentor to the provider, then call OpenAI as you normally would. + + + ```python Python + from traceai_openai import OpenAIInstrumentor + from openai import OpenAI + + OpenAIInstrumentor().instrument(tracer_provider=trace_provider) + + os.environ["OPENAI_API_KEY"] = "YOUR_OPENAI_API_KEY" + client = OpenAI() + completion = client.chat.completions.create( + model="gpt-4o", + messages=[{"role": "user", "content": "Write a one-sentence bedtime story about a unicorn."}], + ) + print(completion.choices[0].message.content) + ``` + ```typescript JS/TS + import { OpenAIInstrumentation } from "@traceai/openai"; + import { OpenAI } from "openai"; + + new OpenAIInstrumentation({}); + + const client = new OpenAI({ apiKey: process.env.OPENAI_API_KEY }); + const completion = await client.chat.completions.create({ + model: "gpt-4o", + messages: [{ role: "user", content: "Write a one-sentence bedtime story about a unicorn." }], + }); + console.log(completion.choices[0].message.content); + ``` + + + Expected terminal output (the model text varies): + + ```text + Under a sky of silver stars, a gentle unicorn dipped its horn into a moonlit + pool and wished every sleeping child sweet dreams. + ``` + + + +--- + +## Confirm the trace + +Open **Observe → your project → Tracing**. Within a few seconds you should see one new trace row for the request, showing **Status OK**, the **gpt-4o** model, the **latency**, and the **token count**. Click it to read the prompt, the completion, and the span timing. + +Observe trace explorer with one new OpenAI trace showing OK status, model, latency, and token columns +*Your request, now a trace. If the row is here with an OK status, instrumentation is working end to end.* + +--- + +## What you just captured + +That one row is a [trace](/docs/observe/concepts/traces) — the full record of one request. Because this example made a single OpenAI call, the trace holds one [span](/docs/observe/concepts/spans): the `llm` operation, with the model, the prompt and completion text, the prompt/completion/total token counts, and the cost. Click the row to open the span and read each of those fields. + +This is the unit everything else in Observe is built on: + +- A request that calls a **model, a tool, and a retriever** produces a span for each, nested under one trace — so you read the whole path top to bottom. +- Attach a [session and user ID](/docs/traceai/manual-instrumentation/set-session-user-id) and the traces group into conversations and per-user views. +- Add [tags](/docs/observe/features/tags) and you can filter the trace list down to one release or experiment. +- Run a [trace eval](/docs/observe/features/evals) and a quality score attaches to the span. + +You didn't configure any of that here — you sent one instrumented request, and Observe had everything it needed. The rest of the product is just different lenses on these traces. + +--- + +## Not seeing data? + +| Symptom | Fix | +|---|---| +| No trace appears | If this is a short script that exits immediately, the batch exporter may not have flushed — call `trace_provider.force_flush()` before the process ends. | +| Still nothing | Widen the date picker (it defaults to the past 7 days) and turn on **Auto refresh**. | +| Wrong or empty project | Confirm `project_name` matches the project you're viewing, and that `FI_API_KEY`/`FI_SECRET_KEY` are the keys for this workspace. | + +For deeper diagnosis see [No traces appear](/docs/observe/troubleshooting/no-traces-appearing). For custom spans and frameworks beyond OpenAI, see [Set up tracing](/docs/traceai/manual-instrumentation/set-up-tracing) and the [framework integrations](/docs/traceai/auto). + +--- + +## Next steps + + + + Read and debug the traces you started capturing. + + + The mental model the rest of Observe is built on. + + + Score the quality of production responses. + + + Get notified when metrics cross a threshold. + + diff --git a/src/pages/docs/observe/reference/dashboard-metric-definitions.mdx b/src/pages/docs/observe/reference/dashboard-metric-definitions.mdx new file mode 100644 index 00000000..319f5fa8 --- /dev/null +++ b/src/pages/docs/observe/reference/dashboard-metric-definitions.mdx @@ -0,0 +1,98 @@ +--- +title: "Dashboard metrics" +description: "Reference for the metrics, aggregations, and granularities available in Observe dashboard widgets — what each metric measures and how it's rolled up." +slug: "dashboard-metric-definitions" +page_type: "reference" +products: ["Observe"] +audience: ["engineer"] +difficulty: "beginner" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-05-25" +schema_type: "TechArticle" +seo: + title: "Dashboard metrics in Observe" + description: "Reference for the metrics, aggregations, and granularities available in Observe dashboard widgets — what each metric measures and how it's rolled up." + primary_keyword: "dashboard metric definitions observability" + direct_answer: true +geo: + answer_target: "What metrics and aggregations can I plot on an Observe dashboard widget?" + llm_summary: "Each Observe dashboard widget plots one metric — span count, error count, span or LLM response time, token usage, cost, or eval pass-rate — rolled up by an aggregation (sum, average, median, count, distinct count, min/max) over a time granularity (minute to month)." +canonical: "/docs/observe/reference/dashboard-metric-definitions" +related: + - "/docs/observe/features/dashboard" + - "/docs/observe/features/alerts" + - "/docs/observe/troubleshooting/dashboard-numbers-look-wrong" +--- + +## About + +Each [dashboard](/docs/observe/features/dashboard) widget plots one metric, rolled up by an aggregation over a time granularity. This page defines the metrics and the aggregation and granularity options. + +## Metrics + +| Metric | Unit | What it measures | +|---|---|---| +| Span count | count | Number of spans matching the widget's filters. | +| Error count | count | Number of spans/traces with `ERROR` status. | +| Span response time | ms | Latency of spans. | +| LLM response time | ms | Latency of LLM spans specifically. | +| Token usage | tokens | Tokens consumed (prompt + completion). | +| Cost | USD | Computed cost of model calls. | +| Eval pass-rate | % | Share of evaluated spans that passed their eval. | + +The widget editor lists the metrics available for your project. + + + Eval pass-rate depends on eval sampling — it covers only the spans an eval actually ran on, not all traffic. If the number looks off, see [Dashboard numbers look wrong](/docs/observe/troubleshooting/dashboard-numbers-look-wrong). + + +## How each metric is derived + +Every metric is computed from the [spans](/docs/observe/concepts/spans) that match the widget's filters and time bucket — nothing is precomputed. The raw value per bucket is derived as follows; the **aggregation** you pick (next section) is then applied over that bucket. + +| Metric | Derived from | Per-bucket value | +|---|---|---| +| Span count | matching spans | `count(spans)` | +| Error count | spans whose status is `ERROR` | `count(spans where status = ERROR)` | +| Span response time | each span's duration | `end_time − start_time` per span, in ms (then aggregated) | +| LLM response time | duration of `llm` spans only | same as above, restricted to `observation_type = llm` | +| Token usage | per-span token counts | `prompt_tokens + completion_tokens`, summed across matching spans | +| Cost | per-`llm`-span cost | `Σ (tokens × model unit price)` over matching `llm` spans | +| Eval pass-rate | spans an eval ran on | `passed_spans ÷ evaluated_spans × 100` | + +Two consequences fall out of this: + +- **Latency metrics are per span, not per trace.** "Span response time" measures one operation; a whole request's wall-clock time is the root span's duration. To compare requests, filter to root spans. +- **Cost and token usage only populate on `llm` spans.** A tool or retrieval span contributes to span count but adds zero tokens and zero cost, so a token widget unfiltered across all spans still reports only model usage. + +## Aggregations + +| Aggregation | Result | +|---|---| +| Sum | Total across the bucket. | +| Average | Mean across the bucket. | +| Median | 50th percentile. | +| Count | Number of matching records. | +| Distinct count | Number of unique values. | +| Min / Max | Smallest / largest value in the bucket. | + +## Granularity + +Buckets the time axis: **minute, hour, day, week, month**. Available options adjust to the selected time range (a 12-month range won't offer minute granularity). + +## Reading a metric correctly + +A widget's number is always *metric × aggregation × granularity × filters*. "Average LLM response time per hour" and "max LLM response time per day" come from the same spans but answer different questions. If a number looks wrong, see [Dashboard numbers look wrong](/docs/observe/troubleshooting/dashboard-numbers-look-wrong). + +## Related + + + + Build widgets from these metrics. + + + Alert when one of these metrics crosses a threshold. + + diff --git a/src/pages/docs/observe/reference/export-formats.mdx b/src/pages/docs/observe/reference/export-formats.mdx new file mode 100644 index 00000000..7614da57 --- /dev/null +++ b/src/pages/docs/observe/reference/export-formats.mdx @@ -0,0 +1,92 @@ +--- +title: "Export and endpoints" +description: "Reference for getting trace data out of Observe — exporting the current view, the available formats, and the OTLP endpoints traces are sent to (cloud and self-hosted)." +slug: "export-formats" +page_type: "reference" +products: ["Observe"] +audience: ["engineer"] +difficulty: "beginner" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-05-25" +schema_type: "TechArticle" +seo: + title: "Trace export and endpoints in Observe" + description: "Reference for getting trace data out of Observe — exporting the current view, the available formats, and the OTLP endpoints traces are sent to (cloud and self-hosted)." + primary_keyword: "export traces endpoints futureagi" + direct_answer: true +geo: + answer_target: "How do I export traces from Observe, and what endpoints does traceAI send to?" + llm_summary: "Observe exports the current trace-explorer view (filters and time range) to CSV from the download icon. Inbound, the traceAI SDK sends spans over OTLP to FutureAGI — set FI_BASE_URL (HTTP) or FI_GRPC_URL (gRPC), defaulting to FutureAGI cloud, or point them at your own collector when self-hosted." +canonical: "/docs/observe/reference/export-formats" +related: + - "/docs/observe/features/llm-tracing" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/observe/concepts/otel" +--- + +## About + +There are two directions for trace data: **out of** Observe (exporting what you're viewing) and **into** Observe (the OTLP endpoints traceAI sends to). This page covers both. + +## Export from the trace explorer + +The download icon in the [trace explorer](/docs/observe/features/llm-tracing) header exports the **current view** — the traces that match your active filters and time range. + +**CSV is the only export format.** There is no JSON or Parquet download from the trace explorer. + +| Format | Use for | +|---|---| +| CSV | Spreadsheet analysis, sharing, importing elsewhere. | + +The CSV holds one row per span in the current view, with the columns the explorer shows: trace ID, span name, status, model, provider, latency, token counts, cost, timestamp, and any eval scores present on the span. + + + The export reflects the current view, so a very large view may be truncated. Narrow the filters or time range to export a complete slice. + + +## Ingestion endpoints + +traceAI exports spans over OTLP to FutureAGI. The transport and target are environment-driven: + +| Variable | Transport | Default | +|---|---|---| +| `FI_BASE_URL` | HTTP collector | FutureAGI cloud collector | +| `FI_GRPC_URL` | gRPC collector | FutureAGI cloud collector | + +When unset, both variables default to the FutureAGI cloud collector, so a cloud project needs no endpoint configuration — only `FI_API_KEY` and `FI_SECRET_KEY`. + +- **Cloud:** leave the defaults; set `FI_API_KEY` and `FI_SECRET_KEY`. +- **Self-hosted:** point `FI_BASE_URL` / `FI_GRPC_URL` at your own collector host so spans stay in your network. + +Choose the transport with `transport=Transport.HTTP` (default) or `Transport.GRPC` in `register()`. See [Set up tracing](/docs/traceai/manual-instrumentation/set-up-tracing). + +### Endpoint contract + +The ingestion endpoint is an **OTLP/traces** receiver — you don't call it directly; the traceAI SDK's exporter does. The contract: + +| Aspect | Detail | +|---|---| +| Protocol | OTLP over HTTP (protobuf) or gRPC — the [OpenTelemetry](/docs/observe/concepts/otel) standard, not a proprietary API. | +| Operation | Export spans (write-only). There is no read/query endpoint; reading happens in the trace explorer. | +| Auth | `FI_API_KEY` + `FI_SECRET_KEY` from the [keys page](https://app.futureagi.com/dashboard/keys), sent by the SDK on every export. Keys are workspace-scoped. | +| Success | The exporter batches spans and sends them in the background; a successful export returns no payload. Spans appear in Observe within seconds. | +| Errors | A `401` means the keys are wrong for this workspace; a `4xx` means a malformed/oversized batch; transient `5xx`/network errors are retried by the batch exporter. | +| Limits | Spans are sent by the **batch** span processor on an interval, so a short-lived process must call `trace_provider.force_flush()` before exit or the last batch is lost. Very large payloads (huge prompts/outputs) can be dropped — [mask or trim](/docs/traceai/manual-instrumentation/mask-span-attributes) them at the SDK. | +| Versioning | Pin `fi-instrumentation-otel` and each instrumentor to a tested version so a release can't change span shape under you; the wire format follows the OTLP version the SDK ships. | + + + Span input and output can carry customer data before they leave your process. Redact at the SDK with `TraceConfig` or the `FI_HIDE_*` variables — see [Mask span attributes](/docs/traceai/manual-instrumentation/mask-span-attributes). + + +## Related + + + + Filter, then export the current view. + + + Configure the OTLP endpoint and transport. + + diff --git a/src/pages/docs/observe/reference/trace-filter-syntax.mdx b/src/pages/docs/observe/reference/trace-filter-syntax.mdx new file mode 100644 index 00000000..c9bf99eb --- /dev/null +++ b/src/pages/docs/observe/reference/trace-filter-syntax.mdx @@ -0,0 +1,149 @@ +--- +title: "Filter syntax" +description: "Reference for filtering traces in Observe — the three filter modes, every filterable property with its attribute key and operators, and ready-to-paste query expressions." +page_type: "reference" +slug: "trace-filter-syntax" +products: ["Observe"] +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-05-25" +schema_type: "TechArticle" +seo: + title: "Filter syntax" + description: "Reference for filtering traces in Observe — the three filter modes, every filterable property with its attribute key and operators, and ready-to-paste query expressions." + primary_keyword: "trace filter syntax observability" + direct_answer: true +geo: + answer_target: "How do I filter traces in Observe, and what can I filter on?" + llm_summary: "Observe filters traces three ways: plain-language AI search, a Basic property/condition/value builder, and a Query expression for power users. Filterable properties include trace ID, trace name, span name, status, model, node type, span kind, user ID, and provider, plus eval scores and annotation values. Query expressions like `status = ERROR AND model = gpt-4o` can be pasted directly." +canonical: "/docs/observe/reference/trace-filter-syntax" +related: + - "/docs/observe/features/llm-tracing" + - "/docs/observe/features/tags" + - "/docs/observe/concepts/spans" +--- + +## About + +The **Filter** panel in the [trace explorer](/docs/observe/features/llm-tracing) narrows which traces are shown. It offers three modes — plain-language AI search, a Basic property/condition/value builder, and a Query expression for power users. + +This page lists the modes, every property you can filter on (with its underlying attribute key and the operators it accepts), a set of ready-to-paste query expressions, and the Basic operators. + +--- + +## Filter modes + +| Mode | Use it for | +|---|---| +| AI search | Describe what you want in plain English (e.g. *"errors on gpt-4o today"*) and the filter is built for you. | +| Basic | Pick a property, a condition, and a value. Add several; they apply together (AND). | +| Query | Write a filter expression directly, for conditions the Basic builder can't express. | + + + **Query mode uses symbolic operators** (`=`, `!=`, `contains`, `>`, `<`), while **Basic mode uses the word equivalents** (`is`, `is not`, `contains`, `greater than`, `less than`). They mean the same thing — the property tables below list the word forms, and the ready-to-use query examples use the symbols. + + +--- + +## Filterable properties + +Each property maps to a [span](/docs/observe/concepts/spans) attribute key. Use the property name in the Basic builder, or the attribute key in a Query expression. + +| Property | Attribute key | Example value | Operators | +|---|---|---|---| +| Trace ID | `trace.id` | `7f3c1a9b…` | `is`, `is not` | +| Trace Name | `trace.name` | `support_agent.run` | `is`, `is not`, `contains` | +| Span Name | `span.name` | `tool.check_order_status` | `is`, `is not`, `contains` | +| Status | `status` | `OK`, `ERROR` | `is`, `is not` | +| Model | `llm.model_name` | `gpt-4o` | `is`, `is not`, `contains` | +| Node Type | `node.type` | `llm`, `chain`, `tool` | `is`, `is not` | +| Span Kind | `fi.span.kind` | `LLM`, `RETRIEVER`, `TOOL` | `is`, `is not` | +| User ID | `user.id` | `user_8821` | `is`, `is not`, `contains` | +| Provider | `llm.provider` | `openai`, `anthropic` | `is`, `is not` | +| Service / Trace Name | `service.name` | `checkout-service` | `is`, `is not`, `contains` | +| Latency | `latency` | `5000` (ms, numeric) | `greater than`, `less than` | +| Eval score | `eval.score` / `scores.` | `0.5` (numeric) | `greater than`, `less than` | +| Tag | `tag.tags` | `needs-review` (list of strings) | `contains` | + +In addition, **annotation values** attached to a span are filterable — use `is` / `is not` on an annotation value. + + + Property names and attribute keys are case-sensitive in Query mode. Status values are upper-case (`OK`, `ERROR`); model and provider names match what the SDK reported. + + +--- + +## Ready-to-use filters + +Paste any of these into the **Query** tab. Each line finds a common class of trace. + +Errors on a specific model, useful when a provider or model version is misbehaving: + +```text +status = ERROR AND llm.model_name = gpt-4o +``` + +Slow spans — anything over 5 seconds, to find latency outliers: + +```text +latency > 5000 +``` + +Every trace for one end user, when you're debugging a single user's report: + +```text +user.id = user_8821 +``` + +Low-scoring responses, to triage quality regressions from an [eval](/docs/observe/features/llm-tracing): + +```text +eval.score < 0.5 +``` + +Retriever spans only, to inspect a RAG pipeline's retrieval step in isolation: + +```text +fi.span.kind = RETRIEVER +``` + +Traces carrying a specific [tag](/docs/observe/features/tags), to pull back a set you grouped earlier: + +```text +tag.tags contains needs-review +``` + + + Combine conditions with `AND` to narrow, and reuse the same expression as a saved view in the trace explorer so the whole team sees the same slice. + + +--- + +## Basic operators + +Operators available depend on the property's type. The Basic builder shows the valid operators for each property as you select it. + +| Operator | Applies to | +|---|---| +| `is` / `is not` | Exact match (status, model, provider, enums). | +| `contains` | Substring match (names, inputs, user ID). | +| `greater than` / `less than` | Numeric values (latency, tokens, eval score). | + +--- + +## Related + + + + Where filters are applied, stacked, and saved as views. + + + Label traces so you can filter back to a set you grouped earlier. + + + The fields these properties and attribute keys map to. + + diff --git a/src/pages/docs/observe/troubleshooting/alerts-did-not-fire.mdx b/src/pages/docs/observe/troubleshooting/alerts-did-not-fire.mdx new file mode 100644 index 00000000..3a441baf --- /dev/null +++ b/src/pages/docs/observe/troubleshooting/alerts-did-not-fire.mdx @@ -0,0 +1,93 @@ +--- +title: "Alerts not firing" +description: "A metric crossed your threshold but no alert arrived. Usual causes: the monitor's frequency hasn't elapsed, it's muted, the threshold direction is wrong, or notifications are misconfigured." +slug: "alerts-did-not-fire" +page_type: "troubleshooting" +products: ["Observe"] +failure_surface: "dashboard" +symptom: "alert did not fire" +audience: ["engineer"] +difficulty: "beginner" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-05-25" +last_screenshotted: "2026-06-19" +support_escalation: "support@futureagi.com" +schema_type: "TechArticle" +seo: + title: "Alerts not firing in Observe" + description: "A metric crossed your threshold but no alert arrived. Usual causes: the monitor's frequency hasn't elapsed, it's muted, the threshold direction is wrong, or notifications are misconfigured." + primary_keyword: "alert did not fire monitor" + direct_answer: true +geo: + answer_target: "Why did my Observe monitor not fire an alert?" + llm_summary: "When a metric crosses a threshold but no alert arrives, check timing first — a monitor only evaluates on its alert_frequency (min 5, default 60 min) — then whether it's muted, whether the threshold operator and value match the breach, whether a percentage-change baseline has enough history, and whether the email or Slack notification channel is configured correctly." +canonical: "/docs/observe/troubleshooting/alerts-did-not-fire" +related: + - "/docs/observe/features/alerts" + - "/docs/observe/features/dashboard" + - "/docs/observe/reference/dashboard-metric-definitions" +--- + + +## Symptom + +A metric crossed what you thought was the threshold, but no email or Slack arrived. The usual causes are timing (the monitor only evaluates on its schedule), the monitor being muted, the threshold direction or value being set differently than you remember, or the notification channel itself failing. Check the schedule and mute state first — those explain most "missing" alerts. + +- A metric clearly breached the limit but no notification came. +- Alerts used to arrive and stopped. +- The alert log shows nothing for the period you expected. + +Observe alerts list showing an Active 'High latency (p95 > 2s)' monitor with Last Triggered '-' and 0 triggers +*An alert configured but never fired: Status is Active, yet Last Triggered shows '-' and the trigger count is 0. That points to schedule timing, a mute, or a threshold set differently than expected — not a broken metric.* + +--- + +## Quick checks + +- One `alert_frequency` cycle has elapsed since the breach (minimum 5, default 60 minutes). +- The monitor is **not** muted (`is_mute`). +- `threshold_operator` and the critical value match the direction of the breach. +- `notification_emails` and/or `slack_webhook_url` are set and valid. + +## Causes and fixes + +| Cause | What you see | Fix | +|---|---|---| +| Frequency hasn't elapsed | A brief breach between evaluation runs left no alert | A monitor evaluates on `alert_frequency` (minimum 5, default 60 minutes). Lower the frequency if you need faster detection. | +| Monitor is muted | The monitor keeps evaluating but no notification arrives | `is_mute` stops notifications while evaluation continues; unmute it. | +| Threshold direction or value | A spike didn't fire a "less than" monitor (or vice versa) | `threshold_operator` (`Greater than` / `Less than`) and the critical value must match the breach you expect. | +| Percentage-change baseline | A new project never alerts on a percentage-change monitor | A percentage-change monitor needs enough history in its `auto_threshold_time_window` to compute a baseline. | +| Notification channel | The alert log shows a fire, but no email/Slack arrives | Verify `notification_emails` (up to 5) and/or `slack_webhook_url`; a bad webhook silently drops the message. | + +## Diagnostic checks + +Open the monitor and read its **frequency, mute state, operator, threshold value, and notification channels**, then check the alert log: + +- A log entry with no email/Slack points at the notification channel (`notification_emails` / `slack_webhook_url`). +- No log entry at all points at timing (`alert_frequency`), mute (`is_mute`), or the threshold direction. + +## Minimal smoke test + +Set a deliberately easy threshold, wait one `alert_frequency` cycle, and confirm an alert log entry plus the email/Slack message arrive. Then restore the real threshold. + +## Escalate + +If the monitor still won't fire on a confirmed breach, contact support@futureagi.com with the monitor name, its config, and the breach timestamp. + +## Prevent recurrence + +- Match `alert_frequency` to how fast you need to know — don't leave it at 60 if minutes matter. +- Test each notification channel once when you create the monitor. + +## Next steps + + + + How monitors and thresholds are configured. + + + Confirm the metric trend the alert watches. + + diff --git a/src/pages/docs/observe/troubleshooting/dashboard-numbers-look-wrong.mdx b/src/pages/docs/observe/troubleshooting/dashboard-numbers-look-wrong.mdx new file mode 100644 index 00000000..8ed8ff97 --- /dev/null +++ b/src/pages/docs/observe/troubleshooting/dashboard-numbers-look-wrong.mdx @@ -0,0 +1,94 @@ +--- +title: "Dashboard numbers" +description: "A dashboard widget shows a number you didn't expect. Almost always it's the time range, granularity, aggregation, filters, or sampling — not bad data." +slug: "dashboard-numbers-look-wrong" +page_type: "troubleshooting" +products: ["Observe"] +failure_surface: "dashboard" +symptom: "dashboard numbers look wrong" +audience: ["engineer"] +difficulty: "beginner" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-05-25" +last_screenshotted: "2026-06-19" +support_escalation: "support@futureagi.com" +schema_type: "TechArticle" +seo: + title: "Dashboard numbers look wrong in Observe" + description: "A dashboard widget shows a number you didn't expect. Almost always it's the time range, granularity, aggregation, filters, or sampling — not bad data." + primary_keyword: "dashboard numbers wrong observability" + direct_answer: true +geo: + answer_target: "Why does an Observe dashboard widget show a number I didn't expect?" + llm_summary: "When a dashboard widget shows an unexpected number, the data is almost always right and the query is reading it differently than assumed. Check the time range, granularity, aggregation, and filters first — plus eval sampling and timezone — and cross-check one value against the trace explorer for the same window." +canonical: "/docs/observe/troubleshooting/dashboard-numbers-look-wrong" +related: + - "/docs/observe/features/dashboard" + - "/docs/observe/features/llm-tracing" + - "/docs/observe/reference/dashboard-metric-definitions" +--- + + +## Symptom + +A widget shows a number that doesn't match what you expected — cost too low, latency too high, a count that seems off. Almost always the data is right and the *query* is reading it differently than you assumed: the time range, granularity, aggregation, or filters change what a widget reports. Check those four before suspecting the underlying traces. + +- A metric looks far higher or lower than reality. +- Two widgets that "should" match don't. +- A number changed when you only changed the time range or granularity. + +Observe System Metrics dashboard with latency, tokens, traffic, and cost charts over a 30-day range +*The System Metrics view. When a number here looks off, the data is usually right and the query is reading it differently — check the time range, granularity, aggregation, and filters before suspecting the traces.* + +--- + +## Quick checks + +- The widget's **time range** matches the window you have in mind. +- The **granularity** (bucket size) is what you expect — per-hour and per-day give different numbers. +- The **aggregation** (sum / average / median) answers the question you're asking. +- No stray **filter** (model, status, attribute) is silently narrowing the data. + +## Causes and fixes + +| Cause | What you see | Fix | +|---|---|---| +| Time range / granularity | A number changed when you only changed the window or bucket size | A chart reflects the selected window and bucket. Set both to match your expectation — *average latency per hour* and *per day* differ from the same traces. | +| Aggregation mismatch | Two widgets that "should" match don't | Sum vs. average vs. median answer different questions — confirm the widget uses the one you mean. | +| Filters narrowing the data | A metric looks far lower than reality | A widget filter (model, status, attribute) silently excludes traces; clear it to compare against the full set. | +| Eval sampling | An eval-based metric covers fewer spans than total traffic | If a metric is built on evals run at a sampling rate, it covers a *subset* of spans, not all of them. | +| Timezone | An apparent gap or spike at a day boundary | Day boundaries follow the dashboard timezone — the boundary effect, not missing data. | + +## Diagnostic checks + +Open the widget editor and read its **time range, granularity, aggregation, group-by, and filters**. Then cross-check one value against the [trace explorer](/docs/observe/features/llm-tracing) for the exact same window: + +- Apply the same time range and filters in the trace explorer. +- Count the matching traces (or read the latency/cost column) and compare to the widget. +- If the two agree, the widget config — not the data — explains the number. + +## Minimal smoke test + +Set the widget's time range and granularity to match your expectation, clear extra filters, and confirm the value lines up with a trace-explorer count for the same window. They should reconcile within the rounding of the chosen aggregation. + +## Escalate + +If a value still can't be reconciled with the trace list for the same window, contact support@futureagi.com with the dashboard, the widget config, and the window. + +## Prevent recurrence + +- Label widgets with their aggregation and window so readers don't misread them. +- Keep one "all traffic, no filters" reference widget to sanity-check the others. + +## Next steps + + + + How widgets are configured. + + + Cross-check a number against the raw traces. + + diff --git a/src/pages/docs/observe/troubleshooting/missing-attributes.mdx b/src/pages/docs/observe/troubleshooting/missing-attributes.mdx new file mode 100644 index 00000000..f565423c --- /dev/null +++ b/src/pages/docs/observe/troubleshooting/missing-attributes.mdx @@ -0,0 +1,104 @@ +--- +title: "Spans and attributes" +description: "The trace appears but spans are missing or fields like input/output are blank. Usual causes: masking is on, the instrumentor isn't attached, or a custom key isn't indexed." +slug: "missing-attributes" +page_type: "troubleshooting" +products: ["Observe"] +failure_surface: "sdk" +symptom: "spans or attributes missing from trace" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-05-25" +last_screenshotted: "2026-06-19" +support_escalation: "support@futureagi.com" +schema_type: "TechArticle" +seo: + title: "Spans or attributes missing from a trace" + description: "The trace appears but spans are missing or fields like input/output are blank. Usual causes: masking is on, the instrumentor isn't attached, or a custom key isn't indexed." + primary_keyword: "missing spans attributes trace" + direct_answer: true +geo: + answer_target: "Why are spans or attributes missing from a trace in Observe?" + llm_summary: "If a trace shows but spans or fields like input/output are missing, check redaction first (FI_HIDE_INPUTS/OUTPUTS or TraceConfig masking makes blanks expected), then confirm the framework's instrumentor is attached, that custom attributes were set on the active span before it closed, and that you're filtering on indexed semantic-convention keys with supported value types." +canonical: "/docs/observe/troubleshooting/missing-attributes" +related: + - "/docs/observe/features/llm-tracing" + - "/docs/traceai/manual-instrumentation/add-attributes-metadata-tags" + - "/docs/traceai/manual-instrumentation/mask-span-attributes" + - "/docs/observe/concepts/spans" +--- + + +## Symptom + +The trace shows up, but it's incomplete — a nested span is missing, or fields like input and output are blank, or a custom attribute you set isn't there. The usual causes are redaction being switched on (in which case blank is *expected*), the framework's instrumentor not being attached, or an attribute set on the wrong span or after it closed. Check redaction first, because a hidden field is working as designed, not a bug. + +- A span's input/output show as hidden or blank. +- A framework's child spans (e.g. nested LangGraph nodes) don't appear. +- A custom attribute you set isn't on the span, or you can't filter by it. + +Observe trace explorer showing an empty trace list — 'No traces found' — with the attribute columns present but unpopulated +*When spans or attributes aren't arriving, the trace list reads empty like this — the columns exist but nothing fills them. Confirm the instrumentor is attached and that redaction isn't hiding fields before assuming data was dropped.* + +--- + +## Quick checks + +- Redaction is **off** for the fields you expect to see (`FI_HIDE_INPUTS` / `FI_HIDE_OUTPUTS` and `TraceConfig` masking). +- The framework's instrumentor is installed and `instrument()` ran against the same tracer provider. +- Custom attributes are set while the span is **still active**, before its `with` block closes. +- Anything you filter on uses a [semantic-convention](/docs/traceai/manual-instrumentation/semantic-conventions) key with a supported value type. + +## Causes and fixes + +| Cause | What you see | Fix | +|---|---|---| +| Redaction is on (check first) | Input/output render as hidden or blank, but the span is otherwise complete | Confirm whether `FI_HIDE_INPUTS` / `FI_HIDE_OUTPUTS` or `TraceConfig` masking is set — if so, the blank is expected. See [Mask span attributes](/docs/traceai/manual-instrumentation/mask-span-attributes). | +| Instrumentor not attached for that framework | A framework's child spans (e.g. nested LangGraph nodes) never appear | Install and `instrument()` the instrumentor for the missing framework, attached to the provider. | +| Attribute set on the wrong span / after close | A custom attribute you set isn't on the span | Set attributes while the span is active; a value set after the `with` block closes is dropped. | +| Custom key isn't indexed for filtering | The attribute is on the span but you can't filter by it | Use a [semantic-convention](/docs/traceai/manual-instrumentation/semantic-conventions) key where one exists — the UI filters on standard keys. | +| Unsupported value type | The attribute is silently dropped | Attribute values must be string, bool, int, float, or an array of those. | + +## Diagnostic commands + +Print one span's attributes from a span exporter to confirm what actually reached the SDK, so you can tell a redacted field from a missing one: + +```python +from opentelemetry.sdk.trace.export import SimpleSpanProcessor, SpanExporter + +class PrintAttributes(SpanExporter): + def export(self, spans): + for span in spans: + print(span.name, dict(span.attributes)) + +trace_provider.add_span_processor(SimpleSpanProcessor(PrintAttributes())) +``` + +If the attribute prints here but isn't filterable in the UI, the key isn't indexed; if it doesn't print at all, it was set on the wrong span or after close. + +## Minimal smoke test + +Re-run one request, then open the trace, click the span, and check the **attributes** list. The previously-missing span or attribute should now show in the span detail, and you should be able to filter by it for standard keys. + +## Escalate + +If you're still stuck, contact support@futureagi.com with your `project_name`, the trace ID, the framework + instrumentor versions, and the attribute you expected. + +## Prevent recurrence + +- Decide masking deliberately and document it, so blank fields aren't mistaken for bugs. +- Prefer semantic-convention keys for anything you'll filter or evaluate on. + +## Next steps + + + + How attributes get onto spans. + + + Why a field might be intentionally hidden. + + diff --git a/src/pages/docs/observe/troubleshooting/no-traces-appearing.mdx b/src/pages/docs/observe/troubleshooting/no-traces-appearing.mdx new file mode 100644 index 00000000..64482f67 --- /dev/null +++ b/src/pages/docs/observe/troubleshooting/no-traces-appearing.mdx @@ -0,0 +1,115 @@ +--- +title: "No traces appear" +description: "Your app ran but no trace shows up in FutureAGI Observe. The usual causes are an unflushed short-lived process, the wrong project_type, missing FI_API_KEY or FI_SECRET_KEY, or a date-picker window that is too narrow." +slug: "no-traces-appearing" +page_type: "troubleshooting" +products: ["Observe"] +failure_surface: "sdk" +symptom: "no traces appear in observe" +audience: ["engineer"] +difficulty: "beginner" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-17" +support_escalation: "support@futureagi.com" +schema_type: "TechArticle" +seo: + title: "No traces appear in FutureAGI Observe — fixes" + description: "Fix an app that runs but sends no traces to Observe: flush a short-lived process, correct the project_type, set FI_API_KEY and FI_SECRET_KEY, and widen the date picker." + primary_keyword: "no traces appearing futureagi" + direct_answer: true +geo: + answer_target: "Why are no traces appearing in FutureAGI Observe?" + llm_summary: "Traces usually fail to appear because a short-lived process exited before force_flush(), the wrong project_type was set, FI_API_KEY or FI_SECRET_KEY is missing, or the date-picker window is too narrow. Flush the process, fix the keys and project_type, and widen the date range, then resend a request." +canonical: "/docs/observe/troubleshooting/no-traces-appearing" +related: + - "/docs/observe/quickstart" + - "/docs/observe/features/llm-tracing" + - "/docs/traceai/troubleshooting/spans-not-exported" + - "/docs/traceai/manual-instrumentation/set-up-tracing" +--- + +## Symptom + +You instrumented your app with traceAI and ran it, but no trace shows up in the [trace explorer](/docs/observe/features/llm-tracing). Typically: + +- A request ran with no error, but no new row appears in the trace list. +- A short script (a one-off `python app.py`) never produces a trace. +- Traces appeared before but stopped after a code change. + +The most common cause is a short-lived process that exited before its spans flushed; the next most common are the wrong `project_type`, missing keys, or a date window that hides the trace. Work the checks below in order — the first one fixes the large majority of cases. + +## Quick checks + +- The process **stays alive** long enough to export, or calls `force_flush()` before exiting. +- `FI_API_KEY` and `FI_SECRET_KEY` are set to this workspace's keys. +- `register()` is called with the correct `project_type` and `project_name`, **before** the framework client is created. +- The date picker is widened to **Today** (not the default 7-day window) and **Auto refresh** is on. + +## Causes and fixes + +| Cause | What you see | Fix | +|---|---|---| +| Short-lived process not flushed (most common) | A one-off script runs clean but no trace appears; long-running services are fine | Call `trace_provider.force_flush()` before the process ends, or pass `batch=False` to `register()`. | +| Wrong `project_type` | App runs, keys are valid, but traces never land in the project you expect | Set `project_type=ProjectType.OBSERVE` (and the matching `project_name`) in `register()`. | +| Missing `FI_API_KEY` / `FI_SECRET_KEY` | Export fails or is silently dropped; nothing reaches Observe | Set both env vars to this workspace's keys before the app starts. | +| Instrumented after the client was created | Some or all spans never emit because the client wasn't wrapped | Call `register()` and the instrumentor **before** constructing the framework client. | +| Date-picker window too narrow | The trace exists but is filtered out of the view | Widen the date range to **Today** and enable **Auto refresh**. | + +The Observe trace explorer date-range picker, widened so a recent trace falls inside the selected window + +## Diagnostic commands + +Confirm the keys are present in the environment the app actually runs in: + +```bash +env | grep -E "FI_API_KEY|FI_SECRET_KEY" +``` + +Force a flush in a short script so spans are exported before the process exits: + +```python +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType + +trace_provider = register( + project_type=ProjectType.OBSERVE, + project_name="my-project", +) + +# ... run one request ... + +trace_provider.force_flush() +``` + +If spans still never leave the SDK, work through [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +## Minimal smoke test + +Send one request, then open **Observe → your project → Tracing** with **Auto refresh** on and the date range widened to **Today**. A new trace should appear within seconds, **Status OK**, with input, output, latency, and model populated. If it doesn't, recheck the causes above in order. + +## Prevent recurrence + +- Add `trace_provider.force_flush()` to short scripts and job runners. +- Call `register()` + `instrument()` once at startup, before any client is built — see [Set up tracing](/docs/traceai/manual-instrumentation/set-up-tracing). +- Keep `FI_API_KEY`, `FI_SECRET_KEY`, and `project_type` in your startup config so they can't drift per environment. + +If you're still stuck, collect your `project_name`, a request timestamp, your installed `fi-instrumentation-otel` and instrumentor versions, and any stderr, and contact support@futureagi.com. + +## Next steps + + + + Get a first trace flowing end to end. + + + The setup this page diagnoses. + + + When spans never leave the SDK at all. + + + Where traces should appear. + + diff --git a/src/pages/docs/prompt/features/linked-traces.mdx b/src/pages/docs/prompt/features/linked-traces.mdx index ed0869a1..b9a42dc0 100644 --- a/src/pages/docs/prompt/features/linked-traces.mdx +++ b/src/pages/docs/prompt/features/linked-traces.mdx @@ -40,7 +40,7 @@ Once linked, the Prompt Workbench shows aggregated metrics per prompt version al ## How to -To link prompts to traces, you need to associate the prompt used in a generation with the corresponding trace. The process is described in the observability and manual tracing docs: [Log prompt templates](/docs/observe/features/manual-tracing/log-prompt-templates). Once your application sends traces that include the prompt template (or template ID), Future AGI links those traces to the prompt in the Prompt Workbench. +To link prompts to traces, you need to associate the prompt used in a generation with the corresponding trace. The process is described in the observability and manual tracing docs: [Log prompt templates](/docs/traceai/manual-instrumentation/log-prompt-templates). Once your application sends traces that include the prompt template (or template ID), Future AGI links those traces to the prompt in the Prompt Workbench. --- @@ -70,7 +70,7 @@ Compare the same metric across **prompt versions** or **time ranges** to see if Manage and fetch prompts programmatically. - + Set up the trace-to-prompt connection in your application. diff --git a/src/pages/docs/quickstart/setup-mcp-server.mdx b/src/pages/docs/quickstart/setup-mcp-server.mdx index a4d08ace..fed183e7 100644 --- a/src/pages/docs/quickstart/setup-mcp-server.mdx +++ b/src/pages/docs/quickstart/setup-mcp-server.mdx @@ -44,7 +44,7 @@ https://api.futureagi.com/mcp With **Future AGI's MCP Server**, you can use natural language to: - **Run automatic evaluations**: evaluate batch and single inputs on various [evaluation](/docs/cookbook/quickstart/first-eval) metrics, both on local datapoints and large datasets -- **Prototype and observe your agents**: add [observability](/docs/observe/features/quickstart), evaluations while [prototyping](/docs/prototype) and deploying agents into production +- **Prototype and observe your agents**: add [observability](/docs/observe/quickstart), evaluations while [prototyping](/docs/prototype) and deploying agents into production - **Manage datasets**: upload, evaluate, download [datasets](/docs/dataset) and find insights - **Add protection rules**: apply toxicity detection, prompt injection protection, and other guardrails automatically - **Generate synthetic data**: describe your dataset and objective to generate synthetic data @@ -54,5 +54,5 @@ Check out our [blog post](https://futureagi.com/blogs/model-context-protocol-mcp ## Next Steps - [Run your first evaluation](/docs/cookbook/quickstart/first-eval) using natural language through the MCP server -- [Explore the Observe quickstart](/docs/observe/features/quickstart) to add tracing to your project +- [Explore the Observe quickstart](/docs/observe/quickstart) to add tracing to your project - [Learn about Protect](/docs/protect) to set up real-time guardrails for your AI application diff --git a/src/pages/docs/quickstart/setup-observability.mdx b/src/pages/docs/quickstart/setup-observability.mdx index 283a71a7..983290bf 100644 --- a/src/pages/docs/quickstart/setup-observability.mdx +++ b/src/pages/docs/quickstart/setup-observability.mdx @@ -77,7 +77,7 @@ Observe supports auto-instrumentation for OpenAI, Anthropic, LangChain, LlamaInd There are 2 ways to implement tracing in your project: 1. **Auto Instrumentor**: Automatically captures all LLM calls. Recommended for most use cases. - 2. **Manual Tracing**: Gives you full control over what gets traced using OpenTelemetry. [Learn more](/docs/observe/features/manual-tracing/set-up-tracing) + 2. **Manual Tracing**: Gives you full control over what gets traced using OpenTelemetry. [Learn more](/docs/traceai/manual-instrumentation/set-up-tracing) Here's a complete example using auto-instrumentation with OpenAI: @@ -136,5 +136,5 @@ Observe supports auto-instrumentation for OpenAI, Anthropic, LangChain, LlamaInd ## Next Steps - [Add more integrations](/docs/integrations) for Anthropic, LangChain, LlamaIndex, and others -- [Set up manual tracing](/docs/observe/features/manual-tracing/set-up-tracing) for custom spans and attributes -- [Add inline evaluations](/docs/observe/features/manual-tracing/in-line-evals) to evaluate traces as they come in \ No newline at end of file +- [Set up manual tracing](/docs/traceai/manual-instrumentation/set-up-tracing) for custom spans and attributes +- [Add inline evaluations](/docs/traceai/manual-instrumentation/in-line-evals) to evaluate traces as they come in \ No newline at end of file diff --git a/src/pages/docs/sdk/tracing.mdx b/src/pages/docs/sdk/tracing.mdx index cdcb2ef2..e665f2ff 100644 --- a/src/pages/docs/sdk/tracing.mdx +++ b/src/pages/docs/sdk/tracing.mdx @@ -13,7 +13,7 @@ description: "Set up OpenTelemetry tracing across Python, TypeScript, Java, and The pattern is the same across all four languages: call `register()` once to set up the provider, then either auto-instrument your frameworks or use `FITracer` for custom spans. LLM calls, retrieval steps, and agent actions get captured as OpenTelemetry spans and sent to your dashboard. - Requires `FI_API_KEY` and `FI_SECRET_KEY` in your environment. For conceptual background on traces, spans, and attributes, see the [Tracing guide](/docs/tracing/auto). + Requires `FI_API_KEY` and `FI_SECRET_KEY` in your environment. For conceptual background on traces, spans, and attributes, see the [Tracing guide](/docs/traceai/auto). ## Quick Example @@ -1127,7 +1127,7 @@ To remove instrumentation (useful in tests or serverless cleanup): -For per-framework setup guides with full examples, see the [Auto-Instrumentation docs](/docs/tracing/auto). +For per-framework setup guides with full examples, see the [Auto-Instrumentation docs](/docs/traceai/auto). ### Other Languages @@ -1236,10 +1236,10 @@ All languages read from the same set of environment variables: ## Related - + Concepts, manual tracing, and per-framework setup guides. - + Setup guides for all 45+ supported frameworks. diff --git a/src/pages/docs/tracing/auto/anthropic.mdx b/src/pages/docs/traceai/auto/anthropic.mdx similarity index 50% rename from src/pages/docs/tracing/auto/anthropic.mdx rename to src/pages/docs/traceai/auto/anthropic.mdx index 4dcfa9b2..0217b793 100644 --- a/src/pages/docs/tracing/auto/anthropic.mdx +++ b/src/pages/docs/traceai/auto/anthropic.mdx @@ -1,6 +1,31 @@ --- title: "Anthropic Tracing with Future AGI: Auto-Instrumentation" description: "Set up auto-instrumentation for Anthropic Claude with Future AGI tracing. Install traceAI-anthropic to capture LLM spans, inputs, and outputs." +slug: "anthropic" +page_type: "integration" +products: ["traceAI"] +integration_category: "llm-provider" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "Anthropic Tracing with Future AGI: Auto-Instrumentation" + description: "Set up auto-instrumentation for Anthropic Claude with Future AGI tracing. Install traceAI-anthropic to capture LLM spans, inputs, and outputs." + primary_keyword: "anthropic tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace Anthropic with FutureAGI?" + llm_summary: "Set up auto-instrumentation for Anthropic Claude with Future AGI tracing. Install traceAI-anthropic to capture LLM spans, inputs, and outputs." +canonical: "/docs/traceai/auto/anthropic" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" --- ## 1. Installation @@ -157,4 +182,59 @@ const message = await client.messages.create({ }); ``` - \ No newline at end of file + + +--- + +## What spans are produced + +Auto-instrumentation for Anthropic emits an **`llm` span for every model call** — chat, completion, or embedding — carrying the model name, the prompt and completion, prompt/completion/total token counts, and the estimated cost. Tool or function calls add a **`tool` span**. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + + + A short-lived script can exit before the batch exporter flushes. Call `trace_provider.force_flush()` before the process ends so the last spans are sent. + + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the Anthropic spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every Anthropic call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/tracing/auto/autogen.mdx b/src/pages/docs/traceai/auto/autogen.mdx similarity index 61% rename from src/pages/docs/tracing/auto/autogen.mdx rename to src/pages/docs/traceai/auto/autogen.mdx index 4b282b48..8a6d47ee 100644 --- a/src/pages/docs/tracing/auto/autogen.mdx +++ b/src/pages/docs/traceai/auto/autogen.mdx @@ -1,6 +1,31 @@ --- title: "Autogen Tracing with Future AGI: Auto-Instrumentation" description: "Set up auto-instrumentation for Autogen with Future AGI tracing. Install traceAI-autogen to capture multi-agent conversation spans automatically." +slug: "autogen" +page_type: "integration" +products: ["traceAI"] +integration_category: "framework" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "Autogen Tracing with Future AGI: Auto-Instrumentation" + description: "Set up auto-instrumentation for Autogen with Future AGI tracing. Install traceAI-autogen to capture multi-agent conversation spans automatically." + primary_keyword: "autogen tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace Autogen with FutureAGI?" + llm_summary: "Set up auto-instrumentation for Autogen with Future AGI tracing. Install traceAI-autogen to capture multi-agent conversation spans automatically." +canonical: "/docs/traceai/auto/autogen" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" --- ## 1. Installation @@ -147,4 +172,59 @@ with Cache.disk(cache_seed=7) as cache: assistant, message="""Solve the following leetcode problem and also comment on it's time and space complexity:nn""" + LEETCODE_QUESTION ) -``` \ No newline at end of file +``` + +--- + +## What spans are produced + +Auto-instrumentation for Autogen emits a **`chain` or `agent` root span** for the run, with nested **`llm` spans** for each model call, **`tool` spans** for tool/function calls, and **`retriever` spans** for retrieval steps — so the whole request reads as one tree. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + + + A short-lived script can exit before the batch exporter flushes. Call `trace_provider.force_flush()` before the process ends so the last spans are sent. + + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the Autogen spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every Autogen call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/tracing/auto/bedrock.mdx b/src/pages/docs/traceai/auto/bedrock.mdx similarity index 56% rename from src/pages/docs/tracing/auto/bedrock.mdx rename to src/pages/docs/traceai/auto/bedrock.mdx index 066fae6f..0b649acf 100644 --- a/src/pages/docs/tracing/auto/bedrock.mdx +++ b/src/pages/docs/traceai/auto/bedrock.mdx @@ -1,6 +1,31 @@ --- title: "AWS Bedrock Tracing with Future AGI: Auto-Instrumentation" description: "Set up auto-instrumentation for AWS Bedrock with Future AGI tracing. Install traceAI-bedrock to capture model invocation spans and metadata." +slug: "bedrock" +page_type: "integration" +products: ["traceAI"] +integration_category: "llm-provider" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "AWS Bedrock Tracing with Future AGI: Auto-Instrumentation" + description: "Set up auto-instrumentation for AWS Bedrock with Future AGI tracing. Install traceAI-bedrock to capture model invocation spans and metadata." + primary_keyword: "aws bedrock tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace AWS Bedrock with FutureAGI?" + llm_summary: "Set up auto-instrumentation for AWS Bedrock with Future AGI tracing. Install traceAI-bedrock to capture model invocation spans and metadata." +canonical: "/docs/traceai/auto/bedrock" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" --- ## 1. Installation @@ -196,3 +221,58 @@ converseWithClaude(); ``` + +--- + +## What spans are produced + +Auto-instrumentation for AWS Bedrock emits an **`llm` span for every model call** — chat, completion, or embedding — carrying the model name, the prompt and completion, prompt/completion/total token counts, and the estimated cost. Tool or function calls add a **`tool` span**. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + + + A short-lived script can exit before the batch exporter flushes. Call `trace_provider.force_flush()` before the process ends so the last spans are sent. + + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the AWS Bedrock spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every AWS Bedrock call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/traceai/auto/crewai.mdx b/src/pages/docs/traceai/auto/crewai.mdx new file mode 100644 index 00000000..225e4721 --- /dev/null +++ b/src/pages/docs/traceai/auto/crewai.mdx @@ -0,0 +1,176 @@ +--- +title: "CrewAI Tracing with Future AGI: Auto-Instrumentation" +description: "Set up auto-instrumentation for CrewAI with Future AGI tracing. Install traceAI-crewai to capture crew task execution and agent interaction spans." +slug: "crewai" +page_type: "integration" +products: ["traceAI"] +integration_category: "framework" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "CrewAI Tracing with Future AGI: Auto-Instrumentation" + description: "Set up auto-instrumentation for CrewAI with Future AGI tracing. Install traceAI-crewai to capture crew task execution and agent interaction spans." + primary_keyword: "crewai tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace CrewAI with FutureAGI?" + llm_summary: "Set up auto-instrumentation for CrewAI with Future AGI tracing. Install traceAI-crewai to capture crew task execution and agent interaction spans." +canonical: "/docs/traceai/auto/crewai" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" +--- + +## 1. Installation +Install the traceAI and Crew packages + +```bash +pip install traceAI-crewai crewai crewai_tools +``` + +--- + +## 2. Set Environment Variables + +Set up your environment variables to authenticate with both FutureAGI and OpenAI. + +```python +import os + +os.environ["OPENAI_API_KEY"] = "your-openai-api-key" +os.environ["FI_API_KEY"] = "your-futureagi-api-key" +os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" +``` + +--- + +## 3. Initialize Trace Provider +Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . + +```python +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType + +trace_provider = register( + project_type=ProjectType.OBSERVE, + project_name="crewai_project", +) +``` + +--- + +## 4. Instrument your Project +Initialize the Crew AI instrumentor to enable automatic tracing. + +```python +from traceai_crewai import CrewAIInstrumentor + +CrewAIInstrumentor().instrument(tracer_provider=trace_provider) +``` + +--- + +## 5. Run Crew AI +Run your Crew AI application as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. + +```python +from crewai import LLM, Agent, Crew, Process, Task +from crewai_tools import SerperDevTool + +def story_example(): + llm = LLM( + model="gpt-4", + temperature=0.8, + max_tokens=150, + top_p=0.9, + frequency_penalty=0.1, + presence_penalty=0.1, + stop=["END"], + seed=42, + ) + + writer = Agent( + role="Writer", + goal="Write creative stories", + backstory="You are a creative writer with a passion for storytelling", + allow_delegation=False, + llm=llm, + ) + + writing_task = Task( + description="Write a short story about a magical forest", + agent=writer, + expected_output="A short story about a magical forest", + ) + + crew = Crew(agents=[writer], tasks=[writing_task]) + + # Execute the crew + result = crew.kickoff() + print(result) + +if __name__ == "__main__": + story_example() +``` + +--- + +## What spans are produced + +Auto-instrumentation for CrewAI emits a **`chain` or `agent` root span** for the run, with nested **`llm` spans** for each model call, **`tool` spans** for tool/function calls, and **`retriever` spans** for retrieval steps — so the whole request reads as one tree. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + + + A short-lived script can exit before the batch exporter flushes. Call `trace_provider.force_flush()` before the process ends so the last spans are sent. + + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the CrewAI spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every CrewAI call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/traceai/auto/dspy.mdx b/src/pages/docs/traceai/auto/dspy.mdx new file mode 100644 index 00000000..6eb85f30 --- /dev/null +++ b/src/pages/docs/traceai/auto/dspy.mdx @@ -0,0 +1,157 @@ +--- +title: "DSPy Tracing with Future AGI: Auto-Instrumentation" +description: "Set up auto-instrumentation for DSPy with Future AGI tracing. Install traceAI-DSPy to capture program compilation and prediction spans automatically." +slug: "dspy" +page_type: "integration" +products: ["traceAI"] +integration_category: "framework" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "DSPy Tracing with Future AGI: Auto-Instrumentation" + description: "Set up auto-instrumentation for DSPy with Future AGI tracing. Install traceAI-DSPy to capture program compilation and prediction spans automatically." + primary_keyword: "dspy tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace DSPy with FutureAGI?" + llm_summary: "Set up auto-instrumentation for DSPy with Future AGI tracing. Install traceAI-DSPy to capture program compilation and prediction spans automatically." +canonical: "/docs/traceai/auto/dspy" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" +--- + +## 1. Installation +Install the traceAI and dspy package. + +```bash +pip install traceAI-DSPy dspy +``` + +--- + +## 2. Set Environment Variables +Set up your environment variables to authenticate with FutureAGI and OpenAI. + +```python +import os + +os.environ["OPENAI_API_KEY"] = "your-openai-api-key" +os.environ["FI_API_KEY"] = "your-futureagi-api-key" +os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" +``` + +--- + +## 3. Initialize Trace Provider + +Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . + +```python +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType + +trace_provider = register( + project_type=ProjectType.OBSERVE, + project_name="dspy_project", +) +``` + +--- +## 4. Instrument your Project +Initialize the DSPy instrumentor to enable automatic tracing. + +```python +from traceai_dspy import DSPyInstrumentor + +DSPyInstrumentor().instrument(tracer_provider=trace_provider) +``` + +--- + +## 5. Create DSPy Components and Run your application +Run DSPy as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. + +```python +import dspy + +class BasicQA(dspy.Signature): + """Answer questions with short factoid answers.""" + + question = dspy.InputField() + answer = dspy.OutputField(desc="often between 1 and 5 words") + +if __name__ == "__main__": + turbo = dspy.LM(model="openai/gpt-4") + + dspy.settings.configure(lm=turbo) + + # Define the predictor. + generate_answer = dspy.Predict(BasicQA) + + # Call the predictor on a particular input. + pred = generate_answer(question="What is the capital of the united states?") + print(f"Predicted Answer: {pred.answer}") +``` + +--- + +## What spans are produced + +Auto-instrumentation for DSPy emits a **`chain` or `agent` root span** for the run, with nested **`llm` spans** for each model call, **`tool` spans** for tool/function calls, and **`retriever` spans** for retrieval steps — so the whole request reads as one tree. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + + + A short-lived script can exit before the batch exporter flushes. Call `trace_provider.force_flush()` before the process ends so the last spans are sent. + + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the DSPy spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every DSPy call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/traceai/auto/google_adk.mdx b/src/pages/docs/traceai/auto/google_adk.mdx new file mode 100644 index 00000000..10bc15b3 --- /dev/null +++ b/src/pages/docs/traceai/auto/google_adk.mdx @@ -0,0 +1,196 @@ +--- +title: "Google ADK Tracing with Future AGI: Auto-Instrumentation" +description: "Set up auto-instrumentation for Google ADK with Future AGI tracing. Install traceai-google-adk to capture agent and tool execution spans." +slug: "google_adk" +page_type: "integration" +products: ["traceAI"] +integration_category: "framework" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "Google ADK Tracing with Future AGI: Auto-Instrumentation" + description: "Set up auto-instrumentation for Google ADK with Future AGI tracing. Install traceai-google-adk to capture agent and tool execution spans." + primary_keyword: "google adk tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace Google ADK with FutureAGI?" + llm_summary: "Set up auto-instrumentation for Google ADK with Future AGI tracing. Install traceai-google-adk to capture agent and tool execution spans." +canonical: "/docs/traceai/auto/google_adk" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" +--- + +## 1. Installation +Install the traceAI and Google ADK packages. + +```bash +pip install traceai-google-adk +``` + +--- + +## 2. Set Environment Variables +Set up your environment variables to authenticate with both FutureAGI and Google. + +```python +import os + +os.environ["FI_API_KEY"] = "your-futureagi-api-key" +os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" +os.environ["GOOGLE_API_KEY"] = "your-google-api-key" +``` + +--- + +## 3. Initialize Trace Provider +Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . + +```python +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType + +trace_provider = register( + project_type=ProjectType.OBSERVE, + project_name="google_adk", +) +``` + +--- +## 4. Instrument your Project +Instrument your project to enable automatic tracing. + +```python +from traceai_google_adk import GoogleADKInstrumentor + +GoogleADKInstrumentor().instrument(tracer_provider=trace_provider) +``` + +--- +## 5. Interact with Google ADK +Start interacting with Google ADK as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. Here is a sample code using the Google ADK SDK. + +```python +import asyncio + +from google.adk.agents import Agent +from google.adk.runners import InMemoryRunner +from google.genai import types + +def get_weather(city: str) -> dict: + """Retrieves the current weather report for a specified city. + + Args: + city (str): The name of the city for which to retrieve the weather report. + + Returns: + dict: status and result or error msg. + """ + if city.lower() == "new york": + return { + "status": "success", + "report": ( + "The weather in New York is sunny with a temperature of 25 degrees" + " Celsius (77 degrees Fahrenheit)." + ), + } + else: + return { + "status": "error", + "error_message": f"Weather information for '{city}' is not available.", + } + +agent = Agent( + name="test_agent", + model="gemini-2.5-flash-preview-05-20", + description="Agent to answer questions using tools.", + instruction="You must use the available tools to find an answer.", + tools=[get_weather] +) + +async def main(): + app_name = "test_instrumentation" + user_id = "test_user" + session_id = "test_session" + runner = InMemoryRunner(agent=agent, app_name=app_name) + session_service = runner.session_service + await session_service.create_session( + app_name=app_name, + user_id=user_id, + session_id=session_id + ) + async for event in runner.run_async( + user_id=user_id, + session_id=session_id, + new_message=types.Content(role="user", parts=[ + types.Part(text="What is the weather in New York?")] + ) + ): + if event.is_final_response(): + print(event.content.parts[0].text.strip()) + +if __name__ == "__main__": + asyncio.run(main()) +``` + +--- + +## What spans are produced + +Auto-instrumentation for Google ADK emits a **`chain` or `agent` root span** for the run, with nested **`llm` spans** for each model call, **`tool` spans** for tool/function calls, and **`retriever` spans** for retrieval steps — so the whole request reads as one tree. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + + + A short-lived script can exit before the batch exporter flushes. Call `trace_provider.force_flush()` before the process ends so the last spans are sent. + + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the Google ADK spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every Google ADK call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/traceai/auto/google_genai.mdx b/src/pages/docs/traceai/auto/google_genai.mdx new file mode 100644 index 00000000..5979f26c --- /dev/null +++ b/src/pages/docs/traceai/auto/google_genai.mdx @@ -0,0 +1,151 @@ +--- +title: "Google GenAI Tracing with Future AGI: Auto-Instrumentation" +description: "Set up auto-instrumentation for Google GenAI with Future AGI tracing. Install traceAI-google-genai to capture Gemini model interaction spans." +slug: "google_genai" +page_type: "integration" +products: ["traceAI"] +integration_category: "llm-provider" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "Google GenAI Tracing with Future AGI: Auto-Instrumentation" + description: "Set up auto-instrumentation for Google GenAI with Future AGI tracing. Install traceAI-google-genai to capture Gemini model interaction spans." + primary_keyword: "google genai tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace Google GenAI with FutureAGI?" + llm_summary: "Set up auto-instrumentation for Google GenAI with Future AGI tracing. Install traceAI-google-genai to capture Gemini model interaction spans." +canonical: "/docs/traceai/auto/google_genai" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" +--- + +## 1. Installation +Install the traceAI and Google GenAI packages. + +```bash +pip install traceAI-google-genai +``` + +--- + +## 2. Set Environment Variables +Set up your environment variables to authenticate with FutureAGI. + +```python +import os + +os.environ["FI_API_KEY"] = "your-futureagi-api-key" +os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" +``` + +--- + +## 3. Initialize Trace Provider +Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . + +```python +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType + +trace_provider = register( + project_type=ProjectType.OBSERVE, + project_name="google_genai", +) +``` + +--- +## 4. Instrument your Project +Instrument your project to enable automatic tracing. + +```python +from traceai_google_genai import GoogleGenAIInstrumentor + +GoogleGenAIInstrumentor().instrument(tracer_provider=trace_provider) +``` + +--- +## 5. Interact with Google ADK +Start interacting with Google ADK as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. Here is a sample code using the Google ADK SDK. + +```python +from google import genai +from google.genai import types + +client = genai.Client(vertexai=True, project="your_project_name", location="global") + +content = types.Content( + role="user", + parts=[ + types.Part.from_text(text="Hello how are you?"), + ], +) +response = client.models.generate_content( + model="gemini-2.0-flash-001", contents=content +) + +print(response) +``` + +--- + +## What spans are produced + +Auto-instrumentation for Google GenAI emits an **`llm` span for every model call** — chat, completion, or embedding — carrying the model name, the prompt and completion, prompt/completion/total token counts, and the estimated cost. Tool or function calls add a **`tool` span**. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + + + A short-lived script can exit before the batch exporter flushes. Call `trace_provider.force_flush()` before the process ends so the last spans are sent. + + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the Google GenAI spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every Google GenAI call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/traceai/auto/groq.mdx b/src/pages/docs/traceai/auto/groq.mdx new file mode 100644 index 00000000..34c7b805 --- /dev/null +++ b/src/pages/docs/traceai/auto/groq.mdx @@ -0,0 +1,155 @@ +--- +title: "Groq Tracing with Future AGI: Auto-Instrumentation" +description: "Set up auto-instrumentation for Groq with Future AGI tracing. Install traceAI-groq to capture high-speed inference spans and performance data." +slug: "groq" +page_type: "integration" +products: ["traceAI"] +integration_category: "llm-provider" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "Groq Tracing with Future AGI: Auto-Instrumentation" + description: "Set up auto-instrumentation for Groq with Future AGI tracing. Install traceAI-groq to capture high-speed inference spans and performance data." + primary_keyword: "groq tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace Groq with FutureAGI?" + llm_summary: "Set up auto-instrumentation for Groq with Future AGI tracing. Install traceAI-groq to capture high-speed inference spans and performance data." +canonical: "/docs/traceai/auto/groq" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" +--- + +## 1. Installation +Install the traceAI and Groq packages. + +```bash +pip install traceAI-groq +``` + +--- + +## 2. Set Environment Variables +Set up your environment variables to authenticate with both FutureAGI and Groq. + +```python +import os + +os.environ["FI_API_KEY"] = "your-futureagi-api-key" +os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" +os.environ["GROQ_API_KEY"] = "your-groq-api-key" +``` + +--- + +## 3. Initialize Trace Provider +Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . + +```python +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType + +trace_provider = register( + project_type=ProjectType.OBSERVE, + project_name="groq_project", +) +``` + +--- +## 4. Instrument your Project +Instrument your project to enable automatic tracing. + +```python +from traceai_groq import GroqInstrumentor + +GroqInstrumentor().instrument(tracer_provider=trace_provider) +``` + +--- +## 5. Interact with Groq +Interact with Groq as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. + +```python +from groq import Groq + +client = Groq() + +chat_completion = client.chat.completions.create( + messages=[ + { + "role": "system", + "content": "you are a helpful assistant." + }, + { + "role": "user", + "content": "Explain the importance of fast language models", + } + ], + model="llama-3.3-70b-versatile", +) + +print(chat_completion.choices[0].message.content) +``` + +--- + +## What spans are produced + +Auto-instrumentation for Groq emits an **`llm` span for every model call** — chat, completion, or embedding — carrying the model name, the prompt and completion, prompt/completion/total token counts, and the estimated cost. Tool or function calls add a **`tool` span**. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + + + A short-lived script can exit before the batch exporter flushes. Call `trace_provider.force_flush()` before the process ends so the last spans are sent. + + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the Groq spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every Groq call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/traceai/auto/guardrails.mdx b/src/pages/docs/traceai/auto/guardrails.mdx new file mode 100644 index 00000000..a022eec5 --- /dev/null +++ b/src/pages/docs/traceai/auto/guardrails.mdx @@ -0,0 +1,157 @@ +--- +title: "Guardrails AI Tracing with Future AGI: Auto-Instrumentation" +description: "Set up auto-instrumentation for Guardrails AI with Future AGI tracing. Install traceAI-guardrails to trace validation and LLM interaction spans." +slug: "guardrails" +page_type: "integration" +products: ["traceAI"] +integration_category: "framework" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "Guardrails AI Tracing with Future AGI: Auto-Instrumentation" + description: "Set up auto-instrumentation for Guardrails AI with Future AGI tracing. Install traceAI-guardrails to trace validation and LLM interaction spans." + primary_keyword: "guardrails ai tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace Guardrails AI with FutureAGI?" + llm_summary: "Set up auto-instrumentation for Guardrails AI with Future AGI tracing. Install traceAI-guardrails to trace validation and LLM interaction spans." +canonical: "/docs/traceai/auto/guardrails" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" +--- + +## 1. Installation +First install the traceAI package to access the observability framework + +```bash +pip install traceAI-guardrails +``` + +--- + +## 2. Set Environment Variables + +Set up your environment variables to authenticate with both FutureAGI and OpenAI. + +```python +import os + +os.environ["OPENAI_API_KEY"] = "your-openai-api-key" +os.environ["FI_API_KEY"] = "your-futureagi-api-key" +os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" +``` + +--- + +## 3. Initialize Trace Provider + +Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . + +```python +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType + +trace_provider = register( + project_type=ProjectType.EXPERIMENT, + project_name="openai_project", +) +``` + +--- + +## 4. Instrument your Project + +Instrument your Project with OpenAI Agents Instrumentor. This step ensures that all interactions with the OpenAI are tracked and monitored. + +```python +from traceai_guardrails import GuardrailsInstrumentor + +GuardrailsInstrumentor().instrument(tracer_provider=trace_provider) +``` + +--- + +## 5. Interact with OpenAI Agents + +Interact with the OpenAI Agents as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. + +```python +from guardrails import Guard + +guard = Guard() + +result = guard( + messages=[ + { + "role": "user", + "content": "Tell me about OpenAI", + }, + ], + model="gpt-4o" +) + +print(f"{result}") +``` + +--- + +## What spans are produced + +Auto-instrumentation for Guardrails AI emits a **`chain` or `agent` root span** for the run, with nested **`llm` spans** for each model call, **`tool` spans** for tool/function calls, and **`retriever` spans** for retrieval steps — so the whole request reads as one tree. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + + + A short-lived script can exit before the batch exporter flushes. Call `trace_provider.force_flush()` before the process ends so the last spans are sent. + + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the Guardrails AI spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every Guardrails AI call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/traceai/auto/haystack.mdx b/src/pages/docs/traceai/auto/haystack.mdx new file mode 100644 index 00000000..d241de44 --- /dev/null +++ b/src/pages/docs/traceai/auto/haystack.mdx @@ -0,0 +1,178 @@ +--- +title: "Haystack Tracing with Future AGI: Auto-Instrumentation" +description: "Set up auto-instrumentation for Haystack with Future AGI tracing. Install traceAI-haystack to capture document pipeline and retrieval spans." +slug: "haystack" +page_type: "integration" +products: ["traceAI"] +integration_category: "framework" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "Haystack Tracing with Future AGI: Auto-Instrumentation" + description: "Set up auto-instrumentation for Haystack with Future AGI tracing. Install traceAI-haystack to capture document pipeline and retrieval spans." + primary_keyword: "haystack tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace Haystack with FutureAGI?" + llm_summary: "Set up auto-instrumentation for Haystack with Future AGI tracing. Install traceAI-haystack to capture document pipeline and retrieval spans." +canonical: "/docs/traceai/auto/haystack" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" +--- + +## 1. Installation +Install the traceAI and Haystack packages. + +```bash +pip install traceAI-haystack haystack-ai trafilatura +``` + +--- + +## 2. Set Environment Variables +Set up your environment variables to authenticate with FutureAGI. + +```python +import os + +os.environ["OPENAI_API_KEY"] = "your-openai-api-key" +os.environ["FI_API_KEY"] = "your-futureagi-api-key" +os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" +``` + +--- + +## 3. Initialize Trace Provider +Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . + +```python +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType + +trace_provider = register( + project_type=ProjectType.OBSERVE, + project_name="haystack_project", +) +``` + +--- + +## 4. Instrument your Project +Initialize the Haystack instrumentor to enable automatic tracing. + +```python +from traceai_haystack import HaystackInstrumentor + +HaystackInstrumentor().instrument(tracer_provider=trace_provider) +``` + +--- + +## 5. Create Haystack Components +Set up your Haystack components as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. + +```python + +from haystack import Pipeline +from haystack.components.fetchers import LinkContentFetcher +from haystack.components.converters import HTMLToDocument +from haystack.components.builders import ChatPromptBuilder +from haystack.components.generators.chat import OpenAIChatGenerator +from haystack.dataclasses import ChatMessage + +fetcher = LinkContentFetcher() +converter = HTMLToDocument() +prompt_template = [ + ChatMessage.from_user( + """ + According to the contents of this website: + {% for document in documents %} + {{document.content}} + {% endfor %} + Answer the given question: {{query}} + Answer: + """ + ) +] + +prompt_builder = ChatPromptBuilder(template=prompt_template) +llm = OpenAIChatGenerator() + +pipeline = Pipeline() +pipeline.add_component("fetcher", fetcher) +pipeline.add_component("converter", converter) +pipeline.add_component("prompt", prompt_builder) +pipeline.add_component("llm", llm) + +pipeline.connect("fetcher.streams", "converter.sources") +pipeline.connect("converter.documents", "prompt.documents") +pipeline.connect("prompt.prompt", "llm") + +result = pipeline.run({"fetcher": {"urls": ["https://haystack.deepset.ai/overview/quick-start"]}, + "prompt": {"query": "Which components do I need for a RAG pipeline?"}}) + +print(result["llm"]["replies"][0].text) +``` + +--- + +## What spans are produced + +Auto-instrumentation for Haystack emits a **`chain` or `agent` root span** for the run, with nested **`llm` spans** for each model call, **`tool` spans** for tool/function calls, and **`retriever` spans** for retrieval steps — so the whole request reads as one tree. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + + + A short-lived script can exit before the batch exporter flushes. Call `trace_provider.force_flush()` before the process ends so the last spans are sent. + + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the Haystack spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every Haystack call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/traceai/auto/index.mdx b/src/pages/docs/traceai/auto/index.mdx new file mode 100644 index 00000000..7115fc51 --- /dev/null +++ b/src/pages/docs/traceai/auto/index.mdx @@ -0,0 +1,187 @@ +--- +title: "Auto-Instrumentation Integrations: Future AGI Tracing" +description: "Auto-instrumentation integrations for LLM frameworks in Python, JavaScript, and Java. Install a traceAI package to start capturing spans automatically." +slug: "index" +page_type: "overview" +products: ["traceAI"] +audience: ["engineer"] +difficulty: "beginner" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "Auto-Instrumentation Integrations: Future AGI Tracing" + description: "Auto-instrumentation integrations for LLM frameworks in Python, JavaScript, and Java. Install a traceAI package to start capturing spans automatically." + primary_keyword: "traceai auto-instrumentation integrations" + direct_answer: true +geo: + answer_target: "How do I trace Auto-Instrumentation Integrations: Future AGI with FutureAGI?" + llm_summary: "Auto-instrumentation integrations for LLM frameworks in Python, JavaScript, and Java. Install a traceAI package to start capturing spans automatically." +canonical: "/docs/traceai/auto" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" +--- + +## About + +Auto-instrumentation adds tracing to your AI application with minimal code changes. Install the `traceAI` package for your framework, [register](/docs/traceai/quickstart) a trace provider, and FutureAGI captures spans, inputs, outputs, latency, token counts, and cost automatically — no per-call span code. The result is the same [trace](/docs/observe/concepts/traces) you'd build by hand, without editing your business logic. + +Reach for auto-instrumentation whenever you use a supported framework — it's the fastest path and the right default. Drop to [manual instrumentation](/docs/traceai/manual-instrumentation/set-up-tracing) only for code a framework wrapper can't see: your own functions, custom tools, or steps between framework calls. The two compose — auto-instrument the framework, then add manual spans where you need them. + +## How it works + +Python and JS/TS integrations use **instrumentors** that patch the client library at startup, so every call it makes is traced. Java integrations use explicit **`Traced*` wrappers** around your existing clients. Both emit the same OpenTelemetry spans, so a [LangChain](/docs/traceai/auto/langchain) run and a raw [OpenAI](/docs/traceai/auto/openai) call land in Observe with the same shape. The setup is always the same four steps — install the package, set credentials, register a provider with `project_type=OBSERVE`, attach the instrumentor — covered end to end in the [quick start](/docs/traceai/quickstart). After a run, confirm the trace in **Observe → Tracing**; if nothing arrives, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +Pick your framework below. Each page covers install, configuration, the spans it produces, and how to verify them. + +## LLM Providers + + + + `traceAI-openai` + + + `traceAI-anthropic` + + + `traceAI-bedrock` + + + `traceAI-vertexai` + + + `traceAI-google-genai` + + + `traceai-google-adk` + + + `traceAI-groq` + + + `traceAI-mistralai` + + + `traceAI-openai` + + + `traceAI-openai` + + + `traceAI-portkey` + + + +## Frameworks & Agents + + + + `traceAI-langchain` + + + `traceAI-langchain` + + + `traceAI-llamaindex` + + + `traceAI-llamaindex` + + + `traceAI-litellm` + + + `traceAI-crewai` + + + `traceAI-autogen` + + + `traceAI-haystack` + + + `traceAI-DSPy` + + + `traceAI-openai-agents` + + + `traceAI-smolagents` + + + `traceAI-instructor` + + + `traceAI-openai` + + + `traceAI-guardrails` + + + `traceAI-mcp` + + + `@traceai/mastra` + + + `@traceai/vercel` + + + +## Voice & Realtime + + + + `traceAI-livekit` + + + `traceAI-pipecat` + + + +## Java + +The Java SDK uses explicit `Traced*` wrappers instead of instrumentors. Add a Maven/Gradle dependency, wrap your client, and traces flow to FutureAGI. See the [Java overview](/docs/traceai/auto/java) for core setup. + + + + `traceai-spring-boot-starter` + + + `traceai-java-openai` + + + `traceai-java-anthropic` + + + `traceai-java-bedrock` + + + `traceai-java-cohere` + + + `traceai-java-pinecone` + + + Google GenAI, Vertex AI, Azure OpenAI, Ollama, Watsonx + + + Qdrant, Milvus, ChromaDB, Weaviate, and 5 more + + + LangChain4j, Semantic Kernel + + + +## Other + + + + No-code workflow integration + + diff --git a/src/pages/docs/traceai/auto/instructor.mdx b/src/pages/docs/traceai/auto/instructor.mdx new file mode 100644 index 00000000..02d582c4 --- /dev/null +++ b/src/pages/docs/traceai/auto/instructor.mdx @@ -0,0 +1,164 @@ +--- +title: "Instructor Tracing with Future AGI: Auto-Instrumentation" +description: "Set up auto-instrumentation for Instructor with Future AGI tracing. Install traceAI-instructor to capture structured output extraction spans." +slug: "instructor" +page_type: "integration" +products: ["traceAI"] +integration_category: "llm-provider" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "Instructor Tracing with Future AGI: Auto-Instrumentation" + description: "Set up auto-instrumentation for Instructor with Future AGI tracing. Install traceAI-instructor to capture structured output extraction spans." + primary_keyword: "instructor tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace Instructor with FutureAGI?" + llm_summary: "Set up auto-instrumentation for Instructor with Future AGI tracing. Install traceAI-instructor to capture structured output extraction spans." +canonical: "/docs/traceai/auto/instructor" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" +--- + +## 1. Installation +Install the traceAI and other necessary packages. + +```bash +pip install traceAI-instructor instructor +``` + +--- + +## 2. Set Environment Variables +Set up your environment variables to authenticate with FutureAGI. + +```python +import os + +os.environ["OPENAI_API_KEY"] = "your-openai-api-key" +os.environ["FI_API_KEY"] = "your-futureagi-api-key" +os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" +``` + +--- + +## 3. Initialize Trace Provider +Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . + +```python +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType + +trace_provider = register( + project_type=ProjectType.OBSERVE, + project_name="Instructor", +) +``` + +--- + +## 4. Instrument your Project +Use the Instructor Instrumentor to instrument your project. + +```python +from traceai_instructor import InstructorInstrumentor + +InstructorInstrumentor().instrument(tracer_provider=trace_provider) +``` + +--- + +## 5. Run your Instructor application. +Run your Instructor application as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. + +```python +import instructor + +from openai import OpenAI +from pydantic import BaseModel + +# Define the output structure +class UserInfo(BaseModel): + name: str + age: int + +# Patch the OpenAI client +client = instructor.patch(client=OpenAI()) + +user_info = client.chat.completions.create( + model="gpt-3.5-turbo", + response_model=UserInfo, + messages=[ + { + "role": "system", + "content": "Extract the name and age from the text and return them in a structured format.", + }, + {"role": "user", "content": "John Doe is nine years old."}, + ], +) + +print(user_info, type(user_info)) +``` + +--- + +## What spans are produced + +Auto-instrumentation for Instructor emits an **`llm` span for every model call** — chat, completion, or embedding — carrying the model name, the prompt and completion, prompt/completion/total token counts, and the estimated cost. Tool or function calls add a **`tool` span**. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + + + A short-lived script can exit before the batch exporter flushes. Call `trace_provider.force_flush()` before the process ends so the last spans are sent. + + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the Instructor spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every Instructor call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/tracing/auto/java/anthropic.mdx b/src/pages/docs/traceai/auto/java/anthropic.mdx similarity index 53% rename from src/pages/docs/tracing/auto/java/anthropic.mdx rename to src/pages/docs/traceai/auto/java/anthropic.mdx index cfc57002..d4b52177 100644 --- a/src/pages/docs/tracing/auto/java/anthropic.mdx +++ b/src/pages/docs/traceai/auto/java/anthropic.mdx @@ -1,6 +1,31 @@ --- title: "Anthropic Java Tracing: TracedAnthropicClient Setup" description: "Trace Anthropic Messages API calls in Java with TracedAnthropicClient. Uses reflection for cross-version compatibility with the Future AGI Java SDK." +slug: "anthropic" +page_type: "integration" +products: ["traceAI"] +integration_category: "java" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "Anthropic Java Tracing: TracedAnthropicClient Setup" + description: "Trace Anthropic Messages API calls in Java with TracedAnthropicClient. Uses reflection for cross-version compatibility with the Future AGI Java SDK." + primary_keyword: "anthropic java tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace Anthropic Java with FutureAGI?" + llm_summary: "Trace Anthropic Messages API calls in Java with TracedAnthropicClient. Uses reflection for cross-version compatibility with the Future AGI Java SDK." +canonical: "/docs/traceai/auto/java/anthropic" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" --- @@ -12,7 +37,7 @@ description: "Trace Anthropic Messages API calls in Java with TracedAnthropicCli ## Prerequisites -Complete the [Java SDK setup](/docs/tracing/auto/java) first. +Complete the [Java SDK setup](/docs/traceai/auto/java) first. ## Installation @@ -139,3 +164,54 @@ Object original = traced.unwrap(); // Cast back if you need typed access AnthropicClient anthropic = (AnthropicClient) original; ``` + +--- + +## What spans are produced + +Auto-instrumentation for Anthropic Java emits a **`chain` or `agent` root span** for the run, with nested **`llm` spans** for each model call, **`tool` spans** for tool/function calls, and **`retriever` spans** for retrieval steps — so the whole request reads as one tree. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the Anthropic Java spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every Anthropic Java call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/tracing/auto/java/bedrock.mdx b/src/pages/docs/traceai/auto/java/bedrock.mdx similarity index 58% rename from src/pages/docs/tracing/auto/java/bedrock.mdx rename to src/pages/docs/traceai/auto/java/bedrock.mdx index 01b81262..3009c1da 100644 --- a/src/pages/docs/tracing/auto/java/bedrock.mdx +++ b/src/pages/docs/traceai/auto/java/bedrock.mdx @@ -1,6 +1,31 @@ --- title: "AWS Bedrock Java Tracing: TracedBedrockRuntimeClient" description: "Trace AWS Bedrock model invocations in Java with TracedBedrockRuntimeClient. Supports both InvokeModel (raw JSON) and Converse (typed API)." +slug: "bedrock" +page_type: "integration" +products: ["traceAI"] +integration_category: "java" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "AWS Bedrock Java Tracing: TracedBedrockRuntimeClient" + description: "Trace AWS Bedrock model invocations in Java with TracedBedrockRuntimeClient. Supports both InvokeModel (raw JSON) and Converse (typed API)." + primary_keyword: "aws bedrock java tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace AWS Bedrock Java with FutureAGI?" + llm_summary: "Trace AWS Bedrock model invocations in Java with TracedBedrockRuntimeClient. Supports both InvokeModel (raw JSON) and Converse (typed API)." +canonical: "/docs/traceai/auto/java/bedrock" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" --- @@ -12,7 +37,7 @@ description: "Trace AWS Bedrock model invocations in Java with TracedBedrockRunt ## Prerequisites -Complete the [Java SDK setup](/docs/tracing/auto/java) first. +Complete the [Java SDK setup](/docs/traceai/auto/java) first. ## Installation @@ -167,3 +192,54 @@ For `invokeModel`, the raw JSON body is stored in `fi.raw_input` and `fi.raw_out ```java BedrockRuntimeClient original = traced.unwrap(); ``` + +--- + +## What spans are produced + +Auto-instrumentation for AWS Bedrock Java emits a **`chain` or `agent` root span** for the run, with nested **`llm` spans** for each model call, **`tool` spans** for tool/function calls, and **`retriever` spans** for retrieval steps — so the whole request reads as one tree. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the AWS Bedrock Java spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every AWS Bedrock Java call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/tracing/auto/java/cohere.mdx b/src/pages/docs/traceai/auto/java/cohere.mdx similarity index 57% rename from src/pages/docs/tracing/auto/java/cohere.mdx rename to src/pages/docs/traceai/auto/java/cohere.mdx index cfcac7fd..0ca8656a 100644 --- a/src/pages/docs/tracing/auto/java/cohere.mdx +++ b/src/pages/docs/traceai/auto/java/cohere.mdx @@ -1,6 +1,31 @@ --- title: "Cohere Java Tracing: TracedCohereClient Setup" description: "Trace Cohere chat, embedding, and reranking operations in Java with TracedCohereClient. Part of the Future AGI Java SDK for LLM observability." +slug: "cohere" +page_type: "integration" +products: ["traceAI"] +integration_category: "java" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "Cohere Java Tracing: TracedCohereClient Setup" + description: "Trace Cohere chat, embedding, and reranking operations in Java with TracedCohereClient. Part of the Future AGI Java SDK for LLM observability." + primary_keyword: "cohere java tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace Cohere Java with FutureAGI?" + llm_summary: "Trace Cohere chat, embedding, and reranking operations in Java with TracedCohereClient. Part of the Future AGI Java SDK for LLM observability." +canonical: "/docs/traceai/auto/java/cohere" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" --- @@ -12,7 +37,7 @@ description: "Trace Cohere chat, embedding, and reranking operations in Java wit ## Prerequisites -Complete the [Java SDK setup](/docs/tracing/auto/java) first. +Complete the [Java SDK setup](/docs/traceai/auto/java) first. ## Installation @@ -196,3 +221,54 @@ for (var result : response.getResults()) { ```java Cohere original = traced.unwrap(); ``` + +--- + +## What spans are produced + +Auto-instrumentation for Cohere Java emits a **`chain` or `agent` root span** for the run, with nested **`llm` spans** for each model call, **`tool` spans** for tool/function calls, and **`retriever` spans** for retrieval steps — so the whole request reads as one tree. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the Cohere Java spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every Cohere Java call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/tracing/auto/java/frameworks.mdx b/src/pages/docs/traceai/auto/java/frameworks.mdx similarity index 59% rename from src/pages/docs/tracing/auto/java/frameworks.mdx rename to src/pages/docs/traceai/auto/java/frameworks.mdx index 61fcca4f..848c3c92 100644 --- a/src/pages/docs/tracing/auto/java/frameworks.mdx +++ b/src/pages/docs/traceai/auto/java/frameworks.mdx @@ -1,18 +1,43 @@ --- title: "Java Framework Tracing: LangChain4j and Semantic Kernel" description: "Trace LangChain4j and Semantic Kernel operations in Java. Framework-level wrappers that instrument chains, agents, and prompt invocations." +slug: "frameworks" +page_type: "integration" +products: ["traceAI"] +integration_category: "java" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "Java Framework Tracing: LangChain4j and Semantic Kernel" + description: "Trace LangChain4j and Semantic Kernel operations in Java. Framework-level wrappers that instrument chains, agents, and prompt invocations." + primary_keyword: "java framework tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace Java Framework with FutureAGI?" + llm_summary: "Trace LangChain4j and Semantic Kernel operations in Java. Framework-level wrappers that instrument chains, agents, and prompt invocations." +canonical: "/docs/traceai/auto/java/frameworks" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" --- - LangChain4j: `TracedChatLanguageModel` implements `ChatLanguageModel` as a drop-in replacement - Semantic Kernel: `TracedKernel` wraps `Kernel` and traces function invocations and prompt calls - Both support any underlying LLM provider -- For Spring AI, see the [Spring Boot](/docs/tracing/auto/spring-boot) page +- For Spring AI, see the [Spring Boot](/docs/traceai/auto/spring-boot) page ## Prerequisites -Complete the [Java SDK setup](/docs/tracing/auto/java) first. +Complete the [Java SDK setup](/docs/traceai/auto/java) first. --- @@ -192,3 +217,54 @@ Token usage is extracted via reflection from `FunctionResult.getMetadata().getUs For finer-grained tracing, `traceai-java-semantic-kernel` also provides: - `TracedChatCompletionService` - wraps `ChatCompletionService` to trace individual LLM calls within a kernel invocation - `TracedTextEmbeddingGenerationService` - wraps embedding generation + +--- + +## What spans are produced + +Auto-instrumentation for Java Framework emits a **`chain` or `agent` root span** for the run, with nested **`llm` spans** for each model call, **`tool` spans** for tool/function calls, and **`retriever` spans** for retrieval steps — so the whole request reads as one tree. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the Java Framework spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every Java Framework call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/tracing/auto/java/index.mdx b/src/pages/docs/traceai/auto/java/index.mdx similarity index 65% rename from src/pages/docs/tracing/auto/java/index.mdx rename to src/pages/docs/traceai/auto/java/index.mdx index 7f9e1851..a4a5d809 100644 --- a/src/pages/docs/tracing/auto/java/index.mdx +++ b/src/pages/docs/traceai/auto/java/index.mdx @@ -1,6 +1,30 @@ --- title: "Java SDK: TraceAI Setup and Instrumentation with Future AGI" description: "Set up TraceAI for Java applications. Initialize the tracer, configure credentials, and instrument your LLM clients, vector databases, and frameworks." +slug: "index" +page_type: "overview" +products: ["traceAI"] +audience: ["engineer"] +difficulty: "beginner" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "Java SDK: TraceAI Setup and Instrumentation with Future AGI" + description: "Set up TraceAI for Java applications. Initialize the tracer, configure credentials, and instrument your LLM clients, vector databases, and frameworks." + primary_keyword: "java tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace Java with FutureAGI?" + llm_summary: "Set up TraceAI for Java applications. Initialize the tracer, configure credentials, and instrument your LLM clients, vector databases, and frameworks." +canonical: "/docs/traceai/auto/java" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" --- @@ -114,15 +138,7 @@ TraceAI.init(TraceConfig.builder() TraceAI.initFromEnvironment(); ``` -The builder falls back to environment variables for any field you don't set explicitly. So you can mix both: - -```java -TraceAI.init(TraceConfig.builder() - .projectName("my-project") // explicit - .enableConsoleExporter(true) // explicit - // apiKey, secretKey, baseUrl read from env vars - .build()); -``` +The builder falls back to environment variables for any field you don't set explicitly, so you can set only the fields you want in code (e.g. `projectName` and `enableConsoleExporter`) and let credentials come from the environment. ### Getting the tracer @@ -275,31 +291,82 @@ This flushes all pending spans (up to 10 second timeout) and resets the tracer. ## Available integrations - + Auto-configuration via `application.yml`. No manual `TraceAI.init()` needed. - + Chat completions, embeddings, streaming. - + Messages API with reflection-based version compatibility. - + InvokeModel (raw JSON) and Converse (typed API). - + Chat, embeddings, and reranking. - + Query, upsert, delete, fetch with namespace support. - + Google GenAI, Vertex AI, Azure OpenAI, Ollama, Watsonx. - + Qdrant, Milvus, ChromaDB, Weaviate, MongoDB, Redis, pgvector, Azure AI Search, Elasticsearch. - + LangChain4j and Semantic Kernel. + +--- + +## What spans are produced + +Each `Traced*` wrapper emits one [span](/docs/observe/concepts/spans) per call it delegates, tagged with the matching [`FISpanKind`](#fispankind) above — `LLM` for chat/completions, `EMBEDDING` for vectorization, `RETRIEVER`/`VECTOR_DB` for vector-store reads and writes, `RERANKER` for reranking, and `TOOL`/`AGENT`/`CHAIN` for orchestration. Every span carries input, output, token counts, latency, and errors, and rolls up into the request's [trace](/docs/observe/concepts/traces) exactly like the Python and JS/TS SDKs — the wire format is the same OpenTelemetry export. + +--- + +## Verify the trace + +Run your instrumented app, then open **Observe → your project → Tracing**. Within a few seconds (the exporter flushes every 5 s) a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click it to read the spans the wrappers produced. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +To confirm spans are produced without the dashboard, set `enableConsoleExporter(true)` on the config and watch them print to stdout. If the row is in Observe with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short-lived JVM (CLI/test) exited before the 5 s batch flush. | Call `TraceAI.shutdown()` before exit to flush pending spans. | +| Some calls traced, others not | You used the raw client instead of its `Traced*` wrapper. | Route every call through the wrapper (`new TracedOpenAIClient(client)`). | +| Traces go to the wrong place | `projectName` / `baseUrl` point at another project or endpoint. | Set `projectName` (and `baseUrl` for self-hosted) in `TraceConfig`. | +| Auth error on export | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | +| `IllegalStateException` on `getTracer()` | `getTracer()` was called before `init()`. | Call `TraceAI.init(...)` (or `initFromEnvironment()`) at startup first. | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). Full, runnable per-integration examples live in the [traceAI Java repo](https://github.com/future-agi/traceAI). + +--- + +## Related + + + + The unit every `Traced*` call becomes. + + + The end-to-end setup flow (Python example; the Java steps mirror it). + + + Add custom spans with `FITracer` where a wrapper doesn't reach. + + + Read, filter, and debug the traces you produce. + + diff --git a/src/pages/docs/tracing/auto/java/llm-providers.mdx b/src/pages/docs/traceai/auto/java/llm-providers.mdx similarity index 69% rename from src/pages/docs/tracing/auto/java/llm-providers.mdx rename to src/pages/docs/traceai/auto/java/llm-providers.mdx index bf032de4..9e26258c 100644 --- a/src/pages/docs/tracing/auto/java/llm-providers.mdx +++ b/src/pages/docs/traceai/auto/java/llm-providers.mdx @@ -1,6 +1,31 @@ --- title: "Java LLM Provider Tracing: Vertex, Azure, Ollama, Watsonx" description: "Trace Google GenAI, Vertex AI, Azure OpenAI, Ollama, and Watsonx in Java with Future AGI. All providers use the same TracedClient wrapper pattern." +slug: "llm-providers" +page_type: "integration" +products: ["traceAI"] +integration_category: "java" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "Java LLM Provider Tracing: Vertex, Azure, Ollama, Watsonx" + description: "Trace Google GenAI, Vertex AI, Azure OpenAI, Ollama, and Watsonx in Java with Future AGI. All providers use the same TracedClient wrapper pattern." + primary_keyword: "java llm provider tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace Java LLM Provider with FutureAGI?" + llm_summary: "Trace Google GenAI, Vertex AI, Azure OpenAI, Ollama, and Watsonx in Java with Future AGI. All providers use the same TracedClient wrapper pattern." +canonical: "/docs/traceai/auto/java/llm-providers" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" --- @@ -12,7 +37,7 @@ description: "Trace Google GenAI, Vertex AI, Azure OpenAI, Ollama, and Watsonx i ## Prerequisites -Complete the [Java SDK setup](/docs/tracing/auto/java) first. All providers below need `traceai-java-core` and `TraceAI.init()` called before use. +Complete the [Java SDK setup](/docs/traceai/auto/java) first. All providers below need `traceai-java-core` and `TraceAI.init()` called before use. --- @@ -295,3 +320,54 @@ All providers above capture these core attributes: | `llm.token_count.total` | Total token count | | `input.value` / `output.value` | Plain text input/output | | `fi.raw_input` / `fi.raw_output` | Full request/response as JSON | + +--- + +## What spans are produced + +Auto-instrumentation for Java LLM Provider emits a **`chain` or `agent` root span** for the run, with nested **`llm` spans** for each model call, **`tool` spans** for tool/function calls, and **`retriever` spans** for retrieval steps — so the whole request reads as one tree. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the Java LLM Provider spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every Java LLM Provider call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/tracing/auto/java/openai.mdx b/src/pages/docs/traceai/auto/java/openai.mdx similarity index 60% rename from src/pages/docs/tracing/auto/java/openai.mdx rename to src/pages/docs/traceai/auto/java/openai.mdx index d4b977d2..0ac9a8e3 100644 --- a/src/pages/docs/tracing/auto/java/openai.mdx +++ b/src/pages/docs/traceai/auto/java/openai.mdx @@ -1,6 +1,31 @@ --- title: "OpenAI Java Tracing: TracedOpenAIClient Setup" description: "Trace OpenAI chat completions, embeddings, and streaming responses in Java with TracedOpenAIClient. Part of the Future AGI Java observability SDK." +slug: "openai" +page_type: "integration" +products: ["traceAI"] +integration_category: "java" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "OpenAI Java Tracing: TracedOpenAIClient Setup" + description: "Trace OpenAI chat completions, embeddings, and streaming responses in Java with TracedOpenAIClient. Part of the Future AGI Java observability SDK." + primary_keyword: "openai java tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace OpenAI Java with FutureAGI?" + llm_summary: "Trace OpenAI chat completions, embeddings, and streaming responses in Java with TracedOpenAIClient. Part of the Future AGI Java observability SDK." +canonical: "/docs/traceai/auto/java/openai" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" --- @@ -12,7 +37,7 @@ description: "Trace OpenAI chat completions, embeddings, and streaming responses ## Prerequisites -Complete the [Java SDK setup](/docs/tracing/auto/java) first. You need `TraceAI.init()` called before using this wrapper. +Complete the [Java SDK setup](/docs/traceai/auto/java) first. You need `TraceAI.init()` called before using this wrapper. ## Installation @@ -200,3 +225,54 @@ If you need the unwrapped client for operations that aren't traced: ```java OpenAIClient original = traced.unwrap(); ``` + +--- + +## What spans are produced + +Auto-instrumentation for OpenAI Java emits a **`chain` or `agent` root span** for the run, with nested **`llm` spans** for each model call, **`tool` spans** for tool/function calls, and **`retriever` spans** for retrieval steps — so the whole request reads as one tree. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the OpenAI Java spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every OpenAI Java call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/tracing/auto/java/pinecone.mdx b/src/pages/docs/traceai/auto/java/pinecone.mdx similarity index 51% rename from src/pages/docs/tracing/auto/java/pinecone.mdx rename to src/pages/docs/traceai/auto/java/pinecone.mdx index 4c41d460..ce15e881 100644 --- a/src/pages/docs/tracing/auto/java/pinecone.mdx +++ b/src/pages/docs/traceai/auto/java/pinecone.mdx @@ -1,6 +1,31 @@ --- title: "Pinecone Java Tracing: TracedPineconeIndex Setup" description: "Trace Pinecone vector operations in Java with TracedPineconeIndex. Query, upsert, delete, and fetch with full span instrumentation." +slug: "pinecone" +page_type: "integration" +products: ["traceAI"] +integration_category: "java" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "Pinecone Java Tracing: TracedPineconeIndex Setup" + description: "Trace Pinecone vector operations in Java with TracedPineconeIndex. Query, upsert, delete, and fetch with full span instrumentation." + primary_keyword: "pinecone java tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace Pinecone Java with FutureAGI?" + llm_summary: "Trace Pinecone vector operations in Java with TracedPineconeIndex. Query, upsert, delete, and fetch with full span instrumentation." +canonical: "/docs/traceai/auto/java/pinecone" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" --- @@ -12,7 +37,7 @@ description: "Trace Pinecone vector operations in Java with TracedPineconeIndex. ## Prerequisites -Complete the [Java SDK setup](/docs/tracing/auto/java) first. +Complete the [Java SDK setup](/docs/traceai/auto/java) first. ## Installation @@ -175,3 +200,54 @@ var fetched = traced.fetch(List.of("vec-1"), "my-namespace"); ```java Index original = traced.unwrap(); ``` + +--- + +## What spans are produced + +Auto-instrumentation for Pinecone Java emits a **`chain` or `agent` root span** for the run, with nested **`llm` spans** for each model call, **`tool` spans** for tool/function calls, and **`retriever` spans** for retrieval steps — so the whole request reads as one tree. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the Pinecone Java spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every Pinecone Java call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/tracing/auto/java/vector-databases.mdx b/src/pages/docs/traceai/auto/java/vector-databases.mdx similarity index 78% rename from src/pages/docs/tracing/auto/java/vector-databases.mdx rename to src/pages/docs/traceai/auto/java/vector-databases.mdx index 660c6ff7..61ed283d 100644 --- a/src/pages/docs/tracing/auto/java/vector-databases.mdx +++ b/src/pages/docs/traceai/auto/java/vector-databases.mdx @@ -1,6 +1,31 @@ --- title: "Java Vector Database Tracing: Qdrant, Milvus, and More" description: "Trace vector database operations in Java. Qdrant, Milvus, ChromaDB, Weaviate, MongoDB, Redis, pgvector, Azure AI Search, and Elasticsearch." +slug: "vector-databases" +page_type: "integration" +products: ["traceAI"] +integration_category: "java" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "Java Vector Database Tracing: Qdrant, Milvus, and More" + description: "Trace vector database operations in Java. Qdrant, Milvus, ChromaDB, Weaviate, MongoDB, Redis, pgvector, Azure AI Search, and Elasticsearch." + primary_keyword: "java vector database tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace Java Vector Database with FutureAGI?" + llm_summary: "Trace vector database operations in Java. Qdrant, Milvus, ChromaDB, Weaviate, MongoDB, Redis, pgvector, Azure AI Search, and Elasticsearch." +canonical: "/docs/traceai/auto/java/vector-databases" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" --- @@ -12,7 +37,7 @@ description: "Trace vector database operations in Java. Qdrant, Milvus, ChromaDB ## Prerequisites -Complete the [Java SDK setup](/docs/tracing/auto/java) first. For Pinecone, see the [dedicated Pinecone page](/docs/tracing/auto/java/pinecone). +Complete the [Java SDK setup](/docs/traceai/auto/java) first. For Pinecone, see the [dedicated Pinecone page](/docs/traceai/auto/java/pinecone). --- @@ -451,3 +476,54 @@ All vector database wrappers capture: | `embedding.dimensions` | Vector dimensions | | `retriever.top_k` | Number of results requested (search operations) | | `db.vector.results.count` | Number of results returned | + +--- + +## What spans are produced + +Auto-instrumentation for Java Vector Database emits a **`chain` or `agent` root span** for the run, with nested **`llm` spans** for each model call, **`tool` spans** for tool/function calls, and **`retriever` spans** for retrieval steps — so the whole request reads as one tree. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the Java Vector Database spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every Java Vector Database call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/traceai/auto/langchain.mdx b/src/pages/docs/traceai/auto/langchain.mdx new file mode 100644 index 00000000..ff6a928f --- /dev/null +++ b/src/pages/docs/traceai/auto/langchain.mdx @@ -0,0 +1,212 @@ +--- +title: "LangChain Tracing with Future AGI: Auto-Instrumentation" +description: "Set up auto-instrumentation for LangChain with Future AGI tracing. Install traceAI-langchain to capture chain, tool, and LLM call spans." +slug: "langchain" +page_type: "integration" +products: ["traceAI"] +integration_category: "framework" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "LangChain Tracing with Future AGI: Auto-Instrumentation" + description: "Set up auto-instrumentation for LangChain with Future AGI tracing. Install traceAI-langchain to capture chain, tool, and LLM call spans." + primary_keyword: "langchain tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace LangChain with FutureAGI?" + llm_summary: "Set up auto-instrumentation for LangChain with Future AGI tracing. Install traceAI-langchain to capture chain, tool, and LLM call spans." +canonical: "/docs/traceai/auto/langchain" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" +--- + +## 1. Installation +First install the traceAI package and necessary LangChain packages. + + + +```bash Python +pip install traceAI-langchain +pip install langchain_openai +``` + +```bash JS/TS +npm install @traceai/langchain @traceai/fi-core @opentelemetry/instrumentation \ + @langchain/openai @langchain/core +``` + + +--- + +## 2. Set Environment Variables + +Set up your environment variables to authenticate with both FutureAGI and OpenAI. + + + +```python Python +import os + +os.environ["OPENAI_API_KEY"] = "your-openai-api-key" +os.environ["FI_API_KEY"] = "your-futureagi-api-key" +os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" +``` + +```typescript JS/TS +process.env.OPENAI_API_KEY = "your-openai-api-key"; +process.env.FI_API_KEY = "your-futureagi-api-key"; +process.env.FI_SECRET_KEY = "your-futureagi-secret-key"; +``` + + + +--- + +## 3. Initialize Trace Provider +Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . + + + +```python Python +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType + +trace_provider = register( + project_type=ProjectType.OBSERVE, + project_name="langchain_project", +) +``` + +```typescript JS/TS +import { register, ProjectType } from "@traceai/fi-core"; + +const tracerProvider = register({ + project_type: ProjectType.OBSERVE, + project_name: "langchain_project", +}); +``` + + + +--- + +## 4. Instrument your Project +Initialize the LangChain Instrumentor to enable automatic tracing. This step ensures that all interactions with the LangChain are tracked and monitored. + + + +```python Python +from traceai_langchain import LangChainInstrumentor + +LangChainInstrumentor().instrument(tracer_provider=trace_provider) +``` + +```typescript JS/TS +import { LangChainInstrumentation } from "@traceai/langchain"; +import * as CallbackManagerModule from "langchain/callbacks"; + +// Pass the custom tracer provider to the instrumentation +const lcInstrumentation = new LangChainInstrumentation({ + tracerProvider: tracerProvider, +}); + +// Manually instrument the LangChain module +lcInstrumentation.manuallyInstrument(CallbackManagerModule); +``` + + + +--- + +## 5. Create LangChain Components +Set up your LangChain pipeline as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. + + + +```python Python +from langchain_openai import ChatOpenAI +from langchain_core.prompts import ChatPromptTemplate + +prompt = ChatPromptTemplate.from_template("{x} {y} {z}?").partial(x="why is", z="blue") +chain = prompt | ChatOpenAI(model_name="gpt-3.5-turbo") + +result = chain.invoke({"y": "sky"}) + +print(f"Response: {result}") +``` + +```typescript JS/TS +import { ChatOpenAI } from "@langchain/openai"; +import { ChatPromptTemplate } from "@langchain/core/prompts"; + +const prompt = ChatPromptTemplate.fromTemplate("{x} {y} {z}?").partial({ x: "why is", z: "blue" }); +const chain = prompt.pipe(new ChatOpenAI({ model: "gpt-3.5-turbo" })); + +const result = await chain.invoke({ y: "sky" }); +console.log("Response:", result); +``` + + + +--- + +## What spans are produced + +Auto-instrumentation for LangChain emits a **`chain` or `agent` root span** for the run, with nested **`llm` spans** for each model call, **`tool` spans** for tool/function calls, and **`retriever` spans** for retrieval steps — so the whole request reads as one tree. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + + + A short-lived script can exit before the batch exporter flushes. Call `trace_provider.force_flush()` before the process ends so the last spans are sent. + + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the LangChain spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every LangChain call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/traceai/auto/langgraph.mdx b/src/pages/docs/traceai/auto/langgraph.mdx new file mode 100644 index 00000000..a41b567e --- /dev/null +++ b/src/pages/docs/traceai/auto/langgraph.mdx @@ -0,0 +1,176 @@ +--- +title: "LangGraph Tracing with Future AGI: Auto-Instrumentation" +description: "Set up auto-instrumentation for LangGraph with Future AGI tracing. Capture agent graph execution and state transition spans via LangChain instrumentor." +slug: "langgraph" +page_type: "integration" +products: ["traceAI"] +integration_category: "framework" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "LangGraph Tracing with Future AGI: Auto-Instrumentation" + description: "Set up auto-instrumentation for LangGraph with Future AGI tracing. Capture agent graph execution and state transition spans via LangChain instrumentor." + primary_keyword: "langgraph tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace LangGraph with FutureAGI?" + llm_summary: "Set up auto-instrumentation for LangGraph with Future AGI tracing. Capture agent graph execution and state transition spans via LangChain instrumentor." +canonical: "/docs/traceai/auto/langgraph" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" +--- + +Our [LangChainInstrumentor](/docs/traceai/auto/langchain) automatically captures traces for both LangGraph and LangChain. If you've already enabled that instrumentor, you do not need to complete the steps below. + +## 1. Installation +First install the traceAI package and necessary LangChain packages. + +```bash +pip install traceAI-langchain +pip install langgraph +pip install langchain-anthropic +pip install ipython +``` +--- + +## 2. Set Environment Variables + +Set up your environment variables to authenticate with both FutureAGI and Anthropic. + +```python +import os + +os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-api-key" +os.environ["FI_API_KEY"] = "your-futureagi-api-key" +os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" +``` + +--- + +## 3. Initialize Trace Provider +Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . + +```python +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType + +trace_provider = register( + project_type=ProjectType.OBSERVE, + project_name="langgraph_project", +) +``` + +--- + +## 4. Instrument your Project +Initialize the LangChain Instrumentor to enable automatic tracing. Our [LangChainInstrumentor](/docs/traceai/auto/langchain) automatically captures traces for both LangGraph and LangChain. + +```python +from traceai_langchain import LangChainInstrumentor + +LangChainInstrumentor().instrument(tracer_provider=trace_provider) +``` + +--- + +## 5. Create LangGraph Agents +Set up your LangGraph agents as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. + +```python +from typing import Annotated +from typing_extensions import TypedDict +from langgraph.graph import StateGraph, START, END +from langgraph.graph.message import add_messages +from langchain_anthropic import ChatAnthropic +from IPython.display import Image, display + +class State(TypedDict): + messages: Annotated[list, add_messages] + +graph_builder = StateGraph(State) +llm = ChatAnthropic(model="claude-3-5-sonnet-20240620") + +def chatbot(state: State): + return {"messages": [llm.invoke(state["messages"])]} + +graph_builder.add_node("chatbot", chatbot) +graph_builder.add_edge(START, "chatbot") +graph_builder.add_edge("chatbot", END) +graph = graph_builder.compile() + +try: + display(Image(graph.get_graph().draw_mermaid_png())) +except Exception: + pass + +def stream_graph_updates(user_input: str): + for event in graph.stream({"messages": [{"role": "user", "content": user_input}]}): + for value in event.values(): + print("Assistant:", value["messages"][-1].content) + +user_input = "What do you know about LangGraph?" +stream_graph_updates(user_input) +``` + +--- + +## What spans are produced + +Auto-instrumentation for LangGraph emits a **`chain` or `agent` root span** for the run, with nested **`llm` spans** for each model call, **`tool` spans** for tool/function calls, and **`retriever` spans** for retrieval steps — so the whole request reads as one tree. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + + + A short-lived script can exit before the batch exporter flushes. Call `trace_provider.force_flush()` before the process ends so the last spans are sent. + + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the LangGraph spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every LangGraph call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/traceai/auto/litellm.mdx b/src/pages/docs/traceai/auto/litellm.mdx new file mode 100644 index 00000000..43096e55 --- /dev/null +++ b/src/pages/docs/traceai/auto/litellm.mdx @@ -0,0 +1,147 @@ +--- +title: "LiteLLM Tracing with Future AGI: Auto-Instrumentation" +description: "Set up auto-instrumentation for LiteLLM with Future AGI tracing. Install traceAI-litellm to capture spans across multiple LLM provider calls." +slug: "litellm" +page_type: "integration" +products: ["traceAI"] +integration_category: "llm-provider" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "LiteLLM Tracing with Future AGI: Auto-Instrumentation" + description: "Set up auto-instrumentation for LiteLLM with Future AGI tracing. Install traceAI-litellm to capture spans across multiple LLM provider calls." + primary_keyword: "litellm tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace LiteLLM with FutureAGI?" + llm_summary: "Set up auto-instrumentation for LiteLLM with Future AGI tracing. Install traceAI-litellm to capture spans across multiple LLM provider calls." +canonical: "/docs/traceai/auto/litellm" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" +--- + +## 1. Installation +Install the traceAI and litellm packages. + +```bash +pip install traceAI-litellm +pip install litellm +``` + +--- + +## 2. Set Environment Variables +Set up your environment variables to authenticate with both FutureAGI and OpenAI. + +```python +import os + +os.environ["FI_API_KEY"] = "your-futureagi-api-key" +os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" +os.environ["OPENAI_API_KEY"] = "your-openai-api-key" +``` + +--- + +## 3. Initialize Trace Provider +Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . + +```python +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType + +trace_provider = register( + project_type=ProjectType.OBSERVE, + project_name="openai_project", +) +``` + +--- + +## 4. Configure LiteLLM Instrumentation +Initialize the LiteLLM instrumentor to enable automatic tracing. + +```python +from traceai_litellm import LiteLLMInstrumentor + +LiteLLMInstrumentor().instrument(tracer_provider=trace_provider) +``` + +--- + +## 5. Run LiteLLM +Run LiteLLM as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. + +```python +import litellm + +response = litellm.completion( + model="gpt-3.5-turbo", + messages=[{"content": "What's the capital of India?"}], +) + +print(response.choices[0].message.content) +``` + +--- + +## What spans are produced + +Auto-instrumentation for LiteLLM emits an **`llm` span for every model call** — chat, completion, or embedding — carrying the model name, the prompt and completion, prompt/completion/total token counts, and the estimated cost. Tool or function calls add a **`tool` span**. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + + + A short-lived script can exit before the batch exporter flushes. Call `trace_provider.force_flush()` before the process ends so the last spans are sent. + + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the LiteLLM spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every LiteLLM call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/tracing/auto/livekit.mdx b/src/pages/docs/traceai/auto/livekit.mdx similarity index 63% rename from src/pages/docs/tracing/auto/livekit.mdx rename to src/pages/docs/traceai/auto/livekit.mdx index 38bc2f66..f5cc0e22 100644 --- a/src/pages/docs/tracing/auto/livekit.mdx +++ b/src/pages/docs/traceai/auto/livekit.mdx @@ -1,6 +1,31 @@ --- title: "LiveKit Tracing with Future AGI: Voice Agent Observability" description: "Integrate LiveKit with Future AGI for voice agent observability. Trace real-time voice interactions and monitor agent performance with traceAI-livekit." +slug: "livekit" +page_type: "integration" +products: ["traceAI"] +integration_category: "voice" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "LiveKit Tracing with Future AGI: Voice Agent Observability" + description: "Integrate LiveKit with Future AGI for voice agent observability. Trace real-time voice interactions and monitor agent performance with traceAI-livekit." + primary_keyword: "livekit tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace LiveKit with FutureAGI?" + llm_summary: "Integrate LiveKit with Future AGI for voice agent observability. Trace real-time voice interactions and monitor agent performance with traceAI-livekit." +canonical: "/docs/traceai/auto/livekit" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" --- ## 1. Installation @@ -236,4 +261,59 @@ async def entrypoint(ctx: JobContext): if __name__ == "__main__": cli.run_app(server) -``` \ No newline at end of file +``` + +--- + +## What spans are produced + +Auto-instrumentation for LiveKit emits **spans per conversation turn** — an `llm` span for the model plus speech-to-text and text-to-speech steps — grouped under one session so you can replay the whole call. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + + + A short-lived script can exit before the batch exporter flushes. Call `trace_provider.force_flush()` before the process ends so the last spans are sent. + + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the LiveKit spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every LiveKit call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/traceai/auto/llamaindex-workflows.mdx b/src/pages/docs/traceai/auto/llamaindex-workflows.mdx new file mode 100644 index 00000000..d1a86de3 --- /dev/null +++ b/src/pages/docs/traceai/auto/llamaindex-workflows.mdx @@ -0,0 +1,186 @@ +--- +title: "LlamaIndex Workflows Tracing with Future AGI" +description: "Set up auto-instrumentation for LlamaIndex Workflows with Future AGI tracing. Trace workflow agent execution via the LlamaIndex instrumentor." +slug: "llamaindex-workflows" +page_type: "integration" +products: ["traceAI"] +integration_category: "framework" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "LlamaIndex Workflows Tracing with Future AGI" + description: "Set up auto-instrumentation for LlamaIndex Workflows with Future AGI tracing. Trace workflow agent execution via the LlamaIndex instrumentor." + primary_keyword: "llamaindex workflows tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace LlamaIndex Workflows with FutureAGI?" + llm_summary: "Set up auto-instrumentation for LlamaIndex Workflows with Future AGI tracing. Trace workflow agent execution via the LlamaIndex instrumentor." +canonical: "/docs/traceai/auto/llamaindex-workflows" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" +--- + +[LlamaIndex Workflows](https://www.llamaindex.ai/blog/introducing-workflows-beta-a-new-way-to-create-complex-ai-applications-with-llamaindex) are a subset of the LlamaIndex package specifically designed to support agent development. + +Our [LlamaIndexInstrumentor](/docs/traceai/auto/llamaindex) automatically captures traces for LlamaIndex Workflows agents. If you've already enabled that instrumentor, you do not need to complete the steps below. + +## 1. Installation +First install the traceAI and necessary llama-index packages. +```bash +pip install traceAI-llamaindex +pip install llama-index +``` + +--- + +## 2. Set Environment Variables + +Set up your environment variables to authenticate with FutureAGI. + +```python +import os + +os.environ["FI_API_KEY"] = "your-futureagi-api-key" +os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" +os.environ["OPENAI_API_KEY"] = "your-openai-api-key" +``` + +--- + +## 3. Initialize Trace Provider + +Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . + +```python +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType + +trace_provider = register( + project_type=ProjectType.OBSERVE, + project_name="openai_project", +) +``` + +--- + +## 4. Instrument your Project + +Instrument your Project with LlamaIndex Instrumentor. This instrumentor will trace both LlamaIndex Workflows calls, as well as calls to the general LlamaIndex package. + +```python +from traceai_llamaindex import LlamaIndexInstrumentor + +LlamaIndexInstrumentor().instrument(tracer_provider=trace_provider) +``` + +--- + +## 5. Run LlamaIndex Workflows + +Run your LlamaIndex workflows as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. + +```python +import asyncio + +from llama_index.core.workflow import ( + Event, + StartEvent, + StopEvent, + Workflow, + step, +) +from llama_index.llms.openai import OpenAI + +class JokeEvent(Event): + joke: str + +class JokeFlow(Workflow): + llm = OpenAI() + + @step + async def generate_joke(self, ev: StartEvent) -> JokeEvent: + topic = ev.topic + + prompt = f"Write your best joke about {topic}." + response = await self.llm.acomplete(prompt) + return JokeEvent(joke=str(response)) + + @step + async def critique_joke(self, ev: JokeEvent) -> StopEvent: + joke = ev.joke + + prompt = f"Give a thorough analysis and critique of the following joke: {joke}" + response = await self.llm.acomplete(prompt) + return StopEvent(result=str(response)) + +async def main(): + w = JokeFlow(timeout=60, verbose=False) + result = await w.run(topic="pirates") + print(str(result)) + +if __name__ == "__main__": + asyncio.run(main()) +``` + +--- + +## What spans are produced + +Auto-instrumentation for LlamaIndex Workflows emits a **`chain` or `agent` root span** for the run, with nested **`llm` spans** for each model call, **`tool` spans** for tool/function calls, and **`retriever` spans** for retrieval steps — so the whole request reads as one tree. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + + + A short-lived script can exit before the batch exporter flushes. Call `trace_provider.force_flush()` before the process ends so the last spans are sent. + + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the LlamaIndex Workflows spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every LlamaIndex Workflows call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/traceai/auto/llamaindex.mdx b/src/pages/docs/traceai/auto/llamaindex.mdx new file mode 100644 index 00000000..7a170927 --- /dev/null +++ b/src/pages/docs/traceai/auto/llamaindex.mdx @@ -0,0 +1,160 @@ +--- +title: "LlamaIndex Tracing with Future AGI: Auto-Instrumentation" +description: "Set up auto-instrumentation for LlamaIndex with Future AGI tracing. Install traceAI-llamaindex to capture query, retrieval, and response spans." +slug: "llamaindex" +page_type: "integration" +products: ["traceAI"] +integration_category: "framework" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "LlamaIndex Tracing with Future AGI: Auto-Instrumentation" + description: "Set up auto-instrumentation for LlamaIndex with Future AGI tracing. Install traceAI-llamaindex to capture query, retrieval, and response spans." + primary_keyword: "llamaindex tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace LlamaIndex with FutureAGI?" + llm_summary: "Set up auto-instrumentation for LlamaIndex with Future AGI tracing. Install traceAI-llamaindex to capture query, retrieval, and response spans." +canonical: "/docs/traceai/auto/llamaindex" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" +--- + +## 1. Installation +Install the traceAI and Llama Index packages. + +```bash +pip install traceAI-llamaindex +pip install llama-index +``` + +--- + +## 2. Set Environment Variables +Set up your environment variables to authenticate with FutureAGI. + +```python +import os + +os.environ["FI_API_KEY"] = "your-futureagi-api-key" +os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" +os.environ["OPENAI_API_KEY"] = "your-openai-api-key" +``` + +--- + +## 3. Initialize Trace Provider +Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . + +```python +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType + +trace_provider = register( + project_type=ProjectType.OBSERVE, + project_name="llamaindex_project", +) +``` + +--- + +## 4. Instrument your Project +Initialize the Llama Index instrumentor to enable automatic tracing. This step ensures that all interactions with the Llama Index are tracked and monitored. + +```python +from traceai_llamaindex import LlamaIndexInstrumentor + +LlamaIndexInstrumentor().instrument(tracer_provider=trace_provider) +``` + +--- + +## 5. Create Llama Index Components +Set up your Llama Index components as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. + +```python +from llama_index.agent.openai import OpenAIAgent +from llama_index.core import Settings +from llama_index.core.tools import FunctionTool +from llama_index.llms.openai import OpenAI + +def multiply(a: int, b: int) -> int: + """Multiply two integers and return the result.""" + return a * b + +def add(a: int, b: int) -> int: + """Add two integers and return the result.""" + return a + b + +multiply_tool = FunctionTool.from_defaults(fn=multiply) +add_tool = FunctionTool.from_defaults(fn=add) +agent = OpenAIAgent.from_tools([multiply_tool, add_tool]) +Settings.llm = OpenAI(model="gpt-3.5-turbo") + +response = agent.query("What is (121 * 3) + 42?") + +print(response) +``` + +--- + +## What spans are produced + +Auto-instrumentation for LlamaIndex emits a **`chain` or `agent` root span** for the run, with nested **`llm` spans** for each model call, **`tool` spans** for tool/function calls, and **`retriever` spans** for retrieval steps — so the whole request reads as one tree. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + + + A short-lived script can exit before the batch exporter flushes. Call `trace_provider.force_flush()` before the process ends so the last spans are sent. + + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the LlamaIndex spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every LlamaIndex call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/traceai/auto/mastra.mdx b/src/pages/docs/traceai/auto/mastra.mdx new file mode 100644 index 00000000..c0e112ff --- /dev/null +++ b/src/pages/docs/traceai/auto/mastra.mdx @@ -0,0 +1,137 @@ +--- +title: "Mastra Tracing with Future AGI: Auto-Instrumentation" +description: "Set up auto-instrumentation for Mastra with Future AGI tracing. Configure @traceai/mastra to export TypeScript agent spans to Future AGI." +slug: "mastra" +page_type: "integration" +products: ["traceAI"] +integration_category: "framework" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "Mastra Tracing with Future AGI: Auto-Instrumentation" + description: "Set up auto-instrumentation for Mastra with Future AGI tracing. Configure @traceai/mastra to export TypeScript agent spans to Future AGI." + primary_keyword: "mastra tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace Mastra with FutureAGI?" + llm_summary: "Set up auto-instrumentation for Mastra with Future AGI tracing. Configure @traceai/mastra to export TypeScript agent spans to Future AGI." +canonical: "/docs/traceai/auto/mastra" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" +--- + +## 1. Installation +First install the Mastra and traceAI packages. + +```bash JS/TS +npm install @mastra/core @traceai/mastra @traceai/fi-core +``` + +--- + +## 2. Set Environment Variables + +Configure your Future AGI credentials. + +```typescript JS/TS +process.env.FI_API_KEY = "your-futureagi-api-key"; +process.env.FI_SECRET_KEY = "your-futureagi-secret-key"; +``` + +--- + +## 3. Configure Mastra Telemetry Export +Use the custom exporter from `@traceai/mastra` to send traces to Future AGI. You can optionally filter out non-LLM spans using `isFISpan`. + +```typescript JS/TS +import { Mastra } from "@mastra/core"; +import { FITraceExporter, isFISpan } from "@traceai/mastra"; + +export const mastra = new Mastra({ + // ... other config + telemetry: { + serviceName: "traceai-mastra-agent", // customize the service name + enabled: true, + export: { + type: "custom", + exporter: new FITraceExporter({ + url: "https://app.futureagi.com/tracer/v1/traces", + headers: { + "x-api-key": process.env.FI_API_KEY as string, + "x-secret-key": process.env.FI_SECRET_KEY as string, + }, + // Optional: filter out non-LLM/node spans from being sent to Future AGI + spanFilter: isFISpan, + }), + }, + }, +}); +``` + +--- + +## 4. Run your Agent +Once configured, run your Mastra agent as usual. The exporter will automatically send trace data to your Future AGI project. + +--- + +## What spans are produced + +Auto-instrumentation for Mastra emits a **`chain` or `agent` root span** for the run, with nested **`llm` spans** for each model call, **`tool` spans** for tool/function calls, and **`retriever` spans** for retrieval steps — so the whole request reads as one tree. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + + + A short-lived script can exit before the batch exporter flushes. Call `trace_provider.force_flush()` before the process ends so the last spans are sent. + + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the Mastra spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every Mastra call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/tracing/auto/mcp.mdx b/src/pages/docs/traceai/auto/mcp.mdx similarity index 56% rename from src/pages/docs/tracing/auto/mcp.mdx rename to src/pages/docs/traceai/auto/mcp.mdx index 4fd76610..de4fc3eb 100644 --- a/src/pages/docs/tracing/auto/mcp.mdx +++ b/src/pages/docs/traceai/auto/mcp.mdx @@ -1,6 +1,31 @@ --- title: "MCP Tracing with Future AGI: Model Context Protocol Spans" description: "Set up auto-instrumentation for MCP with Future AGI tracing. Install traceAI-mcp to capture Model Context Protocol server and tool call spans." +slug: "mcp" +page_type: "integration" +products: ["traceAI"] +integration_category: "framework" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "MCP Tracing with Future AGI: Model Context Protocol Spans" + description: "Set up auto-instrumentation for MCP with Future AGI tracing. Install traceAI-mcp to capture Model Context Protocol server and tool call spans." + primary_keyword: "mcp tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace MCP with FutureAGI?" + llm_summary: "Set up auto-instrumentation for MCP with Future AGI tracing. Install traceAI-mcp to capture Model Context Protocol server and tool call spans." +canonical: "/docs/traceai/auto/mcp" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" --- ## 1. Installation @@ -171,4 +196,59 @@ if __name__ == "__main__": raise RuntimeError("npx is not installed. Please install it with `npm install -g npx`.") asyncio.run(main()) -``` \ No newline at end of file +``` + +--- + +## What spans are produced + +Auto-instrumentation for MCP emits a **`chain` or `agent` root span** for the run, with nested **`llm` spans** for each model call, **`tool` spans** for tool/function calls, and **`retriever` spans** for retrieval steps — so the whole request reads as one tree. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + + + A short-lived script can exit before the batch exporter flushes. Call `trace_provider.force_flush()` before the process ends so the last spans are sent. + + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the MCP spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every MCP call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/traceai/auto/mistralai.mdx b/src/pages/docs/traceai/auto/mistralai.mdx new file mode 100644 index 00000000..368119a9 --- /dev/null +++ b/src/pages/docs/traceai/auto/mistralai.mdx @@ -0,0 +1,150 @@ +--- +title: "Mistral AI Tracing with Future AGI: Auto-Instrumentation" +description: "Set up auto-instrumentation for Mistral AI with Future AGI tracing. Install traceAI-mistralai to capture model inference spans and metadata." +slug: "mistralai" +page_type: "integration" +products: ["traceAI"] +integration_category: "llm-provider" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "Mistral AI Tracing with Future AGI: Auto-Instrumentation" + description: "Set up auto-instrumentation for Mistral AI with Future AGI tracing. Install traceAI-mistralai to capture model inference spans and metadata." + primary_keyword: "mistral ai tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace Mistral AI with FutureAGI?" + llm_summary: "Set up auto-instrumentation for Mistral AI with Future AGI tracing. Install traceAI-mistralai to capture model inference spans and metadata." +canonical: "/docs/traceai/auto/mistralai" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" +--- + +## 1. Installation +Install the traceAI package to access the observability framework. + +```bash +pip install traceAI-mistralai +``` + +--- + +## 2. Set Environment Variables +Set up your environment variables to authenticate with both FutureAGI and MistralAI . + +```python +import os + +os.environ["FI_API_KEY"] = "your-futureagi-api-key" +os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" +os.environ["MISTRAL_API_KEY"] = "your-mistral-api-key" +``` + +--- + +## 3. Initialize Trace Provider +Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . + +```python +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType + +trace_provider = register( + project_type=ProjectType.OBSERVE, + project_name="mistralai_project", +) +``` + +--- + +## 4. Instrument your Project +Instrument your Project with MistralAI Instrumentor. This step ensures that all interactions with the MistralAI are tracked and monitored. + +```python +from traceai_mistralai import MistralAIInstrumentor + +MistralAIInstrumentor().instrument(tracer_provider=trace_provider) +``` + +--- + +## 5. Create Mistral AI Components +Set up your Mistral AI client and use your application as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. + +```python +from mistralai import Mistral + +client = Mistral(api_key=os.environ["MISTRAL_API_KEY"]) + +response = client.agents.complete( + agent_id="agent_id", + messages=[ + {"role": "user", "content": "plan a vacation for me in Tbilisi"}, + ], +) + +print(response) +``` + +--- + +## What spans are produced + +Auto-instrumentation for Mistral AI emits an **`llm` span for every model call** — chat, completion, or embedding — carrying the model name, the prompt and completion, prompt/completion/total token counts, and the estimated cost. Tool or function calls add a **`tool` span**. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + + + A short-lived script can exit before the batch exporter flushes. Call `trace_provider.force_flush()` before the process ends so the last spans are sent. + + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the Mistral AI spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every Mistral AI call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/traceai/auto/ollama.mdx b/src/pages/docs/traceai/auto/ollama.mdx new file mode 100644 index 00000000..96b11240 --- /dev/null +++ b/src/pages/docs/traceai/auto/ollama.mdx @@ -0,0 +1,158 @@ +--- +title: "Ollama Tracing with Future AGI: Auto-Instrumentation" +description: "Set up auto-instrumentation for Ollama with Future AGI tracing. Use traceAI-openai to capture spans from Ollama's OpenAI-compatible local LLM API." +slug: "ollama" +page_type: "integration" +products: ["traceAI"] +integration_category: "llm-provider" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "Ollama Tracing with Future AGI: Auto-Instrumentation" + description: "Set up auto-instrumentation for Ollama with Future AGI tracing. Use traceAI-openai to capture spans from Ollama's OpenAI-compatible local LLM API." + primary_keyword: "ollama tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace Ollama with FutureAGI?" + llm_summary: "Set up auto-instrumentation for Ollama with Future AGI tracing. Use traceAI-openai to capture spans from Ollama's OpenAI-compatible local LLM API." +canonical: "/docs/traceai/auto/ollama" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" +--- + +## 1. Installation +First install the traceAI package to access the observability framework + +```bash +pip install traceAI-openai +``` + +--- + +## 2. Set Environment Variables + +Set up your environment variables to authenticate with FutureAGI. + +```python +import os + +os.environ["FI_API_KEY"] = "your-futureagi-api-key" +os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" +``` + +--- + +## 3. Initialize Trace Provider + +Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . + +```python +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType + +trace_provider = register( + project_type=ProjectType.OBSERVE, + project_name="OLLAMA 3.2", +) +``` + +--- + +## 4. Instrument your Project + +Use the OpenAI Instrumentor to instrument your project, as the OpenAI Client is utilized for interactions with Ollama. This step guarantees that all interactions are tracked and monitored. If you are using a different client to interact with Ollama, use that client's Instrumentor instead. + +```python +from traceai_openai import OpenAIInstrumentor + +OpenAIInstrumentor().instrument(tracer_provider=trace_provider) +``` + +--- + +## 5. Interact with Ollama + +Interact with the Ollama as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. +Make sure that Ollama is running and accessible from your project. + +```python +from openai import OpenAI + +client = OpenAI( + base_url = 'http://localhost:11434/v1', + api_key='ollama', +) + +response = client.chat.completions.create( + model="llama3.2:1b", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is OpenAI?"}, + ] + ) + +print(response.choices[0].message.content) +``` + +--- + +## What spans are produced + +Auto-instrumentation for Ollama emits an **`llm` span for every model call** — chat, completion, or embedding — carrying the model name, the prompt and completion, prompt/completion/total token counts, and the estimated cost. Tool or function calls add a **`tool` span**. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + + + A short-lived script can exit before the batch exporter flushes. Call `trace_provider.force_flush()` before the process ends so the last spans are sent. + + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the Ollama spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every Ollama call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/tracing/auto/openai.mdx b/src/pages/docs/traceai/auto/openai.mdx similarity index 58% rename from src/pages/docs/tracing/auto/openai.mdx rename to src/pages/docs/traceai/auto/openai.mdx index f29071fb..f8c70844 100644 --- a/src/pages/docs/tracing/auto/openai.mdx +++ b/src/pages/docs/traceai/auto/openai.mdx @@ -1,6 +1,31 @@ --- title: "OpenAI Tracing with Future AGI: Auto-Instrumentation" description: "Set up auto-instrumentation for OpenAI with Future AGI tracing. Install traceAI-openai to capture chat completion, embedding, and tool call spans." +slug: "openai" +page_type: "integration" +products: ["traceAI"] +integration_category: "llm-provider" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "OpenAI Tracing with Future AGI: Auto-Instrumentation" + description: "Set up auto-instrumentation for OpenAI with Future AGI tracing. Install traceAI-openai to capture chat completion, embedding, and tool call spans." + primary_keyword: "openai tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace OpenAI with FutureAGI?" + llm_summary: "Set up auto-instrumentation for OpenAI with Future AGI tracing. Install traceAI-openai to capture chat completion, embedding, and tool call spans." +canonical: "/docs/traceai/auto/openai" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" --- ## 1. Installation @@ -230,4 +255,59 @@ completion = client.chat.completions.create( for chunk in completion: print(chunk.choices[0].delta.content, end="") -``` \ No newline at end of file +``` + +--- + +## What spans are produced + +Auto-instrumentation for OpenAI emits an **`llm` span for every model call** — chat, completion, or embedding — carrying the model name, the prompt and completion, prompt/completion/total token counts, and the estimated cost. Tool or function calls add a **`tool` span**. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + + + A short-lived script can exit before the batch exporter flushes. Call `trace_provider.force_flush()` before the process ends so the last spans are sent. + + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the OpenAI spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every OpenAI call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/traceai/auto/openai_agents.mdx b/src/pages/docs/traceai/auto/openai_agents.mdx new file mode 100644 index 00000000..0dcda873 --- /dev/null +++ b/src/pages/docs/traceai/auto/openai_agents.mdx @@ -0,0 +1,148 @@ +--- +title: "OpenAI Agents SDK Tracing with Future AGI" +description: "Set up auto-instrumentation for OpenAI Agents SDK with Future AGI tracing. Install traceAI-openai-agents to capture agent workflow spans." +slug: "openai_agents" +page_type: "integration" +products: ["traceAI"] +integration_category: "framework" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "OpenAI Agents SDK Tracing with Future AGI" + description: "Set up auto-instrumentation for OpenAI Agents SDK with Future AGI tracing. Install traceAI-openai-agents to capture agent workflow spans." + primary_keyword: "openai agents tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace OpenAI Agents with FutureAGI?" + llm_summary: "Set up auto-instrumentation for OpenAI Agents SDK with Future AGI tracing. Install traceAI-openai-agents to capture agent workflow spans." +canonical: "/docs/traceai/auto/openai_agents" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" +--- + +## 1. Installation +First install the traceAI package to access the observability framework + +```bash +pip install traceAI-openai-agents +``` + +--- + +## 2. Set Environment Variables + +Set up your environment variables to authenticate with both FutureAGI and OpenAI. + +```python +import os + +os.environ["OPENAI_API_KEY"] = "your-openai-api-key" +os.environ["FI_API_KEY"] = "your-futureagi-api-key" +os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" +``` + +--- + +## 3. Initialize Trace Provider + +Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . + +```python +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType + +trace_provider = register( + project_type=ProjectType.EXPERIMENT, + project_name="openai_project", +) +``` + +--- + +## 4. Instrument your Project + +Instrument your Project with OpenAI Agents Instrumentor. This step ensures that all interactions with the OpenAI are tracked and monitored. + +```python +from traceai_openai_agents import OpenAIAgentsInstrumentor + +OpenAIAgentsInstrumentor().instrument(tracer_provider=trace_provider) +``` + +--- + +## 5. Interact with OpenAI Agents + +Interact with the OpenAI Agents as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. + +```python +from agents import Agent, Runner + +agent = Agent(name="Assistant", instructions="You are a helpful assistant") +result = Runner.run_sync(agent, "Write a haiku about recursion in programming.") + +print(result.final_output) +``` + +--- + +## What spans are produced + +Auto-instrumentation for OpenAI Agents SDK emits a **`chain` or `agent` root span** for the run, with nested **`llm` spans** for each model call, **`tool` spans** for tool/function calls, and **`retriever` spans** for retrieval steps — so the whole request reads as one tree. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + + + A short-lived script can exit before the batch exporter flushes. Call `trace_provider.force_flush()` before the process ends so the last spans are sent. + + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the OpenAI Agents SDK spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every OpenAI Agents SDK call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/tracing/auto/pipecat.mdx b/src/pages/docs/traceai/auto/pipecat.mdx similarity index 70% rename from src/pages/docs/tracing/auto/pipecat.mdx rename to src/pages/docs/traceai/auto/pipecat.mdx index 13130a7e..b65dab77 100644 --- a/src/pages/docs/tracing/auto/pipecat.mdx +++ b/src/pages/docs/traceai/auto/pipecat.mdx @@ -1,6 +1,31 @@ --- title: "Pipecat Tracing with Future AGI: Voice Pipeline Spans" description: "Set up auto-instrumentation for Pipecat voice apps with Future AGI tracing. Install traceAI-pipecat to capture voice pipeline and processing spans." +slug: "pipecat" +page_type: "integration" +products: ["traceAI"] +integration_category: "voice" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "Pipecat Tracing with Future AGI: Voice Pipeline Spans" + description: "Set up auto-instrumentation for Pipecat voice apps with Future AGI tracing. Install traceAI-pipecat to capture voice pipeline and processing spans." + primary_keyword: "pipecat tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace Pipecat with FutureAGI?" + llm_summary: "Set up auto-instrumentation for Pipecat voice apps with Future AGI tracing. Install traceAI-pipecat to capture voice pipeline and processing spans." +canonical: "/docs/traceai/auto/pipecat" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" --- ## Overview @@ -275,4 +300,59 @@ Base class for mapped span exporters. 3. **Data not being sent to FutureAGI** - Ensure that you have set the `FI_API_KEY` and `FI_SECRET_KEY` environment variables - - Ensure that the `set_global_tracer_provider` in the `register` function is set to `True` \ No newline at end of file + - Ensure that the `set_global_tracer_provider` in the `register` function is set to `True` + +--- + +## What spans are produced + +Auto-instrumentation for Pipecat emits **spans per conversation turn** — an `llm` span for the model plus speech-to-text and text-to-speech steps — grouped under one session so you can replay the whole call. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + + + A short-lived script can exit before the batch exporter flushes. Call `trace_provider.force_flush()` before the process ends so the last spans are sent. + + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the Pipecat spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every Pipecat call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/traceai/auto/portkey.mdx b/src/pages/docs/traceai/auto/portkey.mdx new file mode 100644 index 00000000..3bab8b62 --- /dev/null +++ b/src/pages/docs/traceai/auto/portkey.mdx @@ -0,0 +1,146 @@ +--- +title: "Portkey Tracing with Future AGI: Auto-Instrumentation" +description: "Set up auto-instrumentation for Portkey with Future AGI tracing. Install traceAI-portkey to capture routed LLM call spans and gateway metrics." +slug: "portkey" +page_type: "integration" +products: ["traceAI"] +integration_category: "llm-provider" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "Portkey Tracing with Future AGI: Auto-Instrumentation" + description: "Set up auto-instrumentation for Portkey with Future AGI tracing. Install traceAI-portkey to capture routed LLM call spans and gateway metrics." + primary_keyword: "portkey tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace Portkey with FutureAGI?" + llm_summary: "Set up auto-instrumentation for Portkey with Future AGI tracing. Install traceAI-portkey to capture routed LLM call spans and gateway metrics." +canonical: "/docs/traceai/auto/portkey" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" +--- + +## 1. Installation +Install the traceAI and Portkey packages. + +```bash +pip install portkey_ai traceAI-portkey +``` + +--- + +## 2. Set Environment Variables +Set up your environment variables to authenticate with both FutureAGI and Portkey. + +```python +import os + +os.environ["FI_API_KEY"] = "your-futureagi-api-key" +os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" +os.environ["PORTKEY_VIRTUAL_KEY"] = "your-portkey-virtual-key" +``` + +--- + +## 3. Initialize Trace Provider +Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . + +```python +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType + +trace_provider = register( + project_type=ProjectType.OBSERVE, + project_name="portkey_project", +) +``` + +--- +## 4. Instrument your Project +Instrument your project to enable automatic tracing. + +```python +from traceai_portkey import PortkeyInstrumentor + +PortkeyInstrumentor().instrument(tracer_provider=trace_provider) +``` + +--- +## 5. Interact with Portkey +Interact with Portkey as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. + +```python +from portkey_ai import Portkey + +client = Portkey(virtual_key=os.environ["PORTKEY_VIRTUAL_KEY"]) + +completion = client.chat.completions.create( + model="gpt-4o", + messages=[{"role": "user", "content": "Write a 6-word story about a robot who discovers music."}] +) + +print(completion.choices[0].message.content) +``` + +--- + +## What spans are produced + +Auto-instrumentation for Portkey emits an **`llm` span for every model call** — chat, completion, or embedding — carrying the model name, the prompt and completion, prompt/completion/total token counts, and the estimated cost. Tool or function calls add a **`tool` span**. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + + + A short-lived script can exit before the batch exporter flushes. Call `trace_provider.force_flush()` before the process ends so the last spans are sent. + + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the Portkey spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every Portkey call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/traceai/auto/promptflow.mdx b/src/pages/docs/traceai/auto/promptflow.mdx new file mode 100644 index 00000000..fa079215 --- /dev/null +++ b/src/pages/docs/traceai/auto/promptflow.mdx @@ -0,0 +1,234 @@ +--- +title: "Prompt Flow Tracing with Future AGI: Auto-Instrumentation" +description: "Set up auto-instrumentation for Prompt Flow with Future AGI tracing. Use traceAI-openai to capture prompt flow execution and LLM call spans." +slug: "promptflow" +page_type: "integration" +products: ["traceAI"] +integration_category: "framework" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "Prompt Flow Tracing with Future AGI: Auto-Instrumentation" + description: "Set up auto-instrumentation for Prompt Flow with Future AGI tracing. Use traceAI-openai to capture prompt flow execution and LLM call spans." + primary_keyword: "prompt flow tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace Prompt Flow with FutureAGI?" + llm_summary: "Set up auto-instrumentation for Prompt Flow with Future AGI tracing. Use traceAI-openai to capture prompt flow execution and LLM call spans." +canonical: "/docs/traceai/auto/promptflow" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" +--- + +## 1. Installation +First install the traceAI and promptflow packages. + +```bash +pip install traceAI-openai promptflow promptflow-tools +``` + +--- + +## 2. Set Environment Variables + +Set up your environment variables to authenticate with both FutureAGI and OpenAI services. + +```python +import os + +os.environ["OPENAI_API_KEY"] = "your-openai-api-key" +os.environ["FI_API_KEY"] = "your-futureagi-api-key" +os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" +``` + +--- + +## 3. Initialize Trace Provider + +Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . + +```python +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType + +trace_provider = register( + project_type=ProjectType.OBSERVE, + project_name="promptflow", +) +``` + +--- + +## 4. Instrument your Project + +Instrument your Project with OpenAI Instrumentor. This step ensures that all interactions with the PromptFlow are tracked and monitored. + +```python +from traceai_openai import OpenAIInstrumentor + +OpenAIInstrumentor().instrument(tracer_provider=trace_provider) +``` + +--- +## 5. Prepare the `chat.prompty` File + +Create a `chat.prompty` file in the same directory as your script with the following content: + +```yaml +--- +name: Basic Chat +model: + api: chat + configuration: + type: azure_openai + azure_deployment: gpt-4o + parameters: + temperature: 0.2 + max_tokens: 1024 +inputs: + question: + type: string + chat_history: + type: list +sample: + question: "What is Prompt flow?" + chat_history: [] +--- + +system: +You are a helpful assistant. + +{% for item in chat_history %} +{{item.role}}: +{{item.content}} +{% endfor %} + +user: +{{question}} +``` + +This will ensure that users have the necessary configuration to create the `chat.prompty` file and use it with the `ChatFlow` class. + +--- + +## 6. Create a Flow + +Create a Flow as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. + +```python +from pathlib import Path +from promptflow.core import OpenAIModelConfiguration, Prompty + +BASE_DIR = Path(__file__).absolute().parent + +class ChatFlow: + def __init__(self, model_config: OpenAIModelConfiguration, max_total_token=4096): + self.model_config = model_config + self.max_total_token = max_total_token + + def __call__( + self, + question: str = "What's Azure Machine Learning?", + chat_history: list = [], + ) -> str: + """Flow entry function.""" + + prompty = Prompty.load( + source=BASE_DIR / "chat.prompty", + model={"configuration": self.model_config}, + ) + + output = prompty(question=question, chat_history=chat_history) + + return output +``` + +--- + +## 7. Execute the Flow + +```python +from promptflow.client import PFClient +from promptflow.connections import OpenAIConnection + +pf = PFClient() + +connection = OpenAIConnection( + name="open_ai_connection", + base_url="https://api.openai.com/v1", + api_key=os.environ["OPENAI_API_KEY"], +) + +conn = pf.connections.create_or_update(connection) + +config = OpenAIModelConfiguration( + connection="open_ai_connection", model="gpt-3.5-turbo" +) + +chat_flow = ChatFlow(config) +result = chat_flow(question="What is ChatGPT? Please explain with concise statement") +print(result) +``` + +--- + +## What spans are produced + +Auto-instrumentation for Prompt Flow emits a **`chain` or `agent` root span** for the run, with nested **`llm` spans** for each model call, **`tool` spans** for tool/function calls, and **`retriever` spans** for retrieval steps — so the whole request reads as one tree. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + + + A short-lived script can exit before the batch exporter flushes. Call `trace_provider.force_flush()` before the process ends so the last spans are sent. + + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the Prompt Flow spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every Prompt Flow call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/traceai/auto/smol_agents.mdx b/src/pages/docs/traceai/auto/smol_agents.mdx new file mode 100644 index 00000000..78a5ee63 --- /dev/null +++ b/src/pages/docs/traceai/auto/smol_agents.mdx @@ -0,0 +1,170 @@ +--- +title: "Smol Agents Tracing with Future AGI: Auto-Instrumentation" +description: "Set up auto-instrumentation for Smol Agents with Future AGI tracing. Install traceAI-smolagents to capture lightweight agent execution spans." +slug: "smol_agents" +page_type: "integration" +products: ["traceAI"] +integration_category: "framework" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "Smol Agents Tracing with Future AGI: Auto-Instrumentation" + description: "Set up auto-instrumentation for Smol Agents with Future AGI tracing. Install traceAI-smolagents to capture lightweight agent execution spans." + primary_keyword: "smol agents tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace Smol Agents with FutureAGI?" + llm_summary: "Set up auto-instrumentation for Smol Agents with Future AGI tracing. Install traceAI-smolagents to capture lightweight agent execution spans." +canonical: "/docs/traceai/auto/smol_agents" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" +--- + +## 1. Installation +First install the traceAI and necessary dependencies. + +```bash +pip install traceAI-smolagents smolagents +``` + +--- + +## 2. Set Environment Variables + +Set up your environment variables to authenticate with both FutureAGI and OpenAI. + +```python +import os + +os.environ["OPENAI_API_KEY"] = "your-openai-api-key" +os.environ["FI_API_KEY"] = "your-futureagi-api-key" +os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" +``` + +--- + +## 3. Initialize Trace Provider + +Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . + +```python +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType + +trace_provider = register( + project_type=ProjectType.OBSERVE, + project_name="smolagents", +) +``` + +--- + +## 4. Instrument your Project + +Instrument your Project with SmolagentsInstrumentor . This step ensures that all interactions with the Agents are tracked and monitored. + +```python +from traceai_smolagents import SmolagentsInstrumentor + +SmolagentsInstrumentor().instrument(tracer_provider=trace_provider) +``` + +--- + +## 5. Interact with Smol Agents + +Interact with you Smol Agents as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. + +```python +from smolagents import ( + CodeAgent, + DuckDuckGoSearchTool, + OpenAIServerModel, + ToolCallingAgent, +) + +model = OpenAIServerModel(model_id="gpt-4o") +agent = ToolCallingAgent( + tools=[DuckDuckGoSearchTool()], + model=model, + max_steps=3, + name="search", + description=( + "This is an agent that can do web search. " + "When solving a task, ask him directly first, he gives good answers. " + "Then you can double check." + ), +) +manager_agent = CodeAgent( + tools=[DuckDuckGoSearchTool()], + model=model, + managed_agents=[agent], +) +manager_agent.run( + "How many seconds would it take for a leopard at full speed to run through Pont des Arts? " + "ASK YOUR MANAGED AGENT FOR LEOPARD SPEED FIRST" +) +``` + +--- + +## What spans are produced + +Auto-instrumentation for Smol Agents emits a **`chain` or `agent` root span** for the run, with nested **`llm` spans** for each model call, **`tool` spans** for tool/function calls, and **`retriever` spans** for retrieval steps — so the whole request reads as one tree. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + + + A short-lived script can exit before the batch exporter flushes. Call `trace_provider.force_flush()` before the process ends so the last spans are sent. + + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the Smol Agents spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every Smol Agents call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/tracing/auto/spring-boot.mdx b/src/pages/docs/traceai/auto/spring-boot.mdx similarity index 74% rename from src/pages/docs/tracing/auto/spring-boot.mdx rename to src/pages/docs/traceai/auto/spring-boot.mdx index 9faa9acd..fad8e0bc 100644 --- a/src/pages/docs/tracing/auto/spring-boot.mdx +++ b/src/pages/docs/traceai/auto/spring-boot.mdx @@ -1,6 +1,31 @@ --- title: "Spring Boot Tracing with Future AGI and Spring AI" description: "Add tracing to Spring Boot apps with Spring AI. Configure application.yml, wrap your ChatModel and EmbeddingModel, and traces are collected automatically." +slug: "spring-boot" +page_type: "integration" +products: ["traceAI"] +integration_category: "java" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "Spring Boot Tracing with Future AGI and Spring AI" + description: "Add tracing to Spring Boot apps with Spring AI. Configure application.yml, wrap your ChatModel and EmbeddingModel, and traces are collected automatically." + primary_keyword: "spring boot tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace Spring Boot with FutureAGI?" + llm_summary: "Add tracing to Spring Boot apps with Spring AI. Configure application.yml, wrap your ChatModel and EmbeddingModel, and traces are collected automatically." +canonical: "/docs/traceai/auto/spring-boot" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" --- @@ -337,3 +362,54 @@ The `provider` string you pass to `TracedChatModel` / `TracedEmbeddingModel` is | `spring-ai-mistral-ai-spring-boot-starter` | `"mistral"` | Just swap the Spring AI dependency and change the provider string. The tracing wrapper doesn't care which provider is underneath. + +--- + +## What spans are produced + +Auto-instrumentation for Spring Boot emits a **`chain` or `agent` root span** for the run, with nested **`llm` spans** for each model call, **`tool` spans** for tool/function calls, and **`retriever` spans** for retrieval steps — so the whole request reads as one tree. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the Spring Boot spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every Spring Boot call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/traceai/auto/togetherai.mdx b/src/pages/docs/traceai/auto/togetherai.mdx new file mode 100644 index 00000000..0f4fb2ca --- /dev/null +++ b/src/pages/docs/traceai/auto/togetherai.mdx @@ -0,0 +1,158 @@ +--- +title: "Together AI Tracing with Future AGI: Auto-Instrumentation" +description: "Set up auto-instrumentation for Together AI with Future AGI tracing. Use traceAI-openai to capture inference spans from Together AI models." +slug: "togetherai" +page_type: "integration" +products: ["traceAI"] +integration_category: "llm-provider" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "Together AI Tracing with Future AGI: Auto-Instrumentation" + description: "Set up auto-instrumentation for Together AI with Future AGI tracing. Use traceAI-openai to capture inference spans from Together AI models." + primary_keyword: "together ai tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace Together AI with FutureAGI?" + llm_summary: "Set up auto-instrumentation for Together AI with Future AGI tracing. Use traceAI-openai to capture inference spans from Together AI models." +canonical: "/docs/traceai/auto/togetherai" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" +--- + +## 1. Installation +First install the traceAI package to access the observability framework + +```bash +pip install traceAI-openai +``` + +--- + +## 2. Set Environment Variables + +Set up your environment variables to authenticate with both FutureAGI and OpenAI services. + +```python +import os + +os.environ["TOGETHER_API_KEY"] = "your-together-api-key" +os.environ["FI_API_KEY"] = "your-futureagi-api-key" +os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" +``` + +--- + +## 3. Initialize Trace Provider + +Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . + +```python +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType + +trace_provider = register( + project_type=ProjectType.OBSERVE, + project_name="togetherai_project", +) +``` + +--- + +## 4. Instrument your Project + +Use the OpenAI Instrumentor to instrument your project, as the OpenAI Client is utilized for interactions with Together AI. This step guarantees that all interactions are tracked and monitored. If you are using a different client to interact with Together AI, use that client's Instrumentor instead. + +```python +from traceai_openai import OpenAIInstrumentor + +OpenAIInstrumentor().instrument(tracer_provider=trace_provider) +``` + +--- + +## 5. Interact with Together AI + +Interact with the Together AI through OpenAI Client. Our OpenAI Instrumentor will automatically trace and send the telemetry data to our platform. + +```python +import openai + +client = openai.OpenAI( + api_key=os.environ.get("TOGETHER_API_KEY"), + base_url="https://api.together.xyz/v1", +) + +response = client.chat.completions.create( + model="meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", + messages=[ + {"role": "system", "content": "You are a travel agent. Be descriptive and helpful."}, + {"role": "user", "content": "Tell me the top 3 things to do in San Francisco"}, + ] +) + +print(response.choices[0].message.content) +``` + +--- + +## What spans are produced + +Auto-instrumentation for Together AI emits an **`llm` span for every model call** — chat, completion, or embedding — carrying the model name, the prompt and completion, prompt/completion/total token counts, and the estimated cost. Tool or function calls add a **`tool` span**. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + + + A short-lived script can exit before the batch exporter flushes. Call `trace_provider.force_flush()` before the process ends so the last spans are sent. + + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the Together AI spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every Together AI call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/traceai/auto/vercel.mdx b/src/pages/docs/traceai/auto/vercel.mdx new file mode 100644 index 00000000..0c301f2a --- /dev/null +++ b/src/pages/docs/traceai/auto/vercel.mdx @@ -0,0 +1,192 @@ +--- +title: "Vercel AI SDK Tracing with Future AGI: Auto-Instrumentation" +description: "Set up auto-instrumentation for Vercel AI SDK with Future AGI tracing. Install @traceai/vercel to capture AI function call spans in Next.js apps." +slug: "vercel" +page_type: "integration" +products: ["traceAI"] +integration_category: "framework" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "Vercel AI SDK Tracing with Future AGI: Auto-Instrumentation" + description: "Set up auto-instrumentation for Vercel AI SDK with Future AGI tracing. Install @traceai/vercel to capture AI function call spans in Next.js apps." + primary_keyword: "vercel ai tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace Vercel AI with FutureAGI?" + llm_summary: "Set up auto-instrumentation for Vercel AI SDK with Future AGI tracing. Install @traceai/vercel to capture AI function call spans in Next.js apps." +canonical: "/docs/traceai/auto/vercel" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" +--- + +## 1. Installation +First install the TraceAI + Vercel packages (and OpenTelemetry peer deps). Pick your favourite package manager: + + + +```bash npm +npm install @traceai/vercel @vercel/otel \ + @opentelemetry/api @opentelemetry/sdk-trace-base \ + @opentelemetry/exporter-trace-otlp-grpc @grpc/grpc-js \ + @ai-sdk/openai +``` + +```bash yarn +yarn add @traceai/vercel @vercel/otel \ + @opentelemetry/api @opentelemetry/sdk-trace-base \ + @opentelemetry/exporter-trace-otlp-grpc @grpc/grpc-js \ + @ai-sdk/openai +``` + +```bash pnpm +pnpm add @traceai/vercel @vercel/otel \ + @opentelemetry/api @opentelemetry/sdk-trace-base \ + @opentelemetry/exporter-trace-otlp-grpc @grpc/grpc-js \ + @ai-sdk/openai +``` + + + +> **Note** Vercel currently supports OpenTelemetry **v1.x**. Avoid installing `@opentelemetry/*` 2.x packages. + +--- + +## 2. Set Environment Variables +Configure your Future AGI credentials (locally via `.env`, or in Vercel **Project → Settings → Environment Variables**). + +```bash +FI_API_KEY= +FI_SECRET_KEY= +``` + +--- + +## 3. Initialise tracing +Create `instrumentation.ts` and import it **once** on the server (e.g. in `_app.tsx` or at the top of your first API route). + +```typescript JS/TS title="instrumentation.ts" +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-ignore : module ships without types +import { registerOTel } from "@vercel/otel"; +import { diag, DiagConsoleLogger, DiagLogLevel } from "@opentelemetry/api"; +import { FISimpleSpanProcessor, isFISpan } from "@traceai/vercel"; +import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc"; +import { Metadata } from "@grpc/grpc-js"; + +// Optional: verbose console logs while testing +diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.DEBUG); + +export function register() { + registerOTel({ + attributes: { + project_name: "vercel-project", + project_type: "observe", + }, + spanProcessors: [ + new FISimpleSpanProcessor({ + exporter: (() => { + const meta = new Metadata(); + meta.set("x-api-key", process.env.FI_API_KEY ?? ""); + meta.set("x-secret-key", process.env.FI_SECRET_KEY ?? ""); + return new OTLPTraceExporter({ url: "grpc://grpc.futureagi.com", metadata: meta }); + })(), + // Export only TraceAI spans (remove if you want everything) + spanFilter: isFISpan, + }), + ], + }); +} +``` + +--- + +## 4. Instrument an API Route +Our instrumentation is automatic. Just **import and call** the `register` function inside each serverless function. + +```typescript JS/TS title="pages/api/story.ts" +import type { NextApiRequest, NextApiResponse } from "next"; +import { register as registerTracing } from "../../instrumentation"; +import { generateText } from "ai"; +import { openai } from "@ai-sdk/openai"; + +export default async function handler(req: NextApiRequest, res: NextApiResponse) { + registerTracing(); // initialise OTEL + exporters + + const result = await generateText({ + model: openai("gpt-4o-mini"), + prompt: "Write a short creative story about a time-traveling detective.", + experimental_telemetry: { isEnabled: true }, // ⇢ creates spans for each call + maxTokens: 300, + }); + + res.status(200).json({ + story: result.text?.trim() ?? "n/a", + }); +} +``` + +That’s it. Deploy to Vercel and watch traces flow into **Observe → Traces** in real time 🎉 + +--- + +## What spans are produced + +Auto-instrumentation for Vercel AI SDK emits a **`chain` or `agent` root span** for the run, with nested **`llm` spans** for each model call, **`tool` spans** for tool/function calls, and **`retriever` spans** for retrieval steps — so the whole request reads as one tree. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + + + A short-lived script can exit before the batch exporter flushes. Call `trace_provider.force_flush()` before the process ends so the last spans are sent. + + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the Vercel AI SDK spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every Vercel AI SDK call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/traceai/auto/vertexai.mdx b/src/pages/docs/traceai/auto/vertexai.mdx new file mode 100644 index 00000000..1a6966c1 --- /dev/null +++ b/src/pages/docs/traceai/auto/vertexai.mdx @@ -0,0 +1,194 @@ +--- +title: "Vertex AI Tracing with Future AGI: Auto-Instrumentation" +description: "Set up auto-instrumentation for Vertex AI with Future AGI tracing. Install traceAI-vertexai to capture Gemini model invocation and response spans." +slug: "vertexai" +page_type: "integration" +products: ["traceAI"] +integration_category: "llm-provider" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-06-19" +schema_type: "TechArticle" +seo: + title: "Vertex AI Tracing with Future AGI: Auto-Instrumentation" + description: "Set up auto-instrumentation for Vertex AI with Future AGI tracing. Install traceAI-vertexai to capture Gemini model invocation and response spans." + primary_keyword: "vertex ai tracing instrumentation" + direct_answer: true +geo: + answer_target: "How do I trace Vertex AI with FutureAGI?" + llm_summary: "Set up auto-instrumentation for Vertex AI with Future AGI tracing. Install traceAI-vertexai to capture Gemini model invocation and response spans." +canonical: "/docs/traceai/auto/vertexai" +related: + - "/docs/observe/concepts/spans" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/quickstart" + - "/docs/traceai/auto" +--- + +## 1. Installation +Install the traceAI and Vertex AI packages. + +```bash +pip install traceAI-vertexai +pip install vertexai +``` + +--- + +## 2. Set Environment Variables + +Set up your environment variables to authenticate with FutureAGI . + +```python +import os + +os.environ["FI_API_KEY"] = "your-futureagi-api-key" +os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" +``` + +--- + +## 3. Initialize Trace Provider +Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . + +```python +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType + +trace_provider = register( + project_type=ProjectType.OBSERVE, + project_name="vertexai_project", + ) +``` +--- + +## 4. Configure Vertex AI Instrumentation +Instrument your Project with VertexAI Instrumentor. This step ensures that all interactions with the VertexAI are tracked and monitored. + +```python +from traceai_vertexai import VertexAIInstrumentor + +VertexAIInstrumentor().instrument(tracer_provider=trace_provider) +``` + +--- + +## 5. Create Vertex AI Components + +Interact with Vertex AI as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. + +```python +import vertexai + +from vertexai.generative_models import FunctionDeclaration, GenerativeModel, Part, Tool + +vertexai.init( + project="project_name", +) + +# Describe a function by specifying its schema (JsonSchema format) +get_current_weather_func = FunctionDeclaration( + name="get_current_weather", + description="Get the current weather in a given location", + parameters={ + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, +) + +# Tool is a collection of related functions +weather_tool = Tool(function_declarations=[get_current_weather_func]) + +# Use tools in chat +chat = GenerativeModel("gemini-1.5-flash", tools=[weather_tool]).start_chat() +``` + +--- +## 6. Execute +Run your Vertex AI application. + +```python +if __name__ == "__main__": + # Send a message to the model. The model will respond with a function call. + for response in chat.send_message( + "What is the weather like in Boston?", stream=True + ): + print(response) + # Then send a function response to the model. The model will use it to answer. + for response in chat.send_message( + Part.from_function_response( + name="get_current_weather", + response={"content": {"weather": "super nice"}}, + ), + stream=True, + ): + print(response) + +``` + +--- + +--- + +## What spans are produced + +Auto-instrumentation for Vertex AI emits an **`llm` span for every model call** — chat, completion, or embedding — carrying the model name, the prompt and completion, prompt/completion/total token counts, and the estimated cost. Tool or function calls add a **`tool` span**. Every span carries the standard [trace](/docs/observe/concepts/traces) and [span](/docs/observe/concepts/spans) attributes, so it groups, filters, and scores the same way as any other traceAI data. + + + A short-lived script can exit before the batch exporter flushes. Call `trace_provider.force_flush()` before the process ends so the last spans are sent. + + +--- + +## Verify the trace + +Run your app, then open **Observe → your project → Tracing**. Within a few seconds a new trace row appears with **Status OK** and the model, latency, and token columns filled in. Click the row to read the Vertex AI spans — input, output, timing, and cost per step. + +```text +Observe > Tracing > (newest row) — Status: OK · Model: · Latency · Tokens +``` + +If the row is there with an OK status, instrumentation is working end to end. See the [trace explorer](/docs/observe/features/llm-tracing) for how to read and filter what you captured. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends. | +| Some calls traced, others not | The client was created before `.instrument()` ran, so it wasn't patched. | Register and instrument at startup, before constructing any client. | +| Traces go to the wrong place | `project_type` was not `OBSERVE`. | Pass `project_type=ProjectType.OBSERVE` to `register`. | +| Auth error on `register` | `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. | Re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For more, see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Related + + + + The unit every Vertex AI call becomes. + + + Add custom spans where auto-instrumentation doesn't reach. + + + The four-step setup, end to end. + + + Read, filter, and debug the traces you produce. + + + diff --git a/src/pages/docs/traceai/index.mdx b/src/pages/docs/traceai/index.mdx new file mode 100644 index 00000000..4e5a3976 --- /dev/null +++ b/src/pages/docs/traceai/index.mdx @@ -0,0 +1,102 @@ +--- +title: "Get started with traceAI" +description: "traceAI is FutureAGI's open instrumentation SDK. It captures your AI app as OpenTelemetry traces — automatically for 30+ frameworks, or manually for full control — and exports them to Observe." +slug: "traceai" +page_type: "overview" +products: ["traceAI"] +audience: ["engineer"] +difficulty: "beginner" +status: "review" +owner: "observability" +docs_owner: "observability" +section_owner: "observability" +reviewers: ["observability-eng"] +last_reviewed: "2026-06-17" +coverage_status: "partial" +schema_type: "TechArticle" +seo: + title: "traceAI — OpenTelemetry instrumentation SDK" + description: "Instrument any AI app with traceAI: auto-instrument 30+ frameworks or add manual spans, all exported as OpenTelemetry traces to FutureAGI Observe." + primary_keyword: "traceai instrumentation sdk opentelemetry" + direct_answer: true +geo: + answer_target: "What is traceAI and how does it relate to Observe?" + llm_summary: "traceAI is FutureAGI's OpenTelemetry-based instrumentation SDK that captures AI applications as traces; Observe is the product that reads, searches, and scores those traces." +canonical: "/docs/traceai" +related: + - "/docs/observe/concepts/traceai" + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/auto" + - "/docs/observe" +--- + +## What is traceAI? + +traceAI is FutureAGI's instrumentation SDK — the layer that turns your running AI application into traces. It builds on [OpenTelemetry](/docs/observe/concepts/otel), so the spans it emits are a standard format, not a proprietary one. You instrument once, either automatically for a supported framework or manually for full control, and every request your app handles is exported as a [trace](/docs/observe/concepts/traces) that [Observe](/docs/observe) reads. traceAI is the SDK that produces the data; Observe is the product that lets you search, replay, score, and alert on it. + +--- + +## Why traceAI + +Three reasons it's worth instrumenting with traceAI rather than rolling your own: + +- **It's standard OpenTelemetry, not a proprietary format.** The spans go to FutureAGI today, but the same instrumentation works with any OTel-compatible backend — so you're not locked in, and a trace looks the same wherever it's read. +- **Auto-instrumentation is one line per framework.** Add the instrumentor for OpenAI, LangChain, LlamaIndex, and others, and your model, tool, and retrieval calls become traces with no span code in your application. +- **Manual instrumentation is there when you need control.** Where an instrumentor doesn't reach — your own functions, custom retrieval, glue logic — you create spans by hand, and they nest into the same traces. You get coverage without giving up precision. + +--- + +## What you can do here + +- **Auto-instrument a framework** — add one instrumentor and capture LangChain, LlamaIndex, CrewAI, OpenAI, Anthropic, and 30+ others with no manual span code. +- **Instrument manually** — create your own spans, attributes, and events when auto-instrumentation doesn't reach your code. +- **Set session, user, and tag attributes** — enrich spans so Observe can group and filter them. +- **Mask sensitive data** — redact payloads before they leave your process. + +--- + +## Get started + + + Install the SDK, register a project, and export your first spans. + + +--- + +## Featured concepts + + + + What the SDK is, and how it differs from the Observe product. + + + The open standard traceAI is built on. + + + +--- + +## Featured how-tos + + + + Auto-instrument 30+ providers, frameworks, and agents. + + + Attach the keys Observe filters and groups on. + + + Wrap your own functions as spans. + + + Redact prompts and outputs before export. + + + +--- + +## Where next + +- **[Observe](/docs/observe)** — read, search, and score the traces traceAI sends. +- **[Semantic conventions](/docs/traceai/manual-instrumentation/semantic-conventions)** — the attribute keys traceAI sets. +- **[Spans not exported](/docs/traceai/troubleshooting/spans-not-exported)** — when traces don't arrive. diff --git a/src/pages/docs/observe/features/manual-tracing/add-attributes-metadata-tags.mdx b/src/pages/docs/traceai/manual-instrumentation/add-attributes-metadata-tags.mdx similarity index 90% rename from src/pages/docs/observe/features/manual-tracing/add-attributes-metadata-tags.mdx rename to src/pages/docs/traceai/manual-instrumentation/add-attributes-metadata-tags.mdx index 6ba8979d..24fc2711 100644 --- a/src/pages/docs/observe/features/manual-tracing/add-attributes-metadata-tags.mdx +++ b/src/pages/docs/traceai/manual-instrumentation/add-attributes-metadata-tags.mdx @@ -1,6 +1,24 @@ --- -title: "Enriching Spans with Attributes, Metadata, and Tags" -description: "Enrich spans with custom attributes, metadata, tags, session IDs, user IDs, and prompt templates beyond what standard auto-instrumentation captures." +title: "Enrich Spans with Attributes, Metadata, and Tags" +description: "Add custom attributes, metadata, tags, session and user IDs, and prompt templates to spans using set_attribute, semantic conventions, or context helpers." +page_type: "how-to" +products: ["traceAI"] +task_intent: "Attach custom data to spans" +audience: "engineer" +difficulty: "intermediate" +time_to_complete: "10 minutes" +status: "review" +owner: "observability" +last_tested: "2026-05-25" +schema_type: "TechArticle" +seo: + primary_query: "add custom attributes to spans" +geo: + direct_answer: true +related: + reference: "/docs/traceai/manual-instrumentation/semantic-conventions" + how_to: "/docs/traceai/manual-instrumentation/set-up-tracing" + concept: "/docs/observe/concepts/spans" --- ## About @@ -19,6 +37,13 @@ A trace with only timing and status tells what happened, but not why. Without at --- +## Prerequisites + +- Tracing set up — see [Set up tracing](/docs/traceai/manual-instrumentation/set-up-tracing). +- The attribute keys you want to set — prefer the [semantic conventions](/docs/traceai/manual-instrumentation/semantic-conventions) so spans stay queryable. + +--- + ## How to @@ -460,6 +485,25 @@ A trace with only timing and status tells what happened, but not why. Without at --- +## Verify + +Set an attribute, run a request, then open the trace: + +- The attribute appears in the span's **attributes** list. +- You can **filter** the trace list by it (for standard keys). + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| Attribute not on the span | Set after the span closed, or on the wrong span. | Set it while the span is active (inside its block). | +| Can't filter by the attribute | Used a custom key the UI doesn't index. | Use a [semantic-convention](/docs/traceai/manual-instrumentation/semantic-conventions) key where one exists. | +| Value dropped | Unsupported value type. | Use string, bool, int, float, or an array of those. | + +--- + ## Key concepts - **`set_attribute()`**:Attaches a key/value pair directly to the active span. Supports strings, numbers, and booleans. Prefix custom attributes with your company name to avoid naming conflicts. @@ -476,16 +520,16 @@ A trace with only timing and status tells what happened, but not why. Without at ## Next Steps - + Register a tracer provider and add instrumentation. - + Use FITracer decorators and context managers for typed spans. - + Group traces into sessions and link them to end users. - + Redact sensitive data with TraceConfig before export. diff --git a/src/pages/docs/observe/features/manual-tracing/add-events-exceptions-status.mdx b/src/pages/docs/traceai/manual-instrumentation/add-events-exceptions-status.mdx similarity index 75% rename from src/pages/docs/observe/features/manual-tracing/add-events-exceptions-status.mdx rename to src/pages/docs/traceai/manual-instrumentation/add-events-exceptions-status.mdx index c1ba95ba..762383cf 100644 --- a/src/pages/docs/observe/features/manual-tracing/add-events-exceptions-status.mdx +++ b/src/pages/docs/traceai/manual-instrumentation/add-events-exceptions-status.mdx @@ -1,6 +1,24 @@ --- -title: "Integrate Events, Exceptions, and Status into Spans" -description: "Add OpenTelemetry events, exceptions, and status codes to spans in Future AGI to capture structured lifecycle information and error diagnostics." +title: "Add Events, Exceptions, and Status to Spans" +description: "Mark key moments with events, flag failures with an ERROR status, and attach full exception details to spans so errors are visible in traces and alerts." +page_type: "how-to" +products: ["traceAI"] +task_intent: "Record events, status, and exceptions on a span" +audience: "engineer" +difficulty: "intermediate" +time_to_complete: "5 minutes" +status: "review" +owner: "observability" +last_tested: "2026-05-25" +schema_type: "TechArticle" +seo: + primary_query: "span events exceptions status opentelemetry" +geo: + direct_answer: true +related: + concept: "/docs/observe/concepts/spans" + how_to: "/docs/traceai/manual-instrumentation/get-current-span-context" + feature: "/docs/observe/features/llm-tracing" --- ## About @@ -21,6 +39,13 @@ Spans capture timing and attributes, but they do not automatically record what h --- +## Prerequisites + +- Tracing set up — see [Set up tracing](/docs/traceai/manual-instrumentation/set-up-tracing). +- Code running inside an active span, so `get_current_span()` returns a real span. + +--- + ## How to @@ -157,6 +182,25 @@ Spans capture timing and attributes, but they do not automatically record what h --- +## Verify + +Trigger the span on both a success and a failure path, then open the trace: + +- Events appear on the span's **Events** tab in the detail panel, with timestamps. +- A failed run shows **ERROR** status on the span, with the exception (type, message, stack trace) recorded under Events. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| Events or status don't show | `get_current_span()` returned a no-op span. | Guard with `is_recording()` and run inside an active span. | +| Error isn't visible in the trace list | Status left at the default. | Call `set_status(Status(StatusCode.ERROR, ...))` on failure. | +| Stack trace missing | `record_exception()` wasn't called. | Pair `record_exception(ex)` with the ERROR status. | + +--- + ## Key concepts - **`add_event()` / `addEvent()`**:Attaches a timestamped message to the span at the moment it's called. Useful for logging discrete actions without creating a new span. @@ -169,16 +213,16 @@ Spans capture timing and attributes, but they do not automatically record what h ## Next Steps - + Register a tracer provider and add instrumentation. - + Attach custom data to spans for filtering and evals. - + Access the active span or tracer at any point in your code. - + Use FITracer decorators and context managers for typed spans. diff --git a/src/pages/docs/observe/features/manual-tracing/advanced-tracing-examples.mdx b/src/pages/docs/traceai/manual-instrumentation/advanced-tracing-examples.mdx similarity index 93% rename from src/pages/docs/observe/features/manual-tracing/advanced-tracing-examples.mdx rename to src/pages/docs/traceai/manual-instrumentation/advanced-tracing-examples.mdx index d3ba6745..71ac058f 100644 --- a/src/pages/docs/observe/features/manual-tracing/advanced-tracing-examples.mdx +++ b/src/pages/docs/traceai/manual-instrumentation/advanced-tracing-examples.mdx @@ -1,6 +1,24 @@ --- -title: "Advanced OTEL Tracing: Context, Decorators, and Sampling" -description: "Explore manual context propagation, custom decorators, and sampling techniques for real-world async, multi-service, and high-volume tracing scenarios." +title: "Advanced Tracing: Context, Decorators, and Sampling" +description: "Manual context propagation, custom decorators, and sampling for async, multi-service, and high-volume tracing scenarios with OpenTelemetry and traceAI." +page_type: "how-to" +products: ["traceAI"] +task_intent: "Apply advanced tracing patterns for async, multi-service, and sampling" +audience: "engineer" +difficulty: "advanced" +time_to_complete: "15 minutes" +status: "review" +owner: "observability" +last_tested: "2026-05-25" +schema_type: "TechArticle" +seo: + primary_query: "advanced opentelemetry tracing async sampling" +geo: + direct_answer: true +related: + how_to: "/docs/traceai/manual-instrumentation/set-up-tracing" + concept: "/docs/observe/concepts/spans" + feature: "/docs/observe/features/llm-tracing" --- ## About @@ -19,6 +37,13 @@ Basic span creation works for synchronous, single-service code. But real applica --- +## Prerequisites + +- Comfortable with [Set up tracing](/docs/traceai/manual-instrumentation/set-up-tracing) and manual spans. +- An async, multi-service, or high-volume scenario where default tracing isn't enough. + +--- + ## How to @@ -661,6 +686,25 @@ Basic span creation works for synchronous, single-service code. But real applica --- +## Verify + +Run the scenario once, then open the trace: + +- Spans created across async boundaries or services share the **same trace** (correct parent/child links). +- With sampling on, the expected fraction of traces is recorded. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| Spans split across multiple traces | Context wasn't propagated across the async/service boundary. | Propagate context explicitly (carrier inject/extract) as shown. | +| Everything or nothing sampled | Sampler misconfigured. | Check the sampler ratio and that it's set on the provider. | +| Lost spans under load | Exporter queue saturated. | Tune the batch processor, or reduce sampling. | + +--- + ## Key concepts - **`attach()` / `detach()`**:Python functions to manually bind a captured context to the current thread or async task. Always call `detach(token)` in a `finally` block to avoid context leaks. @@ -675,16 +719,16 @@ Basic span creation works for synchronous, single-service code. But real applica ## Next Steps - + Register a tracer provider and add instrumentation. - + Use FITracer decorators and context managers for typed spans. - + Record exceptions and set span status for error visibility. - + Access and enrich the active span from anywhere in your code. diff --git a/src/pages/docs/observe/features/manual-tracing/annotating-using-api.mdx b/src/pages/docs/traceai/manual-instrumentation/annotating-using-api.mdx similarity index 86% rename from src/pages/docs/observe/features/manual-tracing/annotating-using-api.mdx rename to src/pages/docs/traceai/manual-instrumentation/annotating-using-api.mdx index b2d1d2b2..da56be94 100644 --- a/src/pages/docs/observe/features/manual-tracing/annotating-using-api.mdx +++ b/src/pages/docs/traceai/manual-instrumentation/annotating-using-api.mdx @@ -1,6 +1,24 @@ --- -title: "Adding Annotations to Spans Using the Bulk API" -description: "Label spans with custom tags, human feedback, and notes using the Future AGI bulk-annotation API for systematic trace enrichment." +title: "Annotate Spans with the Bulk API" +description: "Label spans with tags, human feedback, and notes using the FutureAGI bulk-annotation API for systematic trace enrichment and review." +page_type: "how-to" +products: ["traceAI"] +task_intent: "Add annotations to spans via the bulk API" +audience: "engineer" +difficulty: "intermediate" +time_to_complete: "10 minutes" +status: "review" +owner: "observability" +last_tested: "2026-05-25" +schema_type: "TechArticle" +seo: + primary_query: "annotate spans bulk api" +geo: + direct_answer: true +related: + feature: "/docs/observe/features/llm-tracing" + how_to: "/docs/traceai/manual-instrumentation/set-up-tracing" + concept: "/docs/observe/concepts/spans" --- @@ -22,6 +40,13 @@ Traces show what happened but not whether the result was correct, helpful, or sa --- +## Prerequisites + +- Existing spans in a project to annotate. +- `FI_API_KEY` and `FI_SECRET_KEY` for API access. + +--- + ## How to @@ -275,6 +300,25 @@ Traces show what happened but not whether the result was correct, helpful, or sa --- +## Verify + +Send one annotation request, then open the annotated span: + +- The tag/feedback/note appears under the span's **Annotations** tab. +- A successful call returns a 2xx with the updated annotation record. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| Annotation not visible | Wrong span or trace ID. | Confirm the IDs match the target span. | +| 401 / 403 | Missing or wrong keys. | Send `FI_API_KEY`/`FI_SECRET_KEY` for this workspace. | +| Looking for managed review | This is the bulk API, not the queue. | Use the [Annotations](/docs/annotations) system for queues and the Scores API. | + +--- + ## Key concepts **Response object** @@ -307,16 +351,16 @@ Each element in `result.errors` contains: ## Next Steps - + Register a tracer provider and add instrumentation. - + Attach custom data to spans for filtering and evals. - + Run evaluations directly inside a traced span. - + Browse all supported framework instrumentors. diff --git a/src/pages/docs/observe/features/manual-tracing/create-tool-spans.mdx b/src/pages/docs/traceai/manual-instrumentation/create-tool-spans.mdx similarity index 82% rename from src/pages/docs/observe/features/manual-tracing/create-tool-spans.mdx rename to src/pages/docs/traceai/manual-instrumentation/create-tool-spans.mdx index c78dd39f..8949709a 100644 --- a/src/pages/docs/observe/features/manual-tracing/create-tool-spans.mdx +++ b/src/pages/docs/traceai/manual-instrumentation/create-tool-spans.mdx @@ -1,6 +1,24 @@ --- -title: "Tool Spans Creation: Manually Trace Function Calls" -description: "Manually trace tool functions alongside LLM calls by creating spans that capture inputs, outputs, and key events in Future AGI." +title: "Create Tool Spans Manually" +description: "Trace a tool call as a TOOL span with its function name, arguments, and output, and nest the LLM response span underneath to see the full chain." +page_type: "how-to" +products: ["traceAI"] +task_intent: "Trace a tool/function call and its nested LLM response" +audience: "engineer" +difficulty: "intermediate" +time_to_complete: "10 minutes" +status: "review" +owner: "observability" +last_tested: "2026-05-25" +schema_type: "TechArticle" +seo: + primary_query: "trace tool function calls opentelemetry" +geo: + direct_answer: true +related: + concept: "/docs/observe/concepts/spans" + reference: "/docs/traceai/manual-instrumentation/semantic-conventions" + how_to: "/docs/traceai/manual-instrumentation/set-up-tracing" --- ## About @@ -17,6 +35,13 @@ LLM agents often call external tools (APIs, databases, code interpreters), but t --- +## Prerequisites + +- Tracing set up with a tracer — see [Set up tracing](/docs/traceai/manual-instrumentation/set-up-tracing). +- A tool or function your agent calls that you want visible in traces. + +--- + ## How to @@ -88,7 +113,7 @@ LLM agents often call external tools (APIs, databases, code interpreters), but t # Ensure 'tracer' is defined from the setup section above. # Ensure 'openai_client' is defined, e.g., from openai library - # Placeholder definitions for the example + # Example definitions for this snippet question = "What is the weather like in London?" def example_tool_function(input_args): print(f"Tool received: {input_args}") @@ -150,10 +175,10 @@ LLM agents often call external tools (APIs, databases, code interpreters), but t // import OpenAI from 'openai'; // const openaiClient = new OpenAI(); // Example // const model_version_ts = "gpt-4o"; - // const current_user_message_ts = [{ role: "user", content: "Placeholder" }]; + // const current_user_message_ts = [{ role: "user", content: "example" }]; // const TEMPERATURE_ts = 0.7; - // Placeholder definitions for the example + // Example definitions for this snippet const questionTs = "What is the weather like in Berlin?"; interface ToolArgs { city: string; } const exampleToolFunctionTs = async (inputArgs: ToolArgs): Promise => { @@ -227,6 +252,26 @@ LLM agents often call external tools (APIs, databases, code interpreters), but t --- +## Verify + +Run the tool function once, then open the trace in Observe: + +- A **TOOL** span appears with the function name, arguments, and output attributes. +- An **LLM** span is nested **underneath** it, showing the model input and output. +- The parent/child nesting matches the call order. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| LLM span isn't nested under the tool span | The LLM span was started outside the tool span's active scope. | Create the LLM span **inside** the tool span's `with` / `startActiveSpan` block. | +| Tool attributes missing when the tool fails | Attributes were set after the tool raised. | Set `fi.span.kind`, name, and arguments **before** invoking the tool, as the example does. | +| Spans don't appear | A short script exited before the exporter flushed. | Call `force_flush()` before exit, or use a real `register()` provider instead of the console exporter. | + +--- + ## Key concepts - **`fi.span.kind: "TOOL"`**:Marks the span as a tool call so it renders with the correct icon and label in the Future AGI dashboard. @@ -241,16 +286,16 @@ LLM agents often call external tools (APIs, databases, code interpreters), but t ## Next Steps - + Use FITracer decorators and context managers for typed spans. - + Attach custom data to spans for filtering and evals. - + Record exceptions and set span status for error visibility. - + Register a tracer provider and add instrumentation. diff --git a/src/pages/docs/observe/features/manual-tracing/get-current-span-context.mdx b/src/pages/docs/traceai/manual-instrumentation/get-current-span-context.mdx similarity index 73% rename from src/pages/docs/observe/features/manual-tracing/get-current-span-context.mdx rename to src/pages/docs/traceai/manual-instrumentation/get-current-span-context.mdx index d4157bcf..f5b84ff9 100644 --- a/src/pages/docs/observe/features/manual-tracing/get-current-span-context.mdx +++ b/src/pages/docs/traceai/manual-instrumentation/get-current-span-context.mdx @@ -1,6 +1,24 @@ --- -title: "Get Current Tracer and Span: Access Active Context" -description: "Access the active span or tracer at any point in your code to enrich traces with additional attributes and context in Future AGI." +title: "Get the Current Span or Tracer" +description: "Access the active span or tracer from anywhere in your code to add attributes or start child spans, without passing references through every function." +page_type: "how-to" +products: ["traceAI"] +task_intent: "Read the active span or tracer deep in the call stack" +audience: "engineer" +difficulty: "intermediate" +time_to_complete: "5 minutes" +status: "review" +owner: "observability" +last_tested: "2026-05-25" +schema_type: "TechArticle" +seo: + primary_query: "get current span opentelemetry" +geo: + direct_answer: true +related: + concept: "/docs/observe/concepts/spans" + how_to: "/docs/traceai/manual-instrumentation/set-up-tracing" + feature: "/docs/observe/features/llm-tracing" --- ## About @@ -20,6 +38,13 @@ Spans and tracers are usually created at the top of a request, but the functions --- +## Prerequisites + +- Tracing already set up — see [Set up tracing](/docs/traceai/manual-instrumentation/set-up-tracing). +- Code running **inside an active span** (an auto-instrumented request or a span you started), so there is a current span to read. + +--- + ## How to @@ -122,6 +147,24 @@ Spans and tracers are usually created at the top of a request, but the functions --- +## Verify + +From a helper function, set an attribute on the current span (or start a child span), then run one request and open the trace: + +- The attribute appears on the active span's **attributes** list in the detail panel. +- A tracer-created span shows up as a **child** under the active request. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| Attributes don't appear | `get_current_span()` returned a no-op span — nothing was active. | Call it inside an active span (within a `start_as_current_span` block or an auto-instrumented request). | +| JS: setting an attribute throws | `trace.getSpan(context.active())` returned `undefined`. | Guard with `if (currentSpan)` before setting attributes, as the example does. | + +--- + ## Key concepts - **`trace.get_current_span()`**: Returns the span that is currently active in the context. If no span is active, returns a no-op span. @@ -133,16 +176,16 @@ Spans and tracers are usually created at the top of a request, but the functions ## Next Steps - + Register a tracer provider and add instrumentation. - + Use FITracer decorators and context managers for typed spans. - + Attach custom data to spans for filtering and evals. - + Group traces into sessions and link them to end users. diff --git a/src/pages/docs/observe/features/manual-tracing/in-line-evals.mdx b/src/pages/docs/traceai/manual-instrumentation/in-line-evals.mdx similarity index 67% rename from src/pages/docs/observe/features/manual-tracing/in-line-evals.mdx rename to src/pages/docs/traceai/manual-instrumentation/in-line-evals.mdx index 2aa5b933..6a595157 100644 --- a/src/pages/docs/observe/features/manual-tracing/in-line-evals.mdx +++ b/src/pages/docs/traceai/manual-instrumentation/in-line-evals.mdx @@ -1,6 +1,24 @@ --- -title: "In-line Evaluations: Attach Evals to Spans in Future AGI" -description: "Run evaluations directly inside a traced span so results are automatically attached to that span in the Future AGI dashboard." +title: "Run In-line Evaluations on Spans" +description: "Run an eval inside an active span with trace_eval=True so the score attaches to that span, putting trace data and eval result side by side in the dashboard." +page_type: "how-to" +products: ["fi.evals", "traceAI"] +task_intent: "Attach an eval result to the active span" +audience: "engineer" +difficulty: "intermediate" +time_to_complete: "10 minutes" +status: "review" +owner: "observability" +last_tested: "2026-05-25" +schema_type: "TechArticle" +seo: + primary_query: "inline eval attach to span" +geo: + direct_answer: true +related: + feature: "/docs/observe/features/evals" + how_to: "/docs/traceai/manual-instrumentation/set-up-tracing" + concept: "/docs/observe/concepts/spans" --- ## About @@ -17,6 +35,14 @@ Evaluation results are most useful when they sit next to the data that produced --- +## Prerequisites + +- Tracing set up — see [Set up tracing](/docs/traceai/manual-instrumentation/set-up-tracing). +- `fi.evals` available, with `FI_API_KEY` and `FI_SECRET_KEY` set. +- The eval template you want to run (e.g. `groundedness`). + +--- + ## How to @@ -85,6 +111,25 @@ Evaluation results are most useful when they sit next to the data that produced --- +## Verify + +Run the block, then open the trace: + +- The span shows the eval under its **Evals** tab, labelled with your `custom_eval_name`. +- `print(eval_result1)` shows the score and reason in the terminal. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| Eval not attached to the span | `evaluate()` ran outside an active span. | Call it inside the `with tracer.start_as_current_span(...)` block. | +| `trace_eval` has no effect | No active span, or the global provider isn't set. | Register with `set_global_tracer_provider=True` and run inside a span. | +| Method or parameter not found | Installed SDK version differs from this example. | Confirm the `fi.evals` `Evaluator.evaluate()` signature for your installed version. | + +--- + ## Key concepts - **`trace_eval=True`**:The essential parameter that enables in-line evaluation. It tells the system to find the current active span and attach the evaluation results to it as span attributes. @@ -98,16 +143,16 @@ Evaluation results are most useful when they sit next to the data that produced ## Next Steps - + Register a tracer provider and add instrumentation. - + Use FITracer decorators and context managers for typed spans. - + Attach custom data to spans for filtering and evals. - + Browse all supported framework instrumentors. diff --git a/src/pages/docs/observe/features/manual-tracing/instrument-with-traceai-helpers.mdx b/src/pages/docs/traceai/manual-instrumentation/instrument-with-traceai-helpers.mdx similarity index 85% rename from src/pages/docs/observe/features/manual-tracing/instrument-with-traceai-helpers.mdx rename to src/pages/docs/traceai/manual-instrumentation/instrument-with-traceai-helpers.mdx index 34d4368e..4583b3de 100644 --- a/src/pages/docs/observe/features/manual-tracing/instrument-with-traceai-helpers.mdx +++ b/src/pages/docs/traceai/manual-instrumentation/instrument-with-traceai-helpers.mdx @@ -1,6 +1,24 @@ --- -title: "Instrument with traceAI Helpers for Easier Tracing" -description: "Future AGI's traceAI library offers convenient abstractions that streamline your manual instrumentation process for LLM and agent tracing." +title: "Instrument with traceAI Helpers" +description: "Use FITracer decorators and context managers (@tracer.agent/chain/tool, using_attributes) to create typed spans with less boilerplate than raw OpenTelemetry." +page_type: "how-to" +products: ["traceAI"] +task_intent: "Use FITracer helpers to create typed spans" +audience: "engineer" +difficulty: "intermediate" +time_to_complete: "10 minutes" +status: "review" +owner: "observability" +last_tested: "2026-05-25" +schema_type: "TechArticle" +seo: + primary_query: "fitracer decorators traceai" +geo: + direct_answer: true +related: + how_to: "/docs/traceai/manual-instrumentation/set-up-tracing" + concept: "/docs/observe/concepts/spans" + feature: "/docs/observe/features/llm-tracing" --- ## About @@ -19,6 +37,13 @@ Manual tracing with raw OpenTelemetry means writing a lot of setup code for ever --- +## Prerequisites + +- Tracing set up with a `FITracer` — see [Set up tracing](/docs/traceai/manual-instrumentation/set-up-tracing). +- A function or block you want to trace as an agent, chain, or tool span. + +--- + ## How to @@ -295,6 +320,25 @@ Manual tracing with raw OpenTelemetry means writing a lot of setup code for ever --- +## Verify + +Run a decorated function once, then open the trace: + +- The span shows the **kind** (agent/chain/tool) you used, with input and output captured. +- Nested helpers appear as child spans in the tree. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| Input/output not captured | The decorator wasn't applied, or the call ran outside it. | Decorate with `@tracer.agent/chain/tool`, or run inside the context manager. | +| Wrong span kind | Used the generic tracer instead of the typed helper. | Use the matching `FITracer` helper for the operation. | +| No spans | `FITracer` not built from a registered provider. | Create it from `register(...)` as in setup. | + +--- + ## Key concepts - **`FITracer`**: Future AGI wrapper around the standard OTel tracer. Adds `set_input()` / `set_output()` / `set_tool()` on spans, automatic context injection, and typed decorators (`@tracer.chain`, `@tracer.agent`, `@tracer.tool`, `@tracer.llm`, `@tracer.retriever`). @@ -323,22 +367,22 @@ Manual tracing with raw OpenTelemetry means writing a lot of setup code for ever ## Next Steps - + Register a tracer provider and add instrumentation. - + Attach custom data to spans for filtering and evals. - + Group traces into sessions and link them to end users. - + Redact sensitive data with TraceConfig before export. - + Browse all supported framework instrumentors. - + Register an Observe project and start capturing traces. diff --git a/src/pages/docs/observe/features/manual-tracing/langfuse-integration.mdx b/src/pages/docs/traceai/manual-instrumentation/langfuse-integration.mdx similarity index 74% rename from src/pages/docs/observe/features/manual-tracing/langfuse-integration.mdx rename to src/pages/docs/traceai/manual-instrumentation/langfuse-integration.mdx index 331acd61..9ea8473c 100644 --- a/src/pages/docs/observe/features/manual-tracing/langfuse-integration.mdx +++ b/src/pages/docs/traceai/manual-instrumentation/langfuse-integration.mdx @@ -1,6 +1,24 @@ --- -title: "Langfuse Integration with Future AGI Evaluation Results" -description: "Integrate Future AGI evaluations with Langfuse to attach evaluation scores and results directly to your Langfuse traces." +title: "Attach FutureAGI Evals to Langfuse Traces" +description: "Run FutureAGI evaluations and write the scores back onto your Langfuse traces, so eval results live alongside the traces you already collect in Langfuse." +page_type: "how-to" +products: ["fi.evals"] +task_intent: "Send FutureAGI eval scores to Langfuse traces" +audience: "engineer" +difficulty: "intermediate" +time_to_complete: "10 minutes" +status: "review" +owner: "observability" +last_tested: "2026-05-25" +schema_type: "TechArticle" +seo: + primary_query: "futureagi evals langfuse integration" +geo: + direct_answer: true +related: + feature: "/docs/observe/features/evals" + how_to: "/docs/traceai/manual-instrumentation/set-up-tracing" + concept: "/docs/observe/concepts/traces" --- ## About @@ -17,6 +35,13 @@ Langfuse provides tracing but does not have a built-in evaluation engine. This i --- +## Prerequisites + +- A Langfuse project with traces, and its API keys. +- `fi.evals` available with `FI_API_KEY` and `FI_SECRET_KEY`. + +--- + ## How to @@ -108,6 +133,23 @@ Langfuse provides tracing but does not have a built-in evaluation engine. This i --- +## Verify + +Run an eval and push the score, then open the trace in Langfuse: + +- The FutureAGI score appears as a score on the matching Langfuse trace. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| Score not on the Langfuse trace | Trace/observation ID mismatch. | Map the eval result to the correct Langfuse trace ID. | +| Auth error | Wrong Langfuse or FutureAGI keys. | Recheck both sets of credentials. | + +--- + ## Key concepts - **`platform="langfuse"`**:The essential parameter that directs evaluation results to Langfuse and links them with the current active span. @@ -123,13 +165,13 @@ Langfuse provides tracing but does not have a built-in evaluation engine. This i Learn how to run evaluations using the Future AGI AI Evaluations library. - + Run evaluations directly inside a traced span with Future AGI tracing. - + Register a tracer provider and add instrumentation. - + Browse all supported framework instrumentors. diff --git a/src/pages/docs/observe/features/manual-tracing/log-prompt-templates.mdx b/src/pages/docs/traceai/manual-instrumentation/log-prompt-templates.mdx similarity index 71% rename from src/pages/docs/observe/features/manual-tracing/log-prompt-templates.mdx rename to src/pages/docs/traceai/manual-instrumentation/log-prompt-templates.mdx index c2588f61..954004f5 100644 --- a/src/pages/docs/observe/features/manual-tracing/log-prompt-templates.mdx +++ b/src/pages/docs/traceai/manual-instrumentation/log-prompt-templates.mdx @@ -1,6 +1,24 @@ --- -title: "Logging Prompt Templates and Variables in Future AGI Spans" -description: "Attach prompt template data to spans so Future AGI can surface it in the prompt playground for testing changes without deploying." +title: "Log Prompt Templates and Variables" +description: "Attach prompt template name, version, and variables to spans so FutureAGI surfaces them in the prompt playground for editing and re-running without a deploy." +page_type: "how-to" +products: ["traceAI"] +task_intent: "Attach prompt template data to spans" +audience: "engineer" +difficulty: "intermediate" +time_to_complete: "5 minutes" +status: "review" +owner: "observability" +last_tested: "2026-05-25" +schema_type: "TechArticle" +seo: + primary_query: "log prompt templates llm tracing" +geo: + direct_answer: true +related: + concept: "/docs/observe/concepts/spans" + how_to: "/docs/traceai/manual-instrumentation/add-attributes-metadata-tags" + feature: "/docs/observe/features/llm-tracing" --- ## About @@ -17,6 +35,13 @@ LLM outputs depend entirely on the prompt, but the prompt itself is not captured --- +## Prerequisites + +- Tracing set up — see [Set up tracing](/docs/traceai/manual-instrumentation/set-up-tracing). +- A prompt template (name/version) you want to surface in the playground. + +--- + ## How to @@ -83,6 +108,25 @@ LLM outputs depend entirely on the prompt, but the prompt itself is not captured --- +## Verify + +Run a request inside the `using_attributes` (or `using_prompt_template`) block, then open the trace: + +- The span carries the prompt template attributes (name/label/version and variables). +- The template appears in the **prompt playground**, where you can edit variables and re-run. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| Template not on the span | The LLM call ran outside the `using_*` block. | Make the model call **inside** the context-manager block. | +| Variables missing | Used `using_attributes` name-only. | Use `using_prompt_template` to attach the raw template, version, and variables. | +| Nothing in the playground | The instrumentor wasn't attached. | Confirm `OpenAIInstrumentor().instrument(...)` ran before the call. | + +--- + ## Key concepts - **`using_attributes`**: Context manager that enriches the current OpenTelemetry context with prompt template fields. All spans created by auto-instrumentors within the block carry the template data as span attributes. @@ -112,16 +156,16 @@ LLM outputs depend entirely on the prompt, but the prompt itself is not captured ## Next Steps - + Register a tracer provider and add instrumentation. - + Attach custom data to spans for filtering and evals. - + Use FITracer decorators and context managers for typed spans. - + Group traces into sessions and link them to end users. diff --git a/src/pages/docs/observe/features/manual-tracing/mask-span-attributes.mdx b/src/pages/docs/traceai/manual-instrumentation/mask-span-attributes.mdx similarity index 68% rename from src/pages/docs/observe/features/manual-tracing/mask-span-attributes.mdx rename to src/pages/docs/traceai/manual-instrumentation/mask-span-attributes.mdx index 3865a1ca..da1aa19b 100644 --- a/src/pages/docs/observe/features/manual-tracing/mask-span-attributes.mdx +++ b/src/pages/docs/traceai/manual-instrumentation/mask-span-attributes.mdx @@ -1,6 +1,24 @@ --- title: "Mask Span Attributes: Redact Sensitive Trace Data" -description: "Redact sensitive inputs, outputs, images, and embeddings from spans before export, using environment variables or TraceConfig in code." +description: "Redact inputs, outputs, messages, images, and embeddings from spans before export, using FI_HIDE_* environment variables or TraceConfig in code." +page_type: "how-to" +products: ["traceAI"] +task_intent: "Redact sensitive data from spans before export" +audience: "engineer" +difficulty: "intermediate" +time_to_complete: "5 minutes" +status: "review" +owner: "observability" +last_tested: "2026-05-25" +schema_type: "TechArticle" +seo: + primary_query: "redact pii from llm traces" +geo: + direct_answer: true +related: + concept: "/docs/observe/concepts/spans" + how_to: "/docs/traceai/manual-instrumentation/set-up-tracing" + feature: "/docs/observe/features/llm-tracing" --- ## About @@ -18,6 +36,17 @@ Traces often contain sensitive data: user messages, API responses, PII, or large --- +## Prerequisites + +- Tracing set up — see [Set up tracing](/docs/traceai/manual-instrumentation/set-up-tracing). +- A decision on what's sensitive: inputs, outputs, messages, images, or embeddings. + + + Environment variables apply at **process start** — already-running workers keep the values they loaded, so restart them to pick up a change. Redaction operates on span **payloads**, not span, trace, or session **names**; never put secrets or PII in those names. + + +--- + ## How to @@ -88,6 +117,25 @@ Traces often contain sensitive data: user messages, API responses, PII, or large --- +## Verify + +Set one flag (e.g. `FI_HIDE_INPUTS=true`), restart the app, run a request, and open the trace: + +- The masked field shows as hidden/redacted in the span detail, not as the raw value. +- Unmasked fields still display normally. A redacted field is expected behavior, not a missing trace. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| Data still visible after setting an env var | The worker was already running. | Restart the process; env vars apply at startup. | +| Code and env var disagree | `TraceConfig` takes precedence over env vars. | Set it at one level; precedence is `TraceConfig` > env var > default. | +| Base64 images bloat the payload | Image length not capped. | Set `FI_BASE64_IMAGE_MAX_LENGTH` or `hide_input_images`. | + +--- + ## Key concepts - **`TraceConfig`**:An object accepted by all traceAI auto-instrumentors. Use it to specify masking settings directly in code, scoped to a single instrumentor. @@ -101,16 +149,16 @@ Traces often contain sensitive data: user messages, API responses, PII, or large ## Next Steps - + Register a tracer provider and add instrumentation. - + Attach custom data to spans for filtering and evals. - + Use FITracer decorators and context managers for typed spans. - + Browse all supported framework instrumentors. diff --git a/src/pages/docs/observe/features/manual-tracing/semantic-conventions.mdx b/src/pages/docs/traceai/manual-instrumentation/semantic-conventions.mdx similarity index 96% rename from src/pages/docs/observe/features/manual-tracing/semantic-conventions.mdx rename to src/pages/docs/traceai/manual-instrumentation/semantic-conventions.mdx index ebbcec89..2ad24bdb 100644 --- a/src/pages/docs/observe/features/manual-tracing/semantic-conventions.mdx +++ b/src/pages/docs/traceai/manual-instrumentation/semantic-conventions.mdx @@ -1,6 +1,21 @@ --- -title: "FI Semantic Conventions: Standard Span Attribute Keys" -description: "Use standardized attribute keys for spans to ensure consistent, queryable trace data across LLM models, frameworks, and vendors." +title: "Semantic Conventions: Standard Span Attribute Keys" +description: "The standardized attribute keys for spans — span kinds, message, document, embedding, and tool-call attributes — that keep trace data consistent and queryable." +page_type: "reference" +products: ["traceAI"] +audience: "engineer" +status: "review" +owner: "observability" +last_tested: "2026-05-25" +schema_type: "TechArticle" +seo: + primary_query: "span attribute semantic conventions" +geo: + direct_answer: true +related: + concept: "/docs/observe/concepts/spans" + how_to: "/docs/traceai/manual-instrumentation/add-attributes-metadata-tags" + feature: "/docs/observe/features/llm-tracing" --- ## About @@ -860,7 +875,7 @@ Every LLM provider returns data in a different format. Without a standard set of - **`RerankerAttributes`**: Attribute keys for reranker spans (input/output documents, query, model name, top-k). - **`EmbeddingAttributes`**: Attribute keys for embedding spans (text and vector). - **`ToolCallAttributes`**: Attribute keys for tool call objects generated by an LLM (ID, function name, arguments). -- **`FiSpanKindValues`**: Enumeration of valid values for `fi.span.kind`: `LLM`, `CHAIN`, `RETRIEVER`, `RERANKER`, `EMBEDDING`, `AGENT`, `TOOL`, `GUARDRAIL`, `EVALUATOR`, `UNKNOWN`. +- **`FiSpanKindValues`**: Enumeration of valid values for `fi.span.kind`: `LLM`, `CHAIN`, `RETRIEVER`, `RERANKER`, `EMBEDDING`, `AGENT`, `TOOL`, `GUARDRAIL`, `EVALUATOR`, `UNKNOWN`, `CONVERSATION`, `VECTOR_DB`, `A2A_CLIENT`, `A2A_SERVER`. - **Flattening**: OpenTelemetry span attributes must be simple scalar types or flat lists. Nested objects (such as lists of messages) must be flattened with index prefixes like `llm.input_messages.0.message.role`. --- @@ -868,16 +883,16 @@ Every LLM provider returns data in a different format. Without a standard set of ## Next Steps - + Attach custom data, tags, session IDs, and prompt templates to spans. - + Use FITracer decorators and context managers for typed spans. - + Register a tracer provider and add instrumentation. - + Browse all supported framework instrumentors. diff --git a/src/pages/docs/observe/features/manual-tracing/set-session-user-id.mdx b/src/pages/docs/traceai/manual-instrumentation/set-session-user-id.mdx similarity index 85% rename from src/pages/docs/observe/features/manual-tracing/set-session-user-id.mdx rename to src/pages/docs/traceai/manual-instrumentation/set-session-user-id.mdx index 91f4a21f..9603f900 100644 --- a/src/pages/docs/observe/features/manual-tracing/set-session-user-id.mdx +++ b/src/pages/docs/traceai/manual-instrumentation/set-session-user-id.mdx @@ -1,6 +1,24 @@ --- -title: "Set Session ID and User ID on Spans in Future AGI" -description: "Add SessionID and UserID as span attributes to group and filter traces by conversation session and end user in Future AGI." +title: "Set Session and User IDs on Spans" +description: "Add session.id and user.id to spans so traces group into conversations and roll up per end user — set them directly or with context helpers." +page_type: "how-to" +products: ["traceAI"] +task_intent: "Attach session and user IDs to spans" +audience: "engineer" +difficulty: "intermediate" +time_to_complete: "5 minutes" +status: "review" +owner: "observability" +last_tested: "2026-05-25" +schema_type: "TechArticle" +seo: + primary_query: "set session id user id spans" +geo: + direct_answer: true +related: + concept: "/docs/observe/concepts/sessions-and-users" + feature: "/docs/observe/features/session" + how_to: "/docs/traceai/manual-instrumentation/set-up-tracing" --- ## About @@ -17,6 +35,13 @@ Traces are isolated by default. Without a session or user identifier, there is n --- +## Prerequisites + +- Tracing set up — see [Set up tracing](/docs/traceai/manual-instrumentation/set-up-tracing). +- Stable, non-sensitive session and user identifiers (avoid raw PII). + +--- + ## How to @@ -297,6 +322,25 @@ Traces are isolated by default. Without a session or user identifier, there is n --- +## Verify + +Run a request inside the `using_session` / `using_user` block, then open Observe: + +- The trace appears in the [Sessions](/docs/observe/features/session) view under your `session.id`, and under the user in the [Users](/docs/observe/features/users) view. +- Opening the trace shows `session.id` / `user.id` on the span attributes. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| Trace not grouped into a session or user | The model call ran outside the context block. | Make the call **inside** the `using_session` / `using_user` / `using_attributes` block. | +| JS/TS: IDs not propagating | Baggage not set on the active context. | Use `propagation.createBaggage()` + `context.with()` as shown. | +| Empty grouping | An empty-string ID was passed. | IDs must be non-empty strings. | + +--- + ## Key concepts - **`using_session`**:Context manager that adds `session.id` to the OpenTelemetry context. All spans from traceAI auto-instrumentors within the block will carry this attribute. Input must be a non-empty string. @@ -309,16 +353,16 @@ Traces are isolated by default. Without a session or user identifier, there is n ## Next Steps - + Register a tracer provider and add instrumentation. - + Attach custom data to spans for filtering and evals. - + Use FITracer decorators and context managers for typed spans. - + Redact sensitive data with TraceConfig before export. diff --git a/src/pages/docs/observe/features/manual-tracing/set-up-tracing.mdx b/src/pages/docs/traceai/manual-instrumentation/set-up-tracing.mdx similarity index 63% rename from src/pages/docs/observe/features/manual-tracing/set-up-tracing.mdx rename to src/pages/docs/traceai/manual-instrumentation/set-up-tracing.mdx index 20f8fb08..b1b6f4ce 100644 --- a/src/pages/docs/observe/features/manual-tracing/set-up-tracing.mdx +++ b/src/pages/docs/traceai/manual-instrumentation/set-up-tracing.mdx @@ -1,6 +1,24 @@ --- -title: "Set Up Tracing with Future AGI and OpenTelemetry" -description: "Connect your application to Future AGI by registering a tracer provider and adding instrumentation with auto-instrumentors or manual OpenTelemetry spans." +title: "Set Up Tracing with OpenTelemetry" +description: "Connect your app to FutureAGI: register a tracer provider, add an auto-instrumentor or manual OpenTelemetry spans, and confirm traces arrive in your project." +page_type: "how-to" +products: ["traceAI"] +task_intent: "Instrument an application to send traces to FutureAGI" +audience: "engineer" +difficulty: "beginner" +time_to_complete: "10 minutes" +status: "review" +owner: "observability" +last_tested: "2026-05-25" +schema_type: "TechArticle" +seo: + primary_query: "set up opentelemetry tracing futureagi" +geo: + direct_answer: true +related: + concept: "/docs/observe/concepts/traces" + reference: "/docs/traceai/manual-instrumentation/semantic-conventions" + feature: "/docs/observe/features/llm-tracing" --- ## About @@ -15,7 +33,15 @@ Tracing captures every LLM call, tool invocation, or custom operation in your ap - **Experiment tracking**: Register an Experiment project with eval tags and version names to compare prompt or model changes across runs. - **Custom spans**: Use `FITracer` to manually create spans for operations that auto-instrumentors don't cover. - **Privacy control**: Use `TraceConfig` to redact sensitive inputs, outputs, or messages before they leave your app. -- **Any Python or JS/TS app**: Works with any application via OpenTelemetry. Auto-instrumentors cover 20+ frameworks. +- **Any Python or JS/TS app**: Works with any application via OpenTelemetry. Auto-instrumentors cover a wide range of frameworks — see the [Auto Instrumentation catalog](/docs/traceai/auto). + +--- + +## Prerequisites + +- **Python 3.9+** or **Node 18+**. +- A FutureAGI account with your **`FI_API_KEY`** and **`FI_SECRET_KEY`** from the [dashboard](https://app.futureagi.com/dashboard/keys). +- The core `fi-instrumentation-otel` package plus the instrumentor for your framework (e.g. `traceAI-openai`). --- @@ -142,14 +168,14 @@ Tracing captures every LLM call, tool invocation, or custom operation in your ap | LLM Models | Orchestration | Other | |------------|---------------|-------| - | [OpenAI](/docs/tracing/auto/openai) | [LlamaIndex](/docs/tracing/auto/llamaindex) | [DSPy](/docs/tracing/auto/dspy) | - | [OpenAI Agents SDK](/docs/tracing/auto/openai_agents) | [LlamaIndex Workflows](/docs/tracing/auto/llamaindex-workflows) | [Guardrails AI](/docs/tracing/auto/guardrails) | - | [Vertex AI](/docs/tracing/auto/vertexai) | [LangChain](/docs/tracing/auto/langchain) | [smolagents](/docs/tracing/auto/smol_agents) | - | [AWS Bedrock](/docs/tracing/auto/bedrock) | [LangGraph](/docs/tracing/auto/langgraph) | [Ollama](/docs/tracing/auto/ollama) | - | [Mistral AI](/docs/tracing/auto/mistralai) | [LiteLLM](/docs/tracing/auto/litellm) | [Instructor](/docs/tracing/auto/instructor) | - | [Anthropic](/docs/tracing/auto/anthropic) | [CrewAI](/docs/tracing/auto/crewai) | | - | [Groq](/docs/tracing/auto/groq) | [Haystack](/docs/tracing/auto/haystack) | | - | [Together AI](/docs/tracing/auto/togetherai) | [AutoGen](/docs/tracing/auto/autogen) | | + | [OpenAI](/docs/traceai/auto/openai) | [LlamaIndex](/docs/traceai/auto/llamaindex) | [DSPy](/docs/traceai/auto/dspy) | + | [OpenAI Agents SDK](/docs/traceai/auto/openai_agents) | [LlamaIndex Workflows](/docs/traceai/auto/llamaindex-workflows) | [Guardrails AI](/docs/traceai/auto/guardrails) | + | [Vertex AI](/docs/traceai/auto/vertexai) | [LangChain](/docs/traceai/auto/langchain) | [smolagents](/docs/traceai/auto/smol_agents) | + | [AWS Bedrock](/docs/traceai/auto/bedrock) | [LangGraph](/docs/traceai/auto/langgraph) | [Ollama](/docs/traceai/auto/ollama) | + | [Mistral AI](/docs/traceai/auto/mistralai) | [LiteLLM](/docs/traceai/auto/litellm) | [Instructor](/docs/traceai/auto/instructor) | + | [Anthropic](/docs/traceai/auto/anthropic) | [CrewAI](/docs/traceai/auto/crewai) | | + | [Groq](/docs/traceai/auto/groq) | [Haystack](/docs/traceai/auto/haystack) | | + | [Together AI](/docs/traceai/auto/togetherai) | [AutoGen](/docs/traceai/auto/autogen) | | `FITracer` wraps the standard OTel tracer and adds Future AGI-specific features: automatic input/output capture, context injection, and decorator support. @@ -188,108 +214,55 @@ Tracing captures every LLM call, tool invocation, or custom operation in your ap - Use context managers, nested spans, or decorators for full control over span structure. + Wrap an operation in a span with a context manager — the canonical pattern. The span opens when the block starts and closes automatically when it ends. - - - - - ```python Python - def process_operation(): - with tracer.start_as_current_span("span-name") as span: - # Execute operations tracked by 'span' - print("doing some work...") - # When the 'with' block goes out of scope, 'span' is automatically closed - ``` + + ```python Python + def process_operation(): + with tracer.start_as_current_span("span-name") as span: + # Execute operations tracked by 'span'; the span closes + # automatically when the 'with' block goes out of scope. + print("doing some work...") + ``` + ```javascript JS/TS + function processOperation() { + tracer.startActiveSpan("span-name", (span) => { + // Execute operations tracked by 'span', then end it. + span.setAttribute("operation", "processOperation"); + span.end(); + }); + } + ``` + - ```javascript JS/TS - function processOperation() { - const q1 = () => tracer.startActiveSpan('processOperation', (span) => { - span.setAttribute('operation', 'processOperation'); - span.end(); - }); - - const q2 = () => tracer.startActiveSpan('processChildOperation', (span) => { - span.setAttribute('operation', 'processChildOperation'); - span.end(); - }); - - q1(); - q2(); - } - ``` + Nest spans by opening a child span inside a parent block, or use the decorator form (`@tracer.start_as_current_span("...")` in Python) when a whole function maps to one span. For nested-span, decorator, and async patterns, see [Advanced tracing examples](/docs/traceai/manual-instrumentation/advanced-tracing-examples). + + - - - - +--- - ```python Python - def process_operation(): - with tracer.start_as_current_span("parent") as parent: - # Execute parent-level operations - print("doing some work...") - # Create nested span for sub-operations - with tracer.start_as_current_span("child") as child: - # Execute child-level operations - print("doing some nested work...") - # Child span closes automatically when it's out of scope - ``` +## Verify - ```typescript JS/TS - function processOperation() { - tracer.startActiveSpan("parent", (parentSpan) => { - console.log("doing some work..."); +Send one request through your app, then open the project in the FutureAGI dashboard: - tracer.startActiveSpan("child", (childSpan) => { - console.log("doing some nested work..."); - childSpan.end(); - }); +- A new **trace** appears within a few seconds, named after your top-level operation. +- Opening it shows spans with **input**, **output**, **latency**, **model**, and **token count**. +- For an Observe project, find it under **Observe → your project → Tracing**. - parentSpan.end(); - }); - } - ``` +If the trace is there with those fields, instrumentation is working end to end. - - - - +--- - ```python Python - @tracer.start_as_current_span("process_operation") - def process_operation(): - print("doing some work...") - ``` +## Troubleshooting - ```javascript JS/TS - // JavaScript doesn't have decorators in the same way, but you can achieve similar functionality - const decoratedFunction = (fn) => { - return (...args) => { - return tracer.startActiveSpan("process_operation", (span) => { - try { - const result = fn(...args); - span.end(); - return result; - } catch (error) { - span.recordException(error); - span.end(); - throw error; - } - }); - }; - }; - - const processOperation = decoratedFunction(() => { - console.log("doing some work..."); - }); - ``` +| Symptom | Cause | Fix | +|---|---|---| +| No trace appears | A short script exited before the batch exporter flushed. | Call `trace_provider.force_flush()` before the process ends, or set `batch=False` in `register()`. | +| Spans are missing | The instrumentor ran after the framework client was created. | Call `register()` and `instrument()` **before** constructing the client. | +| Auth or connection error | Wrong or missing keys. | Confirm `FI_API_KEY` and `FI_SECRET_KEY` are set for this workspace. | +| gRPC errors | gRPC transport without its dependency. | Install `fi-instrumentation-otel[grpc]`, or use `Transport.HTTP`. | - - - - - +--- ## Key concepts @@ -305,22 +278,22 @@ Tracing captures every LLM call, tool invocation, or custom operation in your ap ## Next Steps - + Browse all supported framework instrumentors. - + Use TraceAI helpers for sessions, users, and context. - + Attach custom data to spans for filtering and evals. - + Group traces into sessions and link them to end users. - + Redact sensitive data with TraceConfig before export. - + Register an Observe project and start capturing traces. diff --git a/src/pages/docs/traceai/quickstart.mdx b/src/pages/docs/traceai/quickstart.mdx new file mode 100644 index 00000000..b57a4f29 --- /dev/null +++ b/src/pages/docs/traceai/quickstart.mdx @@ -0,0 +1,224 @@ +--- +title: "traceAI quickstart: instrument your app and ship your first trace" +description: "Install the traceAI SDK, register a project, auto-instrument your framework, run one request, and confirm the trace in FutureAGI Observe — in about five minutes." +slug: "quickstart" +page_type: "quickstart" +products: ["traceAI"] +audience: ["engineer"] +difficulty: "beginner" +time_to_complete: "5 minutes" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-05-25" +last_screenshotted: "2026-05-26" +schema_type: "HowTo" +success_artifact: "trace" +api_keys_needed: ["FI_API_KEY", "FI_SECRET_KEY", "OPENAI_API_KEY"] +dashboard_verification_path: "Observe > your project > Tracing" +seo: + title: "traceAI quickstart — instrument your AI app in 5 minutes" + description: "Install the traceAI SDK, register a project, auto-instrument a framework, run one request, and see the trace in FutureAGI Observe." + primary_keyword: "traceai sdk quickstart" + direct_answer: true +geo: + answer_target: "How do I get started with the traceAI SDK?" + llm_summary: "Install fi-instrumentation-otel plus a traceAI instrumentor, call register() with project_type OBSERVE, attach the instrumentor, run one request, and the trace appears in Observe within seconds." +canonical: "/docs/traceai/quickstart" +related: + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/traceai/auto" + - "/docs/observe/quickstart" + - "/docs/observe/concepts/traces" +--- + +## About + +Get traceAI sending data in about five minutes. You'll install the SDK, register a project, auto-instrument one framework, run a normal request, and confirm the trace lands in [Observe](/docs/observe). traceAI is the [SDK](/docs/observe/concepts/traceai) that produces traces; Observe is where you read them. Everything else — manual spans, sessions, tags, evals — builds on the trace you create here. + +--- + +## Prerequisites + +- **Python 3.9+** (or Node 18+ for the TypeScript path). +- A **FutureAGI account** and your **`FI_API_KEY`** + **`FI_SECRET_KEY`** from the [dashboard keys page](https://app.futureagi.com/dashboard/keys). +- An **`OPENAI_API_KEY`** — this quickstart auto-instruments OpenAI as the concrete example. Any other [framework integration](/docs/traceai/auto) follows the same four steps. + + + Pin packages to the version you test against. These commands were last verified on **2026-05-25**; add a version (e.g. `traceAI-openai==`) before shipping to CI so a future release can't silently change behavior. + + +--- + +## Step 1: Install the SDK + +Install the core instrumentation package plus the instrumentor for the framework you use. The core package is the same for every integration; only the second package changes. + + +```bash Python +pip install fi-instrumentation-otel traceAI-openai +``` +```bash JS/TS +npm install @traceai/fi-core @traceai/openai +``` + + +Confirm the install resolved: + +```text +Successfully installed fi-instrumentation-otel traceAI-openai +``` + +--- + +## Step 2: Set credentials + +Read keys from the environment — never hardcode them in source. + + +```python Python +import os +os.environ["FI_API_KEY"] = "YOUR_FI_API_KEY" +os.environ["FI_SECRET_KEY"] = "YOUR_FI_SECRET_KEY" +os.environ["OPENAI_API_KEY"] = "YOUR_OPENAI_API_KEY" +``` +```typescript JS/TS +process.env.FI_API_KEY = "YOUR_FI_API_KEY"; +process.env.FI_SECRET_KEY = "YOUR_FI_SECRET_KEY"; +process.env.OPENAI_API_KEY = "YOUR_OPENAI_API_KEY"; +``` + + +--- + +## Step 3: Register a project + +`register` returns a tracer provider and tells traceAI where to send spans. Set `project_type` to `OBSERVE` so the traces appear in Observe, and give the project a name. + + +```python Python +from fi_instrumentation import register, Transport +from fi_instrumentation.fi_types import ProjectType + +trace_provider = register( + project_type=ProjectType.OBSERVE, + project_name="my-first-project", + transport=Transport.HTTP, +) +``` +```typescript JS/TS +import { register, ProjectType } from "@traceai/fi-core"; + +const traceProvider = register({ + project_type: ProjectType.OBSERVE, + project_name: "my-first-project", +}); +``` + + +Expected: the call returns without error and logs the registered project. If it raises an auth error, your `FI_API_KEY` / `FI_SECRET_KEY` are wrong for this workspace. + +```text +FutureAGI: registered project "my-first-project" (type=OBSERVE) +``` + +--- + +## Step 4: Instrument and run one request + +Attach the instrumentor to the provider, then call your framework as you normally would. Auto-instrumentation patches the client, so no per-call span code is needed. + + +```python Python +from traceai_openai import OpenAIInstrumentor +from openai import OpenAI + +OpenAIInstrumentor().instrument(tracer_provider=trace_provider) + +client = OpenAI() +completion = client.chat.completions.create( + model="gpt-4o", + messages=[{"role": "user", "content": "Write a one-sentence bedtime story about a unicorn."}], +) +print(completion.choices[0].message.content) + +# Short scripts exit before the batch exporter flushes — force it. +trace_provider.force_flush() +``` +```typescript JS/TS +import { OpenAIInstrumentation } from "@traceai/openai"; +import { OpenAI } from "openai"; + +new OpenAIInstrumentation({}); + +const client = new OpenAI({ apiKey: process.env.OPENAI_API_KEY }); +const completion = await client.chat.completions.create({ + model: "gpt-4o", + messages: [{ role: "user", content: "Write a one-sentence bedtime story about a unicorn." }], +}); +console.log(completion.choices[0].message.content); +``` + + +Expected terminal output (the model text varies): + +```text +Under a sky of silver stars, a gentle unicorn dipped its horn into a moonlit +pool and wished every sleeping child sweet dreams. +``` + +The OpenAI call produces **one `llm` span** with the model, prompt, completion, token counts, and cost. Frameworks that orchestrate steps (LangChain, CrewAI, agents) produce a `chain`/`agent` root span with nested `llm`, `tool`, and `retriever` spans — see [framework integrations](/docs/traceai/auto). + +--- + +## Verify the trace + +Open **Observe → your project → Tracing**. Within a few seconds you should see one new trace row showing **Status OK**, the **gpt-4o** model, the **latency**, and the **token count**. Click it to read the prompt, the completion, and the span timing. + +Observe trace explorer with one new OpenAI trace showing OK status, model, latency, and token columns +*Your request, now a trace. If the row is here with an OK status, instrumentation is working end to end.* + +--- + +## What just happened + +Four lines of setup turned a normal OpenAI call into a [trace](/docs/observe/concepts/traces). `register()` built a tracer provider pointed at your Observe project; the instrumentor patched the OpenAI client so every call it makes emits a [span](/docs/observe/concepts/spans); and `force_flush()` made sure the batch exporter sent before the script exited. You wrote no span code — auto-instrumentation did it. + +From here the same SDK scales two ways: + +- **More frameworks, same four steps.** Swap `traceAI-openai` for any other [integration](/docs/traceai/auto) — LangChain, LlamaIndex, CrewAI — and the install → register → instrument → run flow is identical. A framework run produces a `chain`/`agent` root span with nested `llm`, `tool`, and `retriever` spans. +- **Your own code, manually.** Where auto-instrumentation can't reach — a custom function, a tool between framework calls — wrap it in a span yourself with [manual instrumentation](/docs/traceai/manual-instrumentation/set-up-tracing). Auto and manual compose in one trace. + +Enrich any span with a [session, user, or tags](/docs/traceai/manual-instrumentation/add-attributes-metadata-tags) so Observe can group and filter what you send. + +--- + +## Not seeing data? + +| Symptom | Fix | +|---|---| +| No trace appears | A short script can exit before the batch exporter flushes — call `trace_provider.force_flush()` before the process ends (already in Step 4). | +| Still nothing | Widen the date picker (it defaults to the past 7 days) and turn on **Auto refresh**. | +| Wrong or empty project | Confirm `project_name` matches the project you're viewing, and that `FI_API_KEY` / `FI_SECRET_KEY` are the keys for this workspace. | +| `register` 401s | The keys are for a different workspace, or one is missing — re-copy both from the [keys page](https://app.futureagi.com/dashboard/keys). | + +For deeper diagnosis see [Spans not exported](/docs/traceai/troubleshooting/spans-not-exported). + +--- + +## Next steps + + + + Auto-instrument LangChain, LlamaIndex, CrewAI, and 30+ others. + + + Create your own spans when auto-instrumentation doesn't reach your code. + + + The mental model the rest of the docs build on. + + + Read and debug the traces you just started capturing. + + diff --git a/src/pages/docs/traceai/troubleshooting/spans-not-exported.mdx b/src/pages/docs/traceai/troubleshooting/spans-not-exported.mdx new file mode 100644 index 00000000..a1d3a260 --- /dev/null +++ b/src/pages/docs/traceai/troubleshooting/spans-not-exported.mdx @@ -0,0 +1,95 @@ +--- +title: "Spans not exported" +description: "Your app runs but no spans reach FutureAGI. Work through the common causes — unflushed batches, wrong project type, missing credentials, and instrumentor order — with the exact fix for each." +slug: "spans-not-exported" +page_type: "troubleshooting" +products: ["traceAI"] +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-05-25" +schema_type: "TechArticle" +failure_surface: "instrumentation" +seo: + title: "Fix: traceAI spans not exported to FutureAGI" + description: "No spans arriving in Observe? Diagnose unflushed exporters, wrong project type, credential and instrumentor-order issues with a fix for each." + primary_keyword: "traceai spans not exported" + direct_answer: true +geo: + answer_target: "Why are my traceAI spans not showing up in FutureAGI?" + llm_summary: "The usual causes are a short-lived process exiting before the batch exporter flushes, a wrong project type, missing FI credentials, or instrumenting after the client is created — each has a direct fix." +canonical: "/docs/traceai/troubleshooting/spans-not-exported" +related: + - "/docs/traceai/manual-instrumentation/set-up-tracing" + - "/docs/observe/troubleshooting/no-traces-appearing" + - "/docs/observe/concepts/traceai" +--- + + +## Symptom + +Your application runs without errors, but no spans appear in [Observe](/docs/observe) — the trace list stays empty, or new requests never show up. The instrumentation looks wired in, yet nothing is exported. + +--- + +## Quick checks + +- The process **stays alive** long enough to flush, or you call `force_flush()` before it exits. +- `register(...)` ran with `project_type=ProjectType.OBSERVE` **before** any model client was created. +- `FI_API_KEY` and `FI_SECRET_KEY` are set in the environment the app actually runs in. +- The instrumentor (`.instrument(...)`) was attached to the **same** tracer provider `register` returned. + +--- + +## Causes and fixes + +| Cause | What you see | Fix | +|---|---|---| +| **Batch exporter never flushed** | A short script or serverless function exits immediately; spans are still buffered. | Call `trace_provider.force_flush()` (and `shutdown()`) before the process ends. The batch exporter sends on an interval, so a process that exits in under a second loses its buffer. | +| **Wrong project type** | Traces go somewhere, but not to your Observe project. | Pass `project_type=ProjectType.OBSERVE` to `register`. A project registered as another type won't appear in Observe. | +| **Missing or wrong credentials** | Export silently fails or 401s in logs. | Set `FI_API_KEY` / `FI_SECRET_KEY` for this workspace; confirm they match the [keys page](https://app.futureagi.com/dashboard/keys). | +| **Instrumented too late** | The model client was created before `.instrument()` ran, so its methods aren't patched. | Call `register` and `.instrument()` at startup, before constructing any provider client. | +| **Provider not instrumented** | Some calls trace, others don't. | Add the instrumentor for each provider you use, or add [manual spans](/docs/traceai/manual-instrumentation/create-tool-spans) around the un-instrumented code. | + +--- + +## Diagnostic commands + +Confirm spans are being produced at all by adding a console exporter alongside the FutureAGI one: + +```python +from opentelemetry.sdk.trace.export import ConsoleSpanExporter, SimpleSpanProcessor + +trace_provider.add_span_processor(SimpleSpanProcessor(ConsoleSpanExporter())) +``` + +If spans print to your console but don't reach Observe, the problem is **export/credentials**, not instrumentation. If nothing prints, the problem is **instrumentation order or coverage**. + +--- + +## Minimal smoke test + +Run the [quick start](/docs/observe/quickstart) script as-is. If its single OpenAI trace appears, your environment and credentials are correct, and the issue is specific to your app's instrumentation order or process lifetime. + +--- + +## Escalate + +If spans still don't arrive after working through the causes above, contact support@futureagi.com with your `project_name`, the installed SDK versions (`fi-instrumentation-otel` and the instrumentor), and the stderr from the run. + +--- + +## Prevent recurrence + +- Always `force_flush()` in short-lived and serverless entry points. +- Register and instrument in one startup module that runs before anything else. +- Add the console exporter in development so missing spans surface immediately. + +--- + +## Next steps + +- [Set up tracing](/docs/traceai/manual-instrumentation/set-up-tracing) — the correct setup order. +- [No traces appear](/docs/observe/troubleshooting/no-traces-appearing) — the Observe-side view of the same symptom. diff --git a/src/pages/docs/tracing/auto/crewai.mdx b/src/pages/docs/tracing/auto/crewai.mdx deleted file mode 100644 index fc6114a5..00000000 --- a/src/pages/docs/tracing/auto/crewai.mdx +++ /dev/null @@ -1,96 +0,0 @@ ---- -title: "CrewAI Tracing with Future AGI: Auto-Instrumentation" -description: "Set up auto-instrumentation for CrewAI with Future AGI tracing. Install traceAI-crewai to capture crew task execution and agent interaction spans." ---- - -## 1. Installation -Install the traceAI and Crew packages - -```bash -pip install traceAI-crewai crewai crewai_tools -``` - ---- - -## 2. Set Environment Variables - -Set up your environment variables to authenticate with both FutureAGI and OpenAI. - -```python -import os - -os.environ["OPENAI_API_KEY"] = "your-openai-api-key" -os.environ["FI_API_KEY"] = "your-futureagi-api-key" -os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" -``` - ---- - -## 3. Initialize Trace Provider -Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . - -```python -from fi_instrumentation import register -from fi_instrumentation.fi_types import ProjectType - -trace_provider = register( - project_type=ProjectType.OBSERVE, - project_name="crewai_project", -) -``` - ---- - -## 4. Instrument your Project -Initialize the Crew AI instrumentor to enable automatic tracing. - -```python -from traceai_crewai import CrewAIInstrumentor - -CrewAIInstrumentor().instrument(tracer_provider=trace_provider) -``` - ---- - -## 5. Run Crew AI -Run your Crew AI application as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. - -```python -from crewai import LLM, Agent, Crew, Process, Task -from crewai_tools import SerperDevTool - -def story_example(): - llm = LLM( - model="gpt-4", - temperature=0.8, - max_tokens=150, - top_p=0.9, - frequency_penalty=0.1, - presence_penalty=0.1, - stop=["END"], - seed=42, - ) - - writer = Agent( - role="Writer", - goal="Write creative stories", - backstory="You are a creative writer with a passion for storytelling", - allow_delegation=False, - llm=llm, - ) - - writing_task = Task( - description="Write a short story about a magical forest", - agent=writer, - expected_output="A short story about a magical forest", - ) - - crew = Crew(agents=[writer], tasks=[writing_task]) - - # Execute the crew - result = crew.kickoff() - print(result) - -if __name__ == "__main__": - story_example() -``` diff --git a/src/pages/docs/tracing/auto/dspy.mdx b/src/pages/docs/tracing/auto/dspy.mdx deleted file mode 100644 index c23075e1..00000000 --- a/src/pages/docs/tracing/auto/dspy.mdx +++ /dev/null @@ -1,77 +0,0 @@ ---- -title: "DSPy Tracing with Future AGI: Auto-Instrumentation" -description: "Set up auto-instrumentation for DSPy with Future AGI tracing. Install traceAI-DSPy to capture program compilation and prediction spans automatically." ---- - -## 1. Installation -Install the traceAI and dspy package. - -```bash -pip install traceAI-DSPy dspy -``` - ---- - -## 2. Set Environment Variables -Set up your environment variables to authenticate with FutureAGI and OpenAI. - -```python -import os - -os.environ["OPENAI_API_KEY"] = "your-openai-api-key" -os.environ["FI_API_KEY"] = "your-futureagi-api-key" -os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" -``` - ---- - -## 3. Initialize Trace Provider - -Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . - -```python -from fi_instrumentation import register -from fi_instrumentation.fi_types import ProjectType - -trace_provider = register( - project_type=ProjectType.OBSERVE, - project_name="dspy_project", -) -``` - ---- -## 4. Instrument your Project -Initialize the DSPy instrumentor to enable automatic tracing. - -```python -from traceai_dspy import DSPyInstrumentor - -DSPyInstrumentor().instrument(tracer_provider=trace_provider) -``` - ---- - -## 5. Create DSPy Components and Run your application -Run DSPy as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. - -```python -import dspy - -class BasicQA(dspy.Signature): - """Answer questions with short factoid answers.""" - - question = dspy.InputField() - answer = dspy.OutputField(desc="often between 1 and 5 words") - -if __name__ == "__main__": - turbo = dspy.LM(model="openai/gpt-4") - - dspy.settings.configure(lm=turbo) - - # Define the predictor. - generate_answer = dspy.Predict(BasicQA) - - # Call the predictor on a particular input. - pred = generate_answer(question="What is the capital of the united states?") - print(f"Predicted Answer: {pred.answer}") -``` diff --git a/src/pages/docs/tracing/auto/google_adk.mdx b/src/pages/docs/tracing/auto/google_adk.mdx deleted file mode 100644 index cc8889e2..00000000 --- a/src/pages/docs/tracing/auto/google_adk.mdx +++ /dev/null @@ -1,116 +0,0 @@ ---- -title: "Google ADK Tracing with Future AGI: Auto-Instrumentation" -description: "Set up auto-instrumentation for Google ADK with Future AGI tracing. Install traceai-google-adk to capture agent and tool execution spans." ---- - -## 1. Installation -Install the traceAI and Google ADK packages. - -```bash -pip install traceai-google-adk -``` - ---- - -## 2. Set Environment Variables -Set up your environment variables to authenticate with both FutureAGI and Google. - -```python -import os - -os.environ["FI_API_KEY"] = "your-futureagi-api-key" -os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" -os.environ["GOOGLE_API_KEY"] = "your-google-api-key" -``` - ---- - -## 3. Initialize Trace Provider -Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . - -```python -from fi_instrumentation import register -from fi_instrumentation.fi_types import ProjectType - -trace_provider = register( - project_type=ProjectType.OBSERVE, - project_name="google_adk", -) -``` - ---- -## 4. Instrument your Project -Instrument your project to enable automatic tracing. - -```python -from traceai_google_adk import GoogleADKInstrumentor - -GoogleADKInstrumentor().instrument(tracer_provider=trace_provider) -``` - ---- -## 5. Interact with Google ADK -Start interacting with Google ADK as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. Here is a sample code using the Google ADK SDK. - -```python -import asyncio - -from google.adk.agents import Agent -from google.adk.runners import InMemoryRunner -from google.genai import types - -def get_weather(city: str) -> dict: - """Retrieves the current weather report for a specified city. - - Args: - city (str): The name of the city for which to retrieve the weather report. - - Returns: - dict: status and result or error msg. - """ - if city.lower() == "new york": - return { - "status": "success", - "report": ( - "The weather in New York is sunny with a temperature of 25 degrees" - " Celsius (77 degrees Fahrenheit)." - ), - } - else: - return { - "status": "error", - "error_message": f"Weather information for '{city}' is not available.", - } - -agent = Agent( - name="test_agent", - model="gemini-2.5-flash-preview-05-20", - description="Agent to answer questions using tools.", - instruction="You must use the available tools to find an answer.", - tools=[get_weather] -) - -async def main(): - app_name = "test_instrumentation" - user_id = "test_user" - session_id = "test_session" - runner = InMemoryRunner(agent=agent, app_name=app_name) - session_service = runner.session_service - await session_service.create_session( - app_name=app_name, - user_id=user_id, - session_id=session_id - ) - async for event in runner.run_async( - user_id=user_id, - session_id=session_id, - new_message=types.Content(role="user", parts=[ - types.Part(text="What is the weather in New York?")] - ) - ): - if event.is_final_response(): - print(event.content.parts[0].text.strip()) - -if __name__ == "__main__": - asyncio.run(main()) -``` \ No newline at end of file diff --git a/src/pages/docs/tracing/auto/google_genai.mdx b/src/pages/docs/tracing/auto/google_genai.mdx deleted file mode 100644 index 2e93f967..00000000 --- a/src/pages/docs/tracing/auto/google_genai.mdx +++ /dev/null @@ -1,71 +0,0 @@ ---- -title: "Google GenAI Tracing with Future AGI: Auto-Instrumentation" -description: "Set up auto-instrumentation for Google GenAI with Future AGI tracing. Install traceAI-google-genai to capture Gemini model interaction spans." ---- - -## 1. Installation -Install the traceAI and Google GenAI packages. - -```bash -pip install traceAI-google-genai -``` - ---- - -## 2. Set Environment Variables -Set up your environment variables to authenticate with FutureAGI. - -```python -import os - -os.environ["FI_API_KEY"] = "your-futureagi-api-key" -os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" -``` - ---- - -## 3. Initialize Trace Provider -Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . - -```python -from fi_instrumentation import register -from fi_instrumentation.fi_types import ProjectType - -trace_provider = register( - project_type=ProjectType.OBSERVE, - project_name="google_genai", -) -``` - ---- -## 4. Instrument your Project -Instrument your project to enable automatic tracing. - -```python -from traceai_google_genai import GoogleGenAIInstrumentor - -GoogleGenAIInstrumentor().instrument(tracer_provider=trace_provider) -``` - ---- -## 5. Interact with Google ADK -Start interacting with Google ADK as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. Here is a sample code using the Google ADK SDK. - -```python -from google import genai -from google.genai import types - -client = genai.Client(vertexai=True, project="your_project_name", location="global") - -content = types.Content( - role="user", - parts=[ - types.Part.from_text(text="Hello how are you?"), - ], -) -response = client.models.generate_content( - model="gemini-2.0-flash-001", contents=content -) - -print(response) -``` \ No newline at end of file diff --git a/src/pages/docs/tracing/auto/groq.mdx b/src/pages/docs/tracing/auto/groq.mdx deleted file mode 100644 index aa8467f4..00000000 --- a/src/pages/docs/tracing/auto/groq.mdx +++ /dev/null @@ -1,75 +0,0 @@ ---- -title: "Groq Tracing with Future AGI: Auto-Instrumentation" -description: "Set up auto-instrumentation for Groq with Future AGI tracing. Install traceAI-groq to capture high-speed inference spans and performance data." ---- - -## 1. Installation -Install the traceAI and Groq packages. - -```bash -pip install traceAI-groq -``` - ---- - -## 2. Set Environment Variables -Set up your environment variables to authenticate with both FutureAGI and Groq. - -```python -import os - -os.environ["FI_API_KEY"] = "your-futureagi-api-key" -os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" -os.environ["GROQ_API_KEY"] = "your-groq-api-key" -``` - ---- - -## 3. Initialize Trace Provider -Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . - -```python -from fi_instrumentation import register -from fi_instrumentation.fi_types import ProjectType - -trace_provider = register( - project_type=ProjectType.OBSERVE, - project_name="groq_project", -) -``` - ---- -## 4. Instrument your Project -Instrument your project to enable automatic tracing. - -```python -from traceai_groq import GroqInstrumentor - -GroqInstrumentor().instrument(tracer_provider=trace_provider) -``` - ---- -## 5. Interact with Groq -Interact with Groq as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. - -```python -from groq import Groq - -client = Groq() - -chat_completion = client.chat.completions.create( - messages=[ - { - "role": "system", - "content": "you are a helpful assistant." - }, - { - "role": "user", - "content": "Explain the importance of fast language models", - } - ], - model="llama-3.3-70b-versatile", -) - -print(chat_completion.choices[0].message.content) -``` \ No newline at end of file diff --git a/src/pages/docs/tracing/auto/guardrails.mdx b/src/pages/docs/tracing/auto/guardrails.mdx deleted file mode 100644 index 17eca308..00000000 --- a/src/pages/docs/tracing/auto/guardrails.mdx +++ /dev/null @@ -1,77 +0,0 @@ ---- -title: "Guardrails AI Tracing with Future AGI: Auto-Instrumentation" -description: "Set up auto-instrumentation for Guardrails AI with Future AGI tracing. Install traceAI-guardrails to trace validation and LLM interaction spans." ---- - -## 1. Installation -First install the traceAI package to access the observability framework - -```bash -pip install traceAI-guardrails -``` - ---- - -## 2. Set Environment Variables - -Set up your environment variables to authenticate with both FutureAGI and OpenAI. - -```python -import os - -os.environ["OPENAI_API_KEY"] = "your-openai-api-key" -os.environ["FI_API_KEY"] = "your-futureagi-api-key" -os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" -``` - ---- - -## 3. Initialize Trace Provider - -Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . - -```python -from fi_instrumentation import register -from fi_instrumentation.fi_types import ProjectType - -trace_provider = register( - project_type=ProjectType.EXPERIMENT, - project_name="openai_project", -) -``` - ---- - -## 4. Instrument your Project - -Instrument your Project with OpenAI Agents Instrumentor. This step ensures that all interactions with the OpenAI are tracked and monitored. - -```python -from traceai_guardrails import GuardrailsInstrumentor - -GuardrailsInstrumentor().instrument(tracer_provider=trace_provider) -``` - ---- - -## 5. Interact with OpenAI Agents - -Interact with the OpenAI Agents as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. - -```python -from guardrails import Guard - -guard = Guard() - -result = guard( - messages=[ - { - "role": "user", - "content": "Tell me about OpenAI", - }, - ], - model="gpt-4o" -) - -print(f"{result}") -``` \ No newline at end of file diff --git a/src/pages/docs/tracing/auto/haystack.mdx b/src/pages/docs/tracing/auto/haystack.mdx deleted file mode 100644 index f5b49379..00000000 --- a/src/pages/docs/tracing/auto/haystack.mdx +++ /dev/null @@ -1,98 +0,0 @@ ---- -title: "Haystack Tracing with Future AGI: Auto-Instrumentation" -description: "Set up auto-instrumentation for Haystack with Future AGI tracing. Install traceAI-haystack to capture document pipeline and retrieval spans." ---- - -## 1. Installation -Install the traceAI and Haystack packages. - -```bash -pip install traceAI-haystack haystack-ai trafilatura -``` - ---- - -## 2. Set Environment Variables -Set up your environment variables to authenticate with FutureAGI. - -```python -import os - -os.environ["OPENAI_API_KEY"] = "your-openai-api-key" -os.environ["FI_API_KEY"] = "your-futureagi-api-key" -os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" -``` - ---- - -## 3. Initialize Trace Provider -Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . - -```python -from fi_instrumentation import register -from fi_instrumentation.fi_types import ProjectType - -trace_provider = register( - project_type=ProjectType.OBSERVE, - project_name="haystack_project", -) -``` - ---- - -## 4. Instrument your Project -Initialize the Haystack instrumentor to enable automatic tracing. - -```python -from traceai_haystack import HaystackInstrumentor - -HaystackInstrumentor().instrument(tracer_provider=trace_provider) -``` - ---- - -## 5. Create Haystack Components -Set up your Haystack components as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. - -```python - -from haystack import Pipeline -from haystack.components.fetchers import LinkContentFetcher -from haystack.components.converters import HTMLToDocument -from haystack.components.builders import ChatPromptBuilder -from haystack.components.generators.chat import OpenAIChatGenerator -from haystack.dataclasses import ChatMessage - -fetcher = LinkContentFetcher() -converter = HTMLToDocument() -prompt_template = [ - ChatMessage.from_user( - """ - According to the contents of this website: - {% for document in documents %} - {{document.content}} - {% endfor %} - Answer the given question: {{query}} - Answer: - """ - ) -] - -prompt_builder = ChatPromptBuilder(template=prompt_template) -llm = OpenAIChatGenerator() - -pipeline = Pipeline() -pipeline.add_component("fetcher", fetcher) -pipeline.add_component("converter", converter) -pipeline.add_component("prompt", prompt_builder) -pipeline.add_component("llm", llm) - -pipeline.connect("fetcher.streams", "converter.sources") -pipeline.connect("converter.documents", "prompt.documents") -pipeline.connect("prompt.prompt", "llm") - -result = pipeline.run({"fetcher": {"urls": ["https://haystack.deepset.ai/overview/quick-start"]}, - "prompt": {"query": "Which components do I need for a RAG pipeline?"}}) - -print(result["llm"]["replies"][0].text) -``` diff --git a/src/pages/docs/tracing/auto/index.mdx b/src/pages/docs/tracing/auto/index.mdx deleted file mode 100644 index 9eb6e48e..00000000 --- a/src/pages/docs/tracing/auto/index.mdx +++ /dev/null @@ -1,157 +0,0 @@ ---- -title: "Auto-Instrumentation Integrations: Future AGI Tracing" -description: "Auto-instrumentation integrations for LLM frameworks in Python, JavaScript, and Java. Install a traceAI package to start capturing spans automatically." ---- - -## About - -Auto-instrumentation adds tracing to your LLM applications with minimal code changes. Install the relevant `traceAI` package for your framework, register a trace provider, and FutureAGI captures spans, inputs, outputs, latency, and metadata automatically. - -Python and JS/TS integrations use instrumentors that patch client libraries. Java integrations use explicit `Traced*` wrappers around your existing clients. Both produce the same OpenTelemetry spans. - -## LLM Providers - - - - `traceAI-openai` - - - `traceAI-anthropic` - - - `traceAI-bedrock` - - - `traceAI-vertexai` - - - `traceAI-google-genai` - - - `traceai-google-adk` - - - `traceAI-groq` - - - `traceAI-mistralai` - - - `traceAI-openai` - - - `traceAI-openai` - - - `traceAI-portkey` - - - -## Frameworks & Agents - - - - `traceAI-langchain` - - - `traceAI-langchain` - - - `traceAI-llamaindex` - - - `traceAI-llamaindex` - - - `traceAI-litellm` - - - `traceAI-crewai` - - - `traceAI-autogen` - - - `traceAI-haystack` - - - `traceAI-DSPy` - - - `traceAI-openai-agents` - - - `traceAI-smolagents` - - - `traceAI-instructor` - - - `traceAI-openai` - - - `traceAI-guardrails` - - - `traceAI-mcp` - - - `@traceai/mastra` - - - `@traceai/vercel` - - - -## Voice & Realtime - - - - `traceAI-livekit` - - - `traceAI-pipecat` - - - -## Java - -The Java SDK uses explicit `Traced*` wrappers instead of instrumentors. Add a Maven/Gradle dependency, wrap your client, and traces flow to FutureAGI. See the [Java overview](/docs/tracing/auto/java) for core setup. - - - - `traceai-spring-boot-starter` - - - `traceai-java-openai` - - - `traceai-java-anthropic` - - - `traceai-java-bedrock` - - - `traceai-java-cohere` - - - `traceai-java-pinecone` - - - Google GenAI, Vertex AI, Azure OpenAI, Ollama, Watsonx - - - Qdrant, Milvus, ChromaDB, Weaviate, and 5 more - - - LangChain4j, Semantic Kernel - - - -## Other - - - - No-code workflow integration - - diff --git a/src/pages/docs/tracing/auto/instructor.mdx b/src/pages/docs/tracing/auto/instructor.mdx deleted file mode 100644 index d2be7649..00000000 --- a/src/pages/docs/tracing/auto/instructor.mdx +++ /dev/null @@ -1,84 +0,0 @@ ---- -title: "Instructor Tracing with Future AGI: Auto-Instrumentation" -description: "Set up auto-instrumentation for Instructor with Future AGI tracing. Install traceAI-instructor to capture structured output extraction spans." ---- - -## 1. Installation -Install the traceAI and other necessary packages. - -```bash -pip install traceAI-instructor instructor -``` - ---- - -## 2. Set Environment Variables -Set up your environment variables to authenticate with FutureAGI. - -```python -import os - -os.environ["OPENAI_API_KEY"] = "your-openai-api-key" -os.environ["FI_API_KEY"] = "your-futureagi-api-key" -os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" -``` - ---- - -## 3. Initialize Trace Provider -Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . - -```python -from fi_instrumentation import register -from fi_instrumentation.fi_types import ProjectType - -trace_provider = register( - project_type=ProjectType.OBSERVE, - project_name="Instructor", -) -``` - ---- - -## 4. Instrument your Project -Use the Instructor Instrumentor to instrument your project. - -```python -from traceai_instructor import InstructorInstrumentor - -InstructorInstrumentor().instrument(tracer_provider=trace_provider) -``` - ---- - -## 5. Run your Instructor application. -Run your Instructor application as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. - -```python -import instructor - -from openai import OpenAI -from pydantic import BaseModel - -# Define the output structure -class UserInfo(BaseModel): - name: str - age: int - -# Patch the OpenAI client -client = instructor.patch(client=OpenAI()) - -user_info = client.chat.completions.create( - model="gpt-3.5-turbo", - response_model=UserInfo, - messages=[ - { - "role": "system", - "content": "Extract the name and age from the text and return them in a structured format.", - }, - {"role": "user", "content": "John Doe is nine years old."}, - ], -) - -print(user_info, type(user_info)) -``` diff --git a/src/pages/docs/tracing/auto/langchain.mdx b/src/pages/docs/tracing/auto/langchain.mdx deleted file mode 100644 index df66c4de..00000000 --- a/src/pages/docs/tracing/auto/langchain.mdx +++ /dev/null @@ -1,132 +0,0 @@ ---- -title: "LangChain Tracing with Future AGI: Auto-Instrumentation" -description: "Set up auto-instrumentation for LangChain with Future AGI tracing. Install traceAI-langchain to capture chain, tool, and LLM call spans." ---- - -## 1. Installation -First install the traceAI package and necessary LangChain packages. - - - -```bash Python -pip install traceAI-langchain -pip install langchain_openai -``` - -```bash JS/TS -npm install @traceai/langchain @traceai/fi-core @opentelemetry/instrumentation \ - @langchain/openai @langchain/core -``` - - ---- - -## 2. Set Environment Variables - -Set up your environment variables to authenticate with both FutureAGI and OpenAI. - - - -```python Python -import os - -os.environ["OPENAI_API_KEY"] = "your-openai-api-key" -os.environ["FI_API_KEY"] = "your-futureagi-api-key" -os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" -``` - -```typescript JS/TS -process.env.OPENAI_API_KEY = "your-openai-api-key"; -process.env.FI_API_KEY = "your-futureagi-api-key"; -process.env.FI_SECRET_KEY = "your-futureagi-secret-key"; -``` - - - ---- - -## 3. Initialize Trace Provider -Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . - - - -```python Python -from fi_instrumentation import register -from fi_instrumentation.fi_types import ProjectType - -trace_provider = register( - project_type=ProjectType.OBSERVE, - project_name="langchain_project", -) -``` - -```typescript JS/TS -import { register, ProjectType } from "@traceai/fi-core"; - -const tracerProvider = register({ - project_type: ProjectType.OBSERVE, - project_name: "langchain_project", -}); -``` - - - ---- - -## 4. Instrument your Project -Initialize the LangChain Instrumentor to enable automatic tracing. This step ensures that all interactions with the LangChain are tracked and monitored. - - - -```python Python -from traceai_langchain import LangChainInstrumentor - -LangChainInstrumentor().instrument(tracer_provider=trace_provider) -``` - -```typescript JS/TS -import { LangChainInstrumentation } from "@traceai/langchain"; -import * as CallbackManagerModule from "langchain/callbacks"; - -// Pass the custom tracer provider to the instrumentation -const lcInstrumentation = new LangChainInstrumentation({ - tracerProvider: tracerProvider, -}); - -// Manually instrument the LangChain module -lcInstrumentation.manuallyInstrument(CallbackManagerModule); -``` - - - ---- - -## 5. Create LangChain Components -Set up your LangChain pipeline as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. - - - -```python Python -from langchain_openai import ChatOpenAI -from langchain_core.prompts import ChatPromptTemplate - -prompt = ChatPromptTemplate.from_template("{x} {y} {z}?").partial(x="why is", z="blue") -chain = prompt | ChatOpenAI(model_name="gpt-3.5-turbo") - -result = chain.invoke({"y": "sky"}) - -print(f"Response: {result}") -``` - -```typescript JS/TS -import { ChatOpenAI } from "@langchain/openai"; -import { ChatPromptTemplate } from "@langchain/core/prompts"; - -const prompt = ChatPromptTemplate.fromTemplate("{x} {y} {z}?").partial({ x: "why is", z: "blue" }); -const chain = prompt.pipe(new ChatOpenAI({ model: "gpt-3.5-turbo" })); - -const result = await chain.invoke({ y: "sky" }); -console.log("Response:", result); -``` - - \ No newline at end of file diff --git a/src/pages/docs/tracing/auto/langgraph.mdx b/src/pages/docs/tracing/auto/langgraph.mdx deleted file mode 100644 index e78df745..00000000 --- a/src/pages/docs/tracing/auto/langgraph.mdx +++ /dev/null @@ -1,96 +0,0 @@ ---- -title: "LangGraph Tracing with Future AGI: Auto-Instrumentation" -description: "Set up auto-instrumentation for LangGraph with Future AGI tracing. Capture agent graph execution and state transition spans via LangChain instrumentor." ---- - -Our [LangChainInstrumentor](/docs/tracing/auto/langchain) automatically captures traces for both LangGraph and LangChain. If you've already enabled that instrumentor, you do not need to complete the steps below. - -## 1. Installation -First install the traceAI package and necessary LangChain packages. - -```bash -pip install traceAI-langchain -pip install langgraph -pip install langchain-anthropic -pip install ipython -``` ---- - -## 2. Set Environment Variables - -Set up your environment variables to authenticate with both FutureAGI and Anthropic. - -```python -import os - -os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-api-key" -os.environ["FI_API_KEY"] = "your-futureagi-api-key" -os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" -``` - ---- - -## 3. Initialize Trace Provider -Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . - -```python -from fi_instrumentation import register -from fi_instrumentation.fi_types import ProjectType - -trace_provider = register( - project_type=ProjectType.OBSERVE, - project_name="langgraph_project", -) -``` - ---- - -## 4. Instrument your Project -Initialize the LangChain Instrumentor to enable automatic tracing. Our [LangChainInstrumentor](/docs/tracing/auto/langchain) automatically captures traces for both LangGraph and LangChain. - -```python -from traceai_langchain import LangChainInstrumentor - -LangChainInstrumentor().instrument(tracer_provider=trace_provider) -``` - ---- - -## 5. Create LangGraph Agents -Set up your LangGraph agents as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. - -```python -from typing import Annotated -from typing_extensions import TypedDict -from langgraph.graph import StateGraph, START, END -from langgraph.graph.message import add_messages -from langchain_anthropic import ChatAnthropic -from IPython.display import Image, display - -class State(TypedDict): - messages: Annotated[list, add_messages] - -graph_builder = StateGraph(State) -llm = ChatAnthropic(model="claude-3-5-sonnet-20240620") - -def chatbot(state: State): - return {"messages": [llm.invoke(state["messages"])]} - -graph_builder.add_node("chatbot", chatbot) -graph_builder.add_edge(START, "chatbot") -graph_builder.add_edge("chatbot", END) -graph = graph_builder.compile() - -try: - display(Image(graph.get_graph().draw_mermaid_png())) -except Exception: - pass - -def stream_graph_updates(user_input: str): - for event in graph.stream({"messages": [{"role": "user", "content": user_input}]}): - for value in event.values(): - print("Assistant:", value["messages"][-1].content) - -user_input = "What do you know about LangGraph?" -stream_graph_updates(user_input) -``` \ No newline at end of file diff --git a/src/pages/docs/tracing/auto/litellm.mdx b/src/pages/docs/tracing/auto/litellm.mdx deleted file mode 100644 index d3d91f66..00000000 --- a/src/pages/docs/tracing/auto/litellm.mdx +++ /dev/null @@ -1,67 +0,0 @@ ---- -title: "LiteLLM Tracing with Future AGI: Auto-Instrumentation" -description: "Set up auto-instrumentation for LiteLLM with Future AGI tracing. Install traceAI-litellm to capture spans across multiple LLM provider calls." ---- - -## 1. Installation -Install the traceAI and litellm packages. - -```bash -pip install traceAI-litellm -pip install litellm -``` - ---- - -## 2. Set Environment Variables -Set up your environment variables to authenticate with both FutureAGI and OpenAI. - -```python -import os - -os.environ["FI_API_KEY"] = "your-futureagi-api-key" -os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" -os.environ["OPENAI_API_KEY"] = "your-openai-api-key" -``` - ---- - -## 3. Initialize Trace Provider -Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . - -```python -from fi_instrumentation import register -from fi_instrumentation.fi_types import ProjectType - -trace_provider = register( - project_type=ProjectType.OBSERVE, - project_name="openai_project", -) -``` - ---- - -## 4. Configure LiteLLM Instrumentation -Initialize the LiteLLM instrumentor to enable automatic tracing. - -```python -from traceai_litellm import LiteLLMInstrumentor - -LiteLLMInstrumentor().instrument(tracer_provider=trace_provider) -``` - ---- - -## 5. Run LiteLLM -Run LiteLLM as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. - -```python -import litellm - -response = litellm.completion( - model="gpt-3.5-turbo", - messages=[{"content": "What's the capital of India?"}], -) - -print(response.choices[0].message.content) -``` \ No newline at end of file diff --git a/src/pages/docs/tracing/auto/llamaindex-workflows.mdx b/src/pages/docs/tracing/auto/llamaindex-workflows.mdx deleted file mode 100644 index 8b29d5a1..00000000 --- a/src/pages/docs/tracing/auto/llamaindex-workflows.mdx +++ /dev/null @@ -1,106 +0,0 @@ ---- -title: "LlamaIndex Workflows Tracing with Future AGI" -description: "Set up auto-instrumentation for LlamaIndex Workflows with Future AGI tracing. Trace workflow agent execution via the LlamaIndex instrumentor." ---- - -[LlamaIndex Workflows](https://www.llamaindex.ai/blog/introducing-workflows-beta-a-new-way-to-create-complex-ai-applications-with-llamaindex) are a subset of the LlamaIndex package specifically designed to support agent development. - -Our [LlamaIndexInstrumentor](/docs/tracing/auto/llamaindex) automatically captures traces for LlamaIndex Workflows agents. If you've already enabled that instrumentor, you do not need to complete the steps below. - -## 1. Installation -First install the traceAI and necessary llama-index packages. -```bash -pip install traceAI-llamaindex -pip install llama-index -``` - ---- - -## 2. Set Environment Variables - -Set up your environment variables to authenticate with FutureAGI. - -```python -import os - -os.environ["FI_API_KEY"] = "your-futureagi-api-key" -os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" -os.environ["OPENAI_API_KEY"] = "your-openai-api-key" -``` - ---- - -## 3. Initialize Trace Provider - -Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . - -```python -from fi_instrumentation import register -from fi_instrumentation.fi_types import ProjectType - -trace_provider = register( - project_type=ProjectType.OBSERVE, - project_name="openai_project", -) -``` - ---- - -## 4. Instrument your Project - -Instrument your Project with LlamaIndex Instrumentor. This instrumentor will trace both LlamaIndex Workflows calls, as well as calls to the general LlamaIndex package. - -```python -from traceai_llamaindex import LlamaIndexInstrumentor - -LlamaIndexInstrumentor().instrument(tracer_provider=trace_provider) -``` - ---- - -## 5. Run LlamaIndex Workflows - -Run your LlamaIndex workflows as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. - -```python -import asyncio - -from llama_index.core.workflow import ( - Event, - StartEvent, - StopEvent, - Workflow, - step, -) -from llama_index.llms.openai import OpenAI - -class JokeEvent(Event): - joke: str - -class JokeFlow(Workflow): - llm = OpenAI() - - @step - async def generate_joke(self, ev: StartEvent) -> JokeEvent: - topic = ev.topic - - prompt = f"Write your best joke about {topic}." - response = await self.llm.acomplete(prompt) - return JokeEvent(joke=str(response)) - - @step - async def critique_joke(self, ev: JokeEvent) -> StopEvent: - joke = ev.joke - - prompt = f"Give a thorough analysis and critique of the following joke: {joke}" - response = await self.llm.acomplete(prompt) - return StopEvent(result=str(response)) - -async def main(): - w = JokeFlow(timeout=60, verbose=False) - result = await w.run(topic="pirates") - print(str(result)) - -if __name__ == "__main__": - asyncio.run(main()) -``` \ No newline at end of file diff --git a/src/pages/docs/tracing/auto/llamaindex.mdx b/src/pages/docs/tracing/auto/llamaindex.mdx deleted file mode 100644 index 8fd33d9f..00000000 --- a/src/pages/docs/tracing/auto/llamaindex.mdx +++ /dev/null @@ -1,80 +0,0 @@ ---- -title: "LlamaIndex Tracing with Future AGI: Auto-Instrumentation" -description: "Set up auto-instrumentation for LlamaIndex with Future AGI tracing. Install traceAI-llamaindex to capture query, retrieval, and response spans." ---- - -## 1. Installation -Install the traceAI and Llama Index packages. - -```bash -pip install traceAI-llamaindex -pip install llama-index -``` - ---- - -## 2. Set Environment Variables -Set up your environment variables to authenticate with FutureAGI. - -```python -import os - -os.environ["FI_API_KEY"] = "your-futureagi-api-key" -os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" -os.environ["OPENAI_API_KEY"] = "your-openai-api-key" -``` - ---- - -## 3. Initialize Trace Provider -Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . - -```python -from fi_instrumentation import register -from fi_instrumentation.fi_types import ProjectType - -trace_provider = register( - project_type=ProjectType.OBSERVE, - project_name="llamaindex_project", -) -``` - ---- - -## 4. Instrument your Project -Initialize the Llama Index instrumentor to enable automatic tracing. This step ensures that all interactions with the Llama Index are tracked and monitored. - -```python -from traceai_llamaindex import LlamaIndexInstrumentor - -LlamaIndexInstrumentor().instrument(tracer_provider=trace_provider) -``` - ---- - -## 5. Create Llama Index Components -Set up your Llama Index components as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. - -```python -from llama_index.agent.openai import OpenAIAgent -from llama_index.core import Settings -from llama_index.core.tools import FunctionTool -from llama_index.llms.openai import OpenAI - -def multiply(a: int, b: int) -> int: - """Multiply two integers and return the result.""" - return a * b - -def add(a: int, b: int) -> int: - """Add two integers and return the result.""" - return a + b - -multiply_tool = FunctionTool.from_defaults(fn=multiply) -add_tool = FunctionTool.from_defaults(fn=add) -agent = OpenAIAgent.from_tools([multiply_tool, add_tool]) -Settings.llm = OpenAI(model="gpt-3.5-turbo") - -response = agent.query("What is (121 * 3) + 42?") - -print(response) -``` diff --git a/src/pages/docs/tracing/auto/mastra.mdx b/src/pages/docs/tracing/auto/mastra.mdx deleted file mode 100644 index 6e34465a..00000000 --- a/src/pages/docs/tracing/auto/mastra.mdx +++ /dev/null @@ -1,119 +0,0 @@ ---- -title: "Mastra Tracing with Future AGI: Auto-Instrumentation" -description: "Set up auto-instrumentation for Mastra with Future AGI tracing. Configure @traceai/mastra to export TypeScript agent spans to Future AGI." ---- - -Auto-instrument your [Mastra](https://mastra.ai) agents and workflows with Future AGI. -Every agent run, tool call, and LLM interaction is exported to Future AGI for monitoring, -evaluation, and debugging — no manual span code required. - - -This guide targets **Mastra v1** (`@mastra/core` ≥ 1.16). Mastra v1 removed the old -`telemetry:` config key, so the previous `FITraceExporter` setup no longer exports any -spans. Use `createFIObservability` from `@traceai/mastra` as shown below. Still on -Mastra v0.x? See [Legacy (Mastra v0.x)](#legacy-mastra-v0x) at the bottom. - - -## 1. Installation -Install Mastra's observability packages, the OTLP/protobuf exporter, and `@traceai/mastra`. - -```bash JS/TS -npm install @mastra/core @mastra/observability @mastra/otel-exporter \ - @opentelemetry/exporter-trace-otlp-proto @traceai/mastra -``` - ---- - -## 2. Set Environment Variables - -Add your Future AGI credentials to your `.env`. `@traceai/mastra` reads them automatically. - -```bash .env -FI_API_KEY=your-futureagi-api-key -FI_SECRET_KEY=your-futureagi-secret-key -``` - ---- - -## 3. Configure Observability -Wire Future AGI into your Mastra instance with `createFIObservability`. It points the -exporter at Future AGI's collector, authenticates with your keys, and exports traces only -(Future AGI's collector does not accept the OTLP logs signal). - -```typescript JS/TS -import { Mastra } from "@mastra/core"; -import { createFIObservability } from "@traceai/mastra"; - -export const mastra = new Mastra({ - // ... your agents, workflows, etc. - observability: createFIObservability({ - serviceName: "traceai-mastra-agent", // appears in the Future AGI trace list - }), -}); -``` - -No changes are needed to your agent code. Every span is mapped to OpenTelemetry -`gen_ai.*` conventions, given the right span kind (LLM / agent / tool / chain), and -its input/output is captured — so the trace renders fully in Future AGI. - ---- - -## 4. Run your Agent -Run your Mastra agent as usual. Traces appear in your Future AGI project under -**Observability** (service `traceai-mastra-agent`). - -```typescript JS/TS -const agent = mastra.getAgent("yourAgent"); -const result = await agent.generate("What's the weather in Bangalore?"); -``` - - -**Short-lived scripts & serverless.** Spans are batched, so a process that exits -immediately may drop them. Keep a reference to the observability instance and flush -before exit: - -```typescript JS/TS -export const observability = createFIObservability({ serviceName: "traceai-mastra-agent" }); -export const mastra = new Mastra({ observability /* , agents, ... */ }); - -// at the end of your script / request handler: -await observability.shutdown(); // flushes buffered spans -``` - - ---- - -## Configuration Options - -`createFIObservability(options)` accepts: - -| Option | Default | Description | -| --- | --- | --- | -| `serviceName` | `"mastra-app"` | Service name; also the default Future AGI **project** name. | -| `projectName` | `serviceName` (or `FI_PROJECT_NAME`) | Future AGI project the traces are filed under. | -| `projectType` | `"observe"` | `"observe"` for tracing, `"experiment"` for eval runs. | -| `apiKey` | `process.env.FI_API_KEY` | Future AGI API key. | -| `secretKey` | `process.env.FI_SECRET_KEY` | Future AGI secret key. | -| `baseUrl` | `https://api.futureagi.com` | Collector base URL (`/tracer/v1/traces` is appended). | -| `endpoint` | — | Full traces endpoint URL; overrides `baseUrl`. | -| `headers` | — | Extra headers merged into the export request. | -| `excludeSpanTypes` | `[MODEL_CHUNK]` | Mastra span types to drop before export (chunk spans are noise). | -| `timeout` | `30000` | Export request timeout (ms). | -| `batchSize` | — | Spans per batch. | - -If you need to compose the exporter into your own `Observability` config, use -`createFIMastraExporter(options)` instead — it returns a pre-configured exporter you -can drop into `new Observability({ configs: { otel: { exporters: [...] } } })`. - ---- - -## Legacy (Mastra v0.x) - -The old `telemetry:` + `FITraceExporter` integration is deprecated and does **not** work -on Mastra v1. If you are still on Mastra v0.x, import it from the `/legacy` subpath: - -```typescript JS/TS -import { FITraceExporter, isFISpan } from "@traceai/mastra/legacy"; -``` - -We recommend upgrading to Mastra v1 and the `createFIObservability` setup above. diff --git a/src/pages/docs/tracing/auto/mistralai.mdx b/src/pages/docs/tracing/auto/mistralai.mdx deleted file mode 100644 index 41de68f0..00000000 --- a/src/pages/docs/tracing/auto/mistralai.mdx +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: "Mistral AI Tracing with Future AGI: Auto-Instrumentation" -description: "Set up auto-instrumentation for Mistral AI with Future AGI tracing. Install traceAI-mistralai to capture model inference spans and metadata." ---- - -## 1. Installation -Install the traceAI package to access the observability framework. - -```bash -pip install traceAI-mistralai -``` - ---- - -## 2. Set Environment Variables -Set up your environment variables to authenticate with both FutureAGI and MistralAI . - -```python -import os - -os.environ["FI_API_KEY"] = "your-futureagi-api-key" -os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" -os.environ["MISTRAL_API_KEY"] = "your-mistral-api-key" -``` - ---- - -## 3. Initialize Trace Provider -Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . - -```python -from fi_instrumentation import register -from fi_instrumentation.fi_types import ProjectType - -trace_provider = register( - project_type=ProjectType.OBSERVE, - project_name="mistralai_project", -) -``` - ---- - -## 4. Instrument your Project -Instrument your Project with MistralAI Instrumentor. This step ensures that all interactions with the MistralAI are tracked and monitored. - -```python -from traceai_mistralai import MistralAIInstrumentor - -MistralAIInstrumentor().instrument(tracer_provider=trace_provider) -``` - ---- - -## 5. Create Mistral AI Components -Set up your Mistral AI client and use your application as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. - -```python -from mistralai import Mistral - -client = Mistral(api_key=os.environ["MISTRAL_API_KEY"]) - -response = client.agents.complete( - agent_id="agent_id", - messages=[ - {"role": "user", "content": "plan a vacation for me in Tbilisi"}, - ], -) - -print(response) -``` \ No newline at end of file diff --git a/src/pages/docs/tracing/auto/ollama.mdx b/src/pages/docs/tracing/auto/ollama.mdx deleted file mode 100644 index d62219ee..00000000 --- a/src/pages/docs/tracing/auto/ollama.mdx +++ /dev/null @@ -1,78 +0,0 @@ ---- -title: "Ollama Tracing with Future AGI: Auto-Instrumentation" -description: "Set up auto-instrumentation for Ollama with Future AGI tracing. Use traceAI-openai to capture spans from Ollama's OpenAI-compatible local LLM API." ---- - -## 1. Installation -First install the traceAI package to access the observability framework - -```bash -pip install traceAI-openai -``` - ---- - -## 2. Set Environment Variables - -Set up your environment variables to authenticate with FutureAGI. - -```python -import os - -os.environ["FI_API_KEY"] = "your-futureagi-api-key" -os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" -``` - ---- - -## 3. Initialize Trace Provider - -Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . - -```python -from fi_instrumentation import register -from fi_instrumentation.fi_types import ProjectType - -trace_provider = register( - project_type=ProjectType.OBSERVE, - project_name="OLLAMA 3.2", -) -``` - ---- - -## 4. Instrument your Project - -Use the OpenAI Instrumentor to instrument your project, as the OpenAI Client is utilized for interactions with Ollama. This step guarantees that all interactions are tracked and monitored. If you are using a different client to interact with Ollama, use that client's Instrumentor instead. - -```python -from traceai_openai import OpenAIInstrumentor - -OpenAIInstrumentor().instrument(tracer_provider=trace_provider) -``` - ---- - -## 5. Interact with Ollama - -Interact with the Ollama as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. -Make sure that Ollama is running and accessible from your project. - -```python -from openai import OpenAI - -client = OpenAI( - base_url = 'http://localhost:11434/v1', - api_key='ollama', -) - -response = client.chat.completions.create( - model="llama3.2:1b", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What is OpenAI?"}, - ] - ) - -print(response.choices[0].message.content) -``` \ No newline at end of file diff --git a/src/pages/docs/tracing/auto/openai_agents.mdx b/src/pages/docs/tracing/auto/openai_agents.mdx deleted file mode 100644 index 307e1250..00000000 --- a/src/pages/docs/tracing/auto/openai_agents.mdx +++ /dev/null @@ -1,68 +0,0 @@ ---- -title: "OpenAI Agents SDK Tracing with Future AGI" -description: "Set up auto-instrumentation for OpenAI Agents SDK with Future AGI tracing. Install traceAI-openai-agents to capture agent workflow spans." ---- - -## 1. Installation -First install the traceAI package to access the observability framework - -```bash -pip install traceAI-openai-agents -``` - ---- - -## 2. Set Environment Variables - -Set up your environment variables to authenticate with both FutureAGI and OpenAI. - -```python -import os - -os.environ["OPENAI_API_KEY"] = "your-openai-api-key" -os.environ["FI_API_KEY"] = "your-futureagi-api-key" -os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" -``` - ---- - -## 3. Initialize Trace Provider - -Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . - -```python -from fi_instrumentation import register -from fi_instrumentation.fi_types import ProjectType - -trace_provider = register( - project_type=ProjectType.EXPERIMENT, - project_name="openai_project", -) -``` - ---- - -## 4. Instrument your Project - -Instrument your Project with OpenAI Agents Instrumentor. This step ensures that all interactions with the OpenAI are tracked and monitored. - -```python -from traceai_openai_agents import OpenAIAgentsInstrumentor - -OpenAIAgentsInstrumentor().instrument(tracer_provider=trace_provider) -``` - ---- - -## 5. Interact with OpenAI Agents - -Interact with the OpenAI Agents as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. - -```python -from agents import Agent, Runner - -agent = Agent(name="Assistant", instructions="You are a helpful assistant") -result = Runner.run_sync(agent, "Write a haiku about recursion in programming.") - -print(result.final_output) -``` \ No newline at end of file diff --git a/src/pages/docs/tracing/auto/portkey.mdx b/src/pages/docs/tracing/auto/portkey.mdx deleted file mode 100644 index 8e818d4d..00000000 --- a/src/pages/docs/tracing/auto/portkey.mdx +++ /dev/null @@ -1,66 +0,0 @@ ---- -title: "Portkey Tracing with Future AGI: Auto-Instrumentation" -description: "Set up auto-instrumentation for Portkey with Future AGI tracing. Install traceAI-portkey to capture routed LLM call spans and gateway metrics." ---- - -## 1. Installation -Install the traceAI and Portkey packages. - -```bash -pip install portkey_ai traceAI-portkey -``` - ---- - -## 2. Set Environment Variables -Set up your environment variables to authenticate with both FutureAGI and Portkey. - -```python -import os - -os.environ["FI_API_KEY"] = "your-futureagi-api-key" -os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" -os.environ["PORTKEY_VIRTUAL_KEY"] = "your-portkey-virtual-key" -``` - ---- - -## 3. Initialize Trace Provider -Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . - -```python -from fi_instrumentation import register -from fi_instrumentation.fi_types import ProjectType - -trace_provider = register( - project_type=ProjectType.OBSERVE, - project_name="portkey_project", -) -``` - ---- -## 4. Instrument your Project -Instrument your project to enable automatic tracing. - -```python -from traceai_portkey import PortkeyInstrumentor - -PortkeyInstrumentor().instrument(tracer_provider=trace_provider) -``` - ---- -## 5. Interact with Portkey -Interact with Portkey as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. - -```python -from portkey_ai import Portkey - -client = Portkey(virtual_key=os.environ["PORTKEY_VIRTUAL_KEY"]) - -completion = client.chat.completions.create( - model="gpt-4o", - messages=[{"role": "user", "content": "Write a 6-word story about a robot who discovers music."}] -) - -print(completion.choices[0].message.content) -``` \ No newline at end of file diff --git a/src/pages/docs/tracing/auto/promptflow.mdx b/src/pages/docs/tracing/auto/promptflow.mdx deleted file mode 100644 index 8e1b2abf..00000000 --- a/src/pages/docs/tracing/auto/promptflow.mdx +++ /dev/null @@ -1,154 +0,0 @@ ---- -title: "Prompt Flow Tracing with Future AGI: Auto-Instrumentation" -description: "Set up auto-instrumentation for Prompt Flow with Future AGI tracing. Use traceAI-openai to capture prompt flow execution and LLM call spans." ---- - -## 1. Installation -First install the traceAI and promptflow packages. - -```bash -pip install traceAI-openai promptflow promptflow-tools -``` - ---- - -## 2. Set Environment Variables - -Set up your environment variables to authenticate with both FutureAGI and OpenAI services. - -```python -import os - -os.environ["OPENAI_API_KEY"] = "your-openai-api-key" -os.environ["FI_API_KEY"] = "your-futureagi-api-key" -os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" -``` - ---- - -## 3. Initialize Trace Provider - -Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . - -```python -from fi_instrumentation import register -from fi_instrumentation.fi_types import ProjectType - -trace_provider = register( - project_type=ProjectType.OBSERVE, - project_name="promptflow", -) -``` - ---- - -## 4. Instrument your Project - -Instrument your Project with OpenAI Instrumentor. This step ensures that all interactions with the PromptFlow are tracked and monitored. - -```python -from traceai_openai import OpenAIInstrumentor - -OpenAIInstrumentor().instrument(tracer_provider=trace_provider) -``` - ---- -## 5. Prepare the `chat.prompty` File - -Create a `chat.prompty` file in the same directory as your script with the following content: - -```yaml ---- -name: Basic Chat -model: - api: chat - configuration: - type: azure_openai - azure_deployment: gpt-4o - parameters: - temperature: 0.2 - max_tokens: 1024 -inputs: - question: - type: string - chat_history: - type: list -sample: - question: "What is Prompt flow?" - chat_history: [] ---- - -system: -You are a helpful assistant. - -{% for item in chat_history %} -{{item.role}}: -{{item.content}} -{% endfor %} - -user: -{{question}} -``` - -This will ensure that users have the necessary configuration to create the `chat.prompty` file and use it with the `ChatFlow` class. - ---- - -## 6. Create a Flow - -Create a Flow as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. - -```python -from pathlib import Path -from promptflow.core import OpenAIModelConfiguration, Prompty - -BASE_DIR = Path(__file__).absolute().parent - -class ChatFlow: - def __init__(self, model_config: OpenAIModelConfiguration, max_total_token=4096): - self.model_config = model_config - self.max_total_token = max_total_token - - def __call__( - self, - question: str = "What's Azure Machine Learning?", - chat_history: list = [], - ) -> str: - """Flow entry function.""" - - prompty = Prompty.load( - source=BASE_DIR / "chat.prompty", - model={"configuration": self.model_config}, - ) - - output = prompty(question=question, chat_history=chat_history) - - return output -``` - ---- - -## 7. Execute the Flow - -```python -from promptflow.client import PFClient -from promptflow.connections import OpenAIConnection - -pf = PFClient() - -connection = OpenAIConnection( - name="open_ai_connection", - base_url="https://api.openai.com/v1", - api_key=os.environ["OPENAI_API_KEY"], -) - -conn = pf.connections.create_or_update(connection) - -config = OpenAIModelConfiguration( - connection="open_ai_connection", model="gpt-3.5-turbo" -) - -chat_flow = ChatFlow(config) -result = chat_flow(question="What is ChatGPT? Please explain with concise statement") -print(result) -``` \ No newline at end of file diff --git a/src/pages/docs/tracing/auto/smol_agents.mdx b/src/pages/docs/tracing/auto/smol_agents.mdx deleted file mode 100644 index 5ecbe11a..00000000 --- a/src/pages/docs/tracing/auto/smol_agents.mdx +++ /dev/null @@ -1,90 +0,0 @@ ---- -title: "Smol Agents Tracing with Future AGI: Auto-Instrumentation" -description: "Set up auto-instrumentation for Smol Agents with Future AGI tracing. Install traceAI-smolagents to capture lightweight agent execution spans." ---- - -## 1. Installation -First install the traceAI and necessary dependencies. - -```bash -pip install traceAI-smolagents smolagents -``` - ---- - -## 2. Set Environment Variables - -Set up your environment variables to authenticate with both FutureAGI and OpenAI. - -```python -import os - -os.environ["OPENAI_API_KEY"] = "your-openai-api-key" -os.environ["FI_API_KEY"] = "your-futureagi-api-key" -os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" -``` - ---- - -## 3. Initialize Trace Provider - -Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . - -```python -from fi_instrumentation import register -from fi_instrumentation.fi_types import ProjectType - -trace_provider = register( - project_type=ProjectType.OBSERVE, - project_name="smolagents", -) -``` - ---- - -## 4. Instrument your Project - -Instrument your Project with SmolagentsInstrumentor . This step ensures that all interactions with the Agents are tracked and monitored. - -```python -from traceai_smolagents import SmolagentsInstrumentor - -SmolagentsInstrumentor().instrument(tracer_provider=trace_provider) -``` - ---- - -## 5. Interact with Smol Agents - -Interact with you Smol Agents as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. - -```python -from smolagents import ( - CodeAgent, - DuckDuckGoSearchTool, - OpenAIServerModel, - ToolCallingAgent, -) - -model = OpenAIServerModel(model_id="gpt-4o") -agent = ToolCallingAgent( - tools=[DuckDuckGoSearchTool()], - model=model, - max_steps=3, - name="search", - description=( - "This is an agent that can do web search. " - "When solving a task, ask him directly first, he gives good answers. " - "Then you can double check." - ), -) -manager_agent = CodeAgent( - tools=[DuckDuckGoSearchTool()], - model=model, - managed_agents=[agent], -) -manager_agent.run( - "How many seconds would it take for a leopard at full speed to run through Pont des Arts? " - "ASK YOUR MANAGED AGENT FOR LEOPARD SPEED FIRST" -) -``` \ No newline at end of file diff --git a/src/pages/docs/tracing/auto/togetherai.mdx b/src/pages/docs/tracing/auto/togetherai.mdx deleted file mode 100644 index 4f4f5c83..00000000 --- a/src/pages/docs/tracing/auto/togetherai.mdx +++ /dev/null @@ -1,78 +0,0 @@ ---- -title: "Together AI Tracing with Future AGI: Auto-Instrumentation" -description: "Set up auto-instrumentation for Together AI with Future AGI tracing. Use traceAI-openai to capture inference spans from Together AI models." ---- - -## 1. Installation -First install the traceAI package to access the observability framework - -```bash -pip install traceAI-openai -``` - ---- - -## 2. Set Environment Variables - -Set up your environment variables to authenticate with both FutureAGI and OpenAI services. - -```python -import os - -os.environ["TOGETHER_API_KEY"] = "your-together-api-key" -os.environ["FI_API_KEY"] = "your-futureagi-api-key" -os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" -``` - ---- - -## 3. Initialize Trace Provider - -Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . - -```python -from fi_instrumentation import register -from fi_instrumentation.fi_types import ProjectType - -trace_provider = register( - project_type=ProjectType.OBSERVE, - project_name="togetherai_project", -) -``` - ---- - -## 4. Instrument your Project - -Use the OpenAI Instrumentor to instrument your project, as the OpenAI Client is utilized for interactions with Together AI. This step guarantees that all interactions are tracked and monitored. If you are using a different client to interact with Together AI, use that client's Instrumentor instead. - -```python -from traceai_openai import OpenAIInstrumentor - -OpenAIInstrumentor().instrument(tracer_provider=trace_provider) -``` - ---- - -## 5. Interact with Together AI - -Interact with the Together AI through OpenAI Client. Our OpenAI Instrumentor will automatically trace and send the telemetry data to our platform. - -```python -import openai - -client = openai.OpenAI( - api_key=os.environ.get("TOGETHER_API_KEY"), - base_url="https://api.together.xyz/v1", -) - -response = client.chat.completions.create( - model="meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", - messages=[ - {"role": "system", "content": "You are a travel agent. Be descriptive and helpful."}, - {"role": "user", "content": "Tell me the top 3 things to do in San Francisco"}, - ] -) - -print(response.choices[0].message.content) -``` \ No newline at end of file diff --git a/src/pages/docs/tracing/auto/vercel.mdx b/src/pages/docs/tracing/auto/vercel.mdx deleted file mode 100644 index 7ea04eba..00000000 --- a/src/pages/docs/tracing/auto/vercel.mdx +++ /dev/null @@ -1,112 +0,0 @@ ---- -title: "Vercel AI SDK Tracing with Future AGI: Auto-Instrumentation" -description: "Set up auto-instrumentation for Vercel AI SDK with Future AGI tracing. Install @traceai/vercel to capture AI function call spans in Next.js apps." ---- - -## 1. Installation -First install the TraceAI + Vercel packages (and OpenTelemetry peer deps). Pick your favourite package manager: - - - -```bash npm -npm install @traceai/vercel @vercel/otel \ - @opentelemetry/api @opentelemetry/sdk-trace-base \ - @opentelemetry/exporter-trace-otlp-grpc @grpc/grpc-js \ - @ai-sdk/openai -``` - -```bash yarn -yarn add @traceai/vercel @vercel/otel \ - @opentelemetry/api @opentelemetry/sdk-trace-base \ - @opentelemetry/exporter-trace-otlp-grpc @grpc/grpc-js \ - @ai-sdk/openai -``` - -```bash pnpm -pnpm add @traceai/vercel @vercel/otel \ - @opentelemetry/api @opentelemetry/sdk-trace-base \ - @opentelemetry/exporter-trace-otlp-grpc @grpc/grpc-js \ - @ai-sdk/openai -``` - - - -> **Note** Vercel currently supports OpenTelemetry **v1.x**. Avoid installing `@opentelemetry/*` 2.x packages. - ---- - -## 2. Set Environment Variables -Configure your Future AGI credentials (locally via `.env`, or in Vercel **Project → Settings → Environment Variables**). - -```bash -FI_API_KEY= -FI_SECRET_KEY= -``` - ---- - -## 3. Initialise tracing -Create `instrumentation.ts` and import it **once** on the server (e.g. in `_app.tsx` or at the top of your first API route). - -```typescript JS/TS title="instrumentation.ts" -// eslint-disable-next-line @typescript-eslint/ban-ts-comment -// @ts-ignore : module ships without types -import { registerOTel } from "@vercel/otel"; -import { diag, DiagConsoleLogger, DiagLogLevel } from "@opentelemetry/api"; -import { FISimpleSpanProcessor, isFISpan } from "@traceai/vercel"; -import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc"; -import { Metadata } from "@grpc/grpc-js"; - -// Optional: verbose console logs while testing -diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.DEBUG); - -export function register() { - registerOTel({ - attributes: { - project_name: "vercel-project", - project_type: "observe", - }, - spanProcessors: [ - new FISimpleSpanProcessor({ - exporter: (() => { - const meta = new Metadata(); - meta.set("x-api-key", process.env.FI_API_KEY ?? ""); - meta.set("x-secret-key", process.env.FI_SECRET_KEY ?? ""); - return new OTLPTraceExporter({ url: "grpc://grpc.futureagi.com", metadata: meta }); - })(), - // Export only TraceAI spans (remove if you want everything) - spanFilter: isFISpan, - }), - ], - }); -} -``` - ---- - -## 4. Instrument an API Route -Our instrumentation is automatic. Just **import and call** the `register` function inside each serverless function. - -```typescript JS/TS title="pages/api/story.ts" -import type { NextApiRequest, NextApiResponse } from "next"; -import { register as registerTracing } from "../../instrumentation"; -import { generateText } from "ai"; -import { openai } from "@ai-sdk/openai"; - -export default async function handler(req: NextApiRequest, res: NextApiResponse) { - registerTracing(); // initialise OTEL + exporters - - const result = await generateText({ - model: openai("gpt-4o-mini"), - prompt: "Write a short creative story about a time-traveling detective.", - experimental_telemetry: { isEnabled: true }, // ⇢ creates spans for each call - maxTokens: 300, - }); - - res.status(200).json({ - story: result.text?.trim() ?? "n/a", - }); -} -``` - -That’s it. Deploy to Vercel and watch traces flow into **Observe → Traces** in real time 🎉 diff --git a/src/pages/docs/tracing/auto/vertexai.mdx b/src/pages/docs/tracing/auto/vertexai.mdx deleted file mode 100644 index da812f54..00000000 --- a/src/pages/docs/tracing/auto/vertexai.mdx +++ /dev/null @@ -1,114 +0,0 @@ ---- -title: "Vertex AI Tracing with Future AGI: Auto-Instrumentation" -description: "Set up auto-instrumentation for Vertex AI with Future AGI tracing. Install traceAI-vertexai to capture Gemini model invocation and response spans." ---- - -## 1. Installation -Install the traceAI and Vertex AI packages. - -```bash -pip install traceAI-vertexai -pip install vertexai -``` - ---- - -## 2. Set Environment Variables - -Set up your environment variables to authenticate with FutureAGI . - -```python -import os - -os.environ["FI_API_KEY"] = "your-futureagi-api-key" -os.environ["FI_SECRET_KEY"] = "your-futureagi-secret-key" -``` - ---- - -## 3. Initialize Trace Provider -Set up the trace provider to create a new project in FutureAGI, establish telemetry data pipelines . - -```python -from fi_instrumentation import register -from fi_instrumentation.fi_types import ProjectType - -trace_provider = register( - project_type=ProjectType.OBSERVE, - project_name="vertexai_project", - ) -``` ---- - -## 4. Configure Vertex AI Instrumentation -Instrument your Project with VertexAI Instrumentor. This step ensures that all interactions with the VertexAI are tracked and monitored. - -```python -from traceai_vertexai import VertexAIInstrumentor - -VertexAIInstrumentor().instrument(tracer_provider=trace_provider) -``` - ---- - -## 5. Create Vertex AI Components - -Interact with Vertex AI as you normally would. Our Instrumentor will automatically trace and send the telemetry data to our platform. - -```python -import vertexai - -from vertexai.generative_models import FunctionDeclaration, GenerativeModel, Part, Tool - -vertexai.init( - project="project_name", -) - -# Describe a function by specifying its schema (JsonSchema format) -get_current_weather_func = FunctionDeclaration( - name="get_current_weather", - description="Get the current weather in a given location", - parameters={ - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, - }, - "required": ["location"], - }, -) - -# Tool is a collection of related functions -weather_tool = Tool(function_declarations=[get_current_weather_func]) - -# Use tools in chat -chat = GenerativeModel("gemini-1.5-flash", tools=[weather_tool]).start_chat() -``` - ---- -## 6. Execute -Run your Vertex AI application. - -```python -if __name__ == "__main__": - # Send a message to the model. The model will respond with a function call. - for response in chat.send_message( - "What is the weather like in Boston?", stream=True - ): - print(response) - # Then send a function response to the model. The model will use it to answer. - for response in chat.send_message( - Part.from_function_response( - name="get_current_weather", - response={"content": {"weather": "super nice"}}, - ), - stream=True, - ): - print(response) - -``` - ---- \ No newline at end of file diff --git a/src/pages/docs/tracing/concepts/index.mdx b/src/pages/docs/tracing/concepts/index.mdx deleted file mode 100644 index 59d88314..00000000 --- a/src/pages/docs/tracing/concepts/index.mdx +++ /dev/null @@ -1,60 +0,0 @@ ---- -title: "Understanding Observability: LLM Tracing Core Concepts" -description: "Core concepts behind LLM observability in Future AGI: what gets captured, how data is structured, and why monitoring matters for AI applications." ---- - -## About - -LLM observability is the practice of capturing, structuring, and analyzing everything that happens inside your AI application. Every LLM call, retrieval, tool execution, and agent decision is recorded as structured data that you can search, filter, score, and alert on. - -Future AGI's observability stack is built on OpenTelemetry. Your application sends traces to the platform, and everything else (dashboards, evals, sessions, alerts) runs on top of that traced data. Without tracing, there is nothing to observe. - ---- - -## The Tracing Pipeline - -Your app emits **spans** (LLM calls, tool calls, chain steps) via OpenTelemetry or the traceAI SDK. The backend receives them over HTTP or gRPC, groups them into **traces**, and stores them by project. - -``` -Your App → traceAI / OpenTelemetry SDK → OTLP (HTTP or gRPC) → Future AGI Backend → Observe Dashboard -``` - -Each **trace** is one request or execution. Each **span** is one operation (LLM, tool, retriever, etc.) with input, output, timing, and optional cost and tokens. That data powers the entire UI: trace list, span detail, [sessions](/docs/observe/features/session), [evals](/docs/observe/features/evals), and [alerts](/docs/observe/features/alerts). - ---- - -## Key Concepts - -| Concept | What it is | Learn more | -|---|---|---| -| **Traces** | A group of spans representing one complete request flow from input to output. | [What are Traces?](/docs/tracing/concepts/traces) | -| **Spans** | A single operation (LLM call, retrieval, tool execution). Records inputs, outputs, timing, and errors. | [What are Spans?](/docs/tracing/concepts/spans) | -| **OpenTelemetry** | The open standard used to collect and export trace data. | [What is OpenTelemetry?](/docs/tracing/concepts/otel) | -| **traceAI** | Future AGI's instrumentation library that wraps OpenTelemetry for LLM-specific spans. | [What is traceAI?](/docs/tracing/concepts/traceai) | - ---- - -## How It Works - -1. **Instrument your app**: Add a traceAI instrumentor (or use manual spans) to capture LLM calls automatically -2. **Traces flow to the platform**: Data is exported via OTLP to Future AGI's backend -3. **Everything is available in the dashboard**: Trace list, span detail, sessions, evals, and alerts all run on top of traced data - ---- - -## Next Steps - - - - The top-level unit: one request = one trace. - - - The building blocks inside every trace. - - - The open standard powering trace collection. - - - Future AGI's LLM-specific instrumentation library. - - diff --git a/src/pages/docs/tracing/concepts/otel.mdx b/src/pages/docs/tracing/concepts/otel.mdx deleted file mode 100644 index 6b136d3f..00000000 --- a/src/pages/docs/tracing/concepts/otel.mdx +++ /dev/null @@ -1,18 +0,0 @@ ---- -title: "What is OpenTelemetry? Future AGI Tracing Explained" -description: "Learn how Future AGI uses OpenTelemetry for vendor-neutral, high-performance tracing of AI applications with standardized telemetry collection." ---- - -[OpenTelemetry (OTel)](https://opentelemetry.io/) is an open-source observability framework designed for collecting, processing, and exporting traces, metrics, and logs from applications. It provides a standardized way to instrument applications and infrastructure to gain insights into their performance and behavior. - -We use OTel at Future AGI because it's vendor-agnostic, open source, and highly performant. It's a standard that includes batch processing of traces and spans in the magnitude of billions. - -## Why Use It? - -- 🔓 **Vendor-neutral**: Not locked to any specific provider -- 🌐 **Open source**: Free and community-driven -- ⚡ **High performance**: Handles billions of traces efficiently - -OTel collects traces, metrics, and logs to monitor system performance and events. - -You can learn more about how we trace applications using OpenTelemetry on our [traceAI](/docs/tracing/concepts/traceai) page. diff --git a/src/pages/docs/tracing/concepts/spans.mdx b/src/pages/docs/tracing/concepts/spans.mdx deleted file mode 100644 index cd7699f6..00000000 --- a/src/pages/docs/tracing/concepts/spans.mdx +++ /dev/null @@ -1,82 +0,0 @@ ---- -title: "What are Spans? LLM, Tool, and Chain Span Types" -description: "Understand spans in Future AGI tracing. Learn about span types including LLM, tool, chain, retriever, and embedding spans with their attributes." ---- - -Spans are the fundamental units of tracing in observability frameworks, providing structured, event-level data for monitoring, debugging, and performance analysis. A span represents a discrete operation executed within a system, capturing execution timing, hierarchical relationships, and metadata relevant to the operation’s context. - -They are aggregated into traces, which collectively depict the flow of execution across various system components. This document provides an in-depth technical analysis of spans, their attributes, classifications, and their role in system observability. - ---- - -## Structure of Spans - -A span consists of multiple attributes that encapsulate its execution details. These attributes can be categorized into the following sections: - -- **Identification and context** provide the span's unique ID, trace ID, and optional parent span ID, establishing hierarchical relationships. It may also include a project reference for system-wide organization. - -- **Execution details** define the operation recorded, including a descriptive name, span type (e.g., function call, API request, database query), and input/output data. If an operation fails, error metadata captures failure details like error codes, messages, and stack traces. - -- **Timing and performance** track execution efficiency through start and end timestamps, latency measurement, and resource usage, such as computational cost or token consumption for LLM-related spans. - -- **Metadata and custom attributes** provide additional context via tags, annotations, and JSON-based extensible fields. Execution environment details, including host machine, service instance, and deployment version, further enrich observability. - ---- - -## Types of Spans -Spans are categorized based on the type of operation they capture. This classification ensures structured trace analysis and aids in performance monitoring. - -- **Tool Spans** -It tracks operations executed by external tools or functions. It captures essential details, including the tool’s name, description, parameters, and performance metrics, enabling comprehensive monitoring of tool interactions. - -- **Chain Spans** -It represents individual steps in a sequential workflow where data flows through multiple interconnected operations. It facilitates the visualization and analysis of execution pipelines, helping optimize process efficiency and detect bottlenecks. - -- **LLM Spans** -It captures interactions with large language models, recording input prompts, generated completions, token usage, and invocation parameters. These spans provide insights into model performance, response times, and computational costs. - -- **Retriever Spans** -It logs data retrieval operations, such as querying a database or fetching documents from an index. It stores search parameters and results, ensuring traceability and facilitating performance assessment of retrieval mechanisms. - -- **Embedding Spans** -It tracks text-to-vector transformations used in machine learning applications. It records embedding vectors, associated model metadata, and processing details, supporting efficient monitoring of vectorization processes. - -- **Agent Spans** -It documents actions performed by autonomous agents, including decision-making logic and tool interactions. It captures the rationale behind an agent’s choices, providing transparency into automated workflows and AI-driven decision processes. - -- **Reranker Spans** -It logs result reordering or ranking adjustments based on specific scoring criteria. It retains input documents and their updated rankings, facilitating analysis of ranking models and relevance optimization. - -- **Unknown Spans** -It serves as a fallback for operations that do not fit predefined span types. It ensures that all observed activities are recorded, even when their category is not explicitly defined. - -- **Guardrail Spans** -It monitors compliance and enforce safety rules within a system. It captures validation results, applied policies, and compliance status, ensuring adherence to predefined operational constraints. - -- **Evaluator Spans** -It represents assessment activities conducted to measure system performance or model effectiveness. It tracks evaluation metrics, scoring data, and feedback, supporting the continuous improvement of models and workflows. - ---- - -## Span Attributes - -Attributes are key-value pairs that contain metadata that can be used to annotate a span to carry information about the operation it is tracking. - -For example, if a span invokes an LLM, the model name, the invocation parameters, the token count etc. - -### Attribute Rules - -1. **Keys**: Must be non-null string values -2. **Values**: Must be one of the following non-null types: - - String - - Boolean - - Floating point value - - Integer - - Array of any of the above types - -### Semantic Attributes - -Semantic Attributes are standardized naming conventions for common metadata present in typical operations. Using semantic attribute naming is recommended to ensure consistency across systems. - -> See [semantic conventions](/docs/observe/features/manual-tracing/semantic-conventions) for more information. - diff --git a/src/pages/docs/tracing/concepts/traceai.mdx b/src/pages/docs/tracing/concepts/traceai.mdx deleted file mode 100644 index 39c2f188..00000000 --- a/src/pages/docs/tracing/concepts/traceai.mdx +++ /dev/null @@ -1,36 +0,0 @@ ---- -title: "What is traceAI? Future AGI Open-Source Tracing SDK" -description: "Learn about traceAI, Future AGI's open-source package for standardized AI application tracing built on OpenTelemetry with framework-specific instrumentors." ---- - -An OSS package to enable standardized tracing of AI applications and frameworks - -traceAI is a set of conventions and plugins that is complimentary to OpenTelemetry to enable tracing of AI applications. It instruments and monitors different code executions across models, frameworks, and vendors and maps them to a set of standardized attributes for traces and spans. - -traceAI is natively supported by Future AGI, but can be used with any OpenTelemetry-compatible backend as well. traceAI provides a set of instrumentations for popular machine learning SDKs and frameworks in a variety of languages. - -## Python - -| Package | Description | Version | -|---------|-------------|----------| -| `traceAI-openai` | traceAI Instrumentation for OpenAI. | [![PyPI](https://img.shields.io/pypi/v/traceAI-openai)](https://pypi.org/project/traceAI-openai)| -| `traceAI-anthropic` | traceAI Instrumentation for Anthropic. | [![PyPI](https://img.shields.io/pypi/v/traceAI-anthropic)](https://pypi.org/project/traceAI-anthropic)| -| `traceAI-llamaindex` | traceAI Instrumentation for LlamaIndex. | [![PyPI](https://img.shields.io/pypi/v/traceAI-llamaindex)](https://pypi.org/project/traceAI-llamaindex)| -| `traceAI-langchain` | traceAI Instrumentation for LangChain. | [![PyPI](https://img.shields.io/pypi/v/traceAI-langchain)](https://pypi.org/project/traceAI-langchain)| -| `traceAI-mcp` | traceAI Instrumentation for MCP. | [![PyPI](https://img.shields.io/pypi/v/traceAI-mcp)](https://pypi.org/project/traceAI-mcp)| -| `traceAI-mistralai` | traceAI Instrumentation for MistralAI. | [![PyPI](https://img.shields.io/pypi/v/traceAI-mistralai)](https://pypi.org/project/traceAI-mistralai)| -| `traceAI-vertexai` | traceAI Instrumentation for VertexAI. | [![PyPI](https://img.shields.io/pypi/v/traceAI-vertexai)](https://pypi.org/project/traceAI-vertexai)| -| `traceAI-google-genai` | traceAI Instrumentation for Google GenAI. | [![PyPI](https://img.shields.io/pypi/v/traceAI-google-genai)](https://pypi.org/project/traceAI-google-genai)| -| `traceAI-google-adk` | traceAI Instrumentation for Google ADK. | [![PyPI](https://img.shields.io/pypi/v/traceAI-google-adk)](https://pypi.org/project/traceAI-google-adk) -| `traceAI-crewai` | traceAI Instrumentation for CrewAI. | [![PyPI](https://img.shields.io/pypi/v/traceAI-crewai)](https://pypi.org/project/traceAI-crewai)| -| `traceAI-haystack` | traceAI Instrumentation for Haystack. | [![PyPI](https://img.shields.io/pypi/v/traceAI-haystack)](https://pypi.org/project/traceAI-haystack)| -| `traceAI-litellm` | traceAI Instrumentation for liteLLM. | [![PyPI](https://img.shields.io/pypi/v/traceAI-litellm)](https://pypi.org/project/traceAI-litellm)| -| `traceAI-groq` | traceAI Instrumentation for Groq. | [![PyPI](https://img.shields.io/pypi/v/traceAI-groq)](https://pypi.org/project/traceAI-groq)| -| `traceAI-autogen` | traceAI Instrumentation for Autogen. | [![PyPI](https://img.shields.io/pypi/v/traceAI-autogen)](https://pypi.org/project/traceAI-autogen)| -| `traceAI-guardrails` | traceAI Instrumentation for Guardrails. | [![PyPI](https://img.shields.io/pypi/v/traceAI-guardrails)](https://pypi.org/project/traceAI-guardrails)| -| `traceAI-openai-agents` | traceAI Instrumentation for OpenAI Agents. | [![PyPI](https://img.shields.io/pypi/v/traceAI-openai-agents)](https://pypi.org/project/traceAI-openai-agents)| -| `traceAI-smolagents` | traceAI Instrumentation for SmolAgents. | [![PyPI](https://img.shields.io/pypi/v/traceAI-smolagents)](https://pypi.org/project/traceAI-smolagents)| -| `traceAI-dspy` | traceAI Instrumentation for DSPy. | [![PyPI](https://img.shields.io/pypi/v/traceAI-dspy)](https://pypi.org/project/traceAI-dspy)| -| `traceAI-bedrock` | traceAI Instrumentation for AWS Bedrock. | [![PyPI](https://img.shields.io/pypi/v/traceAI-bedrock)](https://pypi.org/project/traceAI-bedrock)| -| `traceAI-portkey` | traceAI Instrumentation for Portkey. | [![PyPI](https://img.shields.io/pypi/v/traceAI-portkey)](https://pypi.org/project/traceAI-portkey)| -| `traceAI-instructor` | traceAI Instrumentation for Instructor. | [![PyPI](https://img.shields.io/pypi/v/traceAI-instructor)](https://pypi.org/project/traceAI-instructor)| \ No newline at end of file diff --git a/src/pages/docs/tracing/concepts/traces.mdx b/src/pages/docs/tracing/concepts/traces.mdx deleted file mode 100644 index 1df11412..00000000 --- a/src/pages/docs/tracing/concepts/traces.mdx +++ /dev/null @@ -1,25 +0,0 @@ ---- -title: "What are Traces? Understanding Trace Structure" -description: "A trace in Future AGI represents the full execution flow of an AI request, composed of spans that capture each LLM call, tool use, and retrieval step." ---- - -## Key Features -1. **Execution Flow:** -A trace captures the entire lifecycle of a request, from initiation to completion. It records the sequence of operations and their interactions, providing a detailed map of the request's journey through the system. -2. **Span Aggregation:** -Traces are composed of multiple spans, each representing a discrete operation. By aggregating these spans, traces offer a structured view of the execution flow, highlighting dependencies and interactions between different components. -3. **Performance Analysis:** -Traces are essential for performance analysis, as they allow teams to measure latency, identify bottlenecks, and optimize system efficiency. By examining the execution flow, teams can pinpoint areas for improvement and ensure optimal performance. -4. **Debugging and Diagnostics:** -Traces provide a detailed execution path, enabling teams to trace unexpected behaviors and diagnose issues effectively. By following the flow of a request, teams can identify the root cause of errors and implement corrective measures. - ---- - -## Use Cases -1. **Dependency Analysis:** Traces help in understanding the dependencies between different operations within a system, allowing teams to optimize workflows and improve efficiency. -2. **Performance Monitoring:** By measuring latency across spans, traces can identify performance bottlenecks and areas for optimization, ensuring that the system operates at peak efficiency. -3. **Error Diagnosis:** Traces provide a detailed execution path, allowing teams to trace unexpected behaviors from input to output and diagnose issues effectively. - ---- - -In summary, traces are a vital component of observability frameworks, providing a structured and comprehensive view of the execution flow within a system. They enable teams to analyze dependencies, monitor performance, and diagnose issues, ensuring the reliability and efficiency of the system. diff --git a/src/pages/index.astro b/src/pages/index.astro index 8e92d422..228f3adf 100644 --- a/src/pages/index.astro +++ b/src/pages/index.astro @@ -36,7 +36,7 @@ const sections = [ color: "blue", href: "/docs/observe", links: [ - { title: "Quickstart", href: "/docs/observe/quickstart" }, + { title: "Quickstart", href: "/docs/observe/features/quickstart" }, { title: "Tracing", href: "/docs/tracing" }, ] }, diff --git a/src/pages/llms-full.txt.ts b/src/pages/llms-full.txt.ts index 23f2d387..a8b323ff 100644 --- a/src/pages/llms-full.txt.ts +++ b/src/pages/llms-full.txt.ts @@ -19,8 +19,7 @@ export const GET: APIRoute = async () => { lines.push('> Complete documentation content for Future AGI — an AI lifecycle platform for building, evaluating, observing, and optimizing AI applications.'); lines.push(''); - // Primary source: curated navigation, in the order humans grouped them. - const seen = new Set(); + // Collect all hrefs from navigation const hrefs: { title: string; href: string }[] = []; for (const tab of tabNavigation) { for (const group of tab.groups) { @@ -28,10 +27,10 @@ export const GET: APIRoute = async () => { } } + // For each page, read the MDX and extract content for (const { title, href } of hrefs) { const content = await readPageContent(href); if (!content) continue; - seen.add(href); lines.push(`---`); lines.push(''); @@ -42,74 +41,11 @@ export const GET: APIRoute = async () => { lines.push(''); } - // Sweep src/pages/docs for any .mdx pages not already emitted. Cookbook - // entries, release notes, FAQs, and pages added without a navigation - // update get their full content included automatically. - const onDisk = await walkDocsPages(DOCS_DIR); - const additional = onDisk - .filter(({ href }) => !seen.has(href)) - .sort((a, b) => a.href.localeCompare(b.href)); - - for (const { href, title } of additional) { - const content = await readPageContent(href); - if (!content) continue; - lines.push(`---`); - lines.push(''); - lines.push(`## ${title}`); - lines.push(`URL: ${SITE}${href}`); - lines.push(''); - lines.push(content); - lines.push(''); - } - return new Response(lines.join('\n'), { headers: { 'Content-Type': 'text/plain; charset=utf-8' }, }); }; -async function walkDocsPages( - dir: string, - prefix = '/docs', -): Promise<{ href: string; title: string }[]> { - const out: { href: string; title: string }[] = []; - let entries; - try { - entries = await fs.readdir(dir, { withFileTypes: true }); - } catch { - return out; - } - for (const entry of entries) { - const full = path.join(dir, entry.name); - if (entry.isDirectory()) { - out.push(...(await walkDocsPages(full, `${prefix}/${entry.name}`))); - continue; - } - if (!entry.name.endsWith('.mdx') && !entry.name.endsWith('.md')) continue; - if (entry.name.startsWith('_')) continue; - const stem = entry.name.replace(/\.(mdx|md)$/, ''); - const href = stem === 'index' ? prefix : `${prefix}/${stem}`; - const title = await readTitle(full).catch(() => null); - out.push({ - href, - title: title || titlecase(stem === 'index' ? path.basename(prefix) : stem), - }); - } - return out; -} - -async function readTitle(file: string): Promise { - const raw = await fs.readFile(file, 'utf-8'); - const match = raw.match(/^title:\s*['"]?([^'"\n]+)['"]?\s*$/m); - return match ? match[1].trim() : null; -} - -function titlecase(slug: string): string { - return slug - .split('-') - .map((p) => p.charAt(0).toUpperCase() + p.slice(1)) - .join(' '); -} - function collectHrefs(items: NavItem[], out: { title: string; href: string }[]) { for (const item of items) { if (item.href) { diff --git a/src/pages/llms.txt.ts b/src/pages/llms.txt.ts index 428eddb4..80bedc50 100644 --- a/src/pages/llms.txt.ts +++ b/src/pages/llms.txt.ts @@ -1,33 +1,23 @@ import type { APIRoute } from 'astro'; -import fs from 'node:fs/promises'; -import path from 'node:path'; import { tabNavigation } from '../lib/navigation'; import type { NavItem } from '../lib/navigation'; const SITE = 'https://docs.futureagi.com'; -const DOCS_DIR = path.join(process.cwd(), 'src/pages/docs'); /** * Generate /llms.txt — a concise, LLM-friendly overview of the documentation. * Follows the llms.txt specification: https://llmstxt.org - * - * Primary source is the curated tabNavigation so the human-grouped section - * headers stay intact. After walking the nav, the script scans - * src/pages/docs for any .mdx pages not already linked and lists them in an - * "Additional Pages" section so cookbook entries, release notes, FAQs, and - * any new pages added without a nav update still show up in /llms.txt. */ -export const GET: APIRoute = async () => { +export const GET: APIRoute = () => { const lines: string[] = []; - const seen = new Set(); + // Title & summary lines.push('# Future AGI Documentation'); lines.push(''); - lines.push( - '> Future AGI is an AI lifecycle platform for building, evaluating, observing, and optimizing AI applications. This documentation covers the Python SDK, platform features, integrations, and API reference.', - ); + lines.push('> Future AGI is an AI lifecycle platform for building, evaluating, observing, and optimizing AI applications. This documentation covers the Python SDK, platform features, integrations, and API reference.'); lines.push(''); + // Key sections with links lines.push('## Docs'); lines.push(''); @@ -35,33 +25,15 @@ export const GET: APIRoute = async () => { for (const group of tab.groups) { lines.push(`### ${group.group}`); lines.push(''); - collectLinks(group.items, lines, seen); + collectLinks(group.items, lines); lines.push(''); } } - // Walk src/pages/docs and emit anything not already linked under - // "Additional Pages". Cookbook entries, release notes, FAQs, and pages - // added without a nav update surface here automatically. - const onDisk = await walkDocsPages(DOCS_DIR); - const additional = onDisk - .filter(({ href }) => !seen.has(href)) - .sort((a, b) => a.href.localeCompare(b.href)); - - if (additional.length > 0) { - lines.push('### Additional Pages'); - lines.push(''); - for (const { href, title } of additional) { - lines.push(`- [${title}](${SITE}${href})`); - } - lines.push(''); - } - + // Optional: pointer to full version lines.push('## Full Documentation'); lines.push(''); - lines.push( - `For the complete documentation with all page content, see [llms-full.txt](${SITE}/llms-full.txt).`, - ); + lines.push(`For the complete documentation with all page content, see [llms-full.txt](${SITE}/llms-full.txt).`); lines.push(''); return new Response(lines.join('\n'), { @@ -69,57 +41,13 @@ export const GET: APIRoute = async () => { }); }; -function collectLinks(items: NavItem[], lines: string[], seen: Set) { +function collectLinks(items: NavItem[], lines: string[]) { for (const item of items) { if (item.href) { lines.push(`- [${item.title}](${SITE}${item.href})`); - seen.add(item.href); } if (item.items) { - collectLinks(item.items, lines, seen); + collectLinks(item.items, lines); } } } - -async function walkDocsPages( - dir: string, - prefix = '/docs', -): Promise<{ href: string; title: string }[]> { - const out: { href: string; title: string }[] = []; - let entries; - try { - entries = await fs.readdir(dir, { withFileTypes: true }); - } catch { - return out; - } - for (const entry of entries) { - const full = path.join(dir, entry.name); - if (entry.isDirectory()) { - out.push(...(await walkDocsPages(full, `${prefix}/${entry.name}`))); - continue; - } - if (!entry.name.endsWith('.mdx') && !entry.name.endsWith('.md')) continue; - if (entry.name.startsWith('_')) continue; - const stem = entry.name.replace(/\.(mdx|md)$/, ''); - const href = stem === 'index' ? prefix : `${prefix}/${stem}`; - const title = await readTitle(full).catch(() => null); - out.push({ - href, - title: title || titlecase(stem === 'index' ? path.basename(prefix) : stem), - }); - } - return out; -} - -async function readTitle(file: string): Promise { - const raw = await fs.readFile(file, 'utf8'); - const match = raw.match(/^title:\s*['"]?([^'"\n]+)['"]?\s*$/m); - return match ? match[1].trim() : null; -} - -function titlecase(slug: string): string { - return slug - .split('-') - .map((p) => p.charAt(0).toUpperCase() + p.slice(1)) - .join(' '); -} diff --git a/src/plugins/vite-docs-transform.mjs b/src/plugins/vite-docs-transform.mjs index dab3550b..2dffacfc 100644 --- a/src/plugins/vite-docs-transform.mjs +++ b/src/plugins/vite-docs-transform.mjs @@ -22,6 +22,7 @@ const COMPONENT_MAP = { CopyButton: '@docs/CopyButton.astro', Expandable: '@docs/Expandable.astro', Icon: '@docs/Icon.astro', + Mermaid: '@docs/Mermaid.astro', Note: '@docs/Note.astro', ParamField: '@docs/ParamField.astro', Prerequisites: '@docs/Prerequisites.astro',