diff --git a/api-reference/auto-monitor-setups/create-an-auto-monitor-setup.mdx b/api-reference/auto-monitor-setups/create-an-auto-monitor-setup.mdx index 30dd363..c8f0d34 100644 --- a/api-reference/auto-monitor-setups/create-an-auto-monitor-setup.mdx +++ b/api-reference/auto-monitor-setups/create-an-auto-monitor-setup.mdx @@ -19,12 +19,36 @@ An auto monitor setup is an asynchronous process that attempts to create a [Trac **Example:** `"my-agent-monitor-1"` - - List of evaluator slugs to run on matched spans. Must contain at least one evaluator. + + List of evaluator slugs to run on matched spans. **Example:** `["answer-relevancy", "toxicity-detector"]` See the full list of available slugs in the [Evaluator Slugs](/evaluators/evaluator-slugs) reference. + + + Either `evaluators` or `evaluator_configs` must be provided on create. If both are provided, `evaluator_configs` wins and this field is ignored. Use `evaluator_configs` when you need to pin a per-evaluator scope. + + + + + List of per-evaluator configurations. Use this instead of `evaluators` when you need to control the granularity at which an individual evaluator runs. + + Each entry has the following fields: + + | Field | Type | Required | Description | + |-------|------|----------|-------------| + | `slug` | string | Yes | Evaluator slug — same set accepted by the `evaluators` array. See the [Evaluator Slugs](/evaluators/evaluator-slugs) reference. | + | `scope` | string | No | Granularity at which the evaluator runs. One of `session`, `trace`, `span`. When omitted, no scope is persisted and the downstream monitor applies its own default. | + + **Example:** + ```json + [ + { "slug": "char-count", "scope": "session" }, + { "slug": "toxicity", "scope": "trace" }, + { "slug": "pii" } + ] + ``` @@ -45,6 +69,8 @@ An auto monitor setup is an asynchronous process that attempts to create a [Trac ## Example Request +Using the simple `evaluators` array (no per-evaluator scope): + ```bash curl -X POST https://api.traceloop.com/v2/auto-monitor-setups \ -H "Authorization: Bearer YOUR_API_KEY" \ @@ -59,12 +85,33 @@ curl -X POST https://api.traceloop.com/v2/auto-monitor-setups \ }' ``` +Using `evaluator_configs` to pin a scope per evaluator: + +```bash +curl -X POST https://api.traceloop.com/v2/auto-monitor-setups \ + -H "Authorization: Bearer YOUR_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "external_id": "my-agent-monitor-1", + "selector": [ + {"key": "gen_ai.system", "value": "openai", "source": "span_attributes"} + ], + "evaluator_configs": [ + {"slug": "char-count", "scope": "session"}, + {"slug": "toxicity", "scope": "trace"}, + {"slug": "pii"} + ] + }' +``` + ## Response ### 201 Created Returns the created auto monitor setup object. The `init_rules` array reflects the stored selector — same shape as the input `selector`. +Each entry in `evaluators` optionally includes a `scope` field (`session`, `trace`, or `span`) when a scope was set for that evaluator via `evaluator_configs`. When no scope was provided at create time, `scope` is omitted from the evaluator object (it is never returned as `null`). + The `status` field indicates the current state of the setup process: | Status | Description | @@ -100,6 +147,7 @@ The `status` field indicates the current state of the setup process: "evaluators": [ { "evaluator_type": "answer-relevancy", + "scope": "trace", "input_schema": [ { "type": "string", "name": "completion", "description": "The LLM response to evaluate" }, { "type": "string", "name": "context", "description": "The context for the answer" }, @@ -129,11 +177,16 @@ The `status` field indicates the current state of the setup process: ### 400 Bad Request -Returned when the request body is invalid (e.g. missing required fields or empty `evaluators` array). +Returned when the request body is invalid. Common validation errors: + +- Neither `evaluators` nor `evaluator_configs` is set on a Create request — `one of evaluators or evaluator_configs is required`. +- An entry in `evaluator_configs` is missing its `slug`. +- An entry in `evaluator_configs` has a `scope` that is not one of `session`, `trace`, or `span`. +- A provided evaluator slug is not recognized — `unknown evaluator slug ""`. ```json { - "error": "invalid input: evaluators must contain at least one item" + "error": "one of evaluators or evaluator_configs is required" } ``` diff --git a/openapi.json b/openapi.json index ebf7bbd..ac52549 100644 --- a/openapi.json +++ b/openapi.json @@ -3010,10 +3010,35 @@ ], "type": "object" }, + "request.AutoMonitorEvaluatorConfig": { + "description": "Per-evaluator configuration that lets you pin the granularity (scope) at which the evaluator runs.", + "properties": { + "slug": { + "description": "Evaluator slug. Same set of values accepted by the `evaluators` array.", + "example": "toxicity", + "type": "string" + }, + "scope": { + "description": "Granularity at which the evaluator runs. When omitted, no scope is persisted and downstream applies its own default.", + "enum": [ + "session", + "trace", + "span" + ], + "example": "trace", + "type": "string" + } + }, + "required": [ + "slug" + ], + "type": "object" + }, "request.CreateAutoMonitorSetupInput": { + "description": "At least one of `evaluators` or `evaluator_configs` must be provided. If both are provided, `evaluator_configs` wins and `evaluators` is ignored.", "properties": { "evaluators": { - "description": "List of evaluator slugs to run on matched spans", + "description": "List of evaluator slugs to run on matched spans. Use `evaluator_configs` instead when you need to set a per-evaluator scope. If both fields are provided, `evaluator_configs` takes precedence and this field is ignored.", "example": [ "hallucination", "toxicity" @@ -3024,6 +3049,18 @@ "minItems": 1, "type": "array" }, + "evaluator_configs": { + "description": "List of per-evaluator configurations. Each entry declares an evaluator slug and an optional scope (`session`, `trace`, or `span`) controlling the granularity at which the evaluator runs. Takes precedence over `evaluators` when both are provided.", + "example": [ + { "slug": "char-count", "scope": "session" }, + { "slug": "toxicity", "scope": "trace" }, + { "slug": "pii" } + ], + "items": { + "$ref": "#/components/schemas/request.AutoMonitorEvaluatorConfig" + }, + "type": "array" + }, "external_id": { "description": "Unique identifier for the auto monitor setup, used to reference it in future requests", "example": "my-agent-monitor-1", @@ -3035,7 +3072,6 @@ } }, "required": [ - "evaluators", "external_id" ], "type": "object" @@ -3835,9 +3871,10 @@ "type": "object" }, "request.UpdateAutoMonitorSetupInput": { + "description": "On update, both `evaluators` and `evaluator_configs` may be empty \u2014 in that case the evaluator list is not modified. If both are provided, `evaluator_configs` wins and `evaluators` is ignored.", "properties": { "evaluators": { - "description": "List of evaluator slugs to run on matched spans", + "description": "List of evaluator slugs to run on matched spans. Use `evaluator_configs` instead when you need to set a per-evaluator scope. If both fields are provided, `evaluator_configs` takes precedence and this field is ignored.", "example": [ "hallucination", "toxicity" @@ -3847,6 +3884,18 @@ }, "type": "array" }, + "evaluator_configs": { + "description": "List of per-evaluator configurations. Each entry declares an evaluator slug and an optional scope (`session`, `trace`, or `span`) controlling the granularity at which the evaluator runs. Takes precedence over `evaluators` when both are provided.", + "example": [ + { "slug": "char-count", "scope": "session" }, + { "slug": "toxicity", "scope": "trace" }, + { "slug": "pii" } + ], + "items": { + "$ref": "#/components/schemas/request.AutoMonitorEvaluatorConfig" + }, + "type": "array" + }, "selector": { "description": "Map of span attributes to filter which spans this monitor applies to.\nKeys are span attribute names (e.g. gen_ai.system, gen_ai.request.model) and\nvalues can be strings, numbers, or booleans.\nExample: {\"gen_ai.system\": \"openai\", \"gen_ai.request.model\": \"gpt-4o\", \"gen_ai.request.max_tokens\": 1000}", "type": "object" @@ -4055,6 +4104,16 @@ "processed_at": { "type": "string" }, + "scope": { + "description": "Granularity at which the evaluator runs. Only present when a scope was set for this evaluator when the setup was created or updated. Omitted (not `null`) when no scope was provided.", + "enum": [ + "session", + "trace", + "span" + ], + "example": "trace", + "type": "string" + }, "status": { "type": "string" }