diff --git a/src/agentevals/api/models.py b/src/agentevals/api/models.py index a1cf74e..3922bc8 100644 --- a/src/agentevals/api/models.py +++ b/src/agentevals/api/models.py @@ -119,6 +119,24 @@ class TraceConversionMetadata(CamelModel): model: str | None = None response_model: str | None = None provider: str | None = None + schema_version: str | None = Field( + default=None, + description=( + "Version segment parsed from the emitting scope's schema URL " + "(the path component after the schema family prefix). Per the " + "OpenTelemetry schema URL spec, this version is only meaningful " + "within its schema family — the host/path prefix before the " + "version. It is comparable across traces only if they share a " + "family, and equals the OpenTelemetry Semantic Conventions " + "version only for the OTel family " + "(https://opentelemetry.io/schemas/). For traces from " + "a custom or mirrored schema family, this is that family's own " + "version number, not an OTel semconv version. The full schema " + "URL (otel.schema_url) is preserved separately wherever the " + "family needs to be disambiguated; this field is informational " + "only and does not drive extraction or alias-resolution logic." + ), + ) start_time: int | None = None user_input_preview: str | None = None final_output_preview: str | None = None diff --git a/src/agentevals/api/otlp_processing.py b/src/agentevals/api/otlp_processing.py index 5dc6038..948e31b 100644 --- a/src/agentevals/api/otlp_processing.py +++ b/src/agentevals/api/otlp_processing.py @@ -19,6 +19,7 @@ OTEL_GENAI_CONVERSATION_ID, OTEL_GENAI_INPUT_MESSAGES, OTEL_GENAI_OUTPUT_MESSAGES, + OTEL_SCHEMA_URL, OTEL_SCOPE, OTEL_SCOPE_VERSION, ) @@ -38,6 +39,7 @@ async def process_traces(body: dict, manager: StreamingTraceManager) -> None: for resource_span in body.get("resourceSpans", []): resource_attrs = resource_span.get("resource", {}).get("attributes", []) metadata = _extract_agentevals_metadata(resource_attrs) + resource_schema_url = resource_span.get("schemaUrl", "") if not metadata.get("conversation_id"): metadata["conversation_id"] = _prescan_conversation_id(resource_span) @@ -46,9 +48,10 @@ async def process_traces(body: dict, manager: StreamingTraceManager) -> None: scope = scope_span.get("scope", {}) scope_name = scope.get("name", "") scope_version = scope.get("version", "") + schema_url = scope_span.get("schemaUrl", "") or resource_schema_url for span_data in scope_span.get("spans", []): - span = _normalize_span(span_data, scope_name, scope_version) + span = _normalize_span(span_data, scope_name, scope_version, schema_url) trace_id = span.get("traceId", "") if not trace_id: @@ -200,12 +203,13 @@ def fix_protobuf_id_fields(data) -> None: _GENAI_EVENT_KEYS = {OTEL_GENAI_INPUT_MESSAGES, OTEL_GENAI_OUTPUT_MESSAGES} -def _normalize_span(span_data: dict, scope_name: str, scope_version: str) -> dict: +def _normalize_span(span_data: dict, scope_name: str, scope_version: str, schema_url: str = "") -> dict: """Normalize an OTLP span for the downstream pipeline. Performs two transformations: - 1. Injects otel.scope.name/version from the scopeSpans level into span - attributes (the pipeline expects them there). + 1. Injects otel.scope.name/version (and otel.schema_url) from the + scopeSpans level into span attributes (the pipeline expects them + there). 2. Promotes gen_ai.input.messages and gen_ai.output.messages from span events to span attributes. Some SDKs (e.g. Strands with OTEL_SEMCONV_STABILITY_OPT_IN=gen_ai_latest_experimental) store @@ -222,6 +226,9 @@ def _normalize_span(span_data: dict, scope_name: str, scope_version: str) -> dic if scope_version and OTEL_SCOPE_VERSION not in existing_keys: attrs.append({"key": OTEL_SCOPE_VERSION, "value": {"stringValue": scope_version}}) existing_keys.add(OTEL_SCOPE_VERSION) + if schema_url and OTEL_SCHEMA_URL not in existing_keys: + attrs.append({"key": OTEL_SCHEMA_URL, "value": {"stringValue": schema_url}}) + existing_keys.add(OTEL_SCHEMA_URL) for event in span.get("events", []): for attr in event.get("attributes", []): diff --git a/src/agentevals/api/routes.py b/src/agentevals/api/routes.py index b54c3cf..eb9e0bc 100644 --- a/src/agentevals/api/routes.py +++ b/src/agentevals/api/routes.py @@ -487,6 +487,7 @@ async def convert_trace_files( meta = TraceConversionMetadata( agent_name=meta_dict.get("agent_name"), model=meta_dict.get("model"), + schema_version=meta_dict.get("schema_version"), start_time=meta_dict.get("start_time"), user_input_preview=meta_dict.get("user_input_preview"), final_output_preview=meta_dict.get("final_output_preview"), diff --git a/src/agentevals/extraction.py b/src/agentevals/extraction.py index 51b0348..141f230 100644 --- a/src/agentevals/extraction.py +++ b/src/agentevals/extraction.py @@ -13,6 +13,7 @@ import json import logging +import re from typing import Any, Protocol, TypedDict, TypeVar from .loader.base import Span, Trace @@ -22,6 +23,7 @@ ADK_SCOPE_VALUE, ADK_TOOL_CALL_ARGS, ADK_TOOL_RESPONSE, + GENAI_ATTRIBUTE_ALIASES, OTEL_ERROR_TYPE, OTEL_GENAI_INPUT_MESSAGES, OTEL_GENAI_OP, @@ -44,6 +46,7 @@ OTEL_GENAI_USAGE_CACHE_READ_TOKENS, OTEL_GENAI_USAGE_INPUT_TOKENS, OTEL_GENAI_USAGE_OUTPUT_TOKENS, + OTEL_SCHEMA_URL, OTEL_SCOPE, ) from .utils.genai_messages import ( @@ -58,6 +61,61 @@ FORMAT_DETECTION_SPAN_LIMIT = 10 +# --------------------------------------------------------------------------- +# Alias-aware attribute resolution +# --------------------------------------------------------------------------- + +# Per the OTel schema URL spec (https://opentelemetry.io/docs/specs/otel/schemas/#schema-url), +# a schema URL has the form `http[s]://server[:port]/path/`. +_SCHEMA_VERSION_RE = re.compile(r"^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$") + + +def resolve_attr(attrs: dict[str, Any], canonical_key: str) -> Any | None: + """Look up *canonical_key* in *attrs*, falling back to older aliased names. + + Resolution order: + 1. The canonical (current/newest) key, if present with a truthy value. + 2. Each alias in ``GENAI_ATTRIBUTE_ALIASES[canonical_key]``, in order, + first truthy value wins. + 3. ``None`` if neither the canonical key nor any alias is present. + + The canonical value always takes precedence over aliases, even if both + are present on the same span. This resolves only against the flat attrs + dict at read time - it never mutates or rewrites stored/transmitted + attributes. + """ + value = attrs.get(canonical_key) + if value: + return value + + for alias in GENAI_ATTRIBUTE_ALIASES.get(canonical_key, []): + alias_value = attrs.get(alias) + if alias_value: + return alias_value + + return None + + +def resolve_schema_version(schema_url: str | None) -> str | None: + """Extract the version segment from a schema URL. + + Per the OTel schema URL spec, the version is the last path segment + (e.g. "1.37.0" in "https://opentelemetry.io/schemas/1.37.0"). + + It's scoped to that URL's schema family and only equals the OTel semconv + version for the official OTel family. Degrades gracefully to None for + missing, empty, or malformed input - never raises. + """ + if not schema_url or not isinstance(schema_url, str): + return None + + last_segment = schema_url.rstrip("/").rsplit("/", 1)[-1] + if _SCHEMA_VERSION_RE.match(last_segment): + return last_segment + + return None + + # --------------------------------------------------------------------------- # Pure extraction functions (operate on flat attribute dicts) # --------------------------------------------------------------------------- @@ -189,18 +247,28 @@ class ExtendedModelInfo(TypedDict): cache_creation_tokens: int cache_read_tokens: int error_type: str | None + schema_version: str | None def extract_extended_model_info_from_attrs(attrs: dict[str, Any]) -> ExtendedModelInfo: """Extract extended model and provider metadata from span attributes. - Uses gen_ai.system as fallback for provider when gen_ai.provider.name is - absent (backward compat with pre-v1.37.0 instrumentors). + Uses the alias-resolving lookup for attributes with known historical + renames (e.g. provider falls back to gen_ai.system when + gen_ai.provider.name - the canonical, current name - is absent, for + backward compat with pre-v1.37.0 instrumentors). + + ``schema_version`` is resolved from the scope's ``schema_url`` (captured + at ingest as the ``otel.schema_url`` attribute) and is ``None`` when + absent or malformed. This is purely informational metadata about which + schema version emitted the span - it is never used to detect whether a + span is GenAI (that remains the sole responsibility of the existing + gen_ai.* presence checks). """ return { "request_model": attrs.get(OTEL_GENAI_REQUEST_MODEL), "response_model": attrs.get(OTEL_GENAI_RESPONSE_MODEL), - "provider": attrs.get(OTEL_GENAI_PROVIDER_NAME) or attrs.get(OTEL_GENAI_SYSTEM), + "provider": resolve_attr(attrs, OTEL_GENAI_PROVIDER_NAME), "finish_reasons": _parse_finish_reasons(attrs.get(OTEL_GENAI_RESPONSE_FINISH_REASONS)), "response_id": attrs.get(OTEL_GENAI_RESPONSE_ID), "temperature": _safe_cast(attrs.get(OTEL_GENAI_REQUEST_TEMPERATURE), float), @@ -208,6 +276,7 @@ def extract_extended_model_info_from_attrs(attrs: dict[str, Any]) -> ExtendedMod "cache_creation_tokens": _safe_cast(attrs.get(OTEL_GENAI_USAGE_CACHE_CREATION_TOKENS), int, 0), "cache_read_tokens": _safe_cast(attrs.get(OTEL_GENAI_USAGE_CACHE_READ_TOKENS), int, 0), "error_type": attrs.get(OTEL_ERROR_TYPE), + "schema_version": resolve_schema_version(attrs.get(OTEL_SCHEMA_URL)), } diff --git a/src/agentevals/loader/otlp.py b/src/agentevals/loader/otlp.py index ed22156..ef26cb2 100644 --- a/src/agentevals/loader/otlp.py +++ b/src/agentevals/loader/otlp.py @@ -8,6 +8,7 @@ from ..trace_attrs import ( OTEL_GENAI_INPUT_MESSAGES, OTEL_GENAI_OUTPUT_MESSAGES, + OTEL_SCHEMA_URL, OTEL_SCOPE, OTEL_SCOPE_VERSION, ) @@ -93,21 +94,23 @@ def _parse_otlp_export(self, data: dict) -> list[Trace]: all_spans = [] for resource_span in resource_spans: resource_attrs = self._extract_attributes(resource_span.get("resource", {}).get("attributes", [])) + resource_schema_url = resource_span.get("schemaUrl", "") scope_spans = resource_span.get("scopeSpans") or resource_span.get("instrumentationLibrarySpans", []) for scope_span in scope_spans: scope = scope_span.get("scope") or scope_span.get("instrumentationLibrary") or {} scope_name = scope.get("name", "") scope_version = scope.get("version", "") + schema_url = scope_span.get("schemaUrl", "") or resource_schema_url for span_data in scope_span.get("spans", []): - span = self._parse_span(span_data, resource_attrs, scope_name, scope_version) + span = self._parse_span(span_data, resource_attrs, scope_name, scope_version, schema_url) all_spans.append(span) return self._build_traces(all_spans) def _parse_otlp_spans(self, spans_data: list[dict]) -> list[Trace]: """Parse flat list of OTLP spans (JSONL format for streaming).""" - all_spans = [self._parse_span(span_data, {}, "", "") for span_data in spans_data] + all_spans = [self._parse_span(span_data, {}, "", "", "") for span_data in spans_data] return self._build_traces(all_spans) _GENAI_EVENT_KEYS = {OTEL_GENAI_INPUT_MESSAGES, OTEL_GENAI_OUTPUT_MESSAGES} @@ -118,6 +121,7 @@ def _parse_span( resource_attrs: dict, scope_name: str, scope_version: str, + schema_url: str = "", ) -> Span: """Convert OTLP span to normalized Span object.""" attributes = self._extract_attributes(span_data.get("attributes", [])) @@ -126,6 +130,8 @@ def _parse_span( attributes[OTEL_SCOPE] = scope_name if scope_version: attributes[OTEL_SCOPE_VERSION] = scope_version + if schema_url: + attributes[OTEL_SCHEMA_URL] = schema_url self._promote_genai_event_attributes(span_data, attributes) diff --git a/src/agentevals/streaming/processor.py b/src/agentevals/streaming/processor.py index 8f6367a..5116028 100644 --- a/src/agentevals/streaming/processor.py +++ b/src/agentevals/streaming/processor.py @@ -154,12 +154,15 @@ def force_flush(self, timeout_millis: int = 30000) -> bool: def _span_to_otlp(self, span: ReadableSpan) -> dict: scope_name = span.instrumentation_scope.name if span.instrumentation_scope else "" scope_version = span.instrumentation_scope.version if span.instrumentation_scope else "" + schema_url = span.instrumentation_scope.schema_url if span.instrumentation_scope else "" attributes = [] if scope_name: attributes.append({"key": "otel.scope.name", "value": {"stringValue": scope_name}}) if scope_version: attributes.append({"key": "otel.scope.version", "value": {"stringValue": scope_version}}) + if schema_url: + attributes.append({"key": "otel.schema_url", "value": {"stringValue": schema_url}}) if span.attributes: for key, value in span.attributes.items(): diff --git a/src/agentevals/trace_attrs.py b/src/agentevals/trace_attrs.py index 6bfc0df..35da68a 100644 --- a/src/agentevals/trace_attrs.py +++ b/src/agentevals/trace_attrs.py @@ -6,12 +6,15 @@ Covers OTel GenAI semantic conventions up to v1.40.0. """ +from typing import Optional + # OTel resource OTEL_SERVICE_NAME = "service.name" # OTel scope OTEL_SCOPE = "otel.scope.name" OTEL_SCOPE_VERSION = "otel.scope.version" +OTEL_SCHEMA_URL = "otel.schema_url" # Google ADK scope value ADK_SCOPE_VALUE = "gcp.vertex.agent" @@ -73,3 +76,8 @@ ADK_TOOL_CALL_ARGS = "gcp.vertex.agent.tool_call_args" ADK_TOOL_RESPONSE = "gcp.vertex.agent.tool_response" ADK_INVOCATION_ID = "gcp.vertex.agent.invocation_id" + +# GenAI attribute alias table +GENAI_ATTRIBUTE_ALIASES: dict[str, list[str]] = { + OTEL_GENAI_PROVIDER_NAME: [OTEL_GENAI_SYSTEM], +} diff --git a/src/agentevals/trace_metrics.py b/src/agentevals/trace_metrics.py index 8009957..a8df510 100644 --- a/src/agentevals/trace_metrics.py +++ b/src/agentevals/trace_metrics.py @@ -185,6 +185,7 @@ def extract_trace_metadata(trace, extractor=None) -> dict[str, Any]: "model": None, "response_model": None, "provider": None, + "schema_version": None, "start_time": None, "user_input_preview": None, "final_output_preview": None, @@ -204,11 +205,17 @@ def extract_trace_metadata(trace, extractor=None) -> dict[str, Any]: if llm_spans: metadata["model"] = llm_spans[0].get_tag(OTEL_GENAI_REQUEST_MODEL) + # NOTE: `ext` is extracted only from `llm_spans[0]` (the first LLM + # span). If a trace mixes instrumentors or different + # instrumentation versions, extended model info that appears only + # on later LLM spans may not be captured here. ext = extract_extended_model_info_from_attrs(llm_spans[0].tags) if ext["response_model"]: metadata["response_model"] = ext["response_model"] if ext["provider"]: metadata["provider"] = ext["provider"] + if ext["schema_version"]: + metadata["schema_version"] = ext["schema_version"] user_text = extract_user_text_from_attrs(llm_spans[0].tags) if user_text: diff --git a/tests/test_api.py b/tests/test_api.py index 1d5af56..59e4f99 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1600,6 +1600,64 @@ def test_tempo_json_upload_auto_detected(self): assert traces[0]["traceId"] == "dd547580319ab0312cee07f1def50dad" assert len(traces[0]["invocations"]) == 1 + def test_convert_otlp_scope_schema_url_propagates_schema_version(self): + payload = _make_otlp_json_payload() + scope_spans = payload["resourceSpans"][0]["scopeSpans"][0] + scope_spans["schemaUrl"] = "https://opentelemetry.io/schemas/1.39.0" + scope_spans["spans"].append( + { + "traceId": scope_spans["spans"][0]["traceId"], + "spanId": "s2", + "parentSpanId": scope_spans["spans"][0]["spanId"], + "name": "call_llm", + "startTimeUnixNano": "1100000", + "endTimeUnixNano": "1500000", + "attributes": [ + { + "key": "gcp.vertex.agent.llm_request", + "value": { + "stringValue": json.dumps({"contents": [{"role": "user", "parts": [{"text": "hello"}]}]}) + }, + }, + { + "key": "gcp.vertex.agent.llm_response", + "value": {"stringValue": json.dumps({"content": {"role": "model", "parts": [{"text": "hi"}]}})}, + }, + ], + } + ) + + body = _assert_envelope( + self.client.post( + "/api/convert", + files={"trace_files": ("trace.json", io.BytesIO(json.dumps(payload).encode()))}, + ) + ) + + metadata = body["data"]["traces"][0]["metadata"] + assert metadata["schemaVersion"] == "1.39.0" + + @patch("agentevals.api.routes.extract_trace_metadata") + def test_convert_metadata_includes_schema_version(self, mock_extract_trace_metadata): + mock_extract_trace_metadata.return_value = { + "agent_name": "agent", + "model": "gpt-4o", + "schema_version": "1.39.0", + "start_time": 123, + "user_input_preview": "hello", + "final_output_preview": "world", + } + + body = _assert_envelope( + self.client.post( + "/api/convert", + files={"trace_files": ("trace.json", io.BytesIO(self._tempo_fixture_bytes()))}, + ) + ) + + metadata = body["data"]["traces"][0]["metadata"] + assert metadata["schemaVersion"] == "1.39.0" + def test_unknown_shape_returns_load_warning(self): unknown = json.dumps({"some_other_shape": []}).encode() resp = self.client.post( diff --git a/tests/test_extraction.py b/tests/test_extraction.py index 52ec812..96544f7 100644 --- a/tests/test_extraction.py +++ b/tests/test_extraction.py @@ -21,6 +21,8 @@ is_invocation_span, is_llm_span, is_tool_span, + resolve_attr, + resolve_schema_version, ) from agentevals.loader.base import Span, Trace from agentevals.trace_attrs import ( @@ -28,6 +30,7 @@ ADK_LLM_RESPONSE, ADK_SCOPE_VALUE, ADK_TOOL_CALL_ARGS, + GENAI_ATTRIBUTE_ALIASES, OTEL_ERROR_TYPE, OTEL_GENAI_AGENT_NAME, OTEL_GENAI_INPUT_MESSAGES, @@ -50,6 +53,7 @@ OTEL_GENAI_USAGE_CACHE_READ_TOKENS, OTEL_GENAI_USAGE_INPUT_TOKENS, OTEL_GENAI_USAGE_OUTPUT_TOKENS, + OTEL_SCHEMA_URL, OTEL_SCOPE, ) @@ -399,6 +403,16 @@ def test_mixed_types(self): def test_empty(self): assert flatten_otlp_attributes([]) == {} + def test_schema_url_flows_through(self): + # otel.schema_url is a plain string attribute; flatten_otlp_attributes + # has no allowlist, so it passes through like any other key. + result = flatten_otlp_attributes( + [ + {"key": OTEL_SCHEMA_URL, "value": {"stringValue": "https://opentelemetry.io/schemas/1.37.0"}}, + ] + ) + assert result == {OTEL_SCHEMA_URL: "https://opentelemetry.io/schemas/1.37.0"} + # --------------------------------------------------------------------------- # Span classification helpers @@ -452,6 +466,15 @@ def test_is_invocation_span_by_keyword(self): def test_is_invocation_span_false(self): assert not is_invocation_span(_span(op="chat gpt-4")) + def test_schema_url_is_never_used_for_genai_detection(self): + # schema_url is a generic OTel field, not a GenAI-specific signal. + # A span carrying only otel.schema_url (no gen_ai.* attributes) must + # not be classified as an LLM/tool span by its presence alone. + span = _span(tags={OTEL_SCHEMA_URL: "https://opentelemetry.io/schemas/1.37.0"}) + assert not is_llm_span(span) + assert not is_tool_span(span) + assert not is_adk_scope(span) + # --------------------------------------------------------------------------- # Extractor detection @@ -661,6 +684,79 @@ def test_find_tool_spans_in(self): assert [s.span_id for s in ext.find_tool_spans_in(root)] == ["tool1"] +# --------------------------------------------------------------------------- +# resolve_attr — alias-aware attribute lookup +# --------------------------------------------------------------------------- + + +class TestResolveAttr: + def test_canonical_present_alias_absent(self): + attrs = {OTEL_GENAI_PROVIDER_NAME: "openai"} + assert resolve_attr(attrs, OTEL_GENAI_PROVIDER_NAME) == "openai" + + def test_canonical_absent_alias_present(self): + attrs = {OTEL_GENAI_SYSTEM: "anthropic"} + assert resolve_attr(attrs, OTEL_GENAI_PROVIDER_NAME) == "anthropic" + + def test_both_present_canonical_wins(self): + attrs = { + OTEL_GENAI_PROVIDER_NAME: "openai", + OTEL_GENAI_SYSTEM: "old_value", + } + assert resolve_attr(attrs, OTEL_GENAI_PROVIDER_NAME) == "openai" + + def test_neither_present_returns_none(self): + assert resolve_attr({}, OTEL_GENAI_PROVIDER_NAME) is None + + def test_unknown_canonical_key_with_no_alias_entry_returns_none(self): + # A key with no entry in GENAI_ATTRIBUTE_ALIASES simply has no + # fallback list; absence should not raise. + assert resolve_attr({}, "gen_ai.some.unmapped.key") is None + + def test_alias_table_contains_expected_seed_entry(self): + # Sanity check that the table wiring itself is intact. + assert OTEL_GENAI_SYSTEM in GENAI_ATTRIBUTE_ALIASES.get(OTEL_GENAI_PROVIDER_NAME, []) + + +# --------------------------------------------------------------------------- +# resolve_schema_version +# --------------------------------------------------------------------------- + + +class TestResolveSchemaVersion: + def test_valid_schema_url(self): + assert resolve_schema_version("https://opentelemetry.io/schemas/1.37.0") == "1.37.0" + + def test_valid_schema_url_with_trailing_slash(self): + assert resolve_schema_version("https://opentelemetry.io/schemas/1.4.0/") == "1.4.0" + + def test_valid_schema_url_with_prerelease(self): + assert resolve_schema_version("https://opentelemetry.io/schemas/1.37.0-rc.1") == "1.37.0-rc.1" + + def test_valid_schema_url_with_build_metadata(self): + assert resolve_schema_version("https://opentelemetry.io/schemas/1.37.0+build.5") == "1.37.0+build.5" + + def test_none_degrades_to_none(self): + assert resolve_schema_version(None) is None + + def test_empty_string_degrades_to_none(self): + assert resolve_schema_version("") is None + + def test_malformed_string_degrades_to_none(self): + assert resolve_schema_version("not-a-schema-url") is None + + def test_non_version_last_path_segment_returns_none(self): + assert resolve_schema_version("https://opentelemetry.io/schemas/1.4.0/extra") is None + + def test_non_string_input_degrades_to_none(self): + # Defensive: attrs values are technically Any; must not raise. + assert resolve_schema_version(123) is None # type: ignore[arg-type] + + def test_never_raises_on_garbage_input(self): + for garbage in (object(), [], {}, 1.5): + assert resolve_schema_version(garbage) is None # type: ignore[arg-type] + + # --------------------------------------------------------------------------- # extract_extended_model_info_from_attrs # --------------------------------------------------------------------------- @@ -793,6 +889,7 @@ def test_full_attribute_set(self): OTEL_GENAI_USAGE_CACHE_CREATION_TOKENS: 2000, OTEL_GENAI_USAGE_CACHE_READ_TOKENS: 5000, OTEL_ERROR_TYPE: None, + OTEL_SCHEMA_URL: "https://opentelemetry.io/schemas/1.37.0", } result = extract_extended_model_info_from_attrs(attrs) assert result["provider"] == "anthropic" @@ -805,6 +902,50 @@ def test_full_attribute_set(self): assert result["cache_creation_tokens"] == 2000 assert result["cache_read_tokens"] == 5000 assert result["error_type"] is None + assert result["schema_version"] == "1.37.0" + + # ----------------------------------------------------------------- + # schema_version / schema_url resolution + # ----------------------------------------------------------------- + + def test_schema_version_newest_attribute_names_only(self): + # Fixture with only newest attribute names (gen_ai.provider.name, + # not gen_ai.system) plus a schema_url; all fields, including + # schema_version, should populate normally. + attrs = { + OTEL_GENAI_PROVIDER_NAME: "openai", + OTEL_GENAI_REQUEST_MODEL: "gpt-4o", + OTEL_SCHEMA_URL: "https://opentelemetry.io/schemas/1.37.0", + } + result = extract_extended_model_info_from_attrs(attrs) + assert result["schema_version"] == "1.37.0" + assert result["provider"] == "openai" + assert result["request_model"] == "gpt-4o" + + def test_schema_version_with_aliased_attribute_names_only(self): + # Older instrumentor: gen_ai.system present, gen_ai.provider.name + # absent. provider should still populate via alias fallback, and + # schema_version should resolve from an older schema_url. + attrs = { + OTEL_GENAI_SYSTEM: "anthropic", + OTEL_SCHEMA_URL: "https://opentelemetry.io/schemas/1.20.0", + } + result = extract_extended_model_info_from_attrs(attrs) + assert result["provider"] == "anthropic" + assert result["schema_version"] == "1.20.0" + + def test_schema_version_missing_schema_url_entirely(self): + # No schema_url at all -> None, with no impact on other fields. + attrs = {OTEL_GENAI_PROVIDER_NAME: "openai", OTEL_GENAI_REQUEST_MODEL: "gpt-4o"} + result = extract_extended_model_info_from_attrs(attrs) + assert result["schema_version"] is None + assert result["provider"] == "openai" + assert result["request_model"] == "gpt-4o" + + def test_schema_version_malformed_schema_url(self): + attrs = {OTEL_SCHEMA_URL: "not-a-real-schema-url"} + result = extract_extended_model_info_from_attrs(attrs) + assert result["schema_version"] is None # --------------------------------------------------------------------------- diff --git a/tests/test_runner.py b/tests/test_runner.py index ae56d8a..3bf8cec 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -6,6 +6,7 @@ from agentevals.config import BuiltinMetricDef, EvalRunConfig from agentevals.converter import convert_traces +from agentevals.extraction import get_extractor from agentevals.loader.base import Span, Trace from agentevals.runner import _evaluate_trace, load_eval_set, run_evaluation from agentevals.trace_metrics import extract_trace_metadata @@ -261,6 +262,39 @@ def testextract_trace_metadata_adk(self): assert metadata["final_output_preview"] is not None assert len(metadata["final_output_preview"]) > 0 + def test_extract_trace_metadata_schema_version_unknown_when_schema_missing(self): + trace = _make_tool_trace(["tool_a"]) + metadata = extract_trace_metadata(trace) + assert metadata["schema_version"] is None + + def test_extract_trace_metadata_schema_version_unknown_when_schema_malformed(self): + trace = _make_tool_trace(["tool_a"]) + extractor = get_extractor(trace) + inv_spans = extractor.find_invocation_spans(trace) + llm_spans = ( + extractor.find_llm_spans_in(inv_spans[0]) + if inv_spans + else [s for s in trace.all_spans if extractor.classify_span(s) == "llm"] + ) + if llm_spans: + llm_spans[0].tags["otel.schema_url"] = "not-a-schema-version" + metadata = extract_trace_metadata(trace) + assert metadata["schema_version"] is None + + def test_extract_trace_metadata_schema_version_from_valid_schema_url(self): + trace = _make_tool_trace(["tool_a"]) + extractor = get_extractor(trace) + inv_spans = extractor.find_invocation_spans(trace) + llm_spans = ( + extractor.find_llm_spans_in(inv_spans[0]) + if inv_spans + else [s for s in trace.all_spans if extractor.classify_span(s) == "llm"] + ) + if llm_spans: + llm_spans[0].tags["otel.schema_url"] = "https://opentelemetry.io/schemas/1.39.0" + metadata = extract_trace_metadata(trace) + assert metadata["schema_version"] == "1.39.0" + class TestTrajectoryMatchType: """Verify trajectory_match_type produces different scores on the same trace.