From a6c3935df15eb123a947800ecbef3b4f5317a406 Mon Sep 17 00:00:00 2001 From: shuningc Date: Mon, 3 Aug 2026 15:08:48 -0700 Subject: [PATCH 1/8] docs(HYBIM-856): align evaluator terminology after HYBIM-949 rename Update SplunkAOEvaluators docstrings and rename stale test_galileo_metrics_* tests to test_evaluators_*. Co-authored-by: Cursor --- src/splunk_ao/schema/metrics.py | 4 ++-- tests/schemas/test_metrics.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/splunk_ao/schema/metrics.py b/src/splunk_ao/schema/metrics.py index c41896ce..17a81e49 100644 --- a/src/splunk_ao/schema/metrics.py +++ b/src/splunk_ao/schema/metrics.py @@ -12,9 +12,9 @@ class SplunkAOEvaluators(StrEnum): - """Built-in Splunk AO metric scorers. + """Built-in Splunk AO evaluators. - Values are human-readable UI labels used for scorer lookup via the API. + Values are human-readable UI labels used for evaluator lookup via the API. Member names follow the convention: base name = LLM version, _luna suffix = SLM version. """ diff --git a/tests/schemas/test_metrics.py b/tests/schemas/test_metrics.py index 667f0b87..571604ed 100644 --- a/tests/schemas/test_metrics.py +++ b/tests/schemas/test_metrics.py @@ -15,14 +15,14 @@ def test_metric_custom_no_version() -> None: assert metric.version is None -def test_galileo_metrics_values_are_nonempty_strings() -> None: +def test_evaluators_values_are_nonempty_strings() -> None: """All SplunkAOEvaluators values are non-empty human-readable strings.""" for member in SplunkAOEvaluators: assert isinstance(member.value, str), f"{member.name} value is not a string" assert len(member.value.strip()) > 0, f"{member.name} has an empty value" -def test_galileo_metrics_is_str_compatible() -> None: +def test_evaluators_is_str_compatible() -> None: """SplunkAOEvaluators members are str-compatible (usable as plain strings).""" member = SplunkAOEvaluators.correctness assert isinstance(member, str) @@ -30,7 +30,7 @@ def test_galileo_metrics_is_str_compatible() -> None: assert member.value == "Correctness" -def test_galileo_metrics_naming_convention() -> None: +def test_evaluators_naming_convention() -> None: """Base names map to LLM versions, _luna suffix maps to SLM versions.""" # LLM versions (base names) should NOT have "(SLM)" in the label assert "(SLM)" not in SplunkAOEvaluators.input_pii.value From debc7fe95b1b0a12b1ec2f81f40cffa01eb203ed Mon Sep 17 00:00:00 2001 From: shuningc Date: Tue, 4 Aug 2026 17:07:57 -0700 Subject: [PATCH 2/8] test: rename stale test_lookup_by_galileo_metrics_enum Complete the evaluator terminology cleanup by renaming the remaining galileo_metrics test identifier in test_experiment.py. Co-authored-by: Cursor --- tests/test_experiment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_experiment.py b/tests/test_experiment.py index 665690cd..deaca027 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -1499,7 +1499,7 @@ def test_returns_none_when_metric_aggregates_empty( @patch("splunk_ao.experiment.experiments_available_columns_projects_project_id_experiments_available_columns_post") @patch("splunk_ao.experiment.SplunkAOConfig") - def test_lookup_by_galileo_metrics_enum( + def test_lookup_by_evaluators_enum( self, mock_config_class: MagicMock, mock_api: MagicMock, From bcf403144c0d40f3d27c72fc87171393b46358da Mon Sep 17 00:00:00 2001 From: shuningc Date: Tue, 4 Aug 2026 17:12:31 -0700 Subject: [PATCH 3/8] docs: align evaluator terminology in docstrings and errors Update user-facing docstrings and error messages to use evaluator/agent stream vocabulary after the SplunkAOEvaluators rename. Keep the public metrics= parameter name unchanged for API compatibility. Co-authored-by: Cursor --- src/splunk_ao/agent_stream.py | 36 +++--- src/splunk_ao/agent_streams.py | 152 +++++++++++++------------ src/splunk_ao/evaluator.py | 16 +-- src/splunk_ao/types.py | 8 +- src/splunk_ao/utils/metrics.py | 18 +-- tests/test_agent_streams_evaluators.py | 4 +- 6 files changed, 118 insertions(+), 116 deletions(-) diff --git a/src/splunk_ao/agent_stream.py b/src/splunk_ao/agent_stream.py index 5ba50dfc..d8ad1d05 100644 --- a/src/splunk_ao/agent_stream.py +++ b/src/splunk_ao/agent_stream.py @@ -77,16 +77,16 @@ class AgentStream(StateManagementMixin): project = Project.get(name="My AI Project") agent_stream = project.create_agent_stream(name="Production Logs") - # Enable metrics on the log stream + # Enable evaluators on the agent stream from splunk_ao.schema.metrics import SplunkAOEvaluators - local_metrics = log_stream.enable_evaluators([ + local_evaluators = agent_stream.enable_evaluators([ SplunkAOEvaluators.correctness, SplunkAOEvaluators.completeness, "context_relevance" ]) - # Refresh log stream state from API - log_stream.refresh() + # Refresh agent stream state from API + agent_stream.refresh() """ created_at: datetime | None @@ -408,21 +408,21 @@ def refresh(self) -> None: def get_metrics(self) -> builtins.list[str]: """ - Get the list of metrics currently enabled on this log stream. + Get the list of evaluators currently enabled on this agent stream. Returns ------- - list[str]: List of metric names currently enabled. + list[str]: List of evaluator names currently enabled. Raises ------ - ValueError: If the log stream lacks required id or project_id attributes. + ValueError: If the agent stream lacks required id or project_id attributes. Examples -------- agent_stream = AgentStream.get(name="Production Logs", project_name="My Project") - current_metrics = log_stream.get_metrics() - print(f"Currently enabled: {current_metrics}") + current_evaluators = agent_stream.get_metrics() + print(f"Currently enabled: {current_evaluators}") """ logger.info(f"AgentStream.get_metrics: id='{self.id}' - started") config = SplunkAOConfig.get() @@ -444,26 +444,26 @@ def set_metrics( self, metrics: builtins.list[SplunkAOEvaluators | Metric | LocalMetricConfig | str] ) -> builtins.list[LocalMetricConfig]: """ - Set (replace) the metrics on this log stream. + Set (replace) the evaluators on this agent stream. - This replaces any existing metrics with the new list. Alias for enable_metrics - with clearer naming intent. + This replaces any existing evaluators with the new list. The ``metrics`` parameter + name is retained for API compatibility. Args: - metrics: List of metrics to set. Supports: + metrics: List of evaluators to set. Supports: - SplunkAOEvaluators enum values (e.g., SplunkAOEvaluators.correctness) - Metric objects (including from Metric.get(id="...")) - LocalMetricConfig objects for custom scoring functions - - String names of built-in metrics + - String names of built-in evaluators Returns ------- - List[LocalMetricConfig]: Local metric configurations that must be + List[LocalMetricConfig]: Local evaluator configurations that must be computed client-side. Raises ------ - ValueError: If any specified metrics are unknown. + ValueError: If any specified evaluators are unknown. Examples -------- @@ -472,7 +472,7 @@ def set_metrics( agent_stream = AgentStream.get(name="Production Logs", project_name="My Project") # Set evaluators (replaces existing) - log_stream.set_metrics([ + agent_stream.set_metrics([ Evaluator.metrics.correctness, Evaluator.metrics.completeness, Evaluator.get(id="evaluator-from-console-uuid"), # From console @@ -483,7 +483,7 @@ def set_metrics( agent_streams_svc = AgentStreams() agent_stream = agent_streams_svc.get(name=self.name, project_id=self.project_id) if agent_stream is None: - raise ValueError(f"Log stream '{self.name}' not found") + raise ValueError(f"Agent stream '{self.name}' not found") result = agent_stream.enable_evaluators(metrics) # Set state to synced after successful operation self._set_state(SyncState.SYNCED) diff --git a/src/splunk_ao/agent_streams.py b/src/splunk_ao/agent_streams.py index 84ad68e6..cf17d1cd 100644 --- a/src/splunk_ao/agent_streams.py +++ b/src/splunk_ao/agent_streams.py @@ -76,7 +76,7 @@ class AgentStream(LogStreamResponse): messages=[{"role": "user", "content": "Hello, world!"}] ) - # Enable metrics on a log stream - RECOMMENDED APPROACH + # Enable evaluators on an agent stream - RECOMMENDED APPROACH from splunk_ao.agent_streams import enable_evaluators from splunk_ao.schema.metrics import SplunkAOEvaluators @@ -84,15 +84,15 @@ class AgentStream(LogStreamResponse): # export SPLUNK_AO_AGENT_STREAM="Production Logs" # export SPLUNK_AO_PROJECT="My AI Project" - # Clean and simple - just pass the metrics! - local_metrics = enable_evaluators([ + # Clean and simple - just pass the evaluators! + local_evaluators = enable_evaluators([ SplunkAOEvaluators.correctness, SplunkAOEvaluators.completeness, "context_relevance" ]) # Alternative: Use explicit parameters - local_metrics = enable_evaluators( + local_evaluators = enable_evaluators( agent_stream_name="Production Logs", project_name="My AI Project", metrics=["correctness", "completeness"] @@ -126,68 +126,69 @@ def enable_evaluators( self, metrics: builtins.list[SplunkAOEvaluators | Metric | LocalMetricConfig | str] ) -> builtins.list[LocalMetricConfig]: """ - Enable metrics directly on this log stream instance. + Enable evaluators directly on this agent stream instance. - This is the most intuitive and clean way to enable metrics when you already have a - AgentStream object. The method leverages the log stream's existing project_id and id + This is the most intuitive and clean way to enable evaluators when you already have a + AgentStream object. The method leverages the agent stream's existing project_id and id attributes, eliminating the need for redundant parameter specification and reducing the potential for errors. This approach is ideal for object-oriented workflows where you're working with AgentStream - instances directly, and it provides the clearest semantic meaning: "enable these metrics - on this specific log stream." + instances directly, and it provides the clearest semantic meaning: "enable these evaluators + on this specific agent stream." Parameters ---------- metrics : builtins.list[Union[SplunkAOEvaluators, Metric, LocalMetricConfig, str]] - List of metrics to enable on this log stream. Supports multiple input formats: + List of evaluators to enable on this agent stream. Parameter name retained for API + compatibility. Supports multiple input formats: - - **SplunkAOEvaluators enum values**: Built-in metrics like `SplunkAOEvaluators.correctness` - - **Metric objects**: Custom metrics with optional version specifications - - **LocalMetricConfig objects**: Client-side metrics with custom scoring functions - - **String names**: Built-in metric names like "correctness" or "toxicity" + - **SplunkAOEvaluators enum values**: Built-in evaluators like `SplunkAOEvaluators.correctness` + - **Metric objects**: Custom evaluators with optional version specifications + - **LocalMetricConfig objects**: Client-side evaluators with custom scoring functions + - **String names**: Built-in evaluator names like "correctness" or "toxicity" Returns ------- builtins.list[LocalMetricConfig] - List of local metric configurations that must be computed client-side. - Server-side metrics are automatically registered with Splunk AO and don't + List of local evaluator configurations that must be computed client-side. + Server-side evaluators are automatically registered with Splunk AO and don't need to be returned since users don't interact with them. Raises ------ ValueError - If this AgentStream instance lacks required `id` or `project_id` attributes - - If any specified metrics are unknown or unavailable - - If there are issues with metric configuration or registration + - If any specified evaluators are unknown or unavailable + - If there are issues with evaluator configuration or registration GalileoHTTPException If there are network or API errors when communicating with Splunk AO services Examples -------- - Basic usage with built-in metrics: + Basic usage with built-in evaluators: ```python from splunk_ao.agent_streams import AgentStreams from splunk_ao.schema.metrics import SplunkAOEvaluators - # Get a log stream first - log_streams = AgentStreams() - agent_stream = log_streams.get(name="Production Logs", project_name="My AI Project") + # Get an agent stream first + agent_streams = AgentStreams() + agent_stream = agent_streams.get(name="Production Logs", project_name="My AI Project") - # Enable metrics directly - clean and intuitive! - local_metrics = log_stream.enable_evaluators([ + # Enable evaluators directly - clean and intuitive! + local_evaluators = agent_stream.enable_evaluators([ SplunkAOEvaluators.correctness, SplunkAOEvaluators.completeness, "context_relevance", "toxicity" ]) - logger.info(f"Server-side metrics enabled automatically") - logger.info(f"Need to process {len(local_metrics)} local metrics") + logger.info("Server-side evaluators enabled automatically") + logger.info(f"Need to process {len(local_evaluators)} local evaluators") ``` - Advanced usage with custom metrics: + Advanced usage with custom evaluators: ```python from splunk_ao.schema.metrics import Metric, LocalMetricConfig @@ -195,16 +196,16 @@ def enable_evaluators( def custom_scorer(trace_or_span): return 0.75 # Your scoring logic - local_metrics = log_stream.enable_evaluators([ + local_evaluators = agent_stream.enable_evaluators([ SplunkAOEvaluators.correctness, "completeness", Metric(name="domain_relevance", version=3), - LocalMetricConfig(name="custom_metric", scorer_fn=custom_scorer) + LocalMetricConfig(name="custom_evaluator", scorer_fn=custom_scorer) ]) - # Process local metrics if any - for local_metric in local_metrics: - logger.info(f"Need to process local metric: {local_metric.name}") + # Process local evaluators if any + for local_evaluator in local_evaluators: + logger.info(f"Need to process local evaluator: {local_evaluator.name}") ``` Notes @@ -217,11 +218,11 @@ def custom_scorer(trace_or_span): **Recommended Usage:** - Use this method when you already have a AgentStream object - - More intuitive than specifying project/log stream names again + - More intuitive than specifying project/agent stream names again - Cleaner object-oriented design pattern """ if not hasattr(self, "id") or not hasattr(self, "project_id"): - raise ValueError("Log stream must have id and project_id to enable metrics") + raise ValueError("Agent stream must have id and project_id to enable evaluators") _, local_metrics = create_metric_configs(self.project_id, self.id, metrics) return local_metrics @@ -484,46 +485,46 @@ def enable_evaluators( metrics: builtins.list[SplunkAOEvaluators | Metric | LocalMetricConfig | str], ) -> builtins.list[LocalMetricConfig]: """ - Enable metrics for a log stream by configuring scorers. + Enable evaluators for an agent stream by configuring scorers. The project name can be provided via the 'project_name' parameter or the SPLUNK_AO_PROJECT environment variable. - The log stream name can be provided via the 'agent_stream_name' parameter or the + The agent stream name can be provided via the 'agent_stream_name' parameter or the SPLUNK_AO_AGENT_STREAM environment variable. Parameters ---------- agent_stream_name : Optional[str], optional - The name of the log stream. Takes precedence over the SPLUNK_AO_AGENT_STREAM environment variable. Defaults to None. + The name of the agent stream. Takes precedence over the SPLUNK_AO_AGENT_STREAM environment variable. Defaults to None. project_name : Optional[str], optional The name of the project. Takes precedence over the SPLUNK_AO_PROJECT environment variable. Defaults to None. metrics : builtins.list[Union[SplunkAOEvaluators, Metric, LocalMetricConfig, str]] - List of metrics to enable. Can include: + List of evaluators to enable. Parameter name retained for API compatibility. Can include: - SplunkAOEvaluators enum values (e.g., SplunkAOEvaluators.correctness) - Metric objects with name and optional version - - LocalMetricConfig objects for custom local metrics - - String names of built-in metrics + - LocalMetricConfig objects for custom local evaluators + - String names of built-in evaluators Returns ------- tuple[builtins.list[ScorerConfig], builtins.list[LocalMetricConfig]] - A tuple containing the configured scorer configs and local metric configs. + A tuple containing the configured scorer configs and local evaluator configs. Raises ------ ValueError - If log stream or project cannot be found, or if metrics are unknown. + If agent stream or project cannot be found, or if evaluators are unknown. Examples -------- ```python - # Enable built-in metrics with explicit parameters + # Enable built-in evaluators with explicit parameters from splunk_ao.agent_streams import AgentStreams from splunk_ao.schema.metrics import SplunkAOEvaluators - log_streams = AgentStreams() - scorer_configs, local_metrics = log_streams.enable_evaluators( + agent_streams = AgentStreams() + scorer_configs, local_evaluators = agent_streams.enable_evaluators( agent_stream_name="Production Logs", project_name="My AI Project", metrics=[ @@ -533,26 +534,26 @@ def enable_evaluators( ], ) - # Enable metrics using environment variables + # Enable evaluators using environment variables # export SPLUNK_AO_AGENT_STREAM="Production Logs" # export SPLUNK_AO_PROJECT="My AI Project" - scorer_configs, local_metrics = log_streams.enable_evaluators( + scorer_configs, local_evaluators = agent_streams.enable_evaluators( metrics=["correctness", "completeness"] ) - # Enable custom metrics with mixed parameters + # Enable custom evaluators with mixed parameters from splunk_ao.schema.metrics import Metric, LocalMetricConfig def custom_scorer(trace_or_span): return 0.85 # Custom scoring logic # export SPLUNK_AO_PROJECT="My AI Project" - scorer_configs, local_metrics = log_streams.enable_evaluators( - agent_stream_name="Production Logs", # Explicit log stream + scorer_configs, local_evaluators = agent_streams.enable_evaluators( + agent_stream_name="Production Logs", # Explicit agent stream # project_name from env var metrics=[ - Metric(name="my_custom_metric", version=2), - LocalMetricConfig(name="local_scorer", scorer_fn=custom_scorer) + Metric(name="my_custom_evaluator", version=2), + LocalMetricConfig(name="local_evaluator", scorer_fn=custom_scorer) ] ) ``` @@ -571,7 +572,7 @@ def custom_scorer(trace_or_span): raise ValueError("agent_stream_name must be provided (or set SPLUNK_AO_AGENT_STREAM env var)") agent_stream = self.get(name=agent_stream_name, project_name=project_obj.name) if not agent_stream: - raise ValueError(f"Log stream '{agent_stream_name}' not found in project '{project_obj.name}'") + raise ValueError(f"Agent stream '{agent_stream_name}' not found in project '{project_obj.name}'") # Use the shared utility function directly _, local_metrics = create_metric_configs(project_obj.id, agent_stream.id, metrics) @@ -689,14 +690,14 @@ def enable_evaluators( metrics: builtins.list[SplunkAOEvaluators | Metric | LocalMetricConfig | str], ) -> builtins.list[LocalMetricConfig]: """ - Enable metrics for a log stream with flexible parameter and environment variable support. + Enable evaluators for an agent stream with flexible parameter and environment variable support. This unified function supports both explicit parameters and environment variable fallbacks, making it perfect for all use cases - from production CI/CD pipelines to development testing. **Flexible Usage Patterns:** - - **Environment-only**: Just pass metrics, names from env vars (production/CI) - - **Explicit parameters**: Specify project/log stream names directly (development) + - **Environment-only**: Just pass evaluators, names from env vars (production/CI) + - **Explicit parameters**: Specify project/agent stream names directly (development) - **Mixed approach**: Combine explicit params with environment fallbacks Environment Variables (Optional Fallbacks) @@ -704,34 +705,35 @@ def enable_evaluators( SPLUNK_AO_PROJECT : str The name of the Splunk AO project (used when project_name not provided) SPLUNK_AO_AGENT_STREAM : str - The name of the log stream (used when agent_stream_name not provided) + The name of the agent stream (used when agent_stream_name not provided) Parameters ---------- agent_stream_name : Optional[str], optional - The name of the log stream. Takes precedence over SPLUNK_AO_AGENT_STREAM environment variable. + The name of the agent stream. Takes precedence over SPLUNK_AO_AGENT_STREAM environment variable. If None, will use SPLUNK_AO_AGENT_STREAM env var. Defaults to None. project_name : Optional[str], optional The name of the project. Takes precedence over SPLUNK_AO_PROJECT environment variable. If None, will use SPLUNK_AO_PROJECT env var. Defaults to None. metrics : builtins.list[Union[SplunkAOEvaluators, Metric, LocalMetricConfig, str]] - List of metrics to enable on the log stream. Can include: + List of evaluators to enable on the agent stream. Parameter name retained for API + compatibility. Can include: - SplunkAOEvaluators enum values (e.g., SplunkAOEvaluators.correctness) - - Metric objects with name and optional version for custom metrics + - Metric objects with name and optional version for custom evaluators - LocalMetricConfig objects for client-side custom scoring functions - - String names of built-in metrics (e.g., "correctness", "toxicity") + - String names of built-in evaluators (e.g., "correctness", "toxicity") Returns ------- builtins.list[LocalMetricConfig] - List of local metric configurations that must be computed client-side. - Server-side metrics are automatically registered with Splunk AO and don't + List of local evaluator configurations that must be computed client-side. + Server-side evaluators are automatically registered with Splunk AO and don't need to be returned since users don't interact with them. Raises ------ ValueError - If log stream or project cannot be found, or if any specified metrics are unknown. + If agent stream or project cannot be found, or if any specified evaluators are unknown. errors.UnexpectedStatus If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. httpx.TimeoutException @@ -740,11 +742,11 @@ def enable_evaluators( Examples -------- ```python - # Enable built-in metrics with explicit parameters + # Enable built-in evaluators with explicit parameters from splunk_ao.agent_streams import enable_evaluators from splunk_ao.schema.metrics import SplunkAOEvaluators - local_metrics = enable_evaluators( + local_evaluators = enable_evaluators( agent_stream_name="Production Logs", project_name="My AI Project", metrics=[ @@ -754,27 +756,27 @@ def enable_evaluators( ], ) - # Enable metrics using environment variables only + # Enable evaluators using environment variables only # export SPLUNK_AO_AGENT_STREAM="Production Logs" # export SPLUNK_AO_PROJECT="My AI Project" - local_metrics = enable_evaluators(metrics=["correctness", "completeness"]) + local_evaluators = enable_evaluators(metrics=["correctness", "completeness"]) - # Enable custom and local metrics with environment variable fallbacks + # Enable custom and local evaluators with environment variable fallbacks from splunk_ao.schema.metrics import Metric, LocalMetricConfig from galileo_core.schemas.logging.step import StepType def response_length_scorer(trace_or_span): - '''Custom metric that scores based on response length''' + '''Custom evaluator that scores based on response length''' if hasattr(trace_or_span, "output") and trace_or_span.output: return min(len(trace_or_span.output) / 100.0, 1.0) # Normalize 0-1 return 0.0 - local_metrics = enable_evaluators( + local_evaluators = enable_evaluators( agent_stream_name="Development Logs", metrics=[ SplunkAOEvaluators.correctness, "toxicity", - Metric(name="my_custom_metric", version=2), + Metric(name="my_custom_evaluator", version=2), LocalMetricConfig( name="response_length", scorer_fn=response_length_scorer, @@ -784,9 +786,9 @@ def response_length_scorer(trace_or_span): ], ) - # Process local metrics - for local_metric in local_metrics: - logger.info(f"Need to process local metric: {local_metric.name}") + # Process local evaluators + for local_evaluator in local_evaluators: + logger.info(f"Need to process local evaluator: {local_evaluator.name}") ``` """ return AgentStreams().enable_evaluators(agent_stream_name=agent_stream_name, project_name=project_name, metrics=metrics) diff --git a/src/splunk_ao/evaluator.py b/src/splunk_ao/evaluator.py index 310db4a2..587b7cfa 100644 --- a/src/splunk_ao/evaluator.py +++ b/src/splunk_ao/evaluator.py @@ -69,16 +69,16 @@ class BuiltInEvaluators: """ def __getattr__(self, name: str) -> SplunkAOEvaluators: - """Allow attribute-style access to built-in metrics.""" - # Try to find the metric by name (enum names match UI-visible names) - for scorer in SplunkAOEvaluators: - if scorer.name == name: - return scorer - raise AttributeError(f"Built-in metric '{name}' not found. Available: {[s.name for s in SplunkAOEvaluators]}") + """Allow attribute-style access to built-in evaluators.""" + # Try to find the evaluator by name (enum names match UI-visible names) + for evaluator in SplunkAOEvaluators: + if evaluator.name == name: + return evaluator + raise AttributeError(f"Built-in evaluator '{name}' not found. Available: {[e.name for e in SplunkAOEvaluators]}") def __dir__(self) -> list[str]: - """Return list of available metric names for autocomplete.""" - return [scorer.name for scorer in SplunkAOEvaluators] + """Return list of available evaluator names for autocomplete.""" + return [evaluator.name for evaluator in SplunkAOEvaluators] # Backwards-compatible alias diff --git a/src/splunk_ao/types.py b/src/splunk_ao/types.py index 778189c7..cf346de8 100644 --- a/src/splunk_ao/types.py +++ b/src/splunk_ao/types.py @@ -8,12 +8,12 @@ from splunk_ao.evaluator import Evaluator from splunk_ao.schema.metrics import LocalMetricConfig, SplunkAOEvaluators -# Unified metric type that accepts all valid metric specifications +# Unified evaluator specification type that accepts all valid evaluator inputs MetricSpec = ( - SplunkAOEvaluators # Built-in scorer enum (e.g., SplunkAOEvaluators.correctness) + SplunkAOEvaluators # Built-in evaluator enum (e.g., SplunkAOEvaluators.correctness) | Evaluator # Custom or local evaluator object - | LocalMetricConfig # Legacy local metric config - | str # String name of built-in metric (e.g., "correctness") + | LocalMetricConfig # Legacy local evaluator config + | str # String name of built-in evaluator (e.g., "correctness") ) __all__ = ["MetricSpec"] diff --git a/src/splunk_ao/utils/metrics.py b/src/splunk_ao/utils/metrics.py index 93b33884..5dd0b415 100644 --- a/src/splunk_ao/utils/metrics.py +++ b/src/splunk_ao/utils/metrics.py @@ -98,12 +98,12 @@ def create_metric_configs( metrics: builtins.list[SplunkAOEvaluators | Metric | LocalMetricConfig | str], ) -> tuple[builtins.list[ScorerConfig], builtins.list[LocalMetricConfig]]: """ - Process metrics and create scorer configurations for experiments or log streams. + Process evaluators and create scorer configurations for experiments or agent streams. - This unified function categorizes metrics into server-side and client-side types, - validates they exist, and registers server-side metrics with Splunk AO. + This unified function categorizes evaluators into server-side and client-side types, + validates they exist, and registers server-side evaluators with Splunk AO. - Metrics can be specified as: + Evaluators can be specified as: - SplunkAOEvaluators enum values (human-readable labels like "Correctness") - Metric objects with name and optional version - UUID strings (scorer IDs for direct lookup) @@ -115,22 +115,22 @@ def create_metric_configs( project_id : str The ID of the project run_id : Optional[str] - The ID of the run (can be experiment ID or log stream ID). + The ID of the run (can be experiment ID or agent stream ID). When None, scorer registration is skipped (trigger=True flow). metrics : list - List of metrics to configure + List of evaluators to configure. Parameter name retained for API compatibility. Returns ------- tuple[list[ScorerConfig], list[LocalMetricConfig]] A tuple containing: - - List of ScorerConfig objects for server-side metrics configured in Splunk AO - - List of LocalMetricConfig objects for client-side metrics to process locally + - List of ScorerConfig objects for server-side evaluators configured in Splunk AO + - List of LocalMetricConfig objects for client-side evaluators to process locally Raises ------ ValueError - If any specified metrics are unknown or don't exist in Splunk AO + If any specified evaluators are unknown or don't exist in Splunk AO """ local_metric_configs: list[LocalMetricConfig] = [] scorer_ids: list[str] = [] diff --git a/tests/test_agent_streams_evaluators.py b/tests/test_agent_streams_evaluators.py index 9247338c..064358c8 100644 --- a/tests/test_agent_streams_evaluators.py +++ b/tests/test_agent_streams_evaluators.py @@ -171,7 +171,7 @@ def test_log_stream_enable_metrics_missing_ids(self) -> None: """Test AgentStream enable_evaluators raises error when IDs are missing.""" agent_stream = AgentStream() # Empty log stream without IDs - with pytest.raises(ValueError, match="Log stream must have id and project_id to enable metrics"): + with pytest.raises(ValueError, match="Agent stream must have id and project_id to enable evaluators"): agent_stream.enable_evaluators(["correctness"]) @patch("splunk_ao.agent_streams.Projects") @@ -294,7 +294,7 @@ def test_logstreams_enable_metrics_logstream_not_found(self, mock_get, mock_proj project_name="Test Project", agent_stream_name="Nonexistent Stream", metrics=["correctness"] ) - assert "Log stream 'Nonexistent Stream' not found" in str(exc_info.value) + assert "Agent stream 'Nonexistent Stream' not found" in str(exc_info.value) @patch.object(AgentStreams, "enable_evaluators") def test_enable_metrics_convenience_function_explicit(self, mock_enable_metrics) -> None: From 5de066bd2606cab12a203cffdfb8339bcc40c76e Mon Sep 17 00:00:00 2001 From: shuningc Date: Tue, 4 Aug 2026 17:14:07 -0700 Subject: [PATCH 4/8] docs: clarify SplunkAOEvaluators values match scorer API labels Document that enum values are resolved via legacy /scorers endpoints while keeping evaluator terminology for the SDK-facing concept. Co-authored-by: Cursor --- src/splunk_ao/schema/metrics.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/splunk_ao/schema/metrics.py b/src/splunk_ao/schema/metrics.py index 17a81e49..915bfdeb 100644 --- a/src/splunk_ao/schema/metrics.py +++ b/src/splunk_ao/schema/metrics.py @@ -14,7 +14,9 @@ class SplunkAOEvaluators(StrEnum): """Built-in Splunk AO evaluators. - Values are human-readable UI labels used for evaluator lookup via the API. + Values are human-readable UI labels, matched against scorer labels by the + API (endpoints still use the legacy ``/scorers`` paths — see + ``docs/domain-entity-rename.md``). Member names follow the convention: base name = LLM version, _luna suffix = SLM version. """ From db40cf833535b99b145c62cfee49fa0d4713407d Mon Sep 17 00:00:00 2001 From: shuningc Date: Tue, 4 Aug 2026 23:16:46 -0700 Subject: [PATCH 5/8] docs: add HYBIM-856 evaluator terminology changes to CHANGELOG Co-authored-by: Cursor --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 17d156bb..b4b87bd2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,13 @@ ### Changed +- **Evaluator terminology alignment in docs and errors** (HYBIM-856): Updated + `SplunkAOEvaluators` docstrings, agent stream/evaluator API docstrings, and + user-visible error messages to use evaluator and agent stream vocabulary after + the HYBIM-949 rename. Enum values are documented as matching scorer labels via + the legacy `/scorers` API paths. The public `metrics=` parameter name is + unchanged for API compatibility. Renamed stale `test_galileo_metrics_*` and + `test_lookup_by_galileo_metrics_enum` test identifiers. - Completed spans are queued immediately in an OpenTelemetry `BatchSpanProcessor` and exported on its configured schedule. - `flush()` and `async_flush()` now drain completed spans without concluding an From bed29b7a53e8156dc252ded4a66a8b1fe4364f14 Mon Sep 17 00:00:00 2001 From: shuningc Date: Thu, 6 Aug 2026 14:27:54 -0700 Subject: [PATCH 6/8] docs: remove internal Jira IDs from CHANGELOG entries Users cannot access internal ticket references; describe changes in product terms instead. Co-authored-by: Cursor --- CHANGELOG.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4b87bd2..0e922632 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### Breaking Changes -- **`monitor_progress()` `job_id` parameter removed** (HYBIM-931): The deprecated +- **`monitor_progress()` `job_id` parameter removed**: The deprecated `job_id` keyword argument of `Experiment.monitor_progress()` has been fully removed. Callers passing `job_id=` must remove that argument. @@ -25,13 +25,13 @@ ### Changed -- **Evaluator terminology alignment in docs and errors** (HYBIM-856): Updated +- **Evaluator terminology alignment in docs and errors**: Updated `SplunkAOEvaluators` docstrings, agent stream/evaluator API docstrings, and - user-visible error messages to use evaluator and agent stream vocabulary after - the HYBIM-949 rename. Enum values are documented as matching scorer labels via - the legacy `/scorers` API paths. The public `metrics=` parameter name is - unchanged for API compatibility. Renamed stale `test_galileo_metrics_*` and - `test_lookup_by_galileo_metrics_enum` test identifiers. + user-visible error messages to use evaluator and agent stream vocabulary following + the `SplunkAOMetrics` → `SplunkAOEvaluators` rename. Enum values are documented + as matching scorer labels via the legacy `/scorers` API paths. The public + `metrics=` parameter name is unchanged for API compatibility. Renamed stale + `test_galileo_metrics_*` and `test_lookup_by_galileo_metrics_enum` test identifiers. - Completed spans are queued immediately in an OpenTelemetry `BatchSpanProcessor` and exported on its configured schedule. - `flush()` and `async_flush()` now drain completed spans without concluding an From 01bdb8b54de7c44822c4bc9d85e65ae1a029972c Mon Sep 17 00:00:00 2001 From: shuningc Date: Thu, 6 Aug 2026 14:29:20 -0700 Subject: [PATCH 7/8] docs: fix AgentStreams.enable_evaluators return docs and examples The method returns a list of LocalMetricConfig, not a tuple. Update the Returns section and examples to match the actual API. Co-authored-by: Cursor --- src/splunk_ao/agent_streams.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/splunk_ao/agent_streams.py b/src/splunk_ao/agent_streams.py index cf17d1cd..b58938c1 100644 --- a/src/splunk_ao/agent_streams.py +++ b/src/splunk_ao/agent_streams.py @@ -508,8 +508,9 @@ def enable_evaluators( Returns ------- - tuple[builtins.list[ScorerConfig], builtins.list[LocalMetricConfig]] - A tuple containing the configured scorer configs and local evaluator configs. + builtins.list[LocalMetricConfig] + Local evaluator configurations that must be computed client-side. + Server-side evaluators are automatically registered with Splunk AO. Raises ------ @@ -524,7 +525,7 @@ def enable_evaluators( from splunk_ao.schema.metrics import SplunkAOEvaluators agent_streams = AgentStreams() - scorer_configs, local_evaluators = agent_streams.enable_evaluators( + local_evaluators = agent_streams.enable_evaluators( agent_stream_name="Production Logs", project_name="My AI Project", metrics=[ @@ -537,7 +538,7 @@ def enable_evaluators( # Enable evaluators using environment variables # export SPLUNK_AO_AGENT_STREAM="Production Logs" # export SPLUNK_AO_PROJECT="My AI Project" - scorer_configs, local_evaluators = agent_streams.enable_evaluators( + local_evaluators = agent_streams.enable_evaluators( metrics=["correctness", "completeness"] ) @@ -548,7 +549,7 @@ def custom_scorer(trace_or_span): return 0.85 # Custom scoring logic # export SPLUNK_AO_PROJECT="My AI Project" - scorer_configs, local_evaluators = agent_streams.enable_evaluators( + local_evaluators = agent_streams.enable_evaluators( agent_stream_name="Production Logs", # Explicit agent stream # project_name from env var metrics=[ From 0808d0faaabc0e21dde43ddd3dedeff0dcd98178 Mon Sep 17 00:00:00 2001 From: shuningc Date: Thu, 6 Aug 2026 14:31:44 -0700 Subject: [PATCH 8/8] docs: use set_metrics in public AgentStream examples The splunk_ao.AgentStream OO class exposes set_metrics, not enable_evaluators. Fix class docstring and domain-entity-rename examples that incorrectly called enable_evaluators on the public AgentStream type. Co-authored-by: Cursor --- docs/domain-entity-rename.md | 5 +++-- src/splunk_ao/agent_stream.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/domain-entity-rename.md b/docs/domain-entity-rename.md index 10f4131b..aec9c41d 100644 --- a/docs/domain-entity-rename.md +++ b/docs/domain-entity-rename.md @@ -46,7 +46,7 @@ streams = AgentStream.list(project_name="my-project") # Enable evaluators on the stream from splunk_ao.schema.metrics import SplunkAOEvaluators -stream.enable_evaluators([SplunkAOEvaluators.correctness, SplunkAOEvaluators.completeness]) +stream.set_metrics([SplunkAOEvaluators.correctness, SplunkAOEvaluators.completeness]) ``` ```python @@ -187,7 +187,8 @@ from splunk_ao.metrics import … → from splunk_ao.evaluators import project.create_log_stream(…) → project.create_agent_stream(…) project.list_log_streams(…) → project.list_agent_streams(…) project.logstreams → project.agent_streams -enable_metrics(…) → enable_evaluators(…) +log_stream.enable_metrics(…) → agent_stream.set_metrics(…) +enable_metrics(…) → enable_evaluators(…) (AgentStreams service / module-level only) delete_metric(…) → delete_evaluator(…) get_metrics(…) → get_evaluators(…) create_custom_llm_metric(…) → create_custom_llm_evaluator(…) diff --git a/src/splunk_ao/agent_stream.py b/src/splunk_ao/agent_stream.py index d8ad1d05..0f0ce90c 100644 --- a/src/splunk_ao/agent_stream.py +++ b/src/splunk_ao/agent_stream.py @@ -79,7 +79,7 @@ class AgentStream(StateManagementMixin): # Enable evaluators on the agent stream from splunk_ao.schema.metrics import SplunkAOEvaluators - local_evaluators = agent_stream.enable_evaluators([ + local_evaluators = agent_stream.set_metrics([ SplunkAOEvaluators.correctness, SplunkAOEvaluators.completeness, "context_relevance"