feat(benchmark): expose demographics and standings breakdown in SDK

docs/mri_advanced.md

@@ -245,6 +245,85 @@ participant = benchmark.participants[0]
 participant.delete()
 ```
 
+## Voter Demographics
+
+Every vote on a benchmark carries the voter's demographics, so you can see *who*
+evaluated your models and how different groups ranked them. Both methods live on
+the benchmark and return a pandas DataFrame, like `get_overall_standings`.
+
+!!! note
+    `ageBucket`, `gender` and `occupation` are **estimated** (inferred from
+    behaviour), not self-declared. `country` and `language` are observed. Each
+    dimension includes an `"unknown"` bucket for votes whose attribute could not
+    be determined.
+
+### Voter composition
+
+`get_demographics` returns one row per `(dimension, value)` with the raw vote
+count and its share of the dimension (shares within a dimension sum to 1). The
+`dimension` column holds `BenchmarkDemographicDimension` values (`AgeBucket`,
+`Gender`, `Occupation`, `Country`, `Language`):
+
+```python
+from rapidata import BenchmarkDemographicDimension
+
+demographics = benchmark.get_demographics()
+# columns: dimension, value, votes, share
+
+countries = demographics[
+    demographics["dimension"] == BenchmarkDemographicDimension.COUNTRY
+]
+print(countries)
+```
+
+### Standings per demographic segment
+
+`get_standings_breakdown` returns one row per `(segment, model)` — how each
+demographic segment of voters ranks the models, with that segment's vote count.
+Useful for spotting where a model is ranked differently by different groups. For
+the overall standings across all voters, use `get_overall_standings`. The
+dimension is a `BenchmarkDemographicDimension`.
+
+```python
+from rapidata import BenchmarkDemographicDimension
+
+breakdown = benchmark.get_standings_breakdown(
+    dimension=BenchmarkDemographicDimension.AGEBUCKET,
+)
+# columns: segment, segment_votes, name, wins, total_matches, score
+```
+
+Segments are **raw vote counts**.
+
+### Filtering standings and matrices by demographics
+
+The demographic dimensions are also filters. `get_overall_standings`,
+`get_win_loss_matrix`, `get_demographics` and `get_standings_breakdown` on the
+benchmark — and `get_standings` / `get_win_loss_matrix` on a leaderboard — all
+accept `country`, `language`, `gender`, `age_bucket`, `occupation` and `run_id`
+(on top of `tags` / `leaderboard_ids`). This computes the result from only the
+votes cast by matching voters, e.g. the standings among young women:
+
+```python
+from rapidata import BenchmarkDemographicDimension, Gender, AgeGroup
+
+standings = benchmark.get_overall_standings(
+    gender=[Gender.FEMALE],
+    age_bucket=[AgeGroup.BETWEEN_18_29],
+    country=["US", "GB"],
+)
+
+# and any of the other read methods, e.g.
+breakdown = benchmark.get_standings_breakdown(
+    dimension=BenchmarkDemographicDimension.COUNTRY,
+    tags=["landscape"],
+    gender=[Gender.FEMALE],
+)
+```
+
+`gender` and `age_bucket` take the SDK's `Gender` / `AgeGroup` enums; `country`,
+`language` and `occupation` take plain values.
+
 ## Win/Loss Matrix
 
 `get_standings` collapses every matchup into one Elo score per model. When you want

src/rapidata/__init__.py

@@ -20,6 +20,7 @@
     RapidataValidationSet,
     Box,
     RapidataResults,
+    BenchmarkDemographicDimension,
     DemographicSelection,
     LabelingSelection,
     EffortSelection,

src/rapidata/rapidata_client/__init__.py

@@ -17,6 +17,9 @@
 from .signal import RapidataSignal, RapidataSignalManager
 from .validation import ValidationSetManager, RapidataValidationSet, Box
 from .results import RapidataResults
+from rapidata.api_client.models.benchmark_demographic_dimension import (
+    BenchmarkDemographicDimension,
+)
 from .selection import (
     DemographicSelection,
     LabelingSelection,

src/rapidata/rapidata_client/benchmark/_vote_filters.py

@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+from typing import NamedTuple, Optional, TYPE_CHECKING
+
+from rapidata.api_client.models.audience_audience_id_jobs_get_job_id_parameter import (
+    AudienceAudienceIdJobsGetJobIdParameter,
+)
+
+if TYPE_CHECKING:
+    from rapidata.rapidata_client.filter.models.gender import Gender
+    from rapidata.rapidata_client.filter.models.age_group import AgeGroup
+
+
+def in_filter(
+    values: Optional[list[str]],
+) -> AudienceAudienceIdJobsGetJobIdParameter:
+    """Wrap a list of accepted values as an ``in`` filter (a no-op when None)."""
+    param = AudienceAudienceIdJobsGetJobIdParameter()
+    param.var_in = values
+    return param
+
+
+class DemographicFilters(NamedTuple):
+    """The per-vote demographic filters shared by the benchmark and leaderboard
+    standings, matrix, demographics and standings-breakdown endpoints.
+
+    Each field is keyed to match the corresponding query-endpoint parameter, so a
+    caller passes them straight through (``country=filters.country`` ...).
+    """
+
+    country: AudienceAudienceIdJobsGetJobIdParameter
+    language: AudienceAudienceIdJobsGetJobIdParameter
+    gender: AudienceAudienceIdJobsGetJobIdParameter
+    age_bucket: AudienceAudienceIdJobsGetJobIdParameter
+    occupation: AudienceAudienceIdJobsGetJobIdParameter
+    run_id: AudienceAudienceIdJobsGetJobIdParameter
+
+
+def demographic_filters(
+    country: Optional[list[str]] = None,
+    language: Optional[list[str]] = None,
+    gender: Optional[list[Gender]] = None,
+    age_bucket: Optional[list[AgeGroup]] = None,
+    occupation: Optional[list[str]] = None,
+    run_id: Optional[str] = None,
+) -> DemographicFilters:
+    """Build the demographic filters for a standings/matrix/demographics call.
+
+    ``gender`` and ``age_bucket`` take the SDK's :class:`Gender` and :class:`AgeGroup`
+    enums and are converted to their backend values; the open-ended dimensions are
+    plain values. Unset filters become no-ops.
+    """
+    return DemographicFilters(
+        country=in_filter(country),
+        language=in_filter(language),
+        gender=in_filter(
+            [g._to_backend_model().value for g in gender] if gender else None
+        ),
+        age_bucket=in_filter(
+            [a._to_backend_model().value for a in age_bucket] if age_bucket else None
+        ),
+        occupation=in_filter(occupation),
+        run_id=in_filter([run_id] if run_id is not None else None),
+    )

src/rapidata/rapidata_client/benchmark/leaderboard/rapidata_leaderboard.py

@@ -10,8 +10,9 @@
     LevelOfDetail,
     ResolvedLevelOfDetail,
 )
-from rapidata.api_client.models.audience_audience_id_jobs_get_job_id_parameter import (
-    AudienceAudienceIdJobsGetJobIdParameter,
+from rapidata.rapidata_client.benchmark._vote_filters import (
+    demographic_filters,
+    in_filter,
 )
 from rapidata.service.openapi_service import OpenAPIService
 from rapidata.api_client.models.update_leaderboard_endpoint_input import (
@@ -21,6 +22,8 @@
 if TYPE_CHECKING:
     import pandas as pd
     from rapidata.rapidata_client.job.rapidata_job import RapidataJob
+    from rapidata.rapidata_client.filter.models.gender import Gender
+    from rapidata.rapidata_client.filter.models.age_group import AgeGroup
 
 
 class RapidataLeaderboard:
@@ -224,23 +227,51 @@ def jobs(self) -> list[RapidataJob]:
 
             return [job_manager.get_job_by_id(job_id) for job_id in job_ids]
 
-    def get_standings(self, tags: Optional[list[str]] = None) -> "pd.DataFrame":
+    def get_standings(
+        self,
+        tags: Optional[list[str]] = None,
+        country: Optional[list[str]] = None,
+        language: Optional[list[str]] = None,
+        gender: Optional[list[Gender]] = None,
+        age_bucket: Optional[list[AgeGroup]] = None,
+        occupation: Optional[list[str]] = None,
+        run_id: Optional[str] = None,
+    ) -> "pd.DataFrame":
         """
         Returns the standings of the leaderboard.
 
+        The demographic filters compute the standings from only the votes cast by
+        matching voters — e.g. the standings among women, or among US voters.
+        ``gender`` and ``age_bucket`` are estimated (inferred); ``country`` and
+        ``language`` are observed.
+
         Args:
             tags: The matchups with these tags should be used to create the standings.
                 If tags are None, all matchups will be considered.
                 If tags are empty, no matchups will be considered.
+            country: Only count votes from these countries (ISO-2 codes).
+            language: Only count votes from these languages.
+            gender: Only count votes from voters of these (estimated) genders.
+            age_bucket: Only count votes from voters in these (estimated) age buckets.
+            occupation: Only count votes from voters of these (estimated) occupations.
+            run_id: Only count votes from this evaluation run.
 
         Returns:
             A pandas DataFrame containing the standings of the leaderboard.
         """
         with tracer.start_as_current_span("RapidataLeaderboard.get_standings"):
-            tags_filter = AudienceAudienceIdJobsGetJobIdParameter()
-            tags_filter.var_in = tags
+            votes = demographic_filters(
+                country, language, gender, age_bucket, occupation, run_id
+            )
             participants = self.__openapi_service.leaderboard.leaderboard_api.leaderboard_leaderboard_id_standings_query_get(
-                leaderboard_id=self.id, tags=tags_filter
+                leaderboard_id=self.id,
+                tags=in_filter(tags),
+                country=votes.country,
+                language=votes.language,
+                gender=votes.gender,
+                age_bucket=votes.age_bucket,
+                occupation=votes.occupation,
+                run_id=votes.run_id,
             )
 
             import pandas as pd
@@ -266,6 +297,12 @@ def get_win_loss_matrix(
         self,
         tags: Optional[list[str]] = None,
         use_weighted_scoring: Optional[bool] = None,
+        country: Optional[list[str]] = None,
+        language: Optional[list[str]] = None,
+        gender: Optional[list[Gender]] = None,
+        age_bucket: Optional[list[AgeGroup]] = None,
+        occupation: Optional[list[str]] = None,
+        run_id: Optional[str] = None,
     ) -> pd.DataFrame:
         """
         Returns the pairwise win/loss matrix for the participants in this leaderboard.
@@ -277,6 +314,10 @@ def get_win_loss_matrix(
         always 0. This is the head-to-head breakdown behind :meth:`get_standings`,
         which collapses the same matchups into a single Elo score per model.
 
+        The demographic filters restrict the matrix to matchups decided by matching
+        voters. ``gender`` and ``age_bucket`` are estimated (inferred); ``country``
+        and ``language`` are observed.
+
         Args:
             tags: Only count matchups carrying one of these prompt tags. If None,
                 every matchup on the leaderboard is included; if an empty list, none are.
@@ -285,19 +326,32 @@ def get_win_loss_matrix(
                 plain win, so cells hold weighted sums (floats) rather than raw counts.
                 If False, cells are raw win counts. When None (default), the server
                 applies the leaderboard's configured default.
+            country: Only count votes from these countries (ISO-2 codes).
+            language: Only count votes from these languages.
+            gender: Only count votes from voters of these (estimated) genders.
+            age_bucket: Only count votes from voters in these (estimated) age buckets.
+            occupation: Only count votes from voters of these (estimated) occupations.
+            run_id: Only count votes from this evaluation run.
 
         Returns:
             A pandas DataFrame indexed by participant name on both axes, where cell
             ``[i, j]`` holds the (optionally weighted) number of wins of the row
             participant over the column participant.
         """
         with tracer.start_as_current_span("RapidataLeaderboard.get_win_loss_matrix"):
-            tags_filter = AudienceAudienceIdJobsGetJobIdParameter()
-            tags_filter.var_in = tags
+            votes = demographic_filters(
+                country, language, gender, age_bucket, occupation, run_id
+            )
             result = self.__openapi_service.leaderboard.leaderboard_api.leaderboard_leaderboard_id_matrix_query_get(
                 leaderboard_id=self.id,
-                tags=tags_filter,
+                tags=in_filter(tags),
                 use_weighted_scoring=use_weighted_scoring,
+                country=votes.country,
+                language=votes.language,
+                gender=votes.gender,
+                age_bucket=votes.age_bucket,
+                occupation=votes.occupation,
+                run_id=votes.run_id,
             )
 
             import pandas as pd