feat(benchmark): support structured prompt tags (value+category) and origin

docs/mri_advanced.md

@@ -50,26 +50,38 @@ benchmark = client.mri.create_new_benchmark(
 
 Tags provide metadata for filtering and organizing benchmark results without showing them to evaluators. These tags can also be set and used in the frontend. To view the frontend, you can use the `view` method of the benchmark or leaderboard.
 
+Each tag is either a plain string or a `Tag` with a `value` and an optional `category` — you can mix both in the same list. Bare strings are wrapped into `Tag(value, category=None)` for you. You can also record where each prompt came from with an `Origin`.
+
 ```python
-# Tags for filtering leaderboard results
+from rapidata import Tag, Origin
+
+# Tags per prompt — plain strings and Tag objects can be mixed
 tags = [
-    ["landscape", "outdoor", "beach"],
-    ["landscape", "outdoor", "mountain"],
-    ["outdoor", "city"],
-    ["indoor", "vehicle"]
+    [Tag("beach", category="scene"), "outdoor"],
+    [Tag("mountain", category="scene"), "outdoor"],
+    [Tag("city", category="scene")],
+    [Tag("vehicle", category="object"), "indoor"],
 ]
 
 benchmark = client.mri.create_new_benchmark(
     name="Tagged Benchmark",
     identifiers=["scene_1", "scene_2", "scene_3", "scene_4"],
     prompts=["A sunny beach", "A mountain landscape", "A city skyline", "A car in a garage"],
-    tags=tags
+    tags=tags,
+    origins=[Origin("coco"), Origin("coco"), Origin("coco"), Origin("coco")],
 )
 
-# Filter leaderboard results by tags
-standings = leaderboard.get_standings(tags=["landscape", "outdoor"])
+# Filter leaderboard results by tag value
+standings = leaderboard.get_standings(tags=["outdoor"])
 ```
 
+!!! note
+    Plain-string tags keep working (e.g. `tags=[["landscape", "outdoor"]]`); they are
+    treated as tags with no category. Read tags back with the values-only
+    `benchmark.tags` (`list[list[str]]`) for backwards compatibility, or the
+    structured `benchmark.structured_tags` (`list[list[Tag]]`) for categories, and
+    prompt origins with `benchmark.origins`.
+
 ### Adding prompts and assets after benchmark creation
 
 If you have already created a benchmark and want to add new prompts and assets after the fact. Note however that these will only take effect for new models.
@@ -80,7 +92,7 @@ benchmark.add_prompts(
     identifiers=["new_style"],
     prompts=["Generate artwork in this new style"],
     prompt_assets=["https://assets.rapidata.ai/new_style_ref.jpg"],
-    tags=[["abstract", "modern"]]
+    tags=[[Tag("abstract", category="style"), "modern"]],
 )
 ```
 

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "rapidata"
-version = "3.17.2"
+version = "3.18.0"
 description = "Rapidata package containing the Rapidata Python Client to interact with the Rapidata Web API in an easy way."
 authors = [{ name = "Rapidata AG", email = "info@rapidata.ai" }]
 requires-python = ">=3.10,<4"

src/rapidata/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "3.17.2"
+__version__ = "3.18.0"
 
 from .rapidata_client import (
     RapidataClient,
@@ -64,6 +64,8 @@
     Gender,
     DeviceFilter,
     DeviceType,
+    Tag,
+    Origin,
     Datapoint,
     ContextManager,
     FailedUploadException,

src/rapidata/api_client/models/create_prompt_for_benchmark_endpoint_input.py

@@ -19,6 +19,8 @@
 from pydantic import BaseModel, ConfigDict, Field, StrictStr
 from typing import Any, ClassVar, Dict, List, Optional
 from rapidata.api_client.models.i_asset_input import IAssetInput
+from rapidata.api_client.models.prompt_origin import PromptOrigin
+from rapidata.api_client.models.prompt_tagging import PromptTagging
 from pydantic import ValidationError
 from rapidata.api_client.lazy_model import LazyValidatedModel
 from typing import Optional, Set
@@ -31,8 +33,9 @@ class CreatePromptForBenchmarkEndpointInput(LazyValidatedModel):
     identifier: StrictStr = Field(description="An identifier associated with the prompt.")
     prompt: Optional[StrictStr] = Field(default=None, description="The prompt text.")
     prompt_asset: Optional[IAssetInput] = Field(default=None, description="The asset associated with the prompt.", alias="promptAsset")
-    tags: Optional[List[StrictStr]] = None
-    __properties: ClassVar[List[str]] = ["identifier", "prompt", "promptAsset", "tags"]
+    tags: Optional[List[PromptTagging]] = Field(default=None, description="The tags associated with the prompt. Each tag has a value and an optional category.")
+    origin: Optional[PromptOrigin] = Field(default=None, description="The origin of the prompt.")
+    __properties: ClassVar[List[str]] = ["identifier", "prompt", "promptAsset", "tags", "origin"]
 
     # model_config is inherited from LazyValidatedModel
 
@@ -72,6 +75,16 @@ def to_dict(self) -> Dict[str, Any]:
         # override the default output from pydantic by calling `to_dict()` of prompt_asset
         if self.prompt_asset:
             _dict['promptAsset'] = self.prompt_asset.to_dict()
+        # override the default output from pydantic by calling `to_dict()` of each item in tags (list)
+        _items = []
+        if self.tags:
+            for _item_tags in self.tags:
+                if _item_tags:
+                    _items.append(_item_tags.to_dict())
+            _dict['tags'] = _items
+        # override the default output from pydantic by calling `to_dict()` of origin
+        if self.origin:
+            _dict['origin'] = self.origin.to_dict()
         # set to None if prompt (nullable) is None
         # and model_fields_set contains the field
         if self.prompt is None and "prompt" in self.model_fields_set:
@@ -82,6 +95,11 @@ def to_dict(self) -> Dict[str, Any]:
         if self.tags is None and "tags" in self.model_fields_set:
             _dict['tags'] = None
 
+        # set to None if origin (nullable) is None
+        # and model_fields_set contains the field
+        if self.origin is None and "origin" in self.model_fields_set:
+            _dict['origin'] = None
+
         return _dict
 
     @classmethod
@@ -97,12 +115,11 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
             "identifier": obj.get("identifier"),
             "prompt": obj.get("prompt"),
             "promptAsset": IAssetInput.from_dict(obj["promptAsset"]) if obj.get("promptAsset") is not None else None,
-            "tags": obj.get("tags")
+            "tags": [PromptTagging.from_dict(_item) for _item in obj["tags"]] if obj.get("tags") is not None else None,
+            "origin": PromptOrigin.from_dict(obj["origin"]) if obj.get("origin") is not None else None
         }
         try:
             _obj = cls.model_validate(_data)
         except ValidationError as _val_error:
             _obj = cls._lazy_construct(_data, _val_error)
         return _obj
-
-

src/rapidata/api_client/models/get_prompts_by_benchmark_endpoint_output.py

@@ -20,6 +20,8 @@
 from pydantic import BaseModel, ConfigDict, Field, StrictStr
 from typing import Any, ClassVar, Dict, List, Optional
 from rapidata.api_client.models.i_asset import IAsset
+from rapidata.api_client.models.prompt_origin import PromptOrigin
+from rapidata.api_client.models.prompt_tagging import PromptTagging
 from pydantic import ValidationError
 from rapidata.api_client.lazy_model import LazyValidatedModel
 from typing import Optional, Set
@@ -35,8 +37,9 @@ class GetPromptsByBenchmarkEndpointOutput(LazyValidatedModel):
     prompt_asset: Optional[IAsset] = Field(default=None, description="The optional asset associated with the prompt.", alias="promptAsset")
     identifier: StrictStr = Field(description="The identifier associated with the prompt.")
     created_at: datetime = Field(description="The timestamp when the prompt was created.", alias="createdAt")
-    tags: List[StrictStr]
-    __properties: ClassVar[List[str]] = ["id", "englishPrompt", "originalPrompt", "promptAsset", "identifier", "createdAt", "tags"]
+    tags: List[PromptTagging]
+    origin: Optional[PromptOrigin] = None
+    __properties: ClassVar[List[str]] = ["id", "englishPrompt", "originalPrompt", "promptAsset", "identifier", "createdAt", "tags", "origin"]
 
     # model_config is inherited from LazyValidatedModel
 
@@ -76,6 +79,16 @@ def to_dict(self) -> Dict[str, Any]:
         # override the default output from pydantic by calling `to_dict()` of prompt_asset
         if self.prompt_asset:
             _dict['promptAsset'] = self.prompt_asset.to_dict()
+        # override the default output from pydantic by calling `to_dict()` of each item in tags (list)
+        _items = []
+        if self.tags:
+            for _item_tags in self.tags:
+                if _item_tags:
+                    _items.append(_item_tags.to_dict())
+            _dict['tags'] = _items
+        # override the default output from pydantic by calling `to_dict()` of origin
+        if self.origin:
+            _dict['origin'] = self.origin.to_dict()
         # set to None if english_prompt (nullable) is None
         # and model_fields_set contains the field
         if self.english_prompt is None and "english_prompt" in self.model_fields_set:
@@ -86,6 +99,11 @@ def to_dict(self) -> Dict[str, Any]:
         if self.original_prompt is None and "original_prompt" in self.model_fields_set:
             _dict['originalPrompt'] = None
 
+        # set to None if origin (nullable) is None
+        # and model_fields_set contains the field
+        if self.origin is None and "origin" in self.model_fields_set:
+            _dict['origin'] = None
+
         return _dict
 
     @classmethod
@@ -104,7 +122,9 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
             "promptAsset": IAsset.from_dict(obj["promptAsset"]) if obj.get("promptAsset") is not None else None,
             "identifier": obj.get("identifier"),
             "createdAt": obj.get("createdAt"),
-            "tags": obj.get("tags")
+            # Wire sends Tag objects; tolerate a bare string (older backend) by wrapping it as an uncategorized tag.
+            "tags": [PromptTagging.from_dict(_item) if isinstance(_item, dict) else PromptTagging(value=_item) for _item in obj["tags"]] if obj.get("tags") is not None else None,
+            "origin": PromptOrigin.from_dict(obj["origin"]) if obj.get("origin") is not None else None
         }
         try:
             _obj = cls.model_validate(_data)

src/rapidata/api_client/models/prompt_origin.py

@@ -0,0 +1,87 @@
+# coding: utf-8
+
+"""
+    Rapidata Asset API
+
+    The API for the Rapidata Asset service
+
+    The version of the OpenAPI document: v1
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+
+from __future__ import annotations
+import pprint
+import re  # noqa: F401
+import json
+
+from pydantic import BaseModel, ConfigDict, Field, StrictStr
+from typing import Any, ClassVar, Dict, List, Optional
+from pydantic import ValidationError
+from rapidata.api_client.lazy_model import LazyValidatedModel
+from typing import Optional, Set
+from typing_extensions import Self
+
+class PromptOrigin(LazyValidatedModel):
+    """
+    PromptOrigin
+    """ # noqa: E501
+    value: StrictStr = Field(description="The origin value.")
+    __properties: ClassVar[List[str]] = ["value"]
+
+    # model_config is inherited from LazyValidatedModel
+
+
+    def to_str(self) -> str:
+        """Returns the string representation of the model using alias"""
+        return pprint.pformat(self.model_dump(by_alias=True))
+
+    def to_json(self) -> str:
+        """Returns the JSON representation of the model using alias"""
+        # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
+        return json.dumps(self.to_dict())
+
+    @classmethod
+    def from_json(cls, json_str: str) -> Optional[Self]:
+        """Create an instance of PromptOrigin from a JSON string"""
+        return cls.from_dict(json.loads(json_str))
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Return the dictionary representation of the model using alias.
+
+        This has the following differences from calling pydantic's
+        `self.model_dump(by_alias=True)`:
+
+        * `None` is only added to the output dict for nullable fields that
+          were set at model initialization. Other fields with value `None`
+          are ignored.
+        """
+        excluded_fields: Set[str] = set([
+        ])
+
+        _dict = self.model_dump(
+            by_alias=True,
+            exclude=excluded_fields,
+            exclude_none=True,
+        )
+        return _dict
+
+    @classmethod
+    def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
+        """Create an instance of PromptOrigin from a dict"""
+        if obj is None:
+            return None
+
+        if not isinstance(obj, dict):
+            return cls.model_validate(obj)
+
+        _data = {
+            "value": obj.get("value")
+        }
+        try:
+            _obj = cls.model_validate(_data)
+        except ValidationError as _val_error:
+            _obj = cls._lazy_construct(_data, _val_error)
+        return _obj

src/rapidata/api_client/models/prompt_tagging.py

@@ -0,0 +1,94 @@
+# coding: utf-8
+
+"""
+    Rapidata Asset API
+
+    The API for the Rapidata Asset service
+
+    The version of the OpenAPI document: v1
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+
+from __future__ import annotations
+import pprint
+import re  # noqa: F401
+import json
+
+from pydantic import BaseModel, ConfigDict, Field, StrictStr
+from typing import Any, ClassVar, Dict, List, Optional
+from pydantic import ValidationError
+from rapidata.api_client.lazy_model import LazyValidatedModel
+from typing import Optional, Set
+from typing_extensions import Self
+
+class PromptTagging(LazyValidatedModel):
+    """
+    PromptTagging
+    """ # noqa: E501
+    value: StrictStr = Field(description="The tag value.")
+    category: Optional[StrictStr] = Field(default=None, description="The optional category the tag belongs to.")
+    __properties: ClassVar[List[str]] = ["value", "category"]
+
+    # model_config is inherited from LazyValidatedModel
+
+
+    def to_str(self) -> str:
+        """Returns the string representation of the model using alias"""
+        return pprint.pformat(self.model_dump(by_alias=True))
+
+    def to_json(self) -> str:
+        """Returns the JSON representation of the model using alias"""
+        # TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
+        return json.dumps(self.to_dict())
+
+    @classmethod
+    def from_json(cls, json_str: str) -> Optional[Self]:
+        """Create an instance of PromptTagging from a JSON string"""
+        return cls.from_dict(json.loads(json_str))
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Return the dictionary representation of the model using alias.
+
+        This has the following differences from calling pydantic's
+        `self.model_dump(by_alias=True)`:
+
+        * `None` is only added to the output dict for nullable fields that
+          were set at model initialization. Other fields with value `None`
+          are ignored.
+        """
+        excluded_fields: Set[str] = set([
+        ])
+
+        _dict = self.model_dump(
+            by_alias=True,
+            exclude=excluded_fields,
+            exclude_none=True,
+        )
+        # set to None if category (nullable) is None
+        # and model_fields_set contains the field
+        if self.category is None and "category" in self.model_fields_set:
+            _dict['category'] = None
+
+        return _dict
+
+    @classmethod
+    def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
+        """Create an instance of PromptTagging from a dict"""
+        if obj is None:
+            return None
+
+        if not isinstance(obj, dict):
+            return cls.model_validate(obj)
+
+        _data = {
+            "value": obj.get("value"),
+            "category": obj.get("category")
+        }
+        try:
+            _obj = cls.model_validate(_data)
+        except ValidationError as _val_error:
+            _obj = cls._lazy_construct(_data, _val_error)
+        return _obj

src/rapidata/rapidata_client/__init__.py

@@ -27,6 +27,7 @@
     RapidataRetrievalMode,
     EffortSelection,
 )
+from .benchmark.prompt_metadata import Origin, Tag
 from .datapoints import Datapoint
 from .context import ContextManager
 from .datapoints.metadata import (