feat(evals): add OpenEval dataset import and export helpers (fix #3549)

src/openai/types/evals/__init__.py

@@ -1,22 +1,7 @@
-# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details.
-
-from __future__ import annotations
-
-from .eval_api_error import EvalAPIError as EvalAPIError
-from .run_list_params import RunListParams as RunListParams
-from .run_create_params import RunCreateParams as RunCreateParams
-from .run_list_response import RunListResponse as RunListResponse
-from .run_cancel_response import RunCancelResponse as RunCancelResponse
-from .run_create_response import RunCreateResponse as RunCreateResponse
-from .run_delete_response import RunDeleteResponse as RunDeleteResponse
-from .run_retrieve_response import RunRetrieveResponse as RunRetrieveResponse
-from .create_eval_jsonl_run_data_source import CreateEvalJSONLRunDataSource as CreateEvalJSONLRunDataSource
-from .create_eval_completions_run_data_source import (
-    CreateEvalCompletionsRunDataSource as CreateEvalCompletionsRunDataSource,
-)
-from .create_eval_jsonl_run_data_source_param import (
-    CreateEvalJSONLRunDataSourceParam as CreateEvalJSONLRunDataSourceParam,
-)
-from .create_eval_completions_run_data_source_param import (
-    CreateEvalCompletionsRunDataSourceParam as CreateEvalCompletionsRunDataSourceParam,
+from .openeval import (
+    OpenEvalItem as OpenEvalItem,
+    from_openeval as from_openeval,
+    to_openeval as to_openeval,
 )
+
+__all__ = ["OpenEvalItem", "from_openeval", "to_openeval"]

src/openai/types/evals/openeval.py

@@ -0,0 +1,62 @@
+# File generated for OpenEval dataset import/export support.
+
+from typing import Any, Dict, List, Optional
+from typing_extensions import TypedDict
+
+__all__ = ["OpenEvalItem", "from_openeval", "to_openeval"]
+
+
+class OpenEvalMessage(TypedDict, total=False):
+    role: str
+    content: str
+
+
+class OpenEvalItem(TypedDict, total=False):
+    id: Optional[str]
+    input: List[OpenEvalMessage]
+    expected_output: Optional[str]
+    metadata: Optional[Dict[str, Any]]
+
+
+def from_openeval(item: OpenEvalItem) -> Dict[str, Any]:
+    """
+    Convert an OpenEval dataset item into OpenAI Chat Completion messages format.
+    """
+    messages: List[Dict[str, Any]] = []
+    for msg in item.get("input", []):
+        messages.append({
+            "role": msg.get("role", "user"),
+            "content": msg.get("content", ""),
+        })
+
+    result: Dict[str, Any] = {"messages": messages}
+    if item.get("id"):
+        result["id"] = item["id"]
+    if item.get("expected_output"):
+        result["expected_output"] = item["expected_output"]
+    if item.get("metadata"):
+        result["metadata"] = item["metadata"]
+    return result
+
+
+def to_openeval(
+    messages: List[Dict[str, Any]],
+    id: Optional[str] = None,
+    expected_output: Optional[str] = None,
+    metadata: Optional[Dict[str, Any]] = None,
+) -> OpenEvalItem:
+    """
+    Export OpenAI Chat Completion messages and metadata into OpenEval dataset item format.
+    """
+    input_messages: List[OpenEvalMessage] = [
+        {"role": str(msg.get("role", "user")), "content": str(msg.get("content", ""))}
+        for msg in messages
+    ]
+    item: OpenEvalItem = {"input": input_messages}
+    if id is not None:
+        item["id"] = id
+    if expected_output is not None:
+        item["expected_output"] = expected_output
+    if metadata is not None:
+        item["metadata"] = metadata
+    return item

tests/test_openeval.py

@@ -0,0 +1,23 @@
+from openai.types.evals import OpenEvalItem, from_openeval, to_openeval
+
+
+def test_from_openeval_conversion() -> None:
+    item: OpenEvalItem = {
+        "id": "eval-1",
+        "input": [{"role": "user", "content": "Hello world"}],
+        "expected_output": "Hi there!",
+        "metadata": {"task": "test"},
+    }
+    converted = from_openeval(item)
+    assert converted["id"] == "eval-1"
+    assert converted["messages"] == [{"role": "user", "content": "Hello world"}]
+    assert converted["expected_output"] == "Hi there!"
+    assert converted["metadata"] == {"task": "test"}
+
+
+def test_to_openeval_conversion() -> None:
+    messages = [{"role": "user", "content": "Hello"}]
+    exported = to_openeval(messages, id="eval-2", expected_output="World")
+    assert exported["id"] == "eval-2"
+    assert exported["input"] == [{"role": "user", "content": "Hello"}]
+    assert exported["expected_output"] == "World"