refactor(eval): consolidate version-specific datasets into prompts.yaml

eval/a2ui_eval/dataset.py

@@ -27,14 +27,23 @@
 SCHEMA_PATH = GIT_ROOT / 'eval' / 'datasets' / 'dataset_schema.json'
 
 
+def _version_to_dir_name(version: str) -> str:
+  """Converts a version string (e.g., '0.9.1') to a directory name (e.g., 'v0_9_1')."""
+  return 'v' + version.replace('.', '_')
+
+
 def load_a2ui_dataset(
-    file_path: str, default_catalog_path: str | None = None
+    file_path: str,
+    default_catalog_path: str | None = None,
+    version: str | None = None,
 ) -> MemoryDataset:
   """Loads A2UI evaluation samples from a YAML file.
 
   Args:
       file_path: The path to the YAML dataset file.
-      default_catalog_path: The default catalog path to use if not specified in the sample.
+      default_catalog_path: The default catalog path to use if not specified in
+        the sample.
+      version: Optional target version string to substitute into catalog paths.
 
   Returns:
       A MemoryDataset containing the resolved samples.
@@ -55,16 +64,17 @@ def load_a2ui_dataset(
 
   samples = []
   for item in data:
+    catalog_path = item.get('catalog') or default_catalog_path or DEFAULT_CATALOG_PATH
+    if version and catalog_path:
+      catalog_path = catalog_path.replace('{version}', _version_to_dir_name(version))
     samples.append(
         Sample(
             input=item['promptText'],
             target=item.get('target') or item.get('description'),
             metadata={
                 'name': item.get('name'),
                 'description': item.get('description'),
-                'catalog': (
-                    item.get('catalog') or default_catalog_path or DEFAULT_CATALOG_PATH
-                ),
+                'catalog': catalog_path,
                 'role_description': (
                     item.get('role_description') or DEFAULT_ROLE_DESCRIPTION
                 ),

eval/a2ui_eval/strategies/__init__.py

@@ -24,9 +24,9 @@
 }
 
 
-def get_solver(strategy: str) -> list[Solver]:
+def get_solver(strategy: str, version: str) -> list[Solver]:
   """Returns the solver chain for the specified evaluation strategy."""
   if strategy not in STRATEGIES:
     raise ValueError(f"Unknown evaluation strategy: {strategy}")
 
-  return STRATEGIES[strategy]()
+  return STRATEGIES[strategy](version)

eval/a2ui_eval/strategies/direct.py

@@ -20,15 +20,15 @@
 
 
 @solver
-def a2ui_system_prompt() -> Solver:
+def a2ui_system_prompt(version: str) -> Solver:
   """Solver to inject A2UI schema and catalog into the system prompt using SDK."""
 
   async def solve(state: TaskState, generate: Generate) -> TaskState:
     catalog_path = state.metadata['catalog']
     resolved_catalog_path = str(GIT_ROOT / catalog_path)
 
     catalog_config = CatalogConfig.from_path('basic_catalog', resolved_catalog_path)
-    manager = A2uiSchemaManager(version='0.9', catalogs=[catalog_config])
+    manager = A2uiSchemaManager(version=version, catalogs=[catalog_config])
 
     role_description = state.metadata['role_description']
     workflow_description = state.metadata['workflow_description']
@@ -45,6 +45,6 @@ async def solve(state: TaskState, generate: Generate) -> TaskState:
   return solve
 
 
-def direct_solver() -> list[Solver]:
+def direct_solver(version: str) -> list[Solver]:
   """Returns the solver chain for the 'direct' evaluation strategy."""
-  return [a2ui_system_prompt(), measured_generate()]
+  return [a2ui_system_prompt(version), measured_generate()]

eval/a2ui_eval/strategies/express.py

@@ -29,15 +29,15 @@
 
 
 @solver
-def a2ui_express_prompt() -> Solver:
+def a2ui_express_prompt(version: str) -> Solver:
   """Solver to inject A2UI Express prompt contract instructions."""
 
   async def solve(state: TaskState, generate: Generate) -> TaskState:
     catalog_path = state.metadata["catalog"]
     resolved_catalog_path = str(GIT_ROOT / catalog_path)
     with open(resolved_catalog_path, "r", encoding="utf-8") as f:
       schema = json.load(f)
-    catalog = Catalog.from_json(schema, spec_version="0.9.1")
+    catalog = Catalog.from_json(schema, spec_version=version)
     generator = ExpressPromptGenerator(catalog)
     prompt = generator.generate_prompt()
     state.messages.insert(0, ChatMessageSystem(content=prompt))
@@ -47,7 +47,7 @@ async def solve(state: TaskState, generate: Generate) -> TaskState:
 
 
 @solver
-def compile_express_dsl() -> Solver:
+def compile_express_dsl(version: str) -> Solver:
   """Solver to compile generated A2UI Express DSL back to standard JSON."""
 
   async def solve(state: TaskState, generate: Generate) -> TaskState:
@@ -59,7 +59,7 @@ async def solve(state: TaskState, generate: Generate) -> TaskState:
 
     # Initialize the catalog schema validator for parsing
     catalog_config = CatalogConfig.from_path("basic_catalog", resolved_catalog_path)
-    manager = A2uiSchemaManager(version="1.0", catalogs=[catalog_config])
+    manager = A2uiSchemaManager(version=version, catalogs=[catalog_config])
     catalog = manager.get_selected_catalog()
     validator = catalog.validator
 
@@ -123,6 +123,10 @@ async def solve(state: TaskState, generate: Generate) -> TaskState:
   return solve
 
 
-def express_solver() -> list[Solver]:
+def express_solver(version: str) -> list[Solver]:
   """Returns the solver chain for the 'express' evaluation strategy."""
-  return [a2ui_express_prompt(), measured_generate(), compile_express_dsl()]
+  return [
+      a2ui_express_prompt(version),
+      measured_generate(),
+      compile_express_dsl(version),
+  ]

eval/a2ui_eval/strategies/subagent_tool.py

@@ -35,11 +35,14 @@ async def execute(input: str) -> str:
     Args:
         input: The UI layout request.
     """
+    version = store().get("version", "0.9.1")
     catalog_path = store().get("catalog")
+    if not catalog_path:
+      raise ValueError("Catalog path is missing from the store.")
     resolved_catalog_path = str(GIT_ROOT / catalog_path)
 
     catalog_config = CatalogConfig.from_path("basic_catalog", resolved_catalog_path)
-    manager = A2uiSchemaManager(version="0.9", catalogs=[catalog_config])
+    manager = A2uiSchemaManager(version=version, catalogs=[catalog_config])
 
     role_description = store().get("role_description")
     workflow_description = store().get("workflow_description")
@@ -82,10 +85,11 @@ async def execute(input: str) -> str:
 
 
 @solver
-def push_metadata_to_store() -> Solver:
+def push_metadata_to_store(version: str) -> Solver:
   """Pushes metadata from the TaskState to the global store for tools to access."""
 
   async def solve(state: TaskState, generate: Generate) -> TaskState:
+    state.store.set("version", version)
     state.store.set("catalog", state.metadata.get("catalog"))
     state.store.set("role_description", state.metadata.get("role_description"))
     state.store.set("workflow_description", state.metadata.get("workflow_description"))
@@ -116,15 +120,15 @@ async def solve(state: TaskState, generate: Generate) -> TaskState:
   return solve
 
 
-def subagent_tool_solver() -> list[Solver]:
+def subagent_tool_solver(version: str) -> list[Solver]:
   """Returns the solver chain for the 'subagent_tool' evaluation strategy."""
   return [
       system_message(
           "You are a helpful assistant. To fulfill UI requests, you MUST delegate to"
           " the `a2ui_specialist` tool."
       ),
       # Tools cannot access TaskState directly, so we must bridge the metadata into the store
-      push_metadata_to_store(),
+      push_metadata_to_store(version),
       use_tools([a2ui_specialist()]),
       measured_generate(),
       extract_subagent_payload(),

eval/datasets/dataset_schema.json

@@ -23,7 +23,7 @@
       },
       "catalog": {
         "type": "string",
-        "description": "Path to the catalog json file. Corresponds to the catalog file path loaded into the A2uiSchemaManager. If omitted, the default is loaded from eval/datasets/defaults.py."
+        "description": "Path to the catalog json file. Corresponds to the catalog file path loaded into the A2uiSchemaManager. If omitted, the default is loaded from eval/datasets/defaults.py. Supports `{version}` placeholder which dynamically resolves to the target version directory name (e.g. 'v0_9_1' or 'v1_0')."
       },
       "role_description": {
         "type": "string",