Decouple AgentServer responses from generated model internals

sdk/agentserver/azure-ai-agentserver-responses/Makefile

@@ -9,15 +9,13 @@
 OUTPUT_DIR  ?= azure/ai/agentserver/responses/models/_generated
 TYPESPEC_DIR ?= type_spec
 TEMP_TSP_DIR := $(TYPESPEC_DIR)/TempTypeSpecFiles
+TEMP_OUTPUT_DIR := .tmp_typeddict_codegen
+TYPESPEC_PYTHON_EMITTER_PATH ?=
 OPENAPI_SPEC ?= $(TEMP_TSP_DIR)/Foundry/openapi.virtual-public-preview.yaml
 VALIDATORS_OUTPUT ?= $(OUTPUT_DIR)/_validators.py
 ROOT_SCHEMAS ?= CreateResponse
 OVERLAY ?= _scripts/validation-overlay.yaml
-TEMP_OUTPUT_DIR := $(OUTPUT_DIR)/.tmp_codegen
-MODEL_PACKAGE_DIR := $(TEMP_OUTPUT_DIR)/azure/ai/agentserver/responses/models
-MODEL_SHIMS_DIR := _scripts/generated_shims
 CONTRACTS_DIR := $(TEMP_TSP_DIR)/sdk-service-agentserver-contracts
-MODEL_BASE    := $(OUTPUT_DIR)/sdk/models/_utils/model_base.py
 
 .PHONY: generate-models generate-validators generate-openapi generate-contracts clean install-typespec-deps
 
@@ -90,7 +88,7 @@ endif
 generate-contracts: generate-models generate-openapi generate-validators
 
 # --------------------------------------------------------------------------
-# generate-models: Compile TypeSpec definitions into Python model classes
+# generate-models: Compile TypeSpec definitions into local TypedDict model contracts
 # --------------------------------------------------------------------------
 ifeq ($(OS),Windows_NT)
 generate-models:
@@ -100,25 +98,14 @@ generate-models:
 	cd /d $(TYPESPEC_DIR) && tsp-client sync
 	@echo Installing TypeSpec dependencies from emitter-package.json...
 	cd /d $(TEMP_TSP_DIR) && npm install --silent
-	@echo Generating Python models...
+	@if not "$(TYPESPEC_PYTHON_EMITTER_PATH)"=="" cd /d $(TEMP_TSP_DIR) && npm install --silent --no-save "$(TYPESPEC_PYTHON_EMITTER_PATH)"
+	python _scripts\finalize_typeddict_generation.py --client-tsp "$(CONTRACTS_DIR)\client.tsp"
+	@echo Generating local Python TypedDict models...
 	@if exist "$(OUTPUT_DIR)" rmdir /s /q "$(OUTPUT_DIR)"
-	cd /d $(TEMP_TSP_DIR) && npx tsp compile sdk-service-agentserver-contracts\client.tsp --emit @azure-tools/typespec-python --option "@azure-tools/typespec-python.emitter-output-dir=$(abspath $(TEMP_OUTPUT_DIR))"
-	@if not exist "$(MODEL_PACKAGE_DIR)" (echo Error: generated model package was not found. 1>&2 && exit /b 1)
-	@if not exist "$(OUTPUT_DIR)\sdk" mkdir "$(OUTPUT_DIR)\sdk"
-	@xcopy /E /I /Y "$(MODEL_PACKAGE_DIR)" "$(OUTPUT_DIR)\sdk\models" >NUL
-	@if exist "$(OUTPUT_DIR)\sdk\models\aio" rmdir /s /q "$(OUTPUT_DIR)\sdk\models\aio"
-	@if exist "$(OUTPUT_DIR)\sdk\models\operations" rmdir /s /q "$(OUTPUT_DIR)\sdk\models\operations"
-	@if exist "$(OUTPUT_DIR)\sdk\models\_client.py" del /q "$(OUTPUT_DIR)\sdk\models\_client.py"
-	@if exist "$(OUTPUT_DIR)\sdk\models\_configuration.py" del /q "$(OUTPUT_DIR)\sdk\models\_configuration.py"
-	@if exist "$(OUTPUT_DIR)\sdk\models\_version.py" del /q "$(OUTPUT_DIR)\sdk\models\_version.py"
-	@copy /Y "$(MODEL_SHIMS_DIR)\sdk_models__init__.py" "$(OUTPUT_DIR)\sdk\models\__init__.py" >NUL
-	@copy /Y "$(MODEL_SHIMS_DIR)\__init__.py" "$(OUTPUT_DIR)\__init__.py" >NUL
-	@copy /Y "$(MODEL_SHIMS_DIR)\_enums.py" "$(OUTPUT_DIR)\_enums.py" >NUL
-	@copy /Y "$(MODEL_SHIMS_DIR)\_models.py" "$(OUTPUT_DIR)\_models.py" >NUL
-	@copy /Y "$(MODEL_SHIMS_DIR)\_patch.py" "$(OUTPUT_DIR)\_patch.py" >NUL
-	@copy /Y "$(MODEL_SHIMS_DIR)\models_patch.py" "$(OUTPUT_DIR)\sdk\models\models\_patch.py" >NUL
-	@REM Patch _deserialize_sequence: reject plain strings so union falls through to str branch
-	@powershell -Command "(Get-Content '$(MODEL_BASE)') -replace 'return type\(obj\)\(_deserialize\(deserializer, entry, module\) for entry in obj\)','if isinstance(obj, str):\n        raise DeserializationError()\n    return type(obj)(_deserialize(deserializer, entry, module) for entry in obj)' | Set-Content '$(MODEL_BASE)'"
+	@if exist "$(TEMP_OUTPUT_DIR)" rmdir /s /q "$(TEMP_OUTPUT_DIR)"
+	cd /d $(TEMP_TSP_DIR) && npx tsp compile sdk-service-agentserver-contracts\client.tsp --emit @azure-tools/typespec-python --option "@azure-tools/typespec-python.emitter-output-dir=$(abspath $(TEMP_OUTPUT_DIR))" --option "@azure-tools/typespec-python.namespace=ai.agentserver.responses.models._generated" --option "@azure-tools/typespec-python.package-name=azure-ai-agentserver-responses" --option "@azure-tools/typespec-python.models-mode=typeddict" --option "@azure-tools/typespec-python.generate-packaging-files=false"
+	python _scripts\finalize_typeddict_generation.py --emitter-output-root "$(TEMP_OUTPUT_DIR)" --generated-root "$(OUTPUT_DIR)"
+	@if not exist "$(OUTPUT_DIR)" (echo Error: generated model package was not found. 1>&2 && exit /b 1)
 	@if exist "$(TEMP_OUTPUT_DIR)" rmdir /s /q "$(TEMP_OUTPUT_DIR)"
 else
 generate-models:
@@ -135,28 +122,19 @@ generate-models:
 	cd $(TYPESPEC_DIR) && tsp-client sync
 	@echo "Installing TypeSpec dependencies from emitter-package.json..."
 	cd $(TEMP_TSP_DIR) && npm install --silent
-	@echo "Generating Python models..."
+	@if [ -n "$(TYPESPEC_PYTHON_EMITTER_PATH)" ]; then \
+		cd $(TEMP_TSP_DIR) && npm install --silent --no-save "$(TYPESPEC_PYTHON_EMITTER_PATH)"; \
+	fi
+	python _scripts/finalize_typeddict_generation.py --client-tsp "$(CONTRACTS_DIR)/client.tsp"
+	@echo "Generating local Python TypedDict models..."
 	rm -rf $(OUTPUT_DIR)
-	cd $(TEMP_TSP_DIR) && npx tsp compile sdk-service-agentserver-contracts/client.tsp --emit @azure-tools/typespec-python --option "@azure-tools/typespec-python.emitter-output-dir=$(abspath $(TEMP_OUTPUT_DIR))"
-	@test -d $(MODEL_PACKAGE_DIR) || { \
+	rm -rf $(TEMP_OUTPUT_DIR)
+	cd $(TEMP_TSP_DIR) && npx tsp compile sdk-service-agentserver-contracts/client.tsp --emit @azure-tools/typespec-python --option "@azure-tools/typespec-python.emitter-output-dir=$(abspath $(TEMP_OUTPUT_DIR))" --option "@azure-tools/typespec-python.namespace=ai.agentserver.responses.models._generated" --option "@azure-tools/typespec-python.package-name=azure-ai-agentserver-responses" --option "@azure-tools/typespec-python.models-mode=typeddict" --option "@azure-tools/typespec-python.generate-packaging-files=false"
+	python _scripts/finalize_typeddict_generation.py --emitter-output-root "$(TEMP_OUTPUT_DIR)" --generated-root "$(OUTPUT_DIR)"
+	@test -d $(OUTPUT_DIR) || { \
 		echo "Error: generated model package was not found." >&2; \
 		exit 1; \
 	}
-	mkdir -p $(OUTPUT_DIR)/sdk
-	cp -R $(MODEL_PACKAGE_DIR) $(OUTPUT_DIR)/sdk/models
-	rm -rf $(OUTPUT_DIR)/sdk/models/aio
-	rm -rf $(OUTPUT_DIR)/sdk/models/operations
-	rm -f $(OUTPUT_DIR)/sdk/models/_client.py
-	rm -f $(OUTPUT_DIR)/sdk/models/_configuration.py
-	rm -f $(OUTPUT_DIR)/sdk/models/_version.py
-	cp $(MODEL_SHIMS_DIR)/sdk_models__init__.py $(OUTPUT_DIR)/sdk/models/__init__.py
-	cp $(MODEL_SHIMS_DIR)/__init__.py $(OUTPUT_DIR)/__init__.py
-	cp $(MODEL_SHIMS_DIR)/_enums.py $(OUTPUT_DIR)/_enums.py
-	cp $(MODEL_SHIMS_DIR)/_models.py $(OUTPUT_DIR)/_models.py
-	cp $(MODEL_SHIMS_DIR)/_patch.py $(OUTPUT_DIR)/_patch.py
-	cp $(MODEL_SHIMS_DIR)/models_patch.py $(OUTPUT_DIR)/sdk/models/models/_patch.py
-	# Patch _deserialize_sequence: reject plain strings so union falls through to str branch
-	sed -i 's/    return type(obj)(_deserialize(deserializer, entry, module) for entry in obj)/    if isinstance(obj, str):\n        raise DeserializationError()\n    return type(obj)(_deserialize(deserializer, entry, module) for entry in obj)/' $(MODEL_BASE)
 	rm -rf $(TEMP_OUTPUT_DIR)
 endif
 

sdk/agentserver/azure-ai-agentserver-responses/_scripts/finalize_typeddict_generation.py

@@ -0,0 +1,211 @@
+#!/usr/bin/env python
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+"""Finalize local OpenAI Responses TypedDict model generation."""
+
+from __future__ import annotations
+
+import argparse
+import re
+import shutil
+from pathlib import Path
+
+
+COPYRIGHT = "# Copyright (c) Microsoft Corporation.\n# Licensed under the MIT license.\n"
+
+STALE_USAGE_REFS = {
+    "OpenAI.OutputItemComputerToolCallOutputResource",
+    "OpenAI.OutputItemCustomToolCall",
+    "OpenAI.OutputItemCustomToolCallOutput",
+    "OpenAI.FunctionToolCallOutputResource",
+}
+
+ROOT_INIT_PREFIX = (
+    "# coding=utf-8\n"
+    "# --------------------------------------------------------------------------\n"
+    "# Copyright (c) Microsoft Corporation. All rights reserved.\n"
+    "# Licensed under the MIT License. See License.txt in the project root for license information.\n"
+    "# Code generated by Microsoft (R) Python Code Generator.\n"
+    "# Changes may cause incorrect behavior and will be lost if the code is regenerated.\n"
+    "# --------------------------------------------------------------------------\n\n"
+    "from ._unions import *  # type: ignore # noqa: F401,F403\n"
+    "from .types import *  # type: ignore # noqa: F401,F403\n"
+)
+
+ENUM_FALLBACK_PREFIX = (
+    "# coding=utf-8\n"
+    "# --------------------------------------------------------------------------\n"
+    "# Copyright (c) Microsoft Corporation. All rights reserved.\n"
+    "# Licensed under the MIT License. See License.txt in the project root for license information.\n"
+    "# --------------------------------------------------------------------------\n\n"
+    "class _EnumMemberFallback(str):\n"
+    "    @property\n"
+    "    def value(self) -> str:\n"
+    "        return str(self)\n\n\n"
+    "_ENUM_VALUES: dict[str, dict[str, str]] = "
+)
+
+ENUM_FALLBACK_SUFFIX = (
+    "\n\n"
+    "def _normalize_enum_value(value: str) -> str:\n"
+    "    return ''.join(ch for ch in value.upper() if ch.isalnum())\n\n\n"
+    "def _member_value(enum_name: str, member_name: str) -> str:\n"
+    "    return _ENUM_VALUES.get(enum_name, {}).get(_normalize_enum_value(member_name), member_name.lower())\n\n\n"
+    "class _EnumFallbackMeta(type):\n"
+    "    def __getattr__(cls, name: str) -> _EnumMemberFallback:\n"
+    "        return _EnumMemberFallback(_member_value(cls.__name__, name))\n\n\n"
+    "class _EnumFallback(str, metaclass=_EnumFallbackMeta):\n"
+    "    pass\n\n\n"
+    "def __getattr__(name: str) -> type[_EnumFallback]:\n"
+    "    return type(name, (_EnumFallback,), {})\n"
+)
+
+
+def _write(path: Path, content: str) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(content, encoding="utf-8")
+
+
+def _remove_pycache(root: Path) -> None:
+    for pycache in root.rglob("__pycache__"):
+        shutil.rmtree(pycache)
+
+
+def patch_client_tsp(client_tsp: Path) -> None:
+    """Remove usage decorators for stale upstream OpenAI symbols before compilation."""
+    lines = client_tsp.read_text(encoding="utf-8").splitlines()
+    patched = [
+        line
+        for line in lines
+        if not (
+            line.strip().startswith("@@usage(")
+            and any(stale_ref in line for stale_ref in STALE_USAGE_REFS)
+        )
+    ]
+    if len(patched) != len(lines):
+        client_tsp.write_text("\n".join(patched) + "\n", encoding="utf-8")
+
+
+def _find_emitted_models_root(emitter_output_root: Path) -> Path:
+    candidates = sorted(
+        path
+        for path in emitter_output_root.rglob("types.py")
+        if path.parent.name == "models"
+        and (path.parent / "_unions.py").exists()
+        and (path.parent / "models").is_dir()
+    )
+    if not candidates:
+        raise FileNotFoundError(f"Could not find emitted TypedDict model package under {emitter_output_root}")
+    if len(candidates) > 1:
+        formatted = "\n".join(str(path.parent) for path in candidates)
+        raise RuntimeError(f"Found multiple emitted model packages:\n{formatted}")
+    return candidates[0].parent
+
+
+def _normalize_enum_value(value: str) -> str:
+    return re.sub(r"[^A-Z0-9]", "", value.upper())
+
+
+def _patch_literal_enum_references(types_py: Path) -> None:
+    text = types_py.read_text(encoding="utf-8")
+    enum_values: dict[str, dict[str, str]] = {}
+    for match in re.finditer(r"(?ms)^([A-Za-z][A-Za-z0-9_]*) = Literal\[(.*?)\]\n\"\"\"", text):
+        enum_name = match.group(1)
+        values = re.findall(r'"([^"]+)"', match.group(2))
+        if values:
+            enum_values[enum_name] = {_normalize_enum_value(value): value for value in values}
+
+    def replace(match: re.Match[str]) -> str:
+        enum_name, member_name = match.groups()
+        values = enum_values.get(enum_name)
+        if not values:
+            return match.group(0)
+        value = values.get(_normalize_enum_value(member_name))
+        return f'"{value}"' if value is not None else match.group(0)
+
+    types_py.write_text(
+        re.sub(r"\b([A-Z][A-Za-z0-9_]*)\.([A-Z][A-Z0-9_]*)\b", replace, text),
+        encoding="utf-8",
+    )
+
+
+def _strip_trailing_whitespace(path: Path) -> None:
+    path.write_text(
+        "\n".join(line.rstrip() for line in path.read_text(encoding="utf-8").splitlines()) + "\n",
+        encoding="utf-8",
+    )
+
+
+def _literal_enum_values(types_py: Path) -> dict[str, dict[str, str]]:
+    text = types_py.read_text(encoding="utf-8")
+    enum_values: dict[str, dict[str, str]] = {}
+    for match in re.finditer(r"(?ms)^([A-Za-z][A-Za-z0-9_]*) = Literal\[(.*?)\]\n\"\"\"", text):
+        enum_name = match.group(1)
+        values = re.findall(r'"([^"]+)"', match.group(2))
+        if values:
+            enum_values[enum_name] = {_normalize_enum_value(value): value for value in values}
+    return enum_values
+
+
+def _build_enum_fallback(types_py: Path) -> str:
+    enum_values = _literal_enum_values(types_py)
+    exact_values: dict[str, dict[str, str]] = {}
+    for enum_name, values in enum_values.items():
+        exact_values[enum_name] = {}
+        for member_key, value in values.items():
+            exact_values[enum_name][member_key] = value
+    return ENUM_FALLBACK_PREFIX + repr(exact_values) + ENUM_FALLBACK_SUFFIX
+
+
+def _build_root_init(types_py: Path) -> str:
+    enum_names = sorted(_literal_enum_values(types_py))
+    if not enum_names:
+        return ROOT_INIT_PREFIX
+    enum_lines = "\n".join(f'{name} = getattr(_generated_enums, "{name}")' for name in enum_names)
+    return (
+        ROOT_INIT_PREFIX
+        + "from .models import _enums as _generated_enums\n\n"
+        + enum_lines
+        + "\n"
+    )
+
+
+def finalize(emitter_output_root: Path, generated_root: Path) -> None:
+    """Copy the real emitted TypedDict package into the local generated package boundary."""
+    emitted_root = _find_emitted_models_root(emitter_output_root)
+
+    if generated_root.exists():
+        shutil.rmtree(generated_root)
+    generated_root.mkdir(parents=True)
+
+    for file_name in ("types.py", "_unions.py", "py.typed"):
+        shutil.copy2(emitted_root / file_name, generated_root / file_name)
+    _patch_literal_enum_references(generated_root / "types.py")
+    _strip_trailing_whitespace(generated_root / "types.py")
+
+    models_root = generated_root / "models"
+    models_root.mkdir()
+    for file_name in ("__init__.py", "_patch.py"):
+        shutil.copy2(emitted_root / "models" / file_name, models_root / file_name)
+
+    _write(generated_root / "__init__.py", _build_root_init(generated_root / "types.py"))
+    _write(generated_root / "_types.py", _build_root_init(generated_root / "types.py"))
+    _write(models_root / "_enums.py", _build_enum_fallback(generated_root / "types.py"))
+    _remove_pycache(generated_root)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--client-tsp", type=Path)
+    parser.add_argument("--emitter-output-root", type=Path)
+    parser.add_argument("--generated-root", type=Path)
+    args = parser.parse_args()
+
+    if args.client_tsp:
+        patch_client_tsp(args.client_tsp)
+    if args.emitter_output_root and args.generated_root:
+        finalize(args.emitter_output_root, args.generated_root)
+
+
+if __name__ == "__main__":
+    main()