From a7bb1cf5a9612ef3d97b53ef525f9f6283641e0f Mon Sep 17 00:00:00 2001 From: Abhinaysai Kamineni <66816045+askmy-stack@users.noreply.github.com> Date: Sat, 11 Jul 2026 14:28:52 -0400 Subject: [PATCH] Detect output schema and parameter default changes in diffs. Completes remaining Phase-1 compatibility rules so agent-visible output and default drift is reported with stable codes instead of coarse schema breaks. Co-authored-by: Cursor --- CHANGELOG.md | 7 ++ docs/change-codes.md | 6 +- src/tool_semantics/diff.py | 186 +++++++++++++++++++++++++++++-------- tests/test_diff.py | 65 +++++++++++++ 4 files changed, 224 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93b1ac7..4520aeb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to Tool-Semantics will be documented in this file. +## [Unreleased] + +### Added +- `tool.output_schema_added` / `removed` / `changed` detection (#8) +- `parameter.default_changed` warning when defaults are added, removed, or changed (#15) +- Default-only schema edits no longer emit `parameter.schema_changed` + ## [0.1.0] — 2026-07-11 ### Added diff --git a/docs/change-codes.md b/docs/change-codes.md index a8bff14..a56e667 100644 --- a/docs/change-codes.md +++ b/docs/change-codes.md @@ -9,11 +9,15 @@ Severities **`breaking`** and **`critical`** fail CI (`compare` exits `1`). | `tool.added` | info | A new tool appeared; selection-collision testing is still pending | | `tool.description_changed` | warning | Description text changed; model tool-selection may drift | | `tool.risk_changed` | warning / critical | Declared risk level changed (critical when escalating from `read_only`) | +| `tool.output_schema_added` | info | A tool gained an `outputSchema` | +| `tool.output_schema_removed` | breaking | A tool lost its `outputSchema` | +| `tool.output_schema_changed` | breaking | A tool's `outputSchema` changed | | `parameter.removed` | breaking | A parameter was removed from a tool | | `parameter.added` | info | An optional parameter was added | | `parameter.added_required` | breaking | A required parameter was added | | `parameter.became_required` | breaking | An optional parameter became required | -| `parameter.schema_changed` | breaking | Parameter JSON Schema changed (non-enum or unstructured diff) | +| `parameter.default_changed` | warning | Parameter `default` added, removed, or changed (does not fail CI alone) | +| `parameter.schema_changed` | breaking | Parameter JSON Schema changed (excluding `default`; non-enum or unstructured) | | `parameter.enum_values_removed` | breaking | One or more enum values were removed | | `parameter.enum_values_added` | info | One or more enum values were added | diff --git a/src/tool_semantics/diff.py b/src/tool_semantics/diff.py index 3c28b8c..38fb9de 100644 --- a/src/tool_semantics/diff.py +++ b/src/tool_semantics/diff.py @@ -52,6 +52,131 @@ def _enum_values(schema: dict[str, object]) -> set[str] | None: return {str(value) for value in values} +def _schema_without_default(schema: dict[str, object]) -> dict[str, object]: + return {key: value for key, value in schema.items() if key != "default"} + + +def _append_output_schema_changes( + report: CompatibilityReport, + tool_name: str, + old_schema: dict[str, object] | None, + new_schema: dict[str, object] | None, +) -> None: + if old_schema == new_schema: + return + if old_schema is None: + report.changes.append( + Change( + severity=Severity.INFO, + code="tool.output_schema_added", + subject=tool_name, + message=f"Output schema was added to '{tool_name}'.", + ) + ) + return + if new_schema is None: + report.changes.append( + Change( + severity=Severity.BREAKING, + code="tool.output_schema_removed", + subject=tool_name, + message=f"Output schema was removed from '{tool_name}'.", + ) + ) + return + report.changes.append( + Change( + severity=Severity.BREAKING, + code="tool.output_schema_changed", + subject=tool_name, + message=f"Output schema changed for '{tool_name}'.", + ) + ) + + +def _append_default_change( + report: CompatibilityReport, + subject: str, + parameter_name: str, + old_schema: dict[str, object], + new_schema: dict[str, object], +) -> None: + old_has = "default" in old_schema + new_has = "default" in new_schema + if not old_has and not new_has: + return + if old_has and new_has and old_schema["default"] == new_schema["default"]: + return + if old_has and new_has: + message = ( + f"Default for '{parameter_name}' changed from " + f"{old_schema['default']!r} to {new_schema['default']!r}." + ) + elif not old_has and new_has: + message = f"Default {new_schema['default']!r} was added to '{parameter_name}'." + else: + message = f"Default {old_schema['default']!r} was removed from '{parameter_name}'." + report.changes.append( + Change( + severity=Severity.WARNING, + code="parameter.default_changed", + subject=subject, + message=message, + ) + ) + + +def _append_schema_changes( + report: CompatibilityReport, + subject: str, + parameter_name: str, + old_schema: dict[str, object], + new_schema: dict[str, object], +) -> None: + old_core = _schema_without_default(old_schema) + new_core = _schema_without_default(new_schema) + if old_core == new_core: + return + old_enum = _enum_values(old_core) + new_enum = _enum_values(new_core) + if old_enum is not None and new_enum is not None and old_enum != new_enum: + removed = sorted(old_enum - new_enum) + added = sorted(new_enum - old_enum) + if removed: + report.changes.append( + Change( + severity=Severity.BREAKING, + code="parameter.enum_values_removed", + subject=subject, + message=( + f"Enum values removed from '{parameter_name}': {', '.join(removed)}." + ), + ) + ) + if added: + report.changes.append( + Change( + severity=Severity.INFO, + code="parameter.enum_values_added", + subject=subject, + message=(f"Enum values added to '{parameter_name}': {', '.join(added)}."), + ) + ) + # Enum-only diffs already covered; residual non-enum keys still need a code. + old_non_enum = {key: value for key, value in old_core.items() if key != "enum"} + new_non_enum = {key: value for key, value in new_core.items() if key != "enum"} + if old_non_enum == new_non_enum: + return + report.changes.append( + Change( + severity=Severity.BREAKING, + code="parameter.schema_changed", + subject=subject, + message=f"Schema changed for parameter '{parameter_name}'.", + ) + ) + + def compare_snapshots( baseline: InterfaceSnapshot, candidate: InterfaceSnapshot, @@ -109,6 +234,12 @@ def compare_snapshots( message=f"Risk level changed from '{old_tool.risk}' to '{new_tool.risk}'.", ) ) + _append_output_schema_changes( + report, + name, + old_tool.output_schema, + new_tool.output_schema, + ) old_params = _parameter_map(old_tool) new_params = _parameter_map(new_tool) @@ -138,50 +269,27 @@ def compare_snapshots( for parameter_name in sorted(old_params.keys() & new_params.keys()): old_parameter = old_params[parameter_name] new_parameter = new_params[parameter_name] - if old_parameter.schema_ != new_parameter.schema_: - old_enum = _enum_values(old_parameter.schema_) - new_enum = _enum_values(new_parameter.schema_) - if old_enum is not None and new_enum is not None and old_enum != new_enum: - removed = sorted(old_enum - new_enum) - added = sorted(new_enum - old_enum) - if removed: - report.changes.append( - Change( - severity=Severity.BREAKING, - code="parameter.enum_values_removed", - subject=f"{name}.{parameter_name}", - message=( - f"Enum values removed from '{parameter_name}': " - f"{', '.join(removed)}." - ), - ) - ) - if added: - report.changes.append( - Change( - severity=Severity.INFO, - code="parameter.enum_values_added", - subject=f"{name}.{parameter_name}", - message=( - f"Enum values added to '{parameter_name}': {', '.join(added)}." - ), - ) - ) - else: - report.changes.append( - Change( - severity=Severity.BREAKING, - code="parameter.schema_changed", - subject=f"{name}.{parameter_name}", - message=f"Schema changed for parameter '{parameter_name}'.", - ) - ) + subject = f"{name}.{parameter_name}" + _append_default_change( + report, + subject, + parameter_name, + old_parameter.schema_, + new_parameter.schema_, + ) + _append_schema_changes( + report, + subject, + parameter_name, + old_parameter.schema_, + new_parameter.schema_, + ) if not old_parameter.required and new_parameter.required: report.changes.append( Change( severity=Severity.BREAKING, code="parameter.became_required", - subject=f"{name}.{parameter_name}", + subject=subject, message=f"Parameter '{parameter_name}' became required.", ) ) diff --git a/tests/test_diff.py b/tests/test_diff.py index 6b11016..d6ccd26 100644 --- a/tests/test_diff.py +++ b/tests/test_diff.py @@ -56,3 +56,68 @@ def test_counts_by_severity() -> None: counts = report.counts_by_severity() assert counts["breaking"] >= 1 assert sum(counts.values()) == len(report.changes) + + +def test_detects_output_schema_added_removed_changed() -> None: + baseline = capture_manifest(Path("examples/github_server_v1.json")) + candidate = capture_manifest(Path("examples/github_server_v1.json")) + search = next(tool for tool in candidate.tools if tool.name == "search_issues") + search.output_schema = {"type": "object", "properties": {"items": {"type": "array"}}} + report = compare_snapshots(baseline, candidate) + assert any(change.code == "tool.output_schema_added" for change in report.changes) + assert report.is_compatible + + baseline_with = capture_manifest(Path("examples/github_server_v1.json")) + candidate_changed = capture_manifest(Path("examples/github_server_v1.json")) + for tool in baseline_with.tools: + if tool.name == "search_issues": + tool.output_schema = {"type": "object", "properties": {"items": {"type": "array"}}} + for tool in candidate_changed.tools: + if tool.name == "search_issues": + tool.output_schema = {"type": "object", "properties": {"total": {"type": "integer"}}} + report_changed = compare_snapshots(baseline_with, candidate_changed) + assert any(change.code == "tool.output_schema_changed" for change in report_changed.changes) + assert not report_changed.is_compatible + + candidate_removed = capture_manifest(Path("examples/github_server_v1.json")) + report_removed = compare_snapshots(baseline_with, candidate_removed) + assert any(change.code == "tool.output_schema_removed" for change in report_removed.changes) + assert not report_removed.is_compatible + + +def test_detects_parameter_default_changed_without_schema_changed() -> None: + baseline = capture_manifest(Path("examples/github_server_v1.json")) + candidate = capture_manifest(Path("examples/github_server_v1.json")) + search = next(tool for tool in candidate.tools if tool.name == "search_issues") + state = next(parameter for parameter in search.parameters if parameter.name == "state") + state.schema_ = {"type": "string", "enum": ["open", "closed"], "default": "closed"} + report = compare_snapshots(baseline, candidate) + assert any(change.code == "parameter.default_changed" for change in report.changes) + assert not any(change.code == "parameter.schema_changed" for change in report.changes) + assert report.is_compatible + + +def test_detects_parameter_default_added_and_removed() -> None: + baseline = capture_manifest(Path("examples/github_server_v1.json")) + candidate = capture_manifest(Path("examples/github_server_v1.json")) + search = next(tool for tool in candidate.tools if tool.name == "search_issues") + state = next(parameter for parameter in search.parameters if parameter.name == "state") + state.schema_ = {"type": "string", "enum": ["open", "closed"]} + report = compare_snapshots(baseline, candidate) + assert any( + change.code == "parameter.default_changed" and "removed" in change.message + for change in report.changes + ) + + baseline_no_default = capture_manifest(Path("examples/github_server_v1.json")) + search_base = next(tool for tool in baseline_no_default.tools if tool.name == "search_issues") + state_base = next( + parameter for parameter in search_base.parameters if parameter.name == "state" + ) + state_base.schema_ = {"type": "string", "enum": ["open", "closed"]} + candidate_added = capture_manifest(Path("examples/github_server_v1.json")) + report_added = compare_snapshots(baseline_no_default, candidate_added) + assert any( + change.code == "parameter.default_changed" and "added" in change.message + for change in report_added.changes + )