-
Notifications
You must be signed in to change notification settings - Fork 304
MCP: Validate tool name uniqueness across BuiltIn and Custom tools #3110
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
Copilot
wants to merge
7
commits into
main
Choose a base branch
from
copilot/handle-duplicate-tool-names
base: main
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
+325
−2
Open
Changes from all commits
Commits
Show all changes
7 commits
Select commit
Hold shift + click to select a range
a890ed6
Initial plan
Copilot 871bf54
Add duplicate MCP tool name detection and validation
Copilot b189128
Fix nullable reference type annotations in McpToolRegistryTests
Copilot a3aaf81
Address code review feedback: fix JsonElement handling and clarify te…
Copilot ec7e9a0
Merge branch 'main' of https://github.com/Azure/data-api-builder into…
souvikghosh04 d2188d8
Minor fixes and refactoring of tests
souvikghosh04 c6b02ac
Merge branch 'main' into copilot/handle-duplicate-tool-names
souvikghosh04 File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,293 @@ | ||
| // Copyright (c) Microsoft Corporation. | ||
| // Licensed under the MIT License. | ||
|
|
||
| #nullable enable | ||
|
|
||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Linq; | ||
| using System.Net; | ||
| using System.Text.Json; | ||
| using System.Threading; | ||
| using System.Threading.Tasks; | ||
| using Azure.DataApiBuilder.Mcp.Core; | ||
| using Azure.DataApiBuilder.Mcp.Model; | ||
| using Azure.DataApiBuilder.Service.Exceptions; | ||
| using Microsoft.VisualStudio.TestTools.UnitTesting; | ||
| using ModelContextProtocol.Protocol; | ||
| using static Azure.DataApiBuilder.Mcp.Model.McpEnums; | ||
|
|
||
| namespace Azure.DataApiBuilder.Service.Tests.Mcp | ||
| { | ||
| /// <summary> | ||
| /// Tests for McpToolRegistry to ensure tool name uniqueness validation. | ||
| /// </summary> | ||
| [TestClass] | ||
| public class McpToolRegistryTests | ||
| { | ||
| /// <summary> | ||
| /// Test that registering multiple tools with unique names succeeds. | ||
| /// </summary> | ||
| [TestMethod] | ||
| public void RegisterTool_WithMultipleUniqueNames_Succeeds() | ||
| { | ||
| // Arrange | ||
| McpToolRegistry registry = new(); | ||
| IMcpTool tool1 = new MockMcpTool("tool_one", ToolType.BuiltIn); | ||
souvikghosh04 marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| IMcpTool tool2 = new MockMcpTool("tool_two", ToolType.Custom); | ||
| IMcpTool tool3 = new MockMcpTool("tool_three", ToolType.BuiltIn); | ||
|
|
||
| // Act & Assert - should not throw | ||
| registry.RegisterTool(tool1); | ||
| registry.RegisterTool(tool2); | ||
| registry.RegisterTool(tool3); | ||
|
|
||
| // Verify all tools were registered | ||
| IEnumerable<Tool> allTools = registry.GetAllTools(); | ||
| Assert.AreEqual(3, allTools.Count()); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Test that registering duplicate tools of the same type throws an exception. | ||
| /// Validates that both built-in and custom tools enforce name uniqueness within their own type. | ||
| /// </summary> | ||
| [DataTestMethod] | ||
| [DataRow(ToolType.BuiltIn, "duplicate_tool", "built-in", DisplayName = "Duplicate Built-In Tools")] | ||
| [DataRow(ToolType.Custom, "my_custom_tool", "custom", DisplayName = "Duplicate Custom Tools")] | ||
| public void RegisterTool_WithDuplicateSameType_ThrowsException( | ||
| ToolType toolType, | ||
| string toolName, | ||
| string expectedToolTypeText) | ||
| { | ||
| // Arrange | ||
| McpToolRegistry registry = new(); | ||
| IMcpTool tool1 = new MockMcpTool(toolName, toolType); | ||
| IMcpTool tool2 = new MockMcpTool(toolName, toolType); | ||
|
|
||
| // Act - Register first tool | ||
| registry.RegisterTool(tool1); | ||
|
|
||
| // Assert - Second registration should throw | ||
| DataApiBuilderException exception = Assert.ThrowsException<DataApiBuilderException>( | ||
| () => registry.RegisterTool(tool2) | ||
| ); | ||
|
|
||
| // Verify exception details | ||
| Assert.IsTrue(exception.Message.Contains($"Duplicate MCP tool name '{toolName}' detected")); | ||
| Assert.IsTrue(exception.Message.Contains($"{expectedToolTypeText} tool with this name is already registered")); | ||
| Assert.IsTrue(exception.Message.Contains($"Cannot register {expectedToolTypeText} tool with the same name")); | ||
| Assert.AreEqual(DataApiBuilderException.SubStatusCodes.ErrorInInitialization, exception.SubStatusCode); | ||
| Assert.AreEqual(HttpStatusCode.ServiceUnavailable, exception.StatusCode); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Test that registering tools with conflicting names across different types throws an exception. | ||
| /// Validates that tool names must be unique across all tool types (built-in and custom). | ||
| /// </summary> | ||
| [DataTestMethod] | ||
| [DataRow("create_record", ToolType.BuiltIn, ToolType.Custom, "built-in", "custom", DisplayName = "Built-In then Custom conflict")] | ||
| [DataRow("read_records", ToolType.BuiltIn, ToolType.Custom, "built-in", "custom", DisplayName = "Built-In then Custom conflict (read_records)")] | ||
| [DataRow("my_stored_proc", ToolType.Custom, ToolType.BuiltIn, "custom", "built-in", DisplayName = "Custom then Built-In conflict")] | ||
| public void RegisterTool_WithCrossTypeConflict_ThrowsException( | ||
| string toolName, | ||
| ToolType firstToolType, | ||
| ToolType secondToolType, | ||
| string expectedExistingType, | ||
| string expectedNewType) | ||
| { | ||
| // Arrange | ||
| McpToolRegistry registry = new(); | ||
| IMcpTool existingTool = new MockMcpTool(toolName, firstToolType); | ||
| IMcpTool conflictingTool = new MockMcpTool(toolName, secondToolType); | ||
|
|
||
| // Act - Register first tool | ||
| registry.RegisterTool(existingTool); | ||
|
|
||
| // Assert - Second tool registration should throw | ||
| DataApiBuilderException exception = Assert.ThrowsException<DataApiBuilderException>( | ||
| () => registry.RegisterTool(conflictingTool) | ||
| ); | ||
|
|
||
| // Verify exception details | ||
| Assert.IsTrue(exception.Message.Contains($"Duplicate MCP tool name '{toolName}' detected")); | ||
| Assert.IsTrue(exception.Message.Contains($"{expectedExistingType} tool with this name is already registered")); | ||
| Assert.IsTrue(exception.Message.Contains($"Cannot register {expectedNewType} tool with the same name")); | ||
| Assert.IsTrue(exception.Message.Contains("Tool names must be unique across all tool types")); | ||
| Assert.AreEqual(DataApiBuilderException.SubStatusCodes.ErrorInInitialization, exception.SubStatusCode); | ||
| Assert.AreEqual(HttpStatusCode.ServiceUnavailable, exception.StatusCode); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Test that tool name comparison is case-sensitive. | ||
| /// Tools with different casing should not be allowed. | ||
| /// </summary> | ||
| [TestMethod] | ||
| public void RegisterTool_WithDifferentCasing_ThrowsException() | ||
| { | ||
| // Arrange | ||
| McpToolRegistry registry = new(); | ||
| IMcpTool tool1 = new MockMcpTool("my_tool", ToolType.BuiltIn); | ||
| IMcpTool tool2 = new MockMcpTool("My_Tool", ToolType.Custom); | ||
|
|
||
| // Act - Register first tool | ||
| registry.RegisterTool(tool1); | ||
|
|
||
| // Assert - Case-insensitive duplicate should throw | ||
| DataApiBuilderException exception = Assert.ThrowsException<DataApiBuilderException>( | ||
| () => registry.RegisterTool(tool2) | ||
| ); | ||
|
|
||
| Assert.IsTrue(exception.Message.Contains("Duplicate MCP tool name")); | ||
| Assert.AreEqual(DataApiBuilderException.SubStatusCodes.ErrorInInitialization, exception.SubStatusCode); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Test that GetAllTools returns all registered tools. | ||
| /// </summary> | ||
| [TestMethod] | ||
| public void GetAllTools_ReturnsAllRegisteredTools() | ||
| { | ||
| // Arrange | ||
| McpToolRegistry registry = new(); | ||
| registry.RegisterTool(new MockMcpTool("tool_a", ToolType.BuiltIn)); | ||
| registry.RegisterTool(new MockMcpTool("tool_b", ToolType.Custom)); | ||
| registry.RegisterTool(new MockMcpTool("tool_c", ToolType.BuiltIn)); | ||
|
|
||
| // Act | ||
| IEnumerable<Tool> allTools = registry.GetAllTools(); | ||
|
|
||
| // Assert | ||
| Assert.AreEqual(3, allTools.Count()); | ||
| Assert.IsTrue(allTools.Any(t => t.Name == "tool_a")); | ||
| Assert.IsTrue(allTools.Any(t => t.Name == "tool_b")); | ||
| Assert.IsTrue(allTools.Any(t => t.Name == "tool_c")); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Test that TryGetTool returns false for non-existent tool. | ||
| /// </summary> | ||
| [TestMethod] | ||
| public void TryGetTool_WithNonExistentName_ReturnsFalse() | ||
| { | ||
| // Arrange | ||
| McpToolRegistry registry = new(); | ||
| registry.RegisterTool(new MockMcpTool("existing_tool", ToolType.BuiltIn)); | ||
|
|
||
| // Act | ||
| bool found = registry.TryGetTool("non_existent_tool", out IMcpTool? tool); | ||
|
|
||
| // Assert | ||
| Assert.IsFalse(found); | ||
| Assert.IsNull(tool); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Test edge case: empty tool name should throw exception. | ||
| /// </summary> | ||
| [TestMethod] | ||
| public void RegisterTool_WithEmptyToolName_ThrowsException() | ||
| { | ||
| // Arrange | ||
| McpToolRegistry registry = new(); | ||
| IMcpTool tool = new MockMcpTool("", ToolType.BuiltIn); | ||
|
|
||
| // Assert - Empty tool names should be rejected | ||
| DataApiBuilderException exception = Assert.ThrowsException<DataApiBuilderException>( | ||
| () => registry.RegisterTool(tool) | ||
| ); | ||
|
|
||
| Assert.IsTrue(exception.Message.Contains("cannot be null, empty, or whitespace")); | ||
| Assert.AreEqual(DataApiBuilderException.SubStatusCodes.ErrorInInitialization, exception.SubStatusCode); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Test realistic scenario with actual built-in tool names. | ||
| /// </summary> | ||
| [TestMethod] | ||
| public void RegisterTool_WithRealisticBuiltInToolNames_DetectsDuplicates() | ||
| { | ||
| // Arrange | ||
| McpToolRegistry registry = new(); | ||
|
|
||
| // Simulate registering built-in tools | ||
| registry.RegisterTool(new MockMcpTool("create_record", ToolType.BuiltIn)); | ||
| registry.RegisterTool(new MockMcpTool("read_records", ToolType.BuiltIn)); | ||
| registry.RegisterTool(new MockMcpTool("update_record", ToolType.BuiltIn)); | ||
| registry.RegisterTool(new MockMcpTool("delete_record", ToolType.BuiltIn)); | ||
| registry.RegisterTool(new MockMcpTool("describe_entities", ToolType.BuiltIn)); | ||
|
|
||
| // Try to register a custom tool with a conflicting name | ||
| IMcpTool customTool = new MockMcpTool("read_records", ToolType.Custom); | ||
|
|
||
| // Assert - Should throw | ||
| DataApiBuilderException exception = Assert.ThrowsException<DataApiBuilderException>( | ||
| () => registry.RegisterTool(customTool) | ||
| ); | ||
|
|
||
| Assert.IsTrue(exception.Message.Contains("read_records")); | ||
| Assert.IsTrue(exception.Message.Contains("built-in tool")); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Test that registering a tool with leading/trailing whitespace in the name is treated as a duplicate of the trimmed name. | ||
| /// Note: during tool registration, the registry should trim whitespace and detect duplicates accordingly. | ||
| /// </summary> | ||
| [TestMethod] | ||
| public void RegisterTool_WithLeadingTrailingWhitespace_DetectsDuplicate() | ||
| { | ||
| // Arrange | ||
| McpToolRegistry registry = new(); | ||
| IMcpTool tool1 = new MockMcpTool("my_tool", ToolType.BuiltIn); | ||
| IMcpTool tool2 = new MockMcpTool(" my_tool ", ToolType.Custom); | ||
|
|
||
| // Act | ||
| registry.RegisterTool(tool1); | ||
|
|
||
| // Assert - trimmed name should collide | ||
| Assert.ThrowsException<DataApiBuilderException>( | ||
| () => registry.RegisterTool(tool2) | ||
| ); | ||
| } | ||
|
|
||
| #region Private helpers | ||
|
|
||
| /// <summary> | ||
| /// Mock implementation of IMcpTool for testing purposes. | ||
| /// </summary> | ||
| private class MockMcpTool : IMcpTool | ||
| { | ||
| private readonly string _toolName; | ||
|
|
||
| public MockMcpTool(string toolName, ToolType toolType) | ||
| { | ||
| _toolName = toolName; | ||
| ToolType = toolType; | ||
| } | ||
|
|
||
| public ToolType ToolType { get; } | ||
|
|
||
| public Tool GetToolMetadata() | ||
| { | ||
| // Create a simple JSON object for the input schema | ||
| using JsonDocument doc = JsonDocument.Parse("{\"type\": \"object\"}"); | ||
| return new Tool | ||
| { | ||
| Name = _toolName, | ||
| Description = $"Mock {ToolType} tool", | ||
| InputSchema = doc.RootElement.Clone() | ||
| }; | ||
| } | ||
|
|
||
| public Task<CallToolResult> ExecuteAsync( | ||
| JsonDocument? arguments, | ||
| IServiceProvider serviceProvider, | ||
| CancellationToken cancellationToken = default) | ||
| { | ||
| // Not used in these tests | ||
| throw new NotImplementedException(); | ||
| } | ||
| } | ||
|
|
||
souvikghosh04 marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| #endregion Private helpers | ||
| } | ||
| } | ||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.