From 4fc2ad4112567deefe3666a72951efc8f1686d2c Mon Sep 17 00:00:00 2001 From: Norman Le Date: Fri, 1 May 2026 17:15:11 -0400 Subject: [PATCH 1/2] feat: add client side tools to bridge and new cas events --- .../src/uipath/core/chat/__init__.py | 2 + .../uipath-core/src/uipath/core/chat/tool.py | 17 ++++++ .../uipath/src/uipath/_cli/_chat/_bridge.py | 61 ++++++++++++++++--- .../uipath/src/uipath/agent/models/agent.py | 12 ++++ 4 files changed, 85 insertions(+), 7 deletions(-) diff --git a/packages/uipath-core/src/uipath/core/chat/__init__.py b/packages/uipath-core/src/uipath/core/chat/__init__.py index d81fa8153..e15e0cc8c 100644 --- a/packages/uipath-core/src/uipath/core/chat/__init__.py +++ b/packages/uipath-core/src/uipath/core/chat/__init__.py @@ -93,6 +93,7 @@ UiPathSessionStartEvent, ) from .tool import ( + UiPathConversationExecutingToolCallEvent, UiPathConversationToolCall, UiPathConversationToolCallConfirmation, UiPathConversationToolCallConfirmationData, @@ -157,6 +158,7 @@ "UiPathConversationCitationData", "UiPathConversationCitation", # Tool + "UiPathConversationExecutingToolCallEvent", "UiPathConversationToolCallStartEvent", "UiPathConversationToolCallEndEvent", "UiPathConversationToolCallConfirmation", diff --git a/packages/uipath-core/src/uipath/core/chat/tool.py b/packages/uipath-core/src/uipath/core/chat/tool.py index 8af5fb604..ab36b2d30 100644 --- a/packages/uipath-core/src/uipath/core/chat/tool.py +++ b/packages/uipath-core/src/uipath/core/chat/tool.py @@ -27,6 +27,8 @@ class UiPathConversationToolCallStartEvent(BaseModel): metadata: dict[str, Any] | None = Field(None, alias="metaData") require_confirmation: bool | None = Field(None, alias="requireConfirmation") input_schema: Any | None = Field(None, alias="inputSchema") + is_client_side_tool: bool | None = Field(None, alias="isClientSideTool") + output_schema: Any | None = Field(None, alias="outputSchema") model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) @@ -43,6 +45,18 @@ class UiPathConversationToolCallEndEvent(BaseModel): model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) +class UiPathConversationExecutingToolCallEvent(BaseModel): + """Signals the client that the tool is about to be executed. + Emitted in all paths. For client-side tools, the client should begin + executing its handler upon receiving this event.""" + + tool_name: str = Field(..., alias="toolName") + timestamp: str | None = None + input: dict[str, Any] | None = None + + model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) + + class UiPathConversationToolCallConfirmationEvent(BaseModel): """Signals a tool call confirmation (approve/reject) from the client.""" @@ -82,6 +96,9 @@ class UiPathConversationToolCallEvent(BaseModel): confirm: UiPathConversationToolCallConfirmationEvent | None = Field( None, alias="confirmToolCall" ) + executing: UiPathConversationExecutingToolCallEvent | None = Field( + None, alias="executingToolCall" + ) meta_event: dict[str, Any] | None = Field(None, alias="metaEvent") error: UiPathConversationErrorEvent | None = Field(None, alias="toolCallError") diff --git a/packages/uipath/src/uipath/_cli/_chat/_bridge.py b/packages/uipath/src/uipath/_cli/_chat/_bridge.py index 2a382a59e..9250e6277 100644 --- a/packages/uipath/src/uipath/_cli/_chat/_bridge.py +++ b/packages/uipath/src/uipath/_cli/_chat/_bridge.py @@ -13,8 +13,11 @@ UiPathConversationEvent, UiPathConversationExchangeEndEvent, UiPathConversationExchangeEvent, + UiPathConversationExecutingToolCallEvent, UiPathConversationMessageEvent, UiPathConversationToolCallConfirmationEvent, + UiPathConversationToolCallEndEvent, + UiPathConversationToolCallEvent, ) from uipath.core.triggers import UiPathResumeTrigger from uipath.runtime.chat import UiPathChatProtocol @@ -124,7 +127,9 @@ def __init__( self._tool_confirmation_event = asyncio.Event() self._tool_confirmation_value: ( - UiPathConversationToolCallConfirmationEvent | None + UiPathConversationToolCallConfirmationEvent + | UiPathConversationToolCallEndEvent + | None ) = None self._current_message_id: str | None = None @@ -378,6 +383,48 @@ async def emit_interrupt_event(self, resume_trigger: UiPathResumeTrigger): """ return None + async def emit_executing_tool_call_event( + self, resume_trigger: UiPathResumeTrigger + ) -> None: + """Emit an executingToolCall event for client-side tool execution. + + Only emits for triggers marked with is_execution_phase=True. + This fires exactly once per client-side tool call — for Path 3 (no confirm) + and for Path 4 (after confirmation, on the execution interrupt). + Confirmation-only interrupts (Paths 2/4 first interrupt) are skipped. + """ + + request = ( + resume_trigger.api_resume.request if resume_trigger.api_resume else None + ) + if not request or not isinstance(request, dict): + return + + if not request.get("is_execution_phase"): + return + + tool_call_id = request.get("tool_call_id") + tool_name = request.get("tool_name") + tool_input = request.get("input") + + if not tool_call_id or not tool_name: + logger.info( + f"emit_executing_tool_call_event: missing tool_call_id or tool_name, skipping. tool_call_id={tool_call_id}, tool_name={tool_name}" + ) + return + + executing_event = UiPathConversationMessageEvent( + message_id=self._current_message_id, + tool_call=UiPathConversationToolCallEvent( + tool_call_id=tool_call_id, + executing=UiPathConversationExecutingToolCallEvent( + tool_name=tool_name, + input=tool_input, + ), + ), + ) + await self.emit_message_event(executing_event) + async def wait_for_resume(self) -> dict[str, Any]: """Wait for a confirmToolCall event to be received.""" self._tool_confirmation_event.clear() @@ -424,13 +471,13 @@ async def _handle_conversation_event( parsed_event.exchange and parsed_event.exchange.message and (tool_call := parsed_event.exchange.message.tool_call) - and (confirm := tool_call.confirm) ): - logger.info( - f"Received confirmToolCall for tool_call_id: {tool_call.tool_call_id}, approved: {confirm.approved}" - ) - self._tool_confirmation_value = confirm - self._tool_confirmation_event.set() + if confirm := tool_call.confirm: + self._tool_confirmation_value = confirm + self._tool_confirmation_event.set() + elif end := tool_call.end: + self._tool_confirmation_value = end + self._tool_confirmation_event.set() except Exception as e: logger.warning(f"Error parsing conversation event: {e}") diff --git a/packages/uipath/src/uipath/agent/models/agent.py b/packages/uipath/src/uipath/agent/models/agent.py index 7694a861e..809912b9f 100644 --- a/packages/uipath/src/uipath/agent/models/agent.py +++ b/packages/uipath/src/uipath/agent/models/agent.py @@ -115,6 +115,7 @@ class AgentToolType(str, CaseInsensitiveEnum): INTEGRATION = "Integration" INTERNAL = "Internal" IXP = "Ixp" + CLIENT_SIDE = "ClientSide" UNKNOWN = "Unknown" # fallback branch discriminator @@ -889,6 +890,15 @@ class AgentInternalToolResourceConfig(BaseAgentToolResourceConfig): ) +class AgentClientSideToolResourceConfig(BaseAgentToolResourceConfig): + """Resource config for client-side tools executed by the client SDK.""" + + type: Literal[AgentToolType.CLIENT_SIDE] = AgentToolType.CLIENT_SIDE + properties: BaseResourceProperties = Field(default_factory=BaseResourceProperties) + output_schema: Optional[Dict[str, Any]] = Field(None, alias="outputSchema") + arguments: Optional[Dict[str, Any]] = Field(default_factory=dict) + + class AgentUnknownToolResourceConfig(BaseAgentToolResourceConfig): """Fallback for unknown tool types (parent normalizer sets type='Unknown').""" @@ -902,6 +912,7 @@ class AgentUnknownToolResourceConfig(BaseAgentToolResourceConfig): AgentIntegrationToolResourceConfig, AgentInternalToolResourceConfig, AgentIxpExtractionResourceConfig, + AgentClientSideToolResourceConfig, AgentUnknownToolResourceConfig, # when parent sets type="Unknown" ], Field(discriminator="type"), @@ -1276,6 +1287,7 @@ def _normalize_resources(v: Dict[str, Any]) -> None: "integration": "Integration", "internal": "Internal", "ixp": "Ixp", + "clientside": "ClientSide", "unknown": "Unknown", } CONTEXT_MODE_MAP = { From 39919b271f08a2bca0f81a8afaa1129c0d8930af Mon Sep 17 00:00:00 2001 From: Norman Le Date: Wed, 13 May 2026 21:36:33 -0400 Subject: [PATCH 2/2] chore: update to use existing interrupt event and update comments --- .../uipath/src/uipath/_cli/_chat/_bridge.py | 34 +-- .../uipath/tests/agent/models/test_agent.py | 205 ++++++++++++++++ packages/uipath/tests/cli/chat/test_bridge.py | 219 ++++++++++++++++++ 3 files changed, 428 insertions(+), 30 deletions(-) diff --git a/packages/uipath/src/uipath/_cli/_chat/_bridge.py b/packages/uipath/src/uipath/_cli/_chat/_bridge.py index 9250e6277..1c8682a9d 100644 --- a/packages/uipath/src/uipath/_cli/_chat/_bridge.py +++ b/packages/uipath/src/uipath/_cli/_chat/_bridge.py @@ -365,35 +365,12 @@ async def emit_exchange_error_event(self, error: Exception) -> None: raise RuntimeError(f"Failed to send exchange error event: {e}") from e async def emit_interrupt_event(self, resume_trigger: UiPathResumeTrigger): - """No-op. - - Tool confirmation — the only interrupt pattern CAS uses today — is - handled end-to-end via ``startToolCall`` with ``requireConfirmation: - true`` paired with ``wait_for_resume()``. This is deliberately - simpler than the old interrupt-based flow: CAS needs - ``requireConfirmation`` on the tool call event itself to render the - confirmation UI, so a parallel ``startInterrupt`` event would be - redundant. - - The only hypothetical reason to put work here is a generic, - non-tool-call agent interrupt (e.g. a coded agent calling - ``interrupt("do you want to continue?")``). Nothing uses that today - and it's not a near-term requirement — the method is kept for - generic flexibility. - """ - return None - - async def emit_executing_tool_call_event( - self, resume_trigger: UiPathResumeTrigger - ) -> None: - """Emit an executingToolCall event for client-side tool execution. + """Emit an executingToolCall event if the trigger is marked with is_execution_phase. - Only emits for triggers marked with is_execution_phase=True. - This fires exactly once per client-side tool call — for Path 3 (no confirm) - and for Path 4 (after confirmation, on the execution interrupt). - Confirmation-only interrupts (Paths 2/4 first interrupt) are skipped. + Called by the runtime loop for every durable interrupt. Emits executingToolCall + for triggers that signal execution is about to begin, The + is_execution_phase marker ensures it fires exactly once per tool call. """ - request = ( resume_trigger.api_resume.request if resume_trigger.api_resume else None ) @@ -408,9 +385,6 @@ async def emit_executing_tool_call_event( tool_input = request.get("input") if not tool_call_id or not tool_name: - logger.info( - f"emit_executing_tool_call_event: missing tool_call_id or tool_name, skipping. tool_call_id={tool_call_id}, tool_name={tool_name}" - ) return executing_event = UiPathConversationMessageEvent( diff --git a/packages/uipath/tests/agent/models/test_agent.py b/packages/uipath/tests/agent/models/test_agent.py index d00e5a42b..4cf405dd7 100644 --- a/packages/uipath/tests/agent/models/test_agent.py +++ b/packages/uipath/tests/agent/models/test_agent.py @@ -3837,3 +3837,208 @@ def test_argument_group_name_recipient_missing_argument_name_raises(self): payload = {"type": 8} with pytest.raises(ValidationError): TypeAdapter(AgentEscalationRecipient).validate_python(payload) + + def test_agent_with_client_side_tool(self): + """Test agent with ClientSide tool resource.""" + + json_data = { + "version": "1.0.0", + "id": "aaaaaaaa-0000-0000-0000-000000000010", + "name": "Agent with ClientSide Tool", + "metadata": {"isConversational": False, "storageVersion": "26.0.0"}, + "messages": [ + {"role": "System", "content": "You are an agentic assistant."}, + ], + "inputSchema": {"type": "object", "properties": {}}, + "outputSchema": { + "type": "object", + "properties": {"content": {"type": "string"}}, + }, + "settings": { + "model": "gpt-4o-2024-11-20", + "maxTokens": 16384, + "temperature": 0, + "engine": "basic-v2", + }, + "resources": [ + { + "$resourceType": "tool", + "id": "cst-0001-0000-0000-000000000001", + "name": "browser_navigate", + "description": "Navigate to a URL in the browser", + "location": "external", + "type": "ClientSide", + "inputSchema": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "The URL to navigate to", + } + }, + "required": ["url"], + }, + "outputSchema": { + "type": "object", + "properties": { + "title": {"type": "string"}, + "content": {"type": "string"}, + }, + }, + "arguments": {"timeout": 30}, + "properties": {}, + "isEnabled": True, + } + ], + "features": [], + } + + config: AgentDefinition = TypeAdapter(AgentDefinition).validate_python( + json_data + ) + + assert config.name == "Agent with ClientSide Tool" + assert len(config.resources) == 1 + + tool = config.resources[0] + assert isinstance(tool, AgentClientSideToolResourceConfig) + assert tool.resource_type == AgentResourceType.TOOL + assert tool.type == AgentToolType.CLIENT_SIDE + assert tool.name == "browser_navigate" + assert tool.description == "Navigate to a URL in the browser" + + # Validate input schema + assert tool.input_schema["type"] == "object" + assert "url" in tool.input_schema["properties"] + assert tool.input_schema["required"] == ["url"] + + # Validate outputSchema alias deserializes to output_schema + assert tool.output_schema is not None + assert tool.output_schema["type"] == "object" + assert "title" in tool.output_schema["properties"] + assert "content" in tool.output_schema["properties"] + + # Validate arguments + assert tool.arguments == {"timeout": 30} + + def test_agent_with_client_side_tool_lowercase_type(self): + """Test that _normalize_resources handles lowercase 'clientside' type.""" + + json_data = { + "version": "1.0.0", + "id": "aaaaaaaa-0000-0000-0000-000000000011", + "name": "Agent with clientside Tool", + "metadata": {"isConversational": False, "storageVersion": "26.0.0"}, + "messages": [ + {"role": "System", "content": "You are an agentic assistant."}, + ], + "inputSchema": {"type": "object", "properties": {}}, + "outputSchema": { + "type": "object", + "properties": {"content": {"type": "string"}}, + }, + "settings": { + "model": "gpt-4o-2024-11-20", + "maxTokens": 16384, + "temperature": 0, + "engine": "basic-v2", + }, + "resources": [ + { + "$resourceType": "tool", + "id": "cst-0002-0000-0000-000000000001", + "name": "clipboard_copy", + "description": "Copy text to clipboard", + "location": "external", + "type": "clientside", + "inputSchema": { + "type": "object", + "properties": { + "text": {"type": "string"}, + }, + }, + "properties": {}, + "isEnabled": True, + } + ], + "features": [], + } + + config: AgentDefinition = TypeAdapter(AgentDefinition).validate_python( + json_data + ) + + tool = config.resources[0] + assert isinstance(tool, AgentClientSideToolResourceConfig) + assert tool.type == AgentToolType.CLIENT_SIDE + assert tool.name == "clipboard_copy" + + # output_schema and arguments should default + assert tool.output_schema is None + assert tool.arguments == {} + + def test_agent_with_client_side_tool_output_schema_alias(self): + """Test that the outputSchema alias correctly maps to output_schema.""" + + json_data = { + "version": "1.0.0", + "id": "aaaaaaaa-0000-0000-0000-000000000012", + "name": "Agent with ClientSide outputSchema alias", + "metadata": {"isConversational": False, "storageVersion": "26.0.0"}, + "messages": [ + {"role": "System", "content": "You are an agentic assistant."}, + ], + "inputSchema": {"type": "object", "properties": {}}, + "outputSchema": { + "type": "object", + "properties": {"content": {"type": "string"}}, + }, + "settings": { + "model": "gpt-4o-2024-11-20", + "maxTokens": 16384, + "temperature": 0, + "engine": "basic-v2", + }, + "resources": [ + { + "$resourceType": "tool", + "id": "cst-0003-0000-0000-000000000001", + "name": "screen_capture", + "description": "Capture a screenshot", + "location": "external", + "type": "ClientSide", + "inputSchema": { + "type": "object", + "properties": { + "region": {"type": "string"}, + }, + }, + "outputSchema": { + "type": "object", + "properties": { + "imageBase64": { + "type": "string", + "description": "Base64-encoded image", + } + }, + "required": ["imageBase64"], + }, + "properties": {}, + "isEnabled": True, + } + ], + "features": [], + } + + config: AgentDefinition = TypeAdapter(AgentDefinition).validate_python( + json_data + ) + + tool = config.resources[0] + assert isinstance(tool, AgentClientSideToolResourceConfig) + + # Access via Python attribute name (snake_case) + assert tool.output_schema is not None + assert tool.output_schema["type"] == "object" + assert "imageBase64" in tool.output_schema["properties"] + assert tool.output_schema["required"] == ["imageBase64"] diff --git a/packages/uipath/tests/cli/chat/test_bridge.py b/packages/uipath/tests/cli/chat/test_bridge.py index 2da4f31ad..f5dba777a 100644 --- a/packages/uipath/tests/cli/chat/test_bridge.py +++ b/packages/uipath/tests/cli/chat/test_bridge.py @@ -1,5 +1,6 @@ """Tests for SocketIOChatBridge and get_chat_bridge.""" +import asyncio import logging from datetime import datetime from typing import Any, cast @@ -351,3 +352,221 @@ async def test_send_with_datetime_does_not_raise(self) -> None: assert parsed_data["message"] == "test message" assert isinstance(parsed_data["timestamp"], str) assert isinstance(parsed_data["nested"]["created_at"], str) + + +class TestEmitInterruptEvent: + """Tests for emit_interrupt_event (executingToolCall emission).""" + + def _make_bridge(self) -> SocketIOChatBridge: + bridge = SocketIOChatBridge( + websocket_url="wss://test.example.com", + websocket_path="/socket.io", + conversation_id="conv-123", + exchange_id="exch-456", + headers={}, + ) + bridge._current_message_id = "msg-100" + return bridge + + def _make_trigger(self, request: dict[str, Any] | None) -> "UiPathResumeTrigger": + from uipath.core.triggers import UiPathApiTrigger, UiPathResumeTrigger + + api_resume = UiPathApiTrigger(request=request) if request is not None else None + return UiPathResumeTrigger(api_resume=api_resume) + + @pytest.mark.anyio + async def test_execution_phase_emits_executing_tool_call( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """An execution-phase trigger emits an executingToolCall event with correct payload.""" + monkeypatch.setenv("CAS_WEBSOCKET_DISABLED", "true") + bridge = self._make_bridge() + await bridge.connect() + + emitted_events: list[Any] = [] + original_emit = bridge.emit_message_event + + async def capture_emit(event: Any) -> None: + emitted_events.append(event) + await original_emit(event) + + bridge.emit_message_event = capture_emit # type: ignore[assignment] + + trigger = self._make_trigger( + { + "is_execution_phase": True, + "tool_call_id": "tc-42", + "tool_name": "my_tool", + "input": {"key": "value"}, + } + ) + + await bridge.emit_interrupt_event(trigger) + + assert len(emitted_events) == 1 + event = emitted_events[0] + assert event.message_id == "msg-100" + assert event.tool_call is not None + assert event.tool_call.tool_call_id == "tc-42" + assert event.tool_call.executing is not None + assert event.tool_call.executing.tool_name == "my_tool" + assert event.tool_call.executing.input == {"key": "value"} + + @pytest.mark.anyio + async def test_non_execution_phase_does_not_emit( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A trigger without is_execution_phase does not emit any event.""" + monkeypatch.setenv("CAS_WEBSOCKET_DISABLED", "true") + bridge = self._make_bridge() + await bridge.connect() + + emitted_events: list[Any] = [] + original_emit = bridge.emit_message_event + + async def capture_emit(event: Any) -> None: + emitted_events.append(event) + await original_emit(event) + + bridge.emit_message_event = capture_emit # type: ignore[assignment] + + trigger = self._make_trigger( + { + "is_execution_phase": False, + "tool_call_id": "tc-42", + "tool_name": "my_tool", + } + ) + + await bridge.emit_interrupt_event(trigger) + + assert len(emitted_events) == 0 + + @pytest.mark.anyio + async def test_missing_tool_call_id_does_not_emit( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A trigger missing tool_call_id does not emit.""" + monkeypatch.setenv("CAS_WEBSOCKET_DISABLED", "true") + bridge = self._make_bridge() + await bridge.connect() + + emitted_events: list[Any] = [] + original_emit = bridge.emit_message_event + + async def capture_emit(event: Any) -> None: + emitted_events.append(event) + await original_emit(event) + + bridge.emit_message_event = capture_emit # type: ignore[assignment] + + trigger = self._make_trigger( + { + "is_execution_phase": True, + "tool_name": "my_tool", + } + ) + + await bridge.emit_interrupt_event(trigger) + + assert len(emitted_events) == 0 + + @pytest.mark.anyio + async def test_no_api_resume_does_not_emit(self) -> None: + """A trigger with no api_resume does not emit.""" + bridge = self._make_bridge() + + emitted_events: list[Any] = [] + + async def capture_emit(event: Any) -> None: + emitted_events.append(event) + + bridge.emit_message_event = capture_emit # type: ignore[assignment] + + trigger = self._make_trigger(None) + + await bridge.emit_interrupt_event(trigger) + + assert len(emitted_events) == 0 + + +class TestWaitForResumeEndToolCall: + """Tests for wait_for_resume unblocking on endToolCall events.""" + + @pytest.mark.anyio + async def test_end_tool_call_unblocks_wait_for_resume(self) -> None: + """Receiving an endToolCall event unblocks wait_for_resume and returns parsed payload.""" + bridge = SocketIOChatBridge( + websocket_url="wss://test.example.com", + websocket_path="/socket.io", + conversation_id="conv-123", + exchange_id="exch-456", + headers={}, + ) + + end_event = { + "conversationId": "conv-123", + "exchange": { + "exchangeId": "exch-456", + "message": { + "messageId": "msg-200", + "toolCall": { + "toolCallId": "tc-99", + "endToolCall": { + "output": {"result": "ok"}, + "isError": False, + }, + }, + }, + }, + } + + async def simulate_end_event() -> None: + await asyncio.sleep(0.05) + await bridge._handle_conversation_event(end_event, "sid-1") + + task = asyncio.create_task(simulate_end_event()) + result = await bridge.wait_for_resume() + await task + + assert result["output"] == {"result": "ok"} + assert result["is_error"] is False + + @pytest.mark.anyio + async def test_confirm_tool_call_unblocks_wait_for_resume(self) -> None: + """Receiving a confirmToolCall event also unblocks wait_for_resume.""" + bridge = SocketIOChatBridge( + websocket_url="wss://test.example.com", + websocket_path="/socket.io", + conversation_id="conv-123", + exchange_id="exch-456", + headers={}, + ) + + confirm_event = { + "conversationId": "conv-123", + "exchange": { + "exchangeId": "exch-456", + "message": { + "messageId": "msg-200", + "toolCall": { + "toolCallId": "tc-99", + "confirmToolCall": { + "approved": True, + "input": {"edited": "data"}, + }, + }, + }, + }, + } + + async def simulate_confirm_event() -> None: + await asyncio.sleep(0.05) + await bridge._handle_conversation_event(confirm_event, "sid-1") + + task = asyncio.create_task(simulate_confirm_event()) + result = await bridge.wait_for_resume() + await task + + assert result["approved"] is True + assert result["input"] == {"edited": "data"}