From 0fde30f7b88e9dfff8f1336f60ada76ca5282f23 Mon Sep 17 00:00:00 2001 From: Greg Holmes Date: Tue, 20 Jan 2026 16:22:03 +0000 Subject: [PATCH 1/3] docs: add human-in-the-loop guides for OpenAI and Anthropic Add two new AI Transport guides demonstrating how to implement human-in-the-loop approval workflows for AI agent tool calls: - openai-human-in-the-loop.mdx: HITL with OpenAI function calling - anthropic-human-in-the-loop.mdx: HITL with Anthropic tool use Both guides cover: - Defining tools requiring human approval - Publishing approval requests with requestId correlation - Subscribing to and processing approval decisions - Progress streaming during long-running tool execution - Handling rejection gracefully --- src/data/nav/aitransport.ts | 8 + .../anthropic-human-in-the-loop.mdx | 471 +++++++++++++++++ .../ai-transport/openai-human-in-the-loop.mdx | 481 ++++++++++++++++++ 3 files changed, 960 insertions(+) create mode 100644 src/pages/docs/guides/ai-transport/anthropic-human-in-the-loop.mdx create mode 100644 src/pages/docs/guides/ai-transport/openai-human-in-the-loop.mdx diff --git a/src/data/nav/aitransport.ts b/src/data/nav/aitransport.ts index 3b879bc7ac..6ca46d288e 100644 --- a/src/data/nav/aitransport.ts +++ b/src/data/nav/aitransport.ts @@ -92,6 +92,10 @@ export default { name: 'OpenAI token streaming - message per response', link: '/docs/guides/ai-transport/openai-message-per-response', }, + { + name: 'OpenAI messaging - human-in-the-loop', + link: '/docs/guides/ai-transport/openai-human-in-the-loop', + }, { name: 'Anthropic token streaming - message per token', link: '/docs/guides/ai-transport/anthropic-message-per-token', @@ -100,6 +104,10 @@ export default { name: 'Anthropic token streaming - message per response', link: '/docs/guides/ai-transport/anthropic-message-per-response', }, + { + name: 'Anthropic messaging - human-in-the-loop', + link: '/docs/guides/ai-transport/anthropic-human-in-the-loop', + }, { name: 'Vercel AI SDK token streaming - message per token', link: '/docs/guides/ai-transport/vercel-message-per-token', diff --git a/src/pages/docs/guides/ai-transport/anthropic-human-in-the-loop.mdx b/src/pages/docs/guides/ai-transport/anthropic-human-in-the-loop.mdx new file mode 100644 index 0000000000..275000c0b7 --- /dev/null +++ b/src/pages/docs/guides/ai-transport/anthropic-human-in-the-loop.mdx @@ -0,0 +1,471 @@ +--- +title: "Guide: Human-in-the-loop approval with Anthropic" +meta_description: "Implement human approval workflows for AI agent tool calls with progress streaming using Anthropic and Ably." +meta_keywords: "AI, human in the loop, HITL, Anthropic, Claude, tool use, approval workflow, AI transport, Ably, realtime, progress streaming" +--- + +This guide shows you how to implement a human-in-the-loop (HITL) approval workflow for AI agent tool calls using Anthropic and Ably. The agent requests human approval before executing sensitive operations and streams progress updates back to the user during long-running tool execution. + +Using Ably for HITL workflows enables reliable, realtime communication between AI agents and human approvers. The request-response pattern ensures approval requests are delivered and decisions are processed, while progress streaming keeps users informed during tool execution. + + + +## Prerequisites + +To follow this guide, you need: +- Node.js 20 or higher +- An Anthropic API key +- An Ably API key + +Useful links: +- [Anthropic tool use guide](https://docs.anthropic.com/en/docs/build-with-claude/tool-use/overview) +- [Ably JavaScript SDK getting started](/docs/getting-started/javascript) + +Create a new NPM package, which will contain the agent and client code: + + +```shell +mkdir ably-anthropic-hitl-example && cd ably-anthropic-hitl-example +npm init -y +``` + + +Install the required packages using NPM: + + +```shell +npm install @anthropic-ai/sdk@^0.71 ably@^2 +``` + + + + +Export your Anthropic API key to the environment, which will be used later in the guide by the Anthropic SDK: + + +```shell +export ANTHROPIC_API_KEY="your_api_key_here" +``` + + +## Step 1: Define a tool requiring approval + +Define an Anthropic tool that represents a sensitive operation requiring human approval. This example uses a file deletion tool that should not execute without explicit user consent. + +Create a new file `agent.mjs` with the following contents: + + +```javascript +import Anthropic from '@anthropic-ai/sdk'; +import Ably from 'ably'; +import crypto from 'crypto'; + +const anthropic = new Anthropic(); + +// Define a tool that requires human approval +const tools = [ + { + name: 'delete_files', + description: 'Delete files matching a pattern. This is a destructive operation requiring human approval.', + input_schema: { + type: 'object', + properties: { + pattern: { + type: 'string', + description: 'File pattern to delete (e.g., "*.log", "temp/*")' + }, + directory: { + type: 'string', + description: 'Directory to search in' + } + }, + required: ['pattern', 'directory'] + } + } +]; +``` + + +Tools that modify data, access sensitive resources, or perform irreversible actions are good candidates for HITL approval workflows. + +## Step 2: Set up Ably channels + +Initialize the Ably client and create a channel for communication between the agent and user. The agent publishes approval requests and progress updates, while the user publishes approval decisions. + +Add the following to your `agent.mjs` file: + + +```javascript +// Initialize Ably Realtime client +const realtime = new Ably.Realtime({ + key: '{{API_KEY}}', + echoMessages: false +}); + +// Create a channel for HITL communication +const channel = realtime.channels.get('ai:{{RANDOM_CHANNEL_NAME}}'); + +// Track pending approval requests +const pendingApprovals = new Map(); +``` + + + + +## Step 3: Request human approval + +When Claude returns a tool use block, check if it requires approval. If so, publish an approval request to the channel and wait for a human decision before executing. + +Add the approval request function to `agent.mjs`: + + +```javascript +async function requestApproval(toolUse) { + const requestId = crypto.randomUUID(); + + // Create a promise that resolves when approval is received + const approvalPromise = new Promise((resolve) => { + pendingApprovals.set(requestId, { toolUse, resolve }); + }); + + // Publish the approval request + await channel.publish('approval-request', { + requestId, + tool: toolUse.name, + parameters: toolUse.input, + message: `The agent wants to delete files matching "${toolUse.input.pattern}" in "${toolUse.input.directory}". Do you approve?` + }); + + console.log(`Approval request sent: ${requestId}`); + return approvalPromise; +} +``` + + +The `requestId` correlates the approval request with the response, enabling the agent to handle multiple concurrent approval flows. + +## Step 4: Subscribe to approval responses + +Set up a subscription to receive approval decisions from human users. When a response arrives, verify the approver and resolve the pending promise. + +Add the subscription handler to `agent.mjs`: + + +```javascript +// Subscribe to approval responses from humans +await channel.subscribe('approval-response', (message) => { + const { requestId, decision } = message.data; + const pending = pendingApprovals.get(requestId); + + if (!pending) { + console.log(`No pending approval for request: ${requestId}`); + return; + } + + // Verify the approver (in production, check clientId or user claims) + const approverId = message.clientId; + console.log(`Received ${decision} from ${approverId || 'anonymous user'}`); + + // Resolve the promise with the decision + pending.resolve({ + approved: decision === 'approved', + approverId, + requestId + }); + + pendingApprovals.delete(requestId); +}); +``` + + + + +## Step 5: Execute tool with progress streaming + +Once approved, execute the tool and stream progress updates back to the user. This keeps the user informed during long-running operations. + +Add the tool execution function to `agent.mjs`: + + +```javascript +async function executeDeleteFiles(parameters, requestId) { + const { pattern, directory } = parameters; + + // Simulate finding files to delete + const files = [ + `${directory}/app.log`, + `${directory}/error.log`, + `${directory}/debug.log`, + `${directory}/access.log` + ].filter(f => f.includes(pattern.replace('*', ''))); + + // Stream progress updates + await channel.publish('tool-progress', { + requestId, + status: 'started', + message: `Found ${files.length} files matching "${pattern}"` + }); + + // Process each file with progress updates + const results = []; + for (let i = 0; i < files.length; i++) { + const file = files[i]; + + // Simulate deletion delay + await new Promise(resolve => setTimeout(resolve, 1000)); + + results.push({ file, deleted: true }); + + // Stream progress for each file + await channel.publish('tool-progress', { + requestId, + status: 'in_progress', + message: `Deleted ${file} (${i + 1}/${files.length})`, + progress: Math.round(((i + 1) / files.length) * 100) + }); + } + + // Stream completion + await channel.publish('tool-progress', { + requestId, + status: 'completed', + message: `Successfully deleted ${results.length} files`, + results + }); + + return results; +} +``` + + +Progress streaming keeps users engaged and informed during operations that take time to complete. + +## Step 6: Process tool calls with HITL + +Integrate the approval workflow into the Anthropic response processing. When a tool use block requires approval, pause execution until a human decision is received. + +Add the main agent loop to `agent.mjs`: + + +```javascript +async function processToolUse(toolUse) { + const { name, input } = toolUse; + + if (name === 'delete_files') { + // Request human approval + const approval = await requestApproval(toolUse); + + if (!approval.approved) { + return { + type: 'tool_result', + tool_use_id: toolUse.id, + content: JSON.stringify({ + success: false, + error: 'Operation rejected by user' + }) + }; + } + + // Execute with progress streaming + const results = await executeDeleteFiles(input, approval.requestId); + + return { + type: 'tool_result', + tool_use_id: toolUse.id, + content: JSON.stringify({ + success: true, + deleted_files: results.length, + results + }) + }; + } + + return { + type: 'tool_result', + tool_use_id: toolUse.id, + content: JSON.stringify({ error: 'Unknown tool' }) + }; +} + +async function runAgent(userMessage) { + console.log(`User: ${userMessage}`); + + // Create initial message with tools + const response = await anthropic.messages.create({ + model: 'claude-sonnet-4-5', + max_tokens: 1024, + tools, + messages: [{ role: 'user', content: userMessage }] + }); + + // Check if Claude wants to use a tool + const toolUseBlocks = response.content.filter(block => block.type === 'tool_use'); + + if (toolUseBlocks.length === 0) { + // No tool use, return the text response + const textBlock = response.content.find(block => block.type === 'text'); + console.log(`Agent: ${textBlock?.text || 'No response'}`); + return; + } + + // Process each tool use block + const toolResults = []; + for (const toolUse of toolUseBlocks) { + console.log(`Tool use: ${toolUse.name}`); + const result = await processToolUse(toolUse); + toolResults.push(result); + } + + // Submit tool results back to Claude + const finalResponse = await anthropic.messages.create({ + model: 'claude-sonnet-4-5', + max_tokens: 1024, + tools, + messages: [ + { role: 'user', content: userMessage }, + { role: 'assistant', content: response.content }, + { role: 'user', content: toolResults } + ] + }); + + const finalText = finalResponse.content.find(block => block.type === 'text'); + console.log(`Agent: ${finalText?.text || 'Task completed'}`); +} + +// Example usage +runAgent('Please clean up the log files in /var/logs by deleting all .log files'); +``` + + +Run the agent: + + +```shell +node agent.mjs +``` + + +The agent will request approval and wait for a human decision before proceeding. + +## Step 7: Create the approval client + +Create a client application that receives approval requests and allows humans to approve or reject them. + +Create a new file `client.mjs` with the following contents: + + +```javascript +import Ably from 'ably'; +import readline from 'readline'; + +// Initialize Ably Realtime client +const realtime = new Ably.Realtime({ + key: '{{API_KEY}}', + clientId: 'human-approver' +}); + +// Get the same channel used by the agent +const channel = realtime.channels.get('ai:{{RANDOM_CHANNEL_NAME}}'); + +// Set up readline for user input +const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout +}); + +// Subscribe to approval requests +await channel.subscribe('approval-request', (message) => { + const request = message.data; + + console.log('\n========================================'); + console.log('APPROVAL REQUEST'); + console.log('========================================'); + console.log(`Tool: ${request.tool}`); + console.log(`Parameters: ${JSON.stringify(request.parameters, null, 2)}`); + console.log(`Message: ${request.message}`); + console.log('========================================'); + + rl.question('Approve this action? (y/n): ', async (answer) => { + const decision = answer.toLowerCase() === 'y' ? 'approved' : 'rejected'; + + await channel.publish('approval-response', { + requestId: request.requestId, + decision + }); + + console.log(`Decision sent: ${decision}`); + }); +}); + +// Subscribe to progress updates +await channel.subscribe('tool-progress', (message) => { + const progress = message.data; + const progressBar = progress.progress ? ` [${progress.progress}%]` : ''; + console.log(`Progress${progressBar}: ${progress.message}`); +}); + +console.log('Waiting for approval requests...'); +``` + + +Run the client in a separate terminal: + + +```shell +node client.mjs +``` + + +Once the client is running, restart your agent. With both the agent and client running, the workflow proceeds as follows: + +1. The agent sends a prompt to Claude that triggers a tool use +2. The agent publishes an approval request to the channel +3. The client displays the request and prompts the user +4. The user approves or rejects the request +5. If approved, the agent executes the tool and streams progress +6. The client displays progress updates in realtime + +## Step 8: Handle rejection gracefully + +When a user rejects an approval request, the agent should handle the rejection gracefully and provide appropriate feedback. + +The rejection is already handled in the `processToolUse` function, which returns an error response to Claude. Claude can then generate an appropriate message for the user. + +You can also publish a rejection notification to inform all subscribers: + + +```javascript +if (!approval.approved) { + // Notify subscribers about the rejection + await channel.publish('tool-progress', { + requestId: approval.requestId, + status: 'rejected', + message: 'Operation was rejected by the user' + }); + + return { + type: 'tool_result', + tool_use_id: toolUse.id, + content: JSON.stringify({ + success: false, + error: 'Operation rejected by user' + }) + }; +} +``` + + +## Next steps + +- Learn more about [human-in-the-loop](/docs/ai-transport/messaging/human-in-the-loop) patterns and verification strategies +- Explore [identifying users and agents](/docs/ai-transport/sessions-identity/identifying-users-and-agents) for secure identity verification +- Understand [sessions and identity](/docs/ai-transport/sessions-identity) in AI-enabled applications +- Learn about [tool calls](/docs/ai-transport/messaging/tool-calls) for agent-to-agent communication diff --git a/src/pages/docs/guides/ai-transport/openai-human-in-the-loop.mdx b/src/pages/docs/guides/ai-transport/openai-human-in-the-loop.mdx new file mode 100644 index 0000000000..33a4260373 --- /dev/null +++ b/src/pages/docs/guides/ai-transport/openai-human-in-the-loop.mdx @@ -0,0 +1,481 @@ +--- +title: "Guide: Human-in-the-loop approval with OpenAI" +meta_description: "Implement human approval workflows for AI agent tool calls with progress streaming using OpenAI and Ably." +meta_keywords: "AI, human in the loop, HITL, OpenAI, tool calls, approval workflow, AI transport, Ably, realtime, progress streaming" +--- + +This guide shows you how to implement a human-in-the-loop (HITL) approval workflow for AI agent tool calls using OpenAI and Ably. The agent requests human approval before executing sensitive operations and streams progress updates back to the user during long-running tool execution. + +Using Ably for HITL workflows enables reliable, realtime communication between AI agents and human approvers. The request-response pattern ensures approval requests are delivered and decisions are processed, while progress streaming keeps users informed during tool execution. + + + +## Prerequisites + +To follow this guide, you need: +- Node.js 20 or higher +- An OpenAI API key +- An Ably API key + +Useful links: +- [OpenAI function calling guide](https://platform.openai.com/docs/guides/function-calling) +- [Ably JavaScript SDK getting started](/docs/getting-started/javascript) + +Create a new NPM package, which will contain the agent and client code: + + +```shell +mkdir ably-openai-hitl-example && cd ably-openai-hitl-example +npm init -y +``` + + +Install the required packages using NPM: + + +```shell +npm install openai@^4 ably@^2 +``` + + + + +Export your OpenAI API key to the environment, which will be used later in the guide by the OpenAI SDK: + + +```shell +export OPENAI_API_KEY="your_api_key_here" +``` + + +## Step 1: Define a tool requiring approval + +Define an OpenAI tool that represents a sensitive operation requiring human approval. This example uses a file deletion tool that should not execute without explicit user consent. + +Create a new file `agent.mjs` with the following contents: + + +```javascript +import OpenAI from 'openai'; +import Ably from 'ably'; +import crypto from 'crypto'; + +const openai = new OpenAI(); + +// Define a tool that requires human approval +const tools = [ + { + type: 'function', + name: 'delete_files', + description: 'Delete files matching a pattern. This is a destructive operation requiring human approval.', + parameters: { + type: 'object', + properties: { + pattern: { + type: 'string', + description: 'File pattern to delete (e.g., "*.log", "temp/*")' + }, + directory: { + type: 'string', + description: 'Directory to search in' + } + }, + required: ['pattern', 'directory'] + } + } +]; +``` + + +Tools that modify data, access sensitive resources, or perform irreversible actions are good candidates for HITL approval workflows. + +## Step 2: Set up Ably channels + +Initialize the Ably client and create a channel for communication between the agent and user. The agent publishes approval requests and progress updates, while the user publishes approval decisions. + +Add the following to your `agent.mjs` file: + + +```javascript +// Initialize Ably Realtime client +const realtime = new Ably.Realtime({ + key: '{{API_KEY}}', + echoMessages: false +}); + +// Wait for connection to be established +await realtime.connection.once('connected'); + +// Create a channel for HITL communication +const channel = realtime.channels.get('ai:{{RANDOM_CHANNEL_NAME}}'); + +// Track pending approval requests +const pendingApprovals = new Map(); +``` + + + + +## Step 3: Request human approval + +When the OpenAI model returns a tool call, check if it requires approval. If so, publish an approval request to the channel and wait for a human decision before executing. + +Add the approval request function to `agent.mjs`: + + +```javascript +async function requestApproval(toolCall) { + const requestId = crypto.randomUUID(); + + // Create a promise that resolves when approval is received + const approvalPromise = new Promise((resolve) => { + pendingApprovals.set(requestId, { toolCall, resolve }); + }); + + // Publish the approval request + const parameters = JSON.parse(toolCall.arguments); + await channel.publish('approval-request', { + requestId, + tool: toolCall.name, + parameters, + message: `The agent wants to delete files matching "${parameters.pattern}" in "${parameters.directory}". Do you approve?` + }); + + console.log(`Approval request sent: ${requestId}`); + return approvalPromise; +} +``` + + +The `requestId` correlates the approval request with the response, enabling the agent to handle multiple concurrent approval flows. + +## Step 4: Subscribe to approval responses + +Set up a subscription to receive approval decisions from human users. When a response arrives, verify the approver and resolve the pending promise. + +Add the subscription handler to `agent.mjs`: + + +```javascript +// Subscribe to approval responses from humans +await channel.subscribe('approval-response', (message) => { + const { requestId, decision } = message.data; + const pending = pendingApprovals.get(requestId); + + if (!pending) { + console.log(`No pending approval for request: ${requestId}`); + return; + } + + // Verify the approver (in production, check clientId or user claims) + const approverId = message.clientId; + console.log(`Received ${decision} from ${approverId || 'anonymous user'}`); + + // Resolve the promise with the decision + pending.resolve({ + approved: decision === 'approved', + approverId, + requestId + }); + + pendingApprovals.delete(requestId); +}); +``` + + + + +## Step 5: Execute tool with progress streaming + +Once approved, execute the tool and stream progress updates back to the user. This keeps the user informed during long-running operations. + +Add the tool execution function to `agent.mjs`: + + +```javascript +async function executeDeleteFiles(parameters, requestId) { + const { pattern, directory } = parameters; + + // Simulate finding files to delete + const files = [ + `${directory}/app.log`, + `${directory}/error.log`, + `${directory}/debug.log`, + `${directory}/access.log` + ].filter(f => f.includes(pattern.replace('*', ''))); + + // Stream progress updates + await channel.publish('tool-progress', { + requestId, + status: 'started', + message: `Found ${files.length} files matching "${pattern}"` + }); + + // Process each file with progress updates + const results = []; + for (let i = 0; i < files.length; i++) { + const file = files[i]; + + // Simulate deletion delay + await new Promise(resolve => setTimeout(resolve, 1000)); + + results.push({ file, deleted: true }); + + // Stream progress for each file + await channel.publish('tool-progress', { + requestId, + status: 'in_progress', + message: `Deleted ${file} (${i + 1}/${files.length})`, + progress: Math.round(((i + 1) / files.length) * 100) + }); + } + + // Stream completion + await channel.publish('tool-progress', { + requestId, + status: 'completed', + message: `Successfully deleted ${results.length} files`, + results + }); + + return results; +} +``` + + +Progress streaming keeps users engaged and informed during operations that take time to complete. + +## Step 6: Process tool calls with HITL + +Integrate the approval workflow into the OpenAI response processing. When a tool call requires approval, pause execution until a human decision is received. + +Add the main agent loop to `agent.mjs`: + + +```javascript +async function processToolCall(toolCall) { + const name = toolCall.name; + const parameters = JSON.parse(toolCall.arguments); + + if (name === 'delete_files') { + // Request human approval + const approval = await requestApproval(toolCall); + + if (!approval.approved) { + return { + call_id: toolCall.call_id, + output: JSON.stringify({ + success: false, + error: 'Operation rejected by user' + }) + }; + } + + // Execute with progress streaming + const results = await executeDeleteFiles(parameters, approval.requestId); + + return { + call_id: toolCall.call_id, + output: JSON.stringify({ + success: true, + deleted_files: results.length, + results + }) + }; + } + + return { + call_id: toolCall.call_id, + output: JSON.stringify({ error: 'Unknown tool' }) + }; +} + +async function runAgent(userMessage) { + console.log(userMessage); + + // Create initial response with tools + const response = await openai.responses.create({ + model: 'gpt-4o', + instructions: 'You are a helpful assistant. When the user asks you to perform file operations like deleting files, use the available tools to do so. Do not ask for confirmation - the tools have their own approval mechanisms.', + input: userMessage, + tools + }); + + // Check if the model wants to call a tool + const toolCalls = response.output.filter(item => item.type === 'function_call'); + + if (toolCalls.length === 0) { + // No tool calls, return the text response + const textOutput = response.output.find(item => item.type === 'message'); + console.log(`Agent: ${textOutput?.content?.[0]?.text || 'No response'}`); + return; + } + + // Process each tool call + const toolResults = []; + for (const toolCall of toolCalls) { + console.log(`Tool call: ${toolCall.name}`); + const result = await processToolCall(toolCall); + toolResults.push(result); + } + + // Submit tool results back to the model + const finalResponse = await openai.responses.create({ + model: 'gpt-4o', + input: [ + { role: 'user', content: userMessage }, + ...toolCalls.map(tc => ({ + type: 'function_call', + id: tc.id, + call_id: tc.call_id, + name: tc.name, + arguments: tc.arguments + })), + ...toolResults.map(tr => ({ + type: 'function_call_output', + call_id: tr.call_id, + output: tr.output + })) + ] + }); + + const finalText = finalResponse.output.find(item => item.type === 'message'); + console.log(`Agent: ${finalText?.content?.[0]?.text || 'Task completed'}`); +} + +// Example usage +runAgent('Please clean up the log files in /var/logs by deleting all .log files'); +``` + + +Run the agent: + + +```shell +node agent.mjs +``` + + +The agent will request approval and wait for a human decision before proceeding. + +## Step 7: Create the approval client + +Create a client application that receives approval requests and allows humans to approve or reject them. + +Create a new file `client.mjs` with the following contents: + + +```javascript +import Ably from 'ably'; +import readline from 'readline'; + +// Initialize Ably Realtime client +const realtime = new Ably.Realtime({ + key: '{{API_KEY}}', + clientId: 'human-approver' +}); + +// Get the same channel used by the agent +const channel = realtime.channels.get('ai:{{RANDOM_CHANNEL_NAME}}'); + +// Set up readline for user input +const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout +}); + +// Subscribe to approval requests +await channel.subscribe('approval-request', (message) => { + const request = message.data; + + console.log('\n========================================'); + console.log('APPROVAL REQUEST'); + console.log('========================================'); + console.log(`Tool: ${request.tool}`); + console.log(`Parameters: ${JSON.stringify(request.parameters, null, 2)}`); + console.log(`Message: ${request.message}`); + console.log('========================================'); + + rl.question('Approve this action? (y/n): ', async (answer) => { + const decision = answer.toLowerCase() === 'y' ? 'approved' : 'rejected'; + + await channel.publish('approval-response', { + requestId: request.requestId, + decision + }); + + console.log(`Decision sent: ${decision}`); + }); +}); + +// Subscribe to progress updates +await channel.subscribe('tool-progress', (message) => { + const progress = message.data; + const progressBar = progress.progress ? ` [${progress.progress}%]` : ''; + console.log(`Progress${progressBar}: ${progress.message}`); +}); + +console.log('Waiting for approval requests...'); +``` + + +Run the client in a separate terminal: + + +```shell +node client.mjs +``` + + +Once the client is running, restart your agent. With both the agent and client running, the workflow proceeds as follows: + +1. The agent sends a prompt to OpenAI that triggers a tool call +2. The agent publishes an approval request to the channel +3. The client displays the request and prompts the user +4. The user approves or rejects the request +5. If approved, the agent executes the tool and streams progress +6. The client displays progress updates in realtime + +## Step 8: Handle rejection gracefully + +When a user rejects an approval request, the agent should handle the rejection gracefully and provide appropriate feedback. + +The rejection is already handled in the `processToolCall` function, which returns an error response to the model. The model can then generate an appropriate message for the user. + +You can also publish a rejection notification to inform all subscribers: + + +```javascript +if (!approval.approved) { + // Notify subscribers about the rejection + await channel.publish('tool-progress', { + requestId: approval.requestId, + status: 'rejected', + message: 'Operation was rejected by the user' + }); + + return { + call_id: toolCall.call_id, + output: JSON.stringify({ + success: false, + error: 'Operation rejected by user' + }) + }; +} +``` + + +## Next steps + +- Learn more about [human-in-the-loop](/docs/ai-transport/messaging/human-in-the-loop) patterns and verification strategies +- Explore [identifying users and agents](/docs/ai-transport/sessions-identity/identifying-users-and-agents) for secure identity verification +- Understand [sessions and identity](/docs/ai-transport/sessions-identity) in AI-enabled applications +- Learn about [tool calls](/docs/ai-transport/messaging/tool-calls) for agent-to-agent communication From 53cdda0e91190585511e298b2861eec04bbd1d46 Mon Sep 17 00:00:00 2001 From: Greg Holmes Date: Mon, 26 Jan 2026 11:09:47 +0000 Subject: [PATCH 2/3] fixup! docs: add human-in-the-loop guides for OpenAI and Anthropic --- .../anthropic-human-in-the-loop.mdx | 489 ++++++++--------- .../ai-transport/openai-human-in-the-loop.mdx | 493 ++++++++---------- 2 files changed, 426 insertions(+), 556 deletions(-) diff --git a/src/pages/docs/guides/ai-transport/anthropic-human-in-the-loop.mdx b/src/pages/docs/guides/ai-transport/anthropic-human-in-the-loop.mdx index 275000c0b7..3b83b6097a 100644 --- a/src/pages/docs/guides/ai-transport/anthropic-human-in-the-loop.mdx +++ b/src/pages/docs/guides/ai-transport/anthropic-human-in-the-loop.mdx @@ -1,12 +1,12 @@ --- title: "Guide: Human-in-the-loop approval with Anthropic" -meta_description: "Implement human approval workflows for AI agent tool calls with progress streaming using Anthropic and Ably." -meta_keywords: "AI, human in the loop, HITL, Anthropic, Claude, tool use, approval workflow, AI transport, Ably, realtime, progress streaming" +meta_description: "Implement human approval workflows for AI agent tool calls using Anthropic and Ably with role-based access control." +meta_keywords: "AI, human in the loop, HITL, Anthropic, Claude, tool use, approval workflow, AI transport, Ably, realtime, RBAC" --- -This guide shows you how to implement a human-in-the-loop (HITL) approval workflow for AI agent tool calls using Anthropic and Ably. The agent requests human approval before executing sensitive operations and streams progress updates back to the user during long-running tool execution. +This guide shows you how to implement a human-in-the-loop (HITL) approval workflow for AI agent tool calls using Anthropic's Claude and Ably. The agent requests human approval before executing sensitive operations, with role-based access control to verify approvers have sufficient permissions. -Using Ably for HITL workflows enables reliable, realtime communication between AI agents and human approvers. The request-response pattern ensures approval requests are delivered and decisions are processed, while progress streaming keeps users informed during tool execution. +Using Ably for HITL workflows enables reliable, realtime communication between Claude-powered agents and human approvers. The request-response pattern ensures approval requests are delivered and decisions are processed with proper authorization checks. -Export your Anthropic API key to the environment, which will be used later in the guide by the Anthropic SDK: +Export your API keys to the environment: ```shell -export ANTHROPIC_API_KEY="your_api_key_here" +export ANTHROPIC_API_KEY="your_anthropic_api_key_here" +export ABLY_API_KEY="your_ably_api_key_here" ``` -## Step 1: Define a tool requiring approval +## Step 1: Initialize the agent -Define an Anthropic tool that represents a sensitive operation requiring human approval. This example uses a file deletion tool that should not execute without explicit user consent. +Set up the agent that will call Claude and request human approval for sensitive operations. This example uses a `publish_blog_post` tool that requires authorization before execution. -Create a new file `agent.mjs` with the following contents: +Initialize the Anthropic and Ably clients, and create a channel for communication between the agent and human approvers. + +Add the following to a new file called `agent.mjs`: ```javascript import Anthropic from '@anthropic-ai/sdk'; import Ably from 'ably'; -import crypto from 'crypto'; const anthropic = new Anthropic(); -// Define a tool that requires human approval -const tools = [ - { - name: 'delete_files', - description: 'Delete files matching a pattern. This is a destructive operation requiring human approval.', - input_schema: { - type: 'object', - properties: { - pattern: { - type: 'string', - description: 'File pattern to delete (e.g., "*.log", "temp/*")' - }, - directory: { - type: 'string', - description: 'Directory to search in' - } - }, - required: ['pattern', 'directory'] - } - } -]; -``` - - -Tools that modify data, access sensitive resources, or perform irreversible actions are good candidates for HITL approval workflows. - -## Step 2: Set up Ably channels - -Initialize the Ably client and create a channel for communication between the agent and user. The agent publishes approval requests and progress updates, while the user publishes approval decisions. - -Add the following to your `agent.mjs` file: - - -```javascript // Initialize Ably Realtime client const realtime = new Ably.Realtime({ - key: '{{API_KEY}}', + key: process.env.ABLY_API_KEY, echoMessages: false }); +// Wait for connection to be established +await realtime.connection.once('connected'); + // Create a channel for HITL communication const channel = realtime.channels.get('ai:{{RANDOM_CHANNEL_NAME}}'); // Track pending approval requests const pendingApprovals = new Map(); + +// Function that executes the approved action +async function publishBlogPost(args) { + const { title } = args; + console.log(`Publishing blog post: ${title}`); + // In production, this would call your CMS API + return { published: true, title }; +} ``` @@ -118,270 +97,267 @@ const pendingApprovals = new Map(); Set [`echoMessages`](/docs/api/realtime-sdk/types#client-options) to `false` on the agent's Ably client to prevent the agent from receiving its own messages, avoiding billing for [echoed messages](/docs/pub-sub/advanced#echo). -## Step 3: Request human approval +Tools that modify data, access sensitive resources, or perform actions with business impact are good candidates for HITL approval workflows. -When Claude returns a tool use block, check if it requires approval. If so, publish an approval request to the channel and wait for a human decision before executing. +## Step 2: Request human approval + +When Claude returns a tool use block, publish an approval request to the channel and wait for a human decision. The tool use ID is passed in the message headers to correlate requests with responses. Add the approval request function to `agent.mjs`: ```javascript -async function requestApproval(toolUse) { - const requestId = crypto.randomUUID(); - - // Create a promise that resolves when approval is received - const approvalPromise = new Promise((resolve) => { - pendingApprovals.set(requestId, { toolUse, resolve }); +async function requestHumanApproval(toolUse) { + const approvalPromise = new Promise((resolve, reject) => { + pendingApprovals.set(toolUse.id, { toolUse, resolve, reject }); }); - // Publish the approval request - await channel.publish('approval-request', { - requestId, - tool: toolUse.name, - parameters: toolUse.input, - message: `The agent wants to delete files matching "${toolUse.input.pattern}" in "${toolUse.input.directory}". Do you approve?` + await channel.publish({ + name: 'approval-request', + data: { + tool: toolUse.name, + arguments: toolUse.input + }, + extras: { + headers: { + toolCallId: toolUse.id + } + } }); - console.log(`Approval request sent: ${requestId}`); + console.log(`Approval request sent for: ${toolUse.name}`); return approvalPromise; } ``` -The `requestId` correlates the approval request with the response, enabling the agent to handle multiple concurrent approval flows. +The `toolUse.id` provided by Anthropic correlates the approval request with the response, enabling the agent to handle multiple concurrent approval flows. -## Step 4: Subscribe to approval responses +## Step 3: Subscribe to approval responses -Set up a subscription to receive approval decisions from human users. When a response arrives, verify the approver and resolve the pending promise. +Set up a subscription to receive approval decisions from human users. When a response arrives, verify the approver has sufficient permissions using role-based access control before resolving the pending promise. Add the subscription handler to `agent.mjs`: ```javascript -// Subscribe to approval responses from humans -await channel.subscribe('approval-response', (message) => { - const { requestId, decision } = message.data; - const pending = pendingApprovals.get(requestId); - - if (!pending) { - console.log(`No pending approval for request: ${requestId}`); - return; +async function subscribeApprovalResponses() { + // Define role hierarchy from lowest to highest privilege + const roleHierarchy = ['editor', 'publisher', 'admin']; + + // Define minimum role required for each tool + const approvalPolicies = { + publish_blog_post: { minRole: 'publisher' } + }; + + function canApprove(approverRole, requiredRole) { + const approverLevel = roleHierarchy.indexOf(approverRole); + const requiredLevel = roleHierarchy.indexOf(requiredRole); + return approverLevel >= requiredLevel; } - // Verify the approver (in production, check clientId or user claims) - const approverId = message.clientId; - console.log(`Received ${decision} from ${approverId || 'anonymous user'}`); + await channel.subscribe('approval-response', async (message) => { + const { decision } = message.data; + const toolCallId = message.extras?.headers?.toolCallId; + const pending = pendingApprovals.get(toolCallId); - // Resolve the promise with the decision - pending.resolve({ - approved: decision === 'approved', - approverId, - requestId - }); + if (!pending) { + console.log(`No pending approval for tool call: ${toolCallId}`); + return; + } - pendingApprovals.delete(requestId); -}); + const policy = approvalPolicies[pending.toolUse.name]; + // Get the trusted role from the JWT user claim + const approverRole = message.extras?.userClaim; + + // Verify the approver's role meets the minimum required + if (!canApprove(approverRole, policy.minRole)) { + console.log(`Insufficient role: ${approverRole} < ${policy.minRole}`); + pending.reject(new Error( + `Approver role '${approverRole}' insufficient for required '${policy.minRole}'` + )); + pendingApprovals.delete(toolCallId); + return; + } + + // Process the decision + if (decision === 'approved') { + console.log(`Approved by ${approverRole}`); + pending.resolve({ approved: true, approverRole }); + } else { + console.log(`Rejected by ${approverRole}`); + pending.reject(new Error(`Action rejected by ${approverRole}`)); + } + pendingApprovals.delete(toolCallId); + }); +} ``` - +The `message.extras.userClaim` contains the role embedded in the approver's JWT token, providing a trusted source for authorization decisions. This ensures only users with sufficient privileges can approve sensitive operations. -## Step 5: Execute tool with progress streaming +## Step 4: Process tool calls -Once approved, execute the tool and stream progress updates back to the user. This keeps the user informed during long-running operations. +Create a function to process tool use blocks by requesting approval and executing the action if approved. -Add the tool execution function to `agent.mjs`: +Add the tool processing function to `agent.mjs`: ```javascript -async function executeDeleteFiles(parameters, requestId) { - const { pattern, directory } = parameters; - - // Simulate finding files to delete - const files = [ - `${directory}/app.log`, - `${directory}/error.log`, - `${directory}/debug.log`, - `${directory}/access.log` - ].filter(f => f.includes(pattern.replace('*', ''))); - - // Stream progress updates - await channel.publish('tool-progress', { - requestId, - status: 'started', - message: `Found ${files.length} files matching "${pattern}"` - }); - - // Process each file with progress updates - const results = []; - for (let i = 0; i < files.length; i++) { - const file = files[i]; - - // Simulate deletion delay - await new Promise(resolve => setTimeout(resolve, 1000)); - - results.push({ file, deleted: true }); - - // Stream progress for each file - await channel.publish('tool-progress', { - requestId, - status: 'in_progress', - message: `Deleted ${file} (${i + 1}/${files.length})`, - progress: Math.round(((i + 1) / files.length) * 100) - }); +async function processToolUse(toolUse) { + if (toolUse.name === 'publish_blog_post') { + await requestHumanApproval(toolUse); + return await publishBlogPost(toolUse.input); } - - // Stream completion - await channel.publish('tool-progress', { - requestId, - status: 'completed', - message: `Successfully deleted ${results.length} files`, - results - }); - - return results; + throw new Error(`Unknown tool: ${toolUse.name}`); } ``` -Progress streaming keeps users engaged and informed during operations that take time to complete. +The function awaits approval before executing. If the approver rejects or has insufficient permissions, the promise rejects and the tool is not executed. -## Step 6: Process tool calls with HITL +## Step 5: Run the agent -Integrate the approval workflow into the Anthropic response processing. When a tool use block requires approval, pause execution until a human decision is received. +Create the main agent loop that sends prompts to Claude and processes any tool use blocks that require approval. -Add the main agent loop to `agent.mjs`: +Add the agent runner to `agent.mjs`: ```javascript -async function processToolUse(toolUse) { - const { name, input } = toolUse; - - if (name === 'delete_files') { - // Request human approval - const approval = await requestApproval(toolUse); - - if (!approval.approved) { - return { - type: 'tool_result', - tool_use_id: toolUse.id, - content: JSON.stringify({ - success: false, - error: 'Operation rejected by user' - }) - }; - } - - // Execute with progress streaming - const results = await executeDeleteFiles(input, approval.requestId); - - return { - type: 'tool_result', - tool_use_id: toolUse.id, - content: JSON.stringify({ - success: true, - deleted_files: results.length, - results - }) - }; - } - - return { - type: 'tool_result', - tool_use_id: toolUse.id, - content: JSON.stringify({ error: 'Unknown tool' }) - }; -} +async function runAgent(prompt) { + await subscribeApprovalResponses(); -async function runAgent(userMessage) { - console.log(`User: ${userMessage}`); + console.log(`User: ${prompt}`); - // Create initial message with tools const response = await anthropic.messages.create({ model: 'claude-sonnet-4-5', max_tokens: 1024, - tools, - messages: [{ role: 'user', content: userMessage }] + tools: [ + { + name: 'publish_blog_post', + description: 'Publish a blog post to the website. Requires human approval.', + input_schema: { + type: 'object', + properties: { + title: { + type: 'string', + description: 'Title of the blog post to publish' + } + }, + required: ['title'] + } + } + ], + messages: [{ role: 'user', content: prompt }] }); - // Check if Claude wants to use a tool const toolUseBlocks = response.content.filter(block => block.type === 'tool_use'); - if (toolUseBlocks.length === 0) { - // No tool use, return the text response - const textBlock = response.content.find(block => block.type === 'text'); - console.log(`Agent: ${textBlock?.text || 'No response'}`); - return; - } - - // Process each tool use block - const toolResults = []; for (const toolUse of toolUseBlocks) { console.log(`Tool use: ${toolUse.name}`); - const result = await processToolUse(toolUse); - toolResults.push(result); + try { + const result = await processToolUse(toolUse); + console.log('Result:', result); + } catch (err) { + console.error('Tool use failed:', err.message); + } } +} - // Submit tool results back to Claude - const finalResponse = await anthropic.messages.create({ - model: 'claude-sonnet-4-5', - max_tokens: 1024, - tools, - messages: [ - { role: 'user', content: userMessage }, - { role: 'assistant', content: response.content }, - { role: 'user', content: toolResults } - ] - }); +runAgent("Publish the blog post called 'Introducing our new API'"); +``` + + +## Step 6: Create the authentication server + +The authentication server issues JWT tokens with embedded role claims. The role claim is trusted by Ably and included in messages, enabling secure role-based authorization. + +Add the following to a new file called `server.mjs`: + + +```javascript +import express from 'express'; +import jwt from 'jsonwebtoken'; - const finalText = finalResponse.content.find(block => block.type === 'text'); - console.log(`Agent: ${finalText?.text || 'Task completed'}`); +const app = express(); + +// Mock authentication - replace with your actual auth logic +function authenticateUser(req, res, next) { + // In production, verify the user's session/credentials + req.user = { id: 'user123', role: 'publisher' }; + next(); +} + +// Return claims to embed in the JWT +function getJWTClaims(user) { + return { + 'ably.channel.*': user.role + }; } -// Example usage -runAgent('Please clean up the log files in /var/logs by deleting all .log files'); +app.get('/api/auth/token', authenticateUser, (req, res) => { + const [keyName, keySecret] = process.env.ABLY_API_KEY.split(':'); + + const token = jwt.sign(getJWTClaims(req.user), keySecret, { + algorithm: 'HS256', + keyid: keyName, + expiresIn: '1h' + }); + + res.type('application/jwt').send(token); +}); + +app.listen(3001, () => { + console.log('Auth server running on http://localhost:3001'); +}); ``` -Run the agent: +The `ably.channel.*` claim embeds the user's role in the JWT. When the user publishes messages, this claim is available as `message.extras.userClaim`, providing a trusted source for authorization. + +Run the server: ```shell -node agent.mjs +node server.mjs ``` -The agent will request approval and wait for a human decision before proceeding. - ## Step 7: Create the approval client -Create a client application that receives approval requests and allows humans to approve or reject them. +The approval client receives approval requests and allows humans to approve or reject them. It authenticates via the server to obtain a JWT with the user's role. -Create a new file `client.mjs` with the following contents: +Add the following to a new file called `client.mjs`: ```javascript import Ably from 'ably'; import readline from 'readline'; -// Initialize Ably Realtime client -const realtime = new Ably.Realtime({ - key: '{{API_KEY}}', - clientId: 'human-approver' -}); - -// Get the same channel used by the agent -const channel = realtime.channels.get('ai:{{RANDOM_CHANNEL_NAME}}'); - -// Set up readline for user input const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); -// Subscribe to approval requests +const realtime = new Ably.Realtime({ + authCallback: async (tokenParams, callback) => { + try { + const response = await fetch('http://localhost:3001/api/auth/token'); + const token = await response.text(); + callback(null, token); + } catch (error) { + callback(error, null); + } + } +}); + +realtime.connection.on('connected', () => { + console.log('Connected to Ably'); + console.log('Waiting for approval requests...\n'); +}); + +const channel = realtime.channels.get('ai:{{RANDOM_CHANNEL_NAME}}'); + await channel.subscribe('approval-request', (message) => { const request = message.data; @@ -389,30 +365,25 @@ await channel.subscribe('approval-request', (message) => { console.log('APPROVAL REQUEST'); console.log('========================================'); console.log(`Tool: ${request.tool}`); - console.log(`Parameters: ${JSON.stringify(request.parameters, null, 2)}`); - console.log(`Message: ${request.message}`); + console.log(`Arguments: ${JSON.stringify(request.arguments, null, 2)}`); console.log('========================================'); rl.question('Approve this action? (y/n): ', async (answer) => { const decision = answer.toLowerCase() === 'y' ? 'approved' : 'rejected'; - await channel.publish('approval-response', { - requestId: request.requestId, - decision + await channel.publish({ + name: 'approval-response', + data: { decision }, + extras: { + headers: { + toolCallId: message.extras?.headers?.toolCallId + } + } }); - console.log(`Decision sent: ${decision}`); + console.log(`Decision sent: ${decision}\n`); }); }); - -// Subscribe to progress updates -await channel.subscribe('tool-progress', (message) => { - const progress = message.data; - const progressBar = progress.progress ? ` [${progress.progress}%]` : ''; - console.log(`Progress${progressBar}: ${progress.message}`); -}); - -console.log('Waiting for approval requests...'); ``` @@ -424,44 +395,14 @@ node client.mjs ``` -Once the client is running, restart your agent. With both the agent and client running, the workflow proceeds as follows: +With the server, client, and agent running, the workflow proceeds as follows: 1. The agent sends a prompt to Claude that triggers a tool use 2. The agent publishes an approval request to the channel 3. The client displays the request and prompts the user 4. The user approves or rejects the request -5. If approved, the agent executes the tool and streams progress -6. The client displays progress updates in realtime - -## Step 8: Handle rejection gracefully - -When a user rejects an approval request, the agent should handle the rejection gracefully and provide appropriate feedback. - -The rejection is already handled in the `processToolUse` function, which returns an error response to Claude. Claude can then generate an appropriate message for the user. - -You can also publish a rejection notification to inform all subscribers: - - -```javascript -if (!approval.approved) { - // Notify subscribers about the rejection - await channel.publish('tool-progress', { - requestId: approval.requestId, - status: 'rejected', - message: 'Operation was rejected by the user' - }); - - return { - type: 'tool_result', - tool_use_id: toolUse.id, - content: JSON.stringify({ - success: false, - error: 'Operation rejected by user' - }) - }; -} -``` - +5. The agent verifies the approver's role meets the minimum requirement +6. If approved and authorized, the agent executes the tool ## Next steps diff --git a/src/pages/docs/guides/ai-transport/openai-human-in-the-loop.mdx b/src/pages/docs/guides/ai-transport/openai-human-in-the-loop.mdx index 33a4260373..263b5c3cef 100644 --- a/src/pages/docs/guides/ai-transport/openai-human-in-the-loop.mdx +++ b/src/pages/docs/guides/ai-transport/openai-human-in-the-loop.mdx @@ -1,12 +1,12 @@ --- title: "Guide: Human-in-the-loop approval with OpenAI" -meta_description: "Implement human approval workflows for AI agent tool calls with progress streaming using OpenAI and Ably." -meta_keywords: "AI, human in the loop, HITL, OpenAI, tool calls, approval workflow, AI transport, Ably, realtime, progress streaming" +meta_description: "Implement human approval workflows for AI agent tool calls using OpenAI and Ably with role-based access control." +meta_keywords: "AI, human in the loop, HITL, OpenAI, tool calls, approval workflow, AI transport, Ably, realtime, RBAC" --- -This guide shows you how to implement a human-in-the-loop (HITL) approval workflow for AI agent tool calls using OpenAI and Ably. The agent requests human approval before executing sensitive operations and streams progress updates back to the user during long-running tool execution. +This guide shows you how to implement a human-in-the-loop (HITL) approval workflow for AI agent tool calls using OpenAI and Ably. The agent requests human approval before executing sensitive operations, with role-based access control to verify approvers have sufficient permissions. -Using Ably for HITL workflows enables reliable, realtime communication between AI agents and human approvers. The request-response pattern ensures approval requests are delivered and decisions are processed, while progress streaming keeps users informed during tool execution. +Using Ably for HITL workflows enables reliable, realtime communication between AI agents and human approvers. The request-response pattern ensures approval requests are delivered and decisions are processed with proper authorization checks. -Export your OpenAI API key to the environment, which will be used later in the guide by the OpenAI SDK: +Export your API keys to the environment: ```shell -export OPENAI_API_KEY="your_api_key_here" +export OPENAI_API_KEY="your_openai_api_key_here" +export ABLY_API_KEY="your_ably_api_key_here" ``` -## Step 1: Define a tool requiring approval +## Step 1: Initialize the agent -Define an OpenAI tool that represents a sensitive operation requiring human approval. This example uses a file deletion tool that should not execute without explicit user consent. +Set up the agent that will call OpenAI and request human approval for sensitive operations. This example uses a `publish_blog_post` tool that requires authorization before execution. -Create a new file `agent.mjs` with the following contents: +Initialize the OpenAI and Ably clients, and create a channel for communication between the agent and human approvers. Add the following to a new file called `agent.mjs`: ```javascript import OpenAI from 'openai'; import Ably from 'ably'; -import crypto from 'crypto'; const openai = new OpenAI(); -// Define a tool that requires human approval -const tools = [ - { - type: 'function', - name: 'delete_files', - description: 'Delete files matching a pattern. This is a destructive operation requiring human approval.', - parameters: { - type: 'object', - properties: { - pattern: { - type: 'string', - description: 'File pattern to delete (e.g., "*.log", "temp/*")' - }, - directory: { - type: 'string', - description: 'Directory to search in' - } - }, - required: ['pattern', 'directory'] - } - } -]; -``` - - -Tools that modify data, access sensitive resources, or perform irreversible actions are good candidates for HITL approval workflows. - -## Step 2: Set up Ably channels - -Initialize the Ably client and create a channel for communication between the agent and user. The agent publishes approval requests and progress updates, while the user publishes approval decisions. - -Add the following to your `agent.mjs` file: - - -```javascript // Initialize Ably Realtime client const realtime = new Ably.Realtime({ - key: '{{API_KEY}}', + key: process.env.ABLY_API_KEY, echoMessages: false }); @@ -115,6 +80,14 @@ const channel = realtime.channels.get('ai:{{RANDOM_CHANNEL_NAME}}'); // Track pending approval requests const pendingApprovals = new Map(); + +// Function that executes the approved action +async function publishBlogPost(args) { + const { title } = JSON.parse(args); + console.log(`Publishing blog post: ${title}`); + // In production, this would call your CMS API + return { published: true, title }; +} ``` @@ -122,277 +95,267 @@ const pendingApprovals = new Map(); Set [`echoMessages`](/docs/api/realtime-sdk/types#client-options) to `false` on the agent's Ably client to prevent the agent from receiving its own messages, avoiding billing for [echoed messages](/docs/pub-sub/advanced#echo). -## Step 3: Request human approval +Tools that modify data, access sensitive resources, or perform actions with business impact are good candidates for HITL approval workflows. -When the OpenAI model returns a tool call, check if it requires approval. If so, publish an approval request to the channel and wait for a human decision before executing. +## Step 2: Request human approval + +When the OpenAI model returns a tool call, publish an approval request to the channel and wait for a human decision. The tool call ID is passed in the message headers to correlate requests with responses. Add the approval request function to `agent.mjs`: ```javascript -async function requestApproval(toolCall) { - const requestId = crypto.randomUUID(); - - // Create a promise that resolves when approval is received - const approvalPromise = new Promise((resolve) => { - pendingApprovals.set(requestId, { toolCall, resolve }); +async function requestHumanApproval(toolCall) { + const approvalPromise = new Promise((resolve, reject) => { + pendingApprovals.set(toolCall.call_id, { toolCall, resolve, reject }); }); - // Publish the approval request - const parameters = JSON.parse(toolCall.arguments); - await channel.publish('approval-request', { - requestId, - tool: toolCall.name, - parameters, - message: `The agent wants to delete files matching "${parameters.pattern}" in "${parameters.directory}". Do you approve?` + await channel.publish({ + name: 'approval-request', + data: { + tool: toolCall.name, + arguments: toolCall.arguments + }, + extras: { + headers: { + toolCallId: toolCall.call_id + } + } }); - console.log(`Approval request sent: ${requestId}`); + console.log(`Approval request sent for: ${toolCall.name}`); return approvalPromise; } ``` -The `requestId` correlates the approval request with the response, enabling the agent to handle multiple concurrent approval flows. +The `toolCall.call_id` provided by OpenAI correlates the approval request with the response, enabling the agent to handle multiple concurrent approval flows. -## Step 4: Subscribe to approval responses +## Step 3: Subscribe to approval responses -Set up a subscription to receive approval decisions from human users. When a response arrives, verify the approver and resolve the pending promise. +Set up a subscription to receive approval decisions from human users. When a response arrives, verify the approver has sufficient permissions using role-based access control before resolving the pending promise. Add the subscription handler to `agent.mjs`: ```javascript -// Subscribe to approval responses from humans -await channel.subscribe('approval-response', (message) => { - const { requestId, decision } = message.data; - const pending = pendingApprovals.get(requestId); - - if (!pending) { - console.log(`No pending approval for request: ${requestId}`); - return; +async function subscribeApprovalResponses() { + // Define role hierarchy from lowest to highest privilege + const roleHierarchy = ['editor', 'publisher', 'admin']; + + // Define minimum role required for each tool + const approvalPolicies = { + publish_blog_post: { minRole: 'publisher' } + }; + + function canApprove(approverRole, requiredRole) { + const approverLevel = roleHierarchy.indexOf(approverRole); + const requiredLevel = roleHierarchy.indexOf(requiredRole); + return approverLevel >= requiredLevel; } - // Verify the approver (in production, check clientId or user claims) - const approverId = message.clientId; - console.log(`Received ${decision} from ${approverId || 'anonymous user'}`); + await channel.subscribe('approval-response', async (message) => { + const { decision } = message.data; + const toolCallId = message.extras?.headers?.toolCallId; + const pending = pendingApprovals.get(toolCallId); - // Resolve the promise with the decision - pending.resolve({ - approved: decision === 'approved', - approverId, - requestId - }); + if (!pending) { + console.log(`No pending approval for tool call: ${toolCallId}`); + return; + } - pendingApprovals.delete(requestId); -}); + const policy = approvalPolicies[pending.toolCall.name]; + // Get the trusted role from the JWT user claim + const approverRole = message.extras?.userClaim; + + // Verify the approver's role meets the minimum required + if (!canApprove(approverRole, policy.minRole)) { + console.log(`Insufficient role: ${approverRole} < ${policy.minRole}`); + pending.reject(new Error( + `Approver role '${approverRole}' insufficient for required '${policy.minRole}'` + )); + pendingApprovals.delete(toolCallId); + return; + } + + // Process the decision + if (decision === 'approved') { + console.log(`Approved by ${approverRole}`); + pending.resolve({ approved: true, approverRole }); + } else { + console.log(`Rejected by ${approverRole}`); + pending.reject(new Error(`Action rejected by ${approverRole}`)); + } + pendingApprovals.delete(toolCallId); + }); +} ``` - +The `message.extras.userClaim` contains the role embedded in the approver's JWT token, providing a trusted source for authorization decisions. This ensures only users with sufficient privileges can approve sensitive operations. -## Step 5: Execute tool with progress streaming +## Step 4: Process tool calls -Once approved, execute the tool and stream progress updates back to the user. This keeps the user informed during long-running operations. +Create a function to process tool calls by requesting approval and executing the action if approved. -Add the tool execution function to `agent.mjs`: +Add the tool processing function to `agent.mjs`: ```javascript -async function executeDeleteFiles(parameters, requestId) { - const { pattern, directory } = parameters; - - // Simulate finding files to delete - const files = [ - `${directory}/app.log`, - `${directory}/error.log`, - `${directory}/debug.log`, - `${directory}/access.log` - ].filter(f => f.includes(pattern.replace('*', ''))); - - // Stream progress updates - await channel.publish('tool-progress', { - requestId, - status: 'started', - message: `Found ${files.length} files matching "${pattern}"` - }); - - // Process each file with progress updates - const results = []; - for (let i = 0; i < files.length; i++) { - const file = files[i]; - - // Simulate deletion delay - await new Promise(resolve => setTimeout(resolve, 1000)); - - results.push({ file, deleted: true }); - - // Stream progress for each file - await channel.publish('tool-progress', { - requestId, - status: 'in_progress', - message: `Deleted ${file} (${i + 1}/${files.length})`, - progress: Math.round(((i + 1) / files.length) * 100) - }); +async function processToolCall(toolCall) { + if (toolCall.name === 'publish_blog_post') { + await requestHumanApproval(toolCall); + return await publishBlogPost(toolCall.arguments); } - - // Stream completion - await channel.publish('tool-progress', { - requestId, - status: 'completed', - message: `Successfully deleted ${results.length} files`, - results - }); - - return results; + throw new Error(`Unknown tool: ${toolCall.name}`); } ``` -Progress streaming keeps users engaged and informed during operations that take time to complete. +The function awaits approval before executing. If the approver rejects or has insufficient permissions, the promise rejects and the tool is not executed. -## Step 6: Process tool calls with HITL +## Step 5: Run the agent -Integrate the approval workflow into the OpenAI response processing. When a tool call requires approval, pause execution until a human decision is received. +Create the main agent loop that sends prompts to OpenAI and processes any tool calls that require approval. -Add the main agent loop to `agent.mjs`: +Add the agent runner to `agent.mjs`: ```javascript -async function processToolCall(toolCall) { - const name = toolCall.name; - const parameters = JSON.parse(toolCall.arguments); - - if (name === 'delete_files') { - // Request human approval - const approval = await requestApproval(toolCall); - - if (!approval.approved) { - return { - call_id: toolCall.call_id, - output: JSON.stringify({ - success: false, - error: 'Operation rejected by user' - }) - }; - } +async function runAgent(prompt) { + await subscribeApprovalResponses(); - // Execute with progress streaming - const results = await executeDeleteFiles(parameters, approval.requestId); - - return { - call_id: toolCall.call_id, - output: JSON.stringify({ - success: true, - deleted_files: results.length, - results - }) - }; - } + console.log(`User: ${prompt}`); - return { - call_id: toolCall.call_id, - output: JSON.stringify({ error: 'Unknown tool' }) - }; -} - -async function runAgent(userMessage) { - console.log(userMessage); - - // Create initial response with tools const response = await openai.responses.create({ model: 'gpt-4o', - instructions: 'You are a helpful assistant. When the user asks you to perform file operations like deleting files, use the available tools to do so. Do not ask for confirmation - the tools have their own approval mechanisms.', - input: userMessage, - tools + input: prompt, + tools: [ + { + type: 'function', + name: 'publish_blog_post', + description: 'Publish a blog post to the website. Requires human approval.', + parameters: { + type: 'object', + properties: { + title: { + type: 'string', + description: 'Title of the blog post to publish' + } + }, + required: ['title'] + } + } + ] }); - // Check if the model wants to call a tool const toolCalls = response.output.filter(item => item.type === 'function_call'); - if (toolCalls.length === 0) { - // No tool calls, return the text response - const textOutput = response.output.find(item => item.type === 'message'); - console.log(`Agent: ${textOutput?.content?.[0]?.text || 'No response'}`); - return; - } - - // Process each tool call - const toolResults = []; for (const toolCall of toolCalls) { console.log(`Tool call: ${toolCall.name}`); - const result = await processToolCall(toolCall); - toolResults.push(result); + try { + const result = await processToolCall(toolCall); + console.log('Result:', result); + } catch (err) { + console.error('Tool call failed:', err.message); + } } +} - // Submit tool results back to the model - const finalResponse = await openai.responses.create({ - model: 'gpt-4o', - input: [ - { role: 'user', content: userMessage }, - ...toolCalls.map(tc => ({ - type: 'function_call', - id: tc.id, - call_id: tc.call_id, - name: tc.name, - arguments: tc.arguments - })), - ...toolResults.map(tr => ({ - type: 'function_call_output', - call_id: tr.call_id, - output: tr.output - })) - ] - }); +runAgent("Publish the blog post called 'Introducing our new API'"); +``` + + +## Step 6: Create the authentication server + +The authentication server issues JWT tokens with embedded role claims. The role claim is trusted by Ably and included in messages, enabling secure role-based authorization. + +Add the following to a new file called `server.mjs`: + + +```javascript +import express from 'express'; +import jwt from 'jsonwebtoken'; - const finalText = finalResponse.output.find(item => item.type === 'message'); - console.log(`Agent: ${finalText?.content?.[0]?.text || 'Task completed'}`); +const app = express(); + +// Mock authentication - replace with your actual auth logic +function authenticateUser(req, res, next) { + // In production, verify the user's session/credentials + req.user = { id: 'user123', role: 'publisher' }; + next(); +} + +// Return claims to embed in the JWT +function getJWTClaims(user) { + return { + 'ably.channel.*': user.role + }; } -// Example usage -runAgent('Please clean up the log files in /var/logs by deleting all .log files'); +app.get('/api/auth/token', authenticateUser, (req, res) => { + const [keyName, keySecret] = process.env.ABLY_API_KEY.split(':'); + + const token = jwt.sign(getJWTClaims(req.user), keySecret, { + algorithm: 'HS256', + keyid: keyName, + expiresIn: '1h' + }); + + res.type('application/jwt').send(token); +}); + +app.listen(3001, () => { + console.log('Auth server running on http://localhost:3001'); +}); ``` -Run the agent: +The `ably.channel.*` claim embeds the user's role in the JWT. When the user publishes messages, this claim is available as `message.extras.userClaim`, providing a trusted source for authorization. + +Run the server: ```shell -node agent.mjs +node server.mjs ``` -The agent will request approval and wait for a human decision before proceeding. - ## Step 7: Create the approval client -Create a client application that receives approval requests and allows humans to approve or reject them. +The approval client receives approval requests and allows humans to approve or reject them. It authenticates via the server to obtain a JWT with the user's role. -Create a new file `client.mjs` with the following contents: +Add the following to a new file called `client.mjs`: ```javascript import Ably from 'ably'; import readline from 'readline'; -// Initialize Ably Realtime client -const realtime = new Ably.Realtime({ - key: '{{API_KEY}}', - clientId: 'human-approver' -}); - -// Get the same channel used by the agent -const channel = realtime.channels.get('ai:{{RANDOM_CHANNEL_NAME}}'); - -// Set up readline for user input const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); -// Subscribe to approval requests +const realtime = new Ably.Realtime({ + authCallback: async (tokenParams, callback) => { + try { + const response = await fetch('http://localhost:3001/api/auth/token'); + const token = await response.text(); + callback(null, token); + } catch (error) { + callback(error, null); + } + } +}); + +realtime.connection.on('connected', () => { + console.log('Connected to Ably'); + console.log('Waiting for approval requests...\n'); +}); + +const channel = realtime.channels.get('ai:{{RANDOM_CHANNEL_NAME}}'); + await channel.subscribe('approval-request', (message) => { const request = message.data; @@ -400,30 +363,25 @@ await channel.subscribe('approval-request', (message) => { console.log('APPROVAL REQUEST'); console.log('========================================'); console.log(`Tool: ${request.tool}`); - console.log(`Parameters: ${JSON.stringify(request.parameters, null, 2)}`); - console.log(`Message: ${request.message}`); + console.log(`Arguments: ${request.arguments}`); console.log('========================================'); rl.question('Approve this action? (y/n): ', async (answer) => { const decision = answer.toLowerCase() === 'y' ? 'approved' : 'rejected'; - await channel.publish('approval-response', { - requestId: request.requestId, - decision + await channel.publish({ + name: 'approval-response', + data: { decision }, + extras: { + headers: { + toolCallId: message.extras?.headers?.toolCallId + } + } }); - console.log(`Decision sent: ${decision}`); + console.log(`Decision sent: ${decision}\n`); }); }); - -// Subscribe to progress updates -await channel.subscribe('tool-progress', (message) => { - const progress = message.data; - const progressBar = progress.progress ? ` [${progress.progress}%]` : ''; - console.log(`Progress${progressBar}: ${progress.message}`); -}); - -console.log('Waiting for approval requests...'); ``` @@ -435,43 +393,14 @@ node client.mjs ``` -Once the client is running, restart your agent. With both the agent and client running, the workflow proceeds as follows: +With the server, client, and agent running, the workflow proceeds as follows: 1. The agent sends a prompt to OpenAI that triggers a tool call 2. The agent publishes an approval request to the channel 3. The client displays the request and prompts the user 4. The user approves or rejects the request -5. If approved, the agent executes the tool and streams progress -6. The client displays progress updates in realtime - -## Step 8: Handle rejection gracefully - -When a user rejects an approval request, the agent should handle the rejection gracefully and provide appropriate feedback. - -The rejection is already handled in the `processToolCall` function, which returns an error response to the model. The model can then generate an appropriate message for the user. - -You can also publish a rejection notification to inform all subscribers: - - -```javascript -if (!approval.approved) { - // Notify subscribers about the rejection - await channel.publish('tool-progress', { - requestId: approval.requestId, - status: 'rejected', - message: 'Operation was rejected by the user' - }); - - return { - call_id: toolCall.call_id, - output: JSON.stringify({ - success: false, - error: 'Operation rejected by user' - }) - }; -} -``` - +5. The agent verifies the approver's role meets the minimum requirement +6. If approved and authorized, the agent executes the tool ## Next steps From 666d601ac50797c5896e459097c490bf1e27d900 Mon Sep 17 00:00:00 2001 From: Greg Holmes Date: Mon, 26 Jan 2026 11:12:22 +0000 Subject: [PATCH 3/3] docs: use 'tool' field name in HITL feature docs for consistency --- src/pages/docs/ai-transport/messaging/human-in-the-loop.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pages/docs/ai-transport/messaging/human-in-the-loop.mdx b/src/pages/docs/ai-transport/messaging/human-in-the-loop.mdx index 98207f1025..1f48af8d6c 100644 --- a/src/pages/docs/ai-transport/messaging/human-in-the-loop.mdx +++ b/src/pages/docs/ai-transport/messaging/human-in-the-loop.mdx @@ -47,7 +47,7 @@ async function requestHumanApproval(toolCall) { await channel.publish({ name: 'approval-request', data: { - name: toolCall.name, + tool: toolCall.name, arguments: toolCall.arguments }, extras: {