diff --git a/src/data/nav/aitransport.ts b/src/data/nav/aitransport.ts index 3b879bc7ac..6ca46d288e 100644 --- a/src/data/nav/aitransport.ts +++ b/src/data/nav/aitransport.ts @@ -92,6 +92,10 @@ export default { name: 'OpenAI token streaming - message per response', link: '/docs/guides/ai-transport/openai-message-per-response', }, + { + name: 'OpenAI messaging - human-in-the-loop', + link: '/docs/guides/ai-transport/openai-human-in-the-loop', + }, { name: 'Anthropic token streaming - message per token', link: '/docs/guides/ai-transport/anthropic-message-per-token', @@ -100,6 +104,10 @@ export default { name: 'Anthropic token streaming - message per response', link: '/docs/guides/ai-transport/anthropic-message-per-response', }, + { + name: 'Anthropic messaging - human-in-the-loop', + link: '/docs/guides/ai-transport/anthropic-human-in-the-loop', + }, { name: 'Vercel AI SDK token streaming - message per token', link: '/docs/guides/ai-transport/vercel-message-per-token', diff --git a/src/pages/docs/ai-transport/messaging/human-in-the-loop.mdx b/src/pages/docs/ai-transport/messaging/human-in-the-loop.mdx index 98207f1025..1f48af8d6c 100644 --- a/src/pages/docs/ai-transport/messaging/human-in-the-loop.mdx +++ b/src/pages/docs/ai-transport/messaging/human-in-the-loop.mdx @@ -47,7 +47,7 @@ async function requestHumanApproval(toolCall) { await channel.publish({ name: 'approval-request', data: { - name: toolCall.name, + tool: toolCall.name, arguments: toolCall.arguments }, extras: { diff --git a/src/pages/docs/guides/ai-transport/anthropic-human-in-the-loop.mdx b/src/pages/docs/guides/ai-transport/anthropic-human-in-the-loop.mdx new file mode 100644 index 0000000000..3b83b6097a --- /dev/null +++ b/src/pages/docs/guides/ai-transport/anthropic-human-in-the-loop.mdx @@ -0,0 +1,412 @@ +--- +title: "Guide: Human-in-the-loop approval with Anthropic" +meta_description: "Implement human approval workflows for AI agent tool calls using Anthropic and Ably with role-based access control." +meta_keywords: "AI, human in the loop, HITL, Anthropic, Claude, tool use, approval workflow, AI transport, Ably, realtime, RBAC" +--- + +This guide shows you how to implement a human-in-the-loop (HITL) approval workflow for AI agent tool calls using Anthropic's Claude and Ably. The agent requests human approval before executing sensitive operations, with role-based access control to verify approvers have sufficient permissions. + +Using Ably for HITL workflows enables reliable, realtime communication between Claude-powered agents and human approvers. The request-response pattern ensures approval requests are delivered and decisions are processed with proper authorization checks. + + + +## Prerequisites + +To follow this guide, you need: +- Node.js 20 or higher +- An Anthropic API key +- An Ably API key + +Useful links: +- [Anthropic tool use guide](https://docs.anthropic.com/en/docs/build-with-claude/tool-use/overview) +- [Ably JavaScript SDK getting started](/docs/getting-started/javascript) + +Create a new NPM package, which will contain the agent, client, and server code: + + +```shell +mkdir ably-anthropic-hitl-example && cd ably-anthropic-hitl-example +npm init -y +``` + + +Install the required packages using NPM: + + +```shell +npm install @anthropic-ai/sdk@^0.71 ably@^2 express jsonwebtoken +``` + + + + +Export your API keys to the environment: + + +```shell +export ANTHROPIC_API_KEY="your_anthropic_api_key_here" +export ABLY_API_KEY="your_ably_api_key_here" +``` + + +## Step 1: Initialize the agent + +Set up the agent that will call Claude and request human approval for sensitive operations. This example uses a `publish_blog_post` tool that requires authorization before execution. + +Initialize the Anthropic and Ably clients, and create a channel for communication between the agent and human approvers. + +Add the following to a new file called `agent.mjs`: + + +```javascript +import Anthropic from '@anthropic-ai/sdk'; +import Ably from 'ably'; + +const anthropic = new Anthropic(); + +// Initialize Ably Realtime client +const realtime = new Ably.Realtime({ + key: process.env.ABLY_API_KEY, + echoMessages: false +}); + +// Wait for connection to be established +await realtime.connection.once('connected'); + +// Create a channel for HITL communication +const channel = realtime.channels.get('ai:{{RANDOM_CHANNEL_NAME}}'); + +// Track pending approval requests +const pendingApprovals = new Map(); + +// Function that executes the approved action +async function publishBlogPost(args) { + const { title } = args; + console.log(`Publishing blog post: ${title}`); + // In production, this would call your CMS API + return { published: true, title }; +} +``` + + + + +Tools that modify data, access sensitive resources, or perform actions with business impact are good candidates for HITL approval workflows. + +## Step 2: Request human approval + +When Claude returns a tool use block, publish an approval request to the channel and wait for a human decision. The tool use ID is passed in the message headers to correlate requests with responses. + +Add the approval request function to `agent.mjs`: + + +```javascript +async function requestHumanApproval(toolUse) { + const approvalPromise = new Promise((resolve, reject) => { + pendingApprovals.set(toolUse.id, { toolUse, resolve, reject }); + }); + + await channel.publish({ + name: 'approval-request', + data: { + tool: toolUse.name, + arguments: toolUse.input + }, + extras: { + headers: { + toolCallId: toolUse.id + } + } + }); + + console.log(`Approval request sent for: ${toolUse.name}`); + return approvalPromise; +} +``` + + +The `toolUse.id` provided by Anthropic correlates the approval request with the response, enabling the agent to handle multiple concurrent approval flows. + +## Step 3: Subscribe to approval responses + +Set up a subscription to receive approval decisions from human users. When a response arrives, verify the approver has sufficient permissions using role-based access control before resolving the pending promise. + +Add the subscription handler to `agent.mjs`: + + +```javascript +async function subscribeApprovalResponses() { + // Define role hierarchy from lowest to highest privilege + const roleHierarchy = ['editor', 'publisher', 'admin']; + + // Define minimum role required for each tool + const approvalPolicies = { + publish_blog_post: { minRole: 'publisher' } + }; + + function canApprove(approverRole, requiredRole) { + const approverLevel = roleHierarchy.indexOf(approverRole); + const requiredLevel = roleHierarchy.indexOf(requiredRole); + return approverLevel >= requiredLevel; + } + + await channel.subscribe('approval-response', async (message) => { + const { decision } = message.data; + const toolCallId = message.extras?.headers?.toolCallId; + const pending = pendingApprovals.get(toolCallId); + + if (!pending) { + console.log(`No pending approval for tool call: ${toolCallId}`); + return; + } + + const policy = approvalPolicies[pending.toolUse.name]; + // Get the trusted role from the JWT user claim + const approverRole = message.extras?.userClaim; + + // Verify the approver's role meets the minimum required + if (!canApprove(approverRole, policy.minRole)) { + console.log(`Insufficient role: ${approverRole} < ${policy.minRole}`); + pending.reject(new Error( + `Approver role '${approverRole}' insufficient for required '${policy.minRole}'` + )); + pendingApprovals.delete(toolCallId); + return; + } + + // Process the decision + if (decision === 'approved') { + console.log(`Approved by ${approverRole}`); + pending.resolve({ approved: true, approverRole }); + } else { + console.log(`Rejected by ${approverRole}`); + pending.reject(new Error(`Action rejected by ${approverRole}`)); + } + pendingApprovals.delete(toolCallId); + }); +} +``` + + +The `message.extras.userClaim` contains the role embedded in the approver's JWT token, providing a trusted source for authorization decisions. This ensures only users with sufficient privileges can approve sensitive operations. + +## Step 4: Process tool calls + +Create a function to process tool use blocks by requesting approval and executing the action if approved. + +Add the tool processing function to `agent.mjs`: + + +```javascript +async function processToolUse(toolUse) { + if (toolUse.name === 'publish_blog_post') { + await requestHumanApproval(toolUse); + return await publishBlogPost(toolUse.input); + } + throw new Error(`Unknown tool: ${toolUse.name}`); +} +``` + + +The function awaits approval before executing. If the approver rejects or has insufficient permissions, the promise rejects and the tool is not executed. + +## Step 5: Run the agent + +Create the main agent loop that sends prompts to Claude and processes any tool use blocks that require approval. + +Add the agent runner to `agent.mjs`: + + +```javascript +async function runAgent(prompt) { + await subscribeApprovalResponses(); + + console.log(`User: ${prompt}`); + + const response = await anthropic.messages.create({ + model: 'claude-sonnet-4-5', + max_tokens: 1024, + tools: [ + { + name: 'publish_blog_post', + description: 'Publish a blog post to the website. Requires human approval.', + input_schema: { + type: 'object', + properties: { + title: { + type: 'string', + description: 'Title of the blog post to publish' + } + }, + required: ['title'] + } + } + ], + messages: [{ role: 'user', content: prompt }] + }); + + const toolUseBlocks = response.content.filter(block => block.type === 'tool_use'); + + for (const toolUse of toolUseBlocks) { + console.log(`Tool use: ${toolUse.name}`); + try { + const result = await processToolUse(toolUse); + console.log('Result:', result); + } catch (err) { + console.error('Tool use failed:', err.message); + } + } +} + +runAgent("Publish the blog post called 'Introducing our new API'"); +``` + + +## Step 6: Create the authentication server + +The authentication server issues JWT tokens with embedded role claims. The role claim is trusted by Ably and included in messages, enabling secure role-based authorization. + +Add the following to a new file called `server.mjs`: + + +```javascript +import express from 'express'; +import jwt from 'jsonwebtoken'; + +const app = express(); + +// Mock authentication - replace with your actual auth logic +function authenticateUser(req, res, next) { + // In production, verify the user's session/credentials + req.user = { id: 'user123', role: 'publisher' }; + next(); +} + +// Return claims to embed in the JWT +function getJWTClaims(user) { + return { + 'ably.channel.*': user.role + }; +} + +app.get('/api/auth/token', authenticateUser, (req, res) => { + const [keyName, keySecret] = process.env.ABLY_API_KEY.split(':'); + + const token = jwt.sign(getJWTClaims(req.user), keySecret, { + algorithm: 'HS256', + keyid: keyName, + expiresIn: '1h' + }); + + res.type('application/jwt').send(token); +}); + +app.listen(3001, () => { + console.log('Auth server running on http://localhost:3001'); +}); +``` + + +The `ably.channel.*` claim embeds the user's role in the JWT. When the user publishes messages, this claim is available as `message.extras.userClaim`, providing a trusted source for authorization. + +Run the server: + + +```shell +node server.mjs +``` + + +## Step 7: Create the approval client + +The approval client receives approval requests and allows humans to approve or reject them. It authenticates via the server to obtain a JWT with the user's role. + +Add the following to a new file called `client.mjs`: + + +```javascript +import Ably from 'ably'; +import readline from 'readline'; + +const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout +}); + +const realtime = new Ably.Realtime({ + authCallback: async (tokenParams, callback) => { + try { + const response = await fetch('http://localhost:3001/api/auth/token'); + const token = await response.text(); + callback(null, token); + } catch (error) { + callback(error, null); + } + } +}); + +realtime.connection.on('connected', () => { + console.log('Connected to Ably'); + console.log('Waiting for approval requests...\n'); +}); + +const channel = realtime.channels.get('ai:{{RANDOM_CHANNEL_NAME}}'); + +await channel.subscribe('approval-request', (message) => { + const request = message.data; + + console.log('\n========================================'); + console.log('APPROVAL REQUEST'); + console.log('========================================'); + console.log(`Tool: ${request.tool}`); + console.log(`Arguments: ${JSON.stringify(request.arguments, null, 2)}`); + console.log('========================================'); + + rl.question('Approve this action? (y/n): ', async (answer) => { + const decision = answer.toLowerCase() === 'y' ? 'approved' : 'rejected'; + + await channel.publish({ + name: 'approval-response', + data: { decision }, + extras: { + headers: { + toolCallId: message.extras?.headers?.toolCallId + } + } + }); + + console.log(`Decision sent: ${decision}\n`); + }); +}); +``` + + +Run the client in a separate terminal: + + +```shell +node client.mjs +``` + + +With the server, client, and agent running, the workflow proceeds as follows: + +1. The agent sends a prompt to Claude that triggers a tool use +2. The agent publishes an approval request to the channel +3. The client displays the request and prompts the user +4. The user approves or rejects the request +5. The agent verifies the approver's role meets the minimum requirement +6. If approved and authorized, the agent executes the tool + +## Next steps + +- Learn more about [human-in-the-loop](/docs/ai-transport/messaging/human-in-the-loop) patterns and verification strategies +- Explore [identifying users and agents](/docs/ai-transport/sessions-identity/identifying-users-and-agents) for secure identity verification +- Understand [sessions and identity](/docs/ai-transport/sessions-identity) in AI-enabled applications +- Learn about [tool calls](/docs/ai-transport/messaging/tool-calls) for agent-to-agent communication diff --git a/src/pages/docs/guides/ai-transport/openai-human-in-the-loop.mdx b/src/pages/docs/guides/ai-transport/openai-human-in-the-loop.mdx new file mode 100644 index 0000000000..263b5c3cef --- /dev/null +++ b/src/pages/docs/guides/ai-transport/openai-human-in-the-loop.mdx @@ -0,0 +1,410 @@ +--- +title: "Guide: Human-in-the-loop approval with OpenAI" +meta_description: "Implement human approval workflows for AI agent tool calls using OpenAI and Ably with role-based access control." +meta_keywords: "AI, human in the loop, HITL, OpenAI, tool calls, approval workflow, AI transport, Ably, realtime, RBAC" +--- + +This guide shows you how to implement a human-in-the-loop (HITL) approval workflow for AI agent tool calls using OpenAI and Ably. The agent requests human approval before executing sensitive operations, with role-based access control to verify approvers have sufficient permissions. + +Using Ably for HITL workflows enables reliable, realtime communication between AI agents and human approvers. The request-response pattern ensures approval requests are delivered and decisions are processed with proper authorization checks. + + + +## Prerequisites + +To follow this guide, you need: +- Node.js 20 or higher +- An OpenAI API key +- An Ably API key + +Useful links: +- [OpenAI function calling guide](https://platform.openai.com/docs/guides/function-calling) +- [Ably JavaScript SDK getting started](/docs/getting-started/javascript) + +Create a new NPM package, which will contain the agent, client, and server code: + + +```shell +mkdir ably-openai-hitl-example && cd ably-openai-hitl-example +npm init -y +``` + + +Install the required packages using NPM: + + +```shell +npm install openai@^4 ably@^2 express jsonwebtoken +``` + + + + +Export your API keys to the environment: + + +```shell +export OPENAI_API_KEY="your_openai_api_key_here" +export ABLY_API_KEY="your_ably_api_key_here" +``` + + +## Step 1: Initialize the agent + +Set up the agent that will call OpenAI and request human approval for sensitive operations. This example uses a `publish_blog_post` tool that requires authorization before execution. + +Initialize the OpenAI and Ably clients, and create a channel for communication between the agent and human approvers. Add the following to a new file called `agent.mjs`: + + +```javascript +import OpenAI from 'openai'; +import Ably from 'ably'; + +const openai = new OpenAI(); + +// Initialize Ably Realtime client +const realtime = new Ably.Realtime({ + key: process.env.ABLY_API_KEY, + echoMessages: false +}); + +// Wait for connection to be established +await realtime.connection.once('connected'); + +// Create a channel for HITL communication +const channel = realtime.channels.get('ai:{{RANDOM_CHANNEL_NAME}}'); + +// Track pending approval requests +const pendingApprovals = new Map(); + +// Function that executes the approved action +async function publishBlogPost(args) { + const { title } = JSON.parse(args); + console.log(`Publishing blog post: ${title}`); + // In production, this would call your CMS API + return { published: true, title }; +} +``` + + + + +Tools that modify data, access sensitive resources, or perform actions with business impact are good candidates for HITL approval workflows. + +## Step 2: Request human approval + +When the OpenAI model returns a tool call, publish an approval request to the channel and wait for a human decision. The tool call ID is passed in the message headers to correlate requests with responses. + +Add the approval request function to `agent.mjs`: + + +```javascript +async function requestHumanApproval(toolCall) { + const approvalPromise = new Promise((resolve, reject) => { + pendingApprovals.set(toolCall.call_id, { toolCall, resolve, reject }); + }); + + await channel.publish({ + name: 'approval-request', + data: { + tool: toolCall.name, + arguments: toolCall.arguments + }, + extras: { + headers: { + toolCallId: toolCall.call_id + } + } + }); + + console.log(`Approval request sent for: ${toolCall.name}`); + return approvalPromise; +} +``` + + +The `toolCall.call_id` provided by OpenAI correlates the approval request with the response, enabling the agent to handle multiple concurrent approval flows. + +## Step 3: Subscribe to approval responses + +Set up a subscription to receive approval decisions from human users. When a response arrives, verify the approver has sufficient permissions using role-based access control before resolving the pending promise. + +Add the subscription handler to `agent.mjs`: + + +```javascript +async function subscribeApprovalResponses() { + // Define role hierarchy from lowest to highest privilege + const roleHierarchy = ['editor', 'publisher', 'admin']; + + // Define minimum role required for each tool + const approvalPolicies = { + publish_blog_post: { minRole: 'publisher' } + }; + + function canApprove(approverRole, requiredRole) { + const approverLevel = roleHierarchy.indexOf(approverRole); + const requiredLevel = roleHierarchy.indexOf(requiredRole); + return approverLevel >= requiredLevel; + } + + await channel.subscribe('approval-response', async (message) => { + const { decision } = message.data; + const toolCallId = message.extras?.headers?.toolCallId; + const pending = pendingApprovals.get(toolCallId); + + if (!pending) { + console.log(`No pending approval for tool call: ${toolCallId}`); + return; + } + + const policy = approvalPolicies[pending.toolCall.name]; + // Get the trusted role from the JWT user claim + const approverRole = message.extras?.userClaim; + + // Verify the approver's role meets the minimum required + if (!canApprove(approverRole, policy.minRole)) { + console.log(`Insufficient role: ${approverRole} < ${policy.minRole}`); + pending.reject(new Error( + `Approver role '${approverRole}' insufficient for required '${policy.minRole}'` + )); + pendingApprovals.delete(toolCallId); + return; + } + + // Process the decision + if (decision === 'approved') { + console.log(`Approved by ${approverRole}`); + pending.resolve({ approved: true, approverRole }); + } else { + console.log(`Rejected by ${approverRole}`); + pending.reject(new Error(`Action rejected by ${approverRole}`)); + } + pendingApprovals.delete(toolCallId); + }); +} +``` + + +The `message.extras.userClaim` contains the role embedded in the approver's JWT token, providing a trusted source for authorization decisions. This ensures only users with sufficient privileges can approve sensitive operations. + +## Step 4: Process tool calls + +Create a function to process tool calls by requesting approval and executing the action if approved. + +Add the tool processing function to `agent.mjs`: + + +```javascript +async function processToolCall(toolCall) { + if (toolCall.name === 'publish_blog_post') { + await requestHumanApproval(toolCall); + return await publishBlogPost(toolCall.arguments); + } + throw new Error(`Unknown tool: ${toolCall.name}`); +} +``` + + +The function awaits approval before executing. If the approver rejects or has insufficient permissions, the promise rejects and the tool is not executed. + +## Step 5: Run the agent + +Create the main agent loop that sends prompts to OpenAI and processes any tool calls that require approval. + +Add the agent runner to `agent.mjs`: + + +```javascript +async function runAgent(prompt) { + await subscribeApprovalResponses(); + + console.log(`User: ${prompt}`); + + const response = await openai.responses.create({ + model: 'gpt-4o', + input: prompt, + tools: [ + { + type: 'function', + name: 'publish_blog_post', + description: 'Publish a blog post to the website. Requires human approval.', + parameters: { + type: 'object', + properties: { + title: { + type: 'string', + description: 'Title of the blog post to publish' + } + }, + required: ['title'] + } + } + ] + }); + + const toolCalls = response.output.filter(item => item.type === 'function_call'); + + for (const toolCall of toolCalls) { + console.log(`Tool call: ${toolCall.name}`); + try { + const result = await processToolCall(toolCall); + console.log('Result:', result); + } catch (err) { + console.error('Tool call failed:', err.message); + } + } +} + +runAgent("Publish the blog post called 'Introducing our new API'"); +``` + + +## Step 6: Create the authentication server + +The authentication server issues JWT tokens with embedded role claims. The role claim is trusted by Ably and included in messages, enabling secure role-based authorization. + +Add the following to a new file called `server.mjs`: + + +```javascript +import express from 'express'; +import jwt from 'jsonwebtoken'; + +const app = express(); + +// Mock authentication - replace with your actual auth logic +function authenticateUser(req, res, next) { + // In production, verify the user's session/credentials + req.user = { id: 'user123', role: 'publisher' }; + next(); +} + +// Return claims to embed in the JWT +function getJWTClaims(user) { + return { + 'ably.channel.*': user.role + }; +} + +app.get('/api/auth/token', authenticateUser, (req, res) => { + const [keyName, keySecret] = process.env.ABLY_API_KEY.split(':'); + + const token = jwt.sign(getJWTClaims(req.user), keySecret, { + algorithm: 'HS256', + keyid: keyName, + expiresIn: '1h' + }); + + res.type('application/jwt').send(token); +}); + +app.listen(3001, () => { + console.log('Auth server running on http://localhost:3001'); +}); +``` + + +The `ably.channel.*` claim embeds the user's role in the JWT. When the user publishes messages, this claim is available as `message.extras.userClaim`, providing a trusted source for authorization. + +Run the server: + + +```shell +node server.mjs +``` + + +## Step 7: Create the approval client + +The approval client receives approval requests and allows humans to approve or reject them. It authenticates via the server to obtain a JWT with the user's role. + +Add the following to a new file called `client.mjs`: + + +```javascript +import Ably from 'ably'; +import readline from 'readline'; + +const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout +}); + +const realtime = new Ably.Realtime({ + authCallback: async (tokenParams, callback) => { + try { + const response = await fetch('http://localhost:3001/api/auth/token'); + const token = await response.text(); + callback(null, token); + } catch (error) { + callback(error, null); + } + } +}); + +realtime.connection.on('connected', () => { + console.log('Connected to Ably'); + console.log('Waiting for approval requests...\n'); +}); + +const channel = realtime.channels.get('ai:{{RANDOM_CHANNEL_NAME}}'); + +await channel.subscribe('approval-request', (message) => { + const request = message.data; + + console.log('\n========================================'); + console.log('APPROVAL REQUEST'); + console.log('========================================'); + console.log(`Tool: ${request.tool}`); + console.log(`Arguments: ${request.arguments}`); + console.log('========================================'); + + rl.question('Approve this action? (y/n): ', async (answer) => { + const decision = answer.toLowerCase() === 'y' ? 'approved' : 'rejected'; + + await channel.publish({ + name: 'approval-response', + data: { decision }, + extras: { + headers: { + toolCallId: message.extras?.headers?.toolCallId + } + } + }); + + console.log(`Decision sent: ${decision}\n`); + }); +}); +``` + + +Run the client in a separate terminal: + + +```shell +node client.mjs +``` + + +With the server, client, and agent running, the workflow proceeds as follows: + +1. The agent sends a prompt to OpenAI that triggers a tool call +2. The agent publishes an approval request to the channel +3. The client displays the request and prompts the user +4. The user approves or rejects the request +5. The agent verifies the approver's role meets the minimum requirement +6. If approved and authorized, the agent executes the tool + +## Next steps + +- Learn more about [human-in-the-loop](/docs/ai-transport/messaging/human-in-the-loop) patterns and verification strategies +- Explore [identifying users and agents](/docs/ai-transport/sessions-identity/identifying-users-and-agents) for secure identity verification +- Understand [sessions and identity](/docs/ai-transport/sessions-identity) in AI-enabled applications +- Learn about [tool calls](/docs/ai-transport/messaging/tool-calls) for agent-to-agent communication