diff --git a/.changeset/tool-requires-confirmation-not-enforced.md b/.changeset/tool-requires-confirmation-not-enforced.md new file mode 100644 index 0000000000..84b918fd42 --- /dev/null +++ b/.changeset/tool-requires-confirmation-not-enforced.md @@ -0,0 +1,28 @@ +--- +"@objectstack/spec": patch +--- + +docs(ai): stop `tool.requiresConfirmation` promising a gate it does not provide (#3715) + +The flag is read by **no execution path** — not the LLM tool set, not +`ToolRegistry.execute`, not `POST /ai/tools/:name/execute`, not the MCP bridge. +Yet the authoring surface actively taught reliance on it: the Studio form +section was titled *"Access & safety"* with helpText *"Ask user to approve +before executing (for destructive actions)"*, and the AI skill doc, MCP guide +and spec README all recommended it for destructive operations. + +The prune-or-wire decision is deliberately **deferred** (#3715 — the field's +shape is likely needed once side-effect tools exist, which `ToolCategory` +already anticipates with `action` / `integration` / `flow`). What changes now +is only the promise: + +- spec `.describe()` carries `[EXPERIMENTAL — not enforced]` + a pointer to the + real gate; +- the form section is renamed *"Declarative metadata (not enforced)"* and both + its fields (this and the already-dead `permissions`) say so, with the enforced + alternative spelled out; +- `skills/objectstack-ai/SKILL.md`, `MCP_GUIDE.md` and `README.md` now point at + the action-level `ai.requiresConfirmation` + approval queue (and note that AI + metadata edits are already gated by draft/publish, ADR-0033). + +No behaviour change: nothing read the flag before and nothing reads it now. diff --git a/content/docs/references/ai/tool.mdx b/content/docs/references/ai/tool.mdx index 252f84433f..726b323ae6 100644 --- a/content/docs/references/ai/tool.mdx +++ b/content/docs/references/ai/tool.mdx @@ -40,7 +40,7 @@ AI tool definition. [READ-ONLY PROJECTION — not an execution entry point] Auth | **parameters** | `Record` | ✅ | JSON Schema for tool parameters | | **outputSchema** | `Record` | optional | [EXPERIMENTAL — not enforced] JSON Schema for tool output. Keys are folded into the tool description only; outputs are not validated (liveness #1878/#1893). | | **objectName** | `string` | optional | Target object name (snake_case) | -| **requiresConfirmation** | `boolean` | ✅ | Require user confirmation before execution | +| **requiresConfirmation** | `boolean` | ✅ | [EXPERIMENTAL — not enforced] Require user confirmation before execution. NOTHING pauses on this flag (#3715) — use the action-level `ai.requiresConfirmation` + approval queue for a real gate. | | **permissions** | `string[]` | optional | Required permission-set capabilities | | **active** | `boolean` | ✅ | Whether the tool is enabled | | **builtIn** | `boolean` | ✅ | Platform built-in tool flag | diff --git a/packages/platform-objects/src/apps/translations/en.metadata-forms.generated.ts b/packages/platform-objects/src/apps/translations/en.metadata-forms.generated.ts index dac4f8ffb8..9960e9a8c0 100644 --- a/packages/platform-objects/src/apps/translations/en.metadata-forms.generated.ts +++ b/packages/platform-objects/src/apps/translations/en.metadata-forms.generated.ts @@ -1651,9 +1651,9 @@ export const enMetadataForms: NonNullable = { label: "Schemas", description: "Inputs the tool accepts and the shape of its output." }, - access_and_safety: { - label: "Access & safety", - description: "Permissions and confirmation requirements." + declarative_metadata_not_enforced: { + label: "Declarative metadata (not enforced)", + description: "Recorded on the tool definition but read by no execution path — see the per-field notes for where the real gates live." } }, fields: { diff --git a/packages/platform-objects/src/apps/translations/es-ES.metadata-forms.generated.ts b/packages/platform-objects/src/apps/translations/es-ES.metadata-forms.generated.ts index f30b10f859..99070244ff 100644 --- a/packages/platform-objects/src/apps/translations/es-ES.metadata-forms.generated.ts +++ b/packages/platform-objects/src/apps/translations/es-ES.metadata-forms.generated.ts @@ -1651,9 +1651,9 @@ export const esESMetadataForms: NonNullable = label: "Esquemas", description: "Entradas que acepta la herramienta y forma de su salida." }, - access_and_safety: { - label: "Acceso y seguridad", - description: "Permisos y requisitos de confirmación." + declarative_metadata_not_enforced: { + label: "Declarative metadata (not enforced)", + description: "Recorded on the tool definition but read by no execution path — see the per-field notes for where the real gates live." } }, fields: { diff --git a/packages/platform-objects/src/apps/translations/ja-JP.metadata-forms.generated.ts b/packages/platform-objects/src/apps/translations/ja-JP.metadata-forms.generated.ts index 5f94301d19..720545ff56 100644 --- a/packages/platform-objects/src/apps/translations/ja-JP.metadata-forms.generated.ts +++ b/packages/platform-objects/src/apps/translations/ja-JP.metadata-forms.generated.ts @@ -1651,9 +1651,9 @@ export const jaJPMetadataForms: NonNullable = label: "スキーマ", description: "ツールが受け取る入力と出力の形状。" }, - access_and_safety: { - label: "アクセスと安全性", - description: "権限と確認要件。" + declarative_metadata_not_enforced: { + label: "Declarative metadata (not enforced)", + description: "Recorded on the tool definition but read by no execution path — see the per-field notes for where the real gates live." } }, fields: { diff --git a/packages/platform-objects/src/apps/translations/zh-CN.metadata-forms.generated.ts b/packages/platform-objects/src/apps/translations/zh-CN.metadata-forms.generated.ts index a521220e5a..132c7a1b87 100644 --- a/packages/platform-objects/src/apps/translations/zh-CN.metadata-forms.generated.ts +++ b/packages/platform-objects/src/apps/translations/zh-CN.metadata-forms.generated.ts @@ -1651,9 +1651,9 @@ export const zhCNMetadataForms: NonNullable = label: "参数 / 返回结构", description: "工具接收的输入与输出形态" }, - access_and_safety: { - label: "访问与安全", - description: "权限与确认要求" + declarative_metadata_not_enforced: { + label: "Declarative metadata (not enforced)", + description: "Recorded on the tool definition but read by no execution path — see the per-field notes for where the real gates live." } }, fields: { diff --git a/packages/spec/README.md b/packages/spec/README.md index 6eeee3d734..ed9c97ae8e 100644 --- a/packages/spec/README.md +++ b/packages/spec/README.md @@ -133,6 +133,9 @@ export const objectStackMCP = MCPServerConfigSchema.parse({ ], handler: 'flows.create_record', sideEffects: 'write', + // NOTE: declarative only — nothing pauses on this flag (#3715). A real + // human-in-the-loop gate is `ai.requiresConfirmation` on the underlying + // action (+ the approval queue), or `approval: 'always'` on an MCP binding. requiresConfirmation: true, }, { diff --git a/packages/spec/docs/MCP_GUIDE.md b/packages/spec/docs/MCP_GUIDE.md index 693c0157eb..862ce47907 100644 --- a/packages/spec/docs/MCP_GUIDE.md +++ b/packages/spec/docs/MCP_GUIDE.md @@ -355,7 +355,12 @@ export const salesAgent = AgentSchema.parse({ - `write`: Modifies data - `delete`: Deletes data -For operations with side effects, consider setting `requiresConfirmation: true` +For operations with side effects, gate them with something that is enforced: +`approval: 'always'` on the MCP tool binding, or `ai.requiresConfirmation` on +the underlying **action** (+ the HITL approval queue). + +> ⚠️ `requiresConfirmation` on the **tool definition** is declared but read by +> no execution path — it produces no pause (#3715). ### 4. Security diff --git a/packages/spec/liveness/tool.json b/packages/spec/liveness/tool.json index b4a1a2f113..d7038e8b6a 100644 --- a/packages/spec/liveness/tool.json +++ b/packages/spec/liveness/tool.json @@ -43,7 +43,7 @@ "evidence": "zero reads off any TOOL definition in any repo; every requiresConfirmation read is action.ai.requiresConfirmation (a different, live property). Ignored by the LLM tool set (vercel-adapter.ts:263-272), ToolRegistry.execute (tool-registry.ts:183-210), POST /ai/tools/:name/execute (tool-routes.ts:91,105) and the MCP bridge (framework packages/mcp/src/mcp-server-runtime.ts:176, which uses a hardcoded destructive-name list)", "authorWarn": true, "authorHint": "NOT ENFORCED — nothing pauses for confirmation on this flag, on any path. To gate a destructive operation, put it behind an action with `ai.requiresConfirmation` + the HITL approval queue, which is the only path that actually stops execution.", - "note": "RE-VERIFIED 2026-07 (#3686 preview-claim sweep): the prior `live` verdict cited only a metadata-admin PREVIEW panel, which echoes what the author typed. SAFETY-SHAPED: tool.form.ts:36-42 places it in a section titled 'Access & safety' with helpText 'Ask user to approve before executing (for destructive actions)', and skills/objectstack-ai/SKILL.md:547 tells authors to rely on it — so the authoring surface actively teaches a guarantee nothing delivers. ADR-0033 (:31, :60) already recorded it as never-enforced and resolved to DELETE the placeholder (the draft/publish workspace is the real approval gate); that deletion was never executed. Prune tracked separately." + "note": "RE-VERIFIED 2026-07 (#3686 preview-claim sweep): the prior `live` verdict cited only a metadata-admin PREVIEW panel, which echoes what the author typed. SAFETY-SHAPED: tool.form.ts:36-42 places it in a section titled 'Access & safety' with helpText 'Ask user to approve before executing (for destructive actions)', and skills/objectstack-ai/SKILL.md:547 tells authors to rely on it — so the authoring surface actively teaches a guarantee nothing delivers. ADR-0033 (:31, :60) already recorded it as never-enforced and resolved to DELETE the placeholder (the draft/publish workspace is the real approval gate); that deletion was never executed. OWNER CALL 2026-07: the prune-or-wire decision is DEFERRED (#3715), but the false promise stops now — the spec `.describe()`, the Studio form section (formerly titled \"Access & safety\", whose copy told authors to use it for destructive operations) and the SKILL.md / MCP_GUIDE / README guidance all now say NOT ENFORCED and point at the action-level gate." }, "permissions": { "status": "dead", diff --git a/packages/spec/src/ai/tool.form.ts b/packages/spec/src/ai/tool.form.ts index deb490a0ba..13b1e5da6a 100644 --- a/packages/spec/src/ai/tool.form.ts +++ b/packages/spec/src/ai/tool.form.ts @@ -34,13 +34,17 @@ export const toolForm = defineForm({ ], }, { - label: 'Access & safety', - description: 'Permissions and confirmation requirements.', + // NOT an enforcement section — both fields are declared-but-dead (#3715, + // liveness/tool.json). The label used to read "Access & safety" with + // copy that told authors to rely on them for destructive operations, + // which is exactly the false promise this section must not make. + label: 'Declarative metadata (not enforced)', + description: 'Recorded on the tool definition but read by no execution path — see the per-field notes for where the real gates live.', collapsible: true, collapsed: true, fields: [ - { field: 'requiresConfirmation', helpText: 'Ask user to approve before executing (for destructive actions)' }, - { field: 'permissions', widget: 'string-tags', helpText: 'Required permissions to use this tool' }, + { field: 'requiresConfirmation', helpText: 'NOT ENFORCED (#3715) — nothing pauses for confirmation on this flag. For a real gate use the action-level `ai.requiresConfirmation` + approval queue; AI metadata edits are already gated by draft/publish.' }, + { field: 'permissions', widget: 'string-tags', helpText: 'NOT ENFORCED — tool invocation is not permission-gated by this list. Gate the underlying action via permission sets (ADR-0066), or restrict the agent that exposes the tool.' }, ], }, ], diff --git a/packages/spec/src/ai/tool.zod.ts b/packages/spec/src/ai/tool.zod.ts index 758c64a42c..5f328abf26 100644 --- a/packages/spec/src/ai/tool.zod.ts +++ b/packages/spec/src/ai/tool.zod.ts @@ -96,8 +96,23 @@ export const ToolSchema = lazySchema(() => z.object({ */ objectName: z.string().regex(/^[a-z_][a-z0-9_]*$/).optional().describe('Target object name (snake_case)'), - /** Whether the tool requires human confirmation before execution */ - requiresConfirmation: z.boolean().default(false).describe('Require user confirmation before execution'), + /** + * Whether the tool requires human confirmation before execution. + * + * ⚠️ EXPERIMENTAL — NOT ENFORCED (#3715). No execution path reads this flag: + * not the LLM tool set (a tool reaches the model as name/description/ + * parameters only), not `ToolRegistry.execute`, not + * `POST /ai/tools/:name/execute`, and not the MCP bridge (which derives + * `destructiveHint` from a hardcoded name list). Setting it `true` on a + * destructive tool produces NO pause. + * + * The enforced human-in-the-loop path is the ACTION-level + * `ai.requiresConfirmation` (+ the approval queue). For AI metadata + * mutations the draft/publish workspace is the real gate (ADR-0033). + * Whether this per-tool flag gets wired to that queue or removed is + * tracked in #3715. + */ + requiresConfirmation: z.boolean().default(false).describe('[EXPERIMENTAL — not enforced] Require user confirmation before execution. NOTHING pauses on this flag (#3715) — use the action-level `ai.requiresConfirmation` + approval queue for a real gate.'), /** Permission-set capabilities required to use this tool */ permissions: z.array(z.string()).optional().describe('Required permission-set capabilities'), diff --git a/skills/objectstack-ai/SKILL.md b/skills/objectstack-ai/SKILL.md index fc61fbdbfe..6416c784a9 100644 --- a/skills/objectstack-ai/SKILL.md +++ b/skills/objectstack-ai/SKILL.md @@ -549,10 +549,14 @@ On validation failure the runtime retries by default has 15+ tools, split it. 3. **Missing guardrails and approval gates.** Define `blockedTopics` (plus the token / time budgets) in agent `guardrails`; for destructive operations put - a human in the loop — `requiresConfirmation: true` on the tool, - `approval: 'always'` on an MCP tool binding, `enableActionApproval: true` - (HITL queue, cloud) for auto-exposed actions. There is no - `requireApprovalFor` field. + a human in the loop with a gate that is **actually enforced** — + `enableActionApproval: true` (HITL queue, cloud) for auto-exposed actions, + `ai.requiresConfirmation` on the **action**, or `approval: 'always'` on an + MCP tool binding. AI metadata edits are already gated: they land as drafts a + human must publish (ADR-0033). + ⚠️ Do **not** rely on `requiresConfirmation` on the **tool** — it is declared + but read by no execution path, so it produces no pause (#3715). + There is no `requireApprovalFor` field. 4. **Ignoring tool descriptions.** The LLM uses tool `description` to decide when to call it. Poor descriptions = wrong tool selection. 5. **Not testing trigger phrases.** Ambiguous trigger phrases cause skill