From 5117bddb4b20fd22d64ee0ed41f7d86610ebaa8d Mon Sep 17 00:00:00 2001 From: tangrongyuan <18532002267@163.com> Date: Tue, 28 Jul 2026 16:46:26 +0800 Subject: [PATCH 01/14] docs: design enterprise FlowLens inspector MCP --- ...lowlens-inspector-mcp-enterprise-design.md | 696 ++++++++++++++++++ 1 file changed, 696 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-28-flowlens-inspector-mcp-enterprise-design.md diff --git a/docs/superpowers/specs/2026-07-28-flowlens-inspector-mcp-enterprise-design.md b/docs/superpowers/specs/2026-07-28-flowlens-inspector-mcp-enterprise-design.md new file mode 100644 index 00000000..0d043dbb --- /dev/null +++ b/docs/superpowers/specs/2026-07-28-flowlens-inspector-mcp-enterprise-design.md @@ -0,0 +1,696 @@ +# FlowLens Inspector MCP 企业级接入设计 + +> 日期：2026-07-28 +> +> 状态：讨论稿已确认，待书面审核后进入实施计划 +> 适用分支：`codex/runlens-agentops` + +## 1. 背景 + +FlowLens 已经能够把 DeerFlow Agent Runtime 中的原始运行事件持久化，并派生出 Timeline、Metrics 和确定性 Diagnostics。当前这些能力主要通过 AgentOps API 和 Dashboard 提供给开发者，但 Agent 本身无法以受控方式查询这些诊断信息。 + +本设计新增一个独立的 **FlowLens Inspector MCP Server**，使 DeerFlow Agent 可以通过标准 MCP 工具查询 Run、Timeline 和 Diagnostics，并结合一个真实的事故调查 Skill 完成故障分析。 + +本次实现不采用仅适合单机演示的 `stdio + 直读 SQLite` 方案，而采用可独立部署的 Streamable HTTP 服务、短期访问令牌、Scope 权限、用户与租户隔离、Gateway 二次鉴权、审计和故障降级。 + +## 2. 目标 + +### 2.1 功能目标 + +1. 提供五个只读 MCP 工具： + - `health_check` + - `list_runs` + - `get_run_diagnostics` + - `get_timeline_window` + - `compare_runs` +2. 允许模型通过 MCP 查询当前用户有权访问的 FlowLens 数据。 +3. 提供 `flowlens-incident-response` Skill，指导模型按标准流程调查 Agent 故障。 +4. 让 MCP 调用自身也进入 FlowLens Timeline，形成可递归观测的完整链路。 +5. 同时支持 SQLite 本地开发配置和 PostgreSQL + Redis 企业部署配置。 + +### 2.2 工程目标 + +1. MCP Server 与 DeerFlow Gateway 独立部署、职责清晰。 +2. 使用 Streamable HTTP，不依赖本地子进程生命周期。 +3. 使用短期 JWT、非对称签名、JWKS、Audience 和 Scope。 +4. 在 MCP 层和 Gateway 层分别完成认证与授权。 +5. 提供分页、超时、有限重试、限流、并发隔离、熔断和显式降级。 +6. 提供结构化审计、服务指标、链路标识和安全脱敏。 +7. 通过单元、契约、集成、安全、故障注入和真实模型测试验收。 + +### 2.3 学习目标 + +用户在实施过程中应能够解释并亲手验证： + +1. MCP Client 如何发现和调用远程工具。 +2. Streamable HTTP 与 `stdio` 传输的边界。 +3. Runtime 如何安全地把用户身份委托给 MCP Server。 +4. Scope、所有权检查和租户隔离如何共同防止越权。 +5. Skill 如何提供调查方法，MCP 如何提供可执行能力。 +6. MCP 调用如何转换为 ToolMessage 并重新进入 Agent Loop。 +7. 企业服务如何处理超时、重试、限流、熔断和审计。 + +## 3. 非目标 + +本次不实现： + +1. 修改、删除或回滚 Run 的写操作。 +2. 通过 MCP 读取完整 Prompt、Memory、附件正文或工具密钥。 +3. 在 MCP Server 内再次调用 LLM 进行不确定归因。 +4. 完整事故工单、告警通知或值班系统。 +5. 外部 OIDC 平台的管理控制台。系统仅实现标准验证接口和可替换配置。 +6. 未经实测的生产吞吐量、SLA 或真实企业用户成果。 + +## 4. 方案决策 + +### 4.1 采用方案 + +采用独立 Streamable HTTP MCP 服务： + +- MCP Server 负责协议、认证、工具授权、输入校验和结果整形。 +- Gateway 继续拥有 AgentOps 数据访问和 Run 所有权判断。 +- MCP Server 通过 Gateway 只读 API 获取数据，不直接执行数据库查询。 +- Gateway 和 MCP Server 独立验证同一个短期平台访问令牌。 +- 本地默认使用 SQLite 和内存限流器。 +- 企业 Compose Profile 使用 PostgreSQL 和 Redis。 + +### 4.2 未采用方案 + +| 方案 | 不采用原因 | +|---|---| +| `stdio + 直读 SQLite` | 无独立服务边界，绕过 Gateway 鉴权，不适合多用户 | +| 固定 API Key + 服务账号 | 无法表达最终用户身份，难以实现 Run 所有权隔离 | +| 首版直接部署完整 Keycloak | 身份平台配置会掩盖 MCP 核心学习目标；当前接口已为后续 OIDC 接入预留 | + +## 5. 总体架构 + +```mermaid +flowchart LR + U["用户"] --> R["DeerFlow Runtime"] + R --> T["Gateway Token Issuer"] + T --> R + R --> C["MCP Client"] + C -->|Streamable HTTP + Bearer Token| M["FlowLens Inspector MCP"] + M --> A["认证与 Scope 校验"] + A --> G["DeerFlow Gateway AgentOps API"] + G --> Z["用户与租户授权"] + Z --> Q["Run / Event / Diagnostics Repository"] + Q --> D[("SQLite 或 PostgreSQL")] + M --> L[("MCP Audit Store")] + C --> J["RunJournal"] + J --> E[("RunEventStore")] +``` + +关键边界： + +1. 模型只决定调用哪个工具及业务参数，不能决定自己的身份。 +2. MCP Client 从 Runtime Context 获取凭据，不把 Token 暴露给模型。 +3. MCP Server 不相信模型传入的 `user_id` 或 `tenant_id`。 +4. Gateway 不相信 MCP 自行声明的身份，而是再次验证 Access Token。 +5. MCP Client 负责把 `mcp.*` 生命周期事件写入当前 RunJournal。 +6. MCP Server 负责写独立安全审计，不把安全审计混入业务 Timeline。 + +## 6. 服务与代码边界 + +新增独立 Python Workspace Package： + +```text +backend/packages/flowlens-inspector-mcp/ +├── pyproject.toml +└── flowlens_inspector_mcp/ + ├── server.py + ├── settings.py + ├── auth/ + │ ├── principal.py + │ ├── verifier.py + │ └── scopes.py + ├── tools/ + │ ├── health.py + │ ├── runs.py + │ ├── diagnostics.py + │ ├── timeline.py + │ └── comparison.py + ├── gateway_client.py + ├── schemas.py + ├── resilience.py + ├── audit.py + └── telemetry.py +``` + +职责约束： + +- `server.py`：创建 FastMCP 应用、注册中间件和工具。 +- `auth/*`：把已验证的 Token 转换为只读 `AccessPrincipal`。 +- `tools/*`：进行参数校验并编排 Gateway 调用，不包含数据库访问。 +- `gateway_client.py`：封装 AgentOps HTTP API、超时和错误映射。 +- `resilience.py`：重试、限流、并发隔离和熔断。 +- `audit.py`：记录脱敏的 append-only 安全审计。 +- `telemetry.py`：指标、日志和 Trace Context。 +- `schemas.py`：稳定的 MCP 输入输出模型。 + +## 7. MCP 工具契约 + +所有工具返回版本化 Pydantic Schema。结果必须有 `schema_version`、`request_id` 和 `generated_at`。 + +### 7.1 `health_check` + +用途：检查 MCP、Gateway 和数据源是否可用。 + +输入：无。 + +输出： + +- 服务版本 +- MCP 状态 +- Gateway 状态 +- 数据源类型 +- 依赖延迟 +- 是否处于降级状态 + +该工具需要 `agentops.health.read`。容器探针 `/health/live` 和 `/health/ready` 不属于 MCP 工具。 + +### 7.2 `list_runs` + +用途：列出当前主体有权访问的 Run。 + +输入： + +- 可选 `thread_id` +- 可选状态 +- 可选时间范围 +- 可选不透明分页游标 +- `limit`，默认 20，最大 100 + +输出： + +- `run_id` +- `thread_id` +- 状态 +- 开始与结束时间 +- 耗时 +- Token 数 +- 工具、Subagent、MCP、Memory 调用摘要 +- 根因摘要 +- 下一页游标 + +### 7.3 `get_run_diagnostics` + +用途：获取一次 Run 的确定性诊断。 + +输入：`run_id`。 + +输出： + +- Run 状态 +- `final` 或 `provisional` +- 主因 +- 伴随原因 +- 置信度 +- 证据事件引用 +- 修复建议 +- 核心 Metrics + +运行中的 Run 返回 `provisional`，不得伪装为最终诊断。 + +### 7.4 `get_timeline_window` + +用途：获取 Timeline 的局部事件窗口。 + +输入： + +- `run_id` +- 可选阶段过滤 +- 可选事件类型过滤 +- 可选中心事件或分页游标 +- `limit`，默认 20，最大 100 + +输出： + +- TimelineItem 列表 +- 事件序号 +- Span 和 Parent Span +- 状态与耗时 +- 脱敏摘要 +- 上一页与下一页游标 + +使用基于事件序号的不透明游标，不使用大偏移量分页。 + +### 7.5 `compare_runs` + +用途：确定性对比两个已授权 Run。 + +输入： + +- `base_run_id` +- `candidate_run_id` + +输出： + +- 耗时差异 +- Token 差异 +- 模型、工具、Subagent、MCP 和 Memory 调用差异 +- 错误与回滚差异 +- 根因差异 +- 显著事件模式 +- 完整或部分结果状态 + +工具不调用 LLM，不生成主观评分。 + +## 8. 身份、认证与授权 + +### 8.1 Access Token + +Gateway 为 Runtime 签发短期平台访问令牌。建议载荷： + +```json +{ + "iss": "deerflow-gateway", + "sub": "user-123", + "tenant_id": "tenant-a", + "aud": "flowlens-platform", + "scope": [ + "agentops.health.read", + "agentops.runs.read", + "agentops.timeline.read", + "agentops.diagnostics.read", + "agentops.runs.compare" + ], + "origin_run_id": "run-456", + "azp": "deerflow-runtime", + "jti": "unique-token-id", + "iat": 1785000000, + "exp": 1785000300 +} +``` + +要求： + +- 默认有效期约 5 分钟。 +- MCP Credential Provider 在过期前静默续签。 +- Token 只存在于运行时内存和 Authorization Header。 +- Token 不进入模型消息、Checkpoint、Timeline、审计正文或普通日志。 + +### 8.2 签名与密钥 + +- 使用 `RS256` 非对称签名。 +- Gateway 持有私钥。 +- MCP Server 和 Gateway 通过 JWKS 公钥验证。 +- Token 必须包含 `kid`，支持密钥轮换。 +- 开发密钥由脚本生成并写入 Git Ignore 路径。 +- 生产配置可切换到外部 OIDC Issuer 和 JWKS URL。 +- JWKS 使用有界缓存；遇到未知 `kid` 时只主动刷新一次，防止恶意请求造成重复拉取。 + +### 8.3 Scope + +| 工具 | Scope | +|---|---| +| `health_check` | `agentops.health.read` | +| `list_runs` | `agentops.runs.read` | +| `get_run_diagnostics` | `agentops.diagnostics.read` | +| `get_timeline_window` | `agentops.timeline.read` | +| `compare_runs` | `agentops.runs.compare` | + +### 8.4 双重鉴权 + +MCP Server： + +1. 验证签名、`iss`、`aud`、`iat`、`nbf`、`exp` 和 `jti` 格式。 +2. 验证工具所需 Scope。 +3. 生成不可变 `AccessPrincipal`。 + +Gateway： + +1. 再次验证 Token。 +2. 从 Token 提取用户和租户，不接受普通 Header 覆盖。 +3. 验证目标 Run 的所有权。 +4. 只返回脱敏数据。 + +错误语义： + +- `401`：Token 缺失、无效或过期。 +- `403`：身份有效但 Scope 不足。 +- `404`：Run 不存在或无权访问，避免泄露资源存在性。 +- `429`：触发限流。 +- `503`：依赖暂时不可用。 + +### 8.5 传输与会话安全 + +- 非本地部署必须使用 HTTPS；TLS 默认在受控 Ingress/Nginx 终止。 +- MCP Server 校验 `Origin` 和 `Host`，只接受配置中的精确允许列表。 +- 本地开发只绑定 `localhost` 或 Docker 内部网络，不暴露到公共网卡。 +- MCP Session ID 必须不可预测，并与已验证的 `AccessPrincipal` 绑定，不能跨用户复用。 +- 限制请求体、响应体和单次批量参数大小，超限请求直接拒绝。 +- MCP Server 发布受保护资源元数据；认证失败响应包含标准 `WWW-Authenticate` 信息。 +- MCP Server 不接收浏览器 Cookie，也不启用宽泛 CORS。 + +## 9. 多租户数据模型 + +AgentOps Run 记录新增 `tenant_id`。新数据必须同时持久化： + +```text +tenant_id + user_id + run_id +``` + +约束： + +1. Repository 的用户查询接口必须显式接收 `tenant_id` 和 `user_id`。 +2. Run 查询建立租户、用户和时间的组合索引。 +3. RunEvent 继续以 `run_id` 归属 Run；读取事件前必须先完成 Run 授权。 +4. 旧数据迁移到 `default` 租户。 +5. SQLite 和 PostgreSQL 执行相同的逻辑迁移测试。 +6. `mcp_audit_events` 独立保存 `tenant_id` 和 `user_id`。 + +## 10. 可靠性设计 + +### 10.1 超时 + +| 操作 | 默认超时 | +|---|---:| +| 建立 HTTP 连接 | 2 秒 | +| `health_check` | 3 秒 | +| `list_runs` | 5 秒 | +| Timeline 与 Diagnostics | 8 秒 | +| `compare_runs` | 10 秒 | +| 单次 MCP 工具总时限 | 12 秒 | + +所有数值均可通过环境变量调整，但必须存在上限。 + +### 10.2 重试 + +- 只重试只读、幂等请求。 +- 网络异常、`502`、`503`、`504` 最多重试 2 次。 +- 使用指数退避和随机抖动。 +- `400`、`403`、`404` 不重试。 +- Token 过期时刷新后只重试一次。 +- 调用方取消后立即停止重试。 + +### 10.3 限流与并发 + +限流键： + +```text +tenant_id + user_id + tool_name +``` + +默认策略： + +- 普通查询每用户每分钟 60 次。 +- Timeline 每分钟 30 次。 +- Run 对比每分钟 10 次。 +- 单个实例最多并发处理 32 个请求。 + +本地使用内存实现；企业 Profile 使用 Redis 实现并执行集成测试。 + +### 10.4 熔断与降级 + +- Gateway 连续失败达到阈值后进入 `OPEN`。 +- 冷却后进入 `HALF_OPEN`，仅允许探测请求。 +- 探测成功后恢复 `CLOSED`。 +- 降级结果必须显式包含 `status=partial` 或 `status=unavailable`。 +- 不允许使用缺失数据生成完整诊断结论。 + +部分结果格式至少包含： + +```json +{ + "status": "partial", + "available_sections": ["metrics"], + "missing_sections": ["diagnostics"], + "retryable": true +} +``` + +## 11. 审计与可观测性 + +### 11.1 FlowLens 运行事件 + +MCP Client 在当前 RunJournal 中写入： + +```text +mcp.session.start +mcp.tool.start +mcp.tool.retry +mcp.tool.end +mcp.tool.error +mcp.session.end +``` + +这些事件用于还原 Agent 执行链，不保存 Access Token。 + +### 11.2 安全审计 + +新增 append-only `mcp_audit_events`： + +- `request_id` +- `trace_id` +- `user_id` +- `tenant_id` +- `origin_run_id` +- `tool_name` +- `target_run_id` +- `status` +- `duration_ms` +- `retry_count` +- `error_code` +- `created_at` + +生产数据库账号对该表只授予 `INSERT` 和必要的受控查询权限，不授予普通业务路径 `UPDATE` 或 `DELETE`。 + +禁止记录： + +- Access Token +- 完整 Prompt +- Memory 正文 +- 工具返回正文 +- 附件内容 +- 密钥和数据库连接串 + +### 11.3 指标 + +服务内部指标至少包括： + +- MCP 请求总数 +- 各工具成功率 +- P50、P95、P99 延迟 +- 超时和重试次数 +- 限流次数 +- Gateway 错误数 +- 当前并发数 +- 熔断器状态 + +`request_id`、`trace_id` 和 `origin_run_id` 贯穿 Runtime、MCP Client、MCP Server 和 Gateway。 + +## 12. Skill 设计 + +新增： + +```text +skills/public/flowlens-incident-response/ +├── SKILL.md +├── references/ +│ ├── diagnostic-playbook.md +│ └── output-format.md +└── evals/ + ├── trigger_eval_set.json + └── evals.json +``` + +### 12.1 Skill 定位 + +Skill 是事故调查 SOP，不是数据源： + +- Skill 决定调查顺序和输出规范。 +- MCP 获取有权限约束的诊断事实。 +- 模型负责工具规划和最终表达。 + +### 12.2 调查流程 + +1. 使用 `list_runs` 定位目标 Run。 +2. 使用 `get_run_diagnostics` 获取初步根因。 +3. 根据证据事件确定 Timeline 窗口。 +4. 使用 `get_timeline_window` 获取上下文。 +5. 必要时使用 `compare_runs` 对比成功基线。 +6. 输出结论、证据链、影响、修复建议和不确定性。 + +### 12.3 Skill 约束 + +- 不根据缺失事件猜测根因。 +- `provisional` 和 `partial` 必须明确说明。 +- 关键结论引用 `run_id` 和事件序号。 +- 不输出 Token、Prompt、Memory、密钥和附件正文。 +- MCP 不可用时明确说明依赖失败。 +- 避免无上限翻页和重复调用同一工具。 + +## 13. 测试策略 + +### 13.1 单元测试 + +- Pydantic 输入输出 Schema +- JWT 校验和错误 Claim +- Scope 映射 +- 不透明游标 +- 脱敏 +- 重试分类 +- 限流 +- 熔断状态转换 +- 审计字段白名单 + +### 13.2 MCP 契约测试 + +- `tools/list` 暴露恰好五个预期工具。 +- 工具名称、描述、输入 Schema 和输出 Schema 稳定。 +- 非法参数由协议层返回明确错误。 + +### 13.3 集成测试 + +- Streamable HTTP Client 到 MCP Server 的真实会话。 +- MCP Server 到 Gateway 的真实 HTTP 调用。 +- SQLite Fixture 下的 Run、Timeline 和 Diagnostics 查询。 +- PostgreSQL 下的 Migration、分页和租户查询。 +- Redis 下的跨实例限流。 + +### 13.4 安全测试 + +- 无 Token +- 过期 Token +- 错误签名 +- 错误 Issuer +- 错误 Audience +- 缺少 Scope +- 跨用户 Run +- 跨租户 Run +- 超大分页参数 +- 日志和审计中无密钥、Token 与正文泄漏 + +### 13.5 故障注入 + +- Gateway 连接超时 +- `502`、`503`、`504` +- 重试耗尽 +- 熔断开启、半开和恢复 +- Redis 不可用 +- PostgreSQL 不可用 +- 调用方取消 +- MCP Server 优雅关闭 + +### 13.6 Skill 与真实模型测试 + +CI 使用确定性输入验证 Skill 的触发和工具顺序，不依赖付费模型。 + +本地手工验收使用真实模型： + +1. 自动加载事故调查 Skill。 +2. 自动选择 FlowLens MCP 工具。 +3. MCP 结果作为 ToolMessage 返回 Agent Loop。 +4. 最终回答包含可核对证据。 +5. AgentOps Timeline 能看到本次 MCP 调用。 + +## 14. Docker 与部署 + +新增 `flowlens-inspector-mcp` 服务。 + +本地默认 Profile： + +- Frontend +- Gateway +- Nginx +- FlowLens Inspector MCP +- SQLite +- 内存限流器 + +企业验证 Profile： + +- Frontend +- Gateway +- Nginx +- FlowLens Inspector MCP +- PostgreSQL +- Redis + +容器要求： + +- 非 root 用户运行。 +- `/health/live` 和 `/health/ready`。 +- 启动失败时不进入 Ready。 +- 关闭时停止接收新请求并等待正在执行的查询。 +- CPU 和内存限制可配置。 +- 私钥和数据库凭据通过环境变量或 Secret 挂载。 +- MCP 服务默认仅暴露在 Docker 内部网络，由 Gateway 或 Nginx 控制入口。 + +## 15. 性能验证 + +新增： + +```text +scripts/seed_agentops_runs.py +scripts/benchmark_inspector_mcp.py +``` + +脚本测量： + +- P50、P95、P99 延迟 +- 每秒请求数 +- Timeline 游标分页 +- SQLite 与 PostgreSQL 差异 +- 并发下的限流和熔断 + +第一次基准测试产生可复现基线。受控基准环境中的独立性能任务对超过基线 20% 的退化发出失败或告警；普通开发机测试不以易波动的绝对延迟作为硬门槛。README 只记录实际测量结果和测试环境，不预先承诺生产 SLA。 + +## 16. 验收场景 + +### 16.1 正常调查 + +用户要求 Agent 分析最近一次 Run。Agent 加载 Skill，调用 `list_runs`、`get_run_diagnostics` 和必要的 Timeline 工具，输出带证据的报告。 + +### 16.2 故障调查 + +通过可控 Fixture 或真实 Agent 操作产生 Tool Error/Timeout。Agent 自动定位主因、相关事件和修复建议。 + +### 16.3 Run 对比 + +Agent 对比一个成功 Run 和一个失败/退化 Run，输出 Metrics 与事件模式差异，不使用主观评分。 + +### 16.4 安全验证 + +使用错误 Scope、其他用户或其他租户的 `run_id` 调用工具，服务分别返回 `403` 或不泄露存在性的 `404`。 + +### 16.5 自观测闭环 + +一次诊断请求的 Timeline 中能够看到 MCP Session、Tool Start、Retry/End/Error，并通过 `request_id` 关联到 MCP 审计。 + +## 17. 完成标准 + +只有同时满足以下条件，功能才视为完成： + +1. 五个工具通过 MCP 契约测试。 +2. Streamable HTTP 全链路在 Docker 中运行。 +3. JWT、Scope、用户和租户隔离测试通过。 +4. SQLite 和 PostgreSQL 数据路径均通过集成测试。 +5. Redis 跨实例限流路径通过企业 Profile 验证。 +6. 超时、重试、熔断和部分降级通过故障注入测试。 +7. MCP 审计中不存在敏感正文或 Token。 +8. Skill 触发、工具选择和报告格式通过测试。 +9. 真实模型能够完成至少一次正常调查和一次故障调查。 +10. AgentOps Timeline 能完整显示 MCP 调用事件。 +11. README 提供架构、配置、启动、验证和演示说明。 +12. 所有新增性能声明均有脚本输出和测试环境作为证据。 + +## 18. 对外表述边界 + +完成后可以表述： + +> 设计并实现独立的 Streamable HTTP MCP 诊断服务，通过短期 JWT、Scope、JWKS、用户/租户隔离与 Gateway 双重鉴权，向 Agent 提供 Run、Timeline、Diagnostics 和 Run 对比能力；结合事故调查 Skill、审计、限流、重试、熔断及 Docker 多数据源配置，完成本地真实模型端到端和故障注入验证。 + +不得表述： + +- 已服务真实企业客户。 +- 已承载未经测试的生产并发量。 +- 已达到没有证据的 SLA。 +- 已部署完整外部 OIDC 平台。 +- 规则诊断能够覆盖所有 Agent 故障。 + +## 19. 标准参考 + +- MCP Transports： +- MCP Authorization： +- MCP Python SDK： From 5605d70694f181020362f46a3c172e6046bf6ad7 Mon Sep 17 00:00:00 2001 From: tangrongyuan <18532002267@163.com> Date: Tue, 28 Jul 2026 17:01:31 +0800 Subject: [PATCH 02/14] docs: plan FlowLens inspector MCP implementation --- .../2026-07-28-flowlens-inspector-mcp.md | 1314 +++++++++++++++++ 1 file changed, 1314 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-28-flowlens-inspector-mcp.md diff --git a/docs/superpowers/plans/2026-07-28-flowlens-inspector-mcp.md b/docs/superpowers/plans/2026-07-28-flowlens-inspector-mcp.md new file mode 100644 index 00000000..49c5b110 --- /dev/null +++ b/docs/superpowers/plans/2026-07-28-flowlens-inspector-mcp.md @@ -0,0 +1,1314 @@ +# FlowLens Inspector MCP Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build an authenticated, tenant-aware Streamable HTTP MCP service that lets DeerFlow agents inspect FlowLens runs through five read-only tools and a real incident-response Skill. + +**Architecture:** Keep Gateway as the owner of AgentOps query authorization and expose FlowLens Inspector as a separate FastMCP resource server. DeerFlow discovers tools with a service credential, exchanges runtime identity for a short-lived delegated JWT at tool-call time, and forwards that JWT through MCP to Gateway; client-side RunJournal events and server-side audit records make the entire call observable. + +**Tech Stack:** Python 3.12, FastMCP/MCP Python SDK 1.27+, Streamable HTTP, FastAPI, PyJWT with RSA/JWKS, Pydantic 2, HTTPX, Tenacity, SQLAlchemy/Alembic, Redis, Prometheus client, Pytest, Ruff, Docker Compose + +## Global Constraints + +- Public product naming remains `FlowLens AgentOps`; the MCP service is `FlowLens Inspector MCP`. +- The five MCP tools are read-only and must never mutate Run, Timeline, Checkpoint, Memory, Prompt, or historical events. +- Tool results never include complete prompts, memory text, attachment bodies, credentials, cookies, authorization headers, or database URLs. +- Core diagnostics and run comparison are deterministic; no MCP tool invokes an LLM. +- Streamable HTTP is the primary transport; `stdio` is not the acceptance path. +- Access tokens use `RS256`, `kid`, JWKS, `iss`, `aud`, `sub`, `tenant_id`, `scope`, `iat`, `exp`, `jti`, and `ver`. +- Delegated access tokens default to 300 seconds and are refreshed before expiry. +- MCP and Gateway both verify the token; Gateway always performs final Run owner and tenant checks. +- Timeline pages default to 20 and cap at 100 MCP items even though the model plan is not cost-constrained. +- HTTP connection timeout is 2 seconds and total MCP tool timeout is capped at 12 seconds. +- Only idempotent dependency failures are retried, at most twice, with backoff and jitter. +- Local mode uses SQLite and an in-memory limiter; enterprise verification uses PostgreSQL and Redis. +- Every behavior change follows RED, GREEN, focused regression, Ruff, then an independently reviewable commit. +- Performance and test-count claims are published only from fresh command output. +- Execution uses Inline Execution with a user-facing checkpoint after each of the four stages; do not dispatch one subagent per task. + +--- + +## File Structure + +### New MCP package + +- Create: `backend/packages/flowlens-inspector-mcp/pyproject.toml` +- Create: `backend/packages/flowlens-inspector-mcp/flowlens_inspector_mcp/__init__.py` +- Create: `backend/packages/flowlens-inspector-mcp/flowlens_inspector_mcp/__main__.py` +- Create: `backend/packages/flowlens-inspector-mcp/flowlens_inspector_mcp/settings.py` +- Create: `backend/packages/flowlens-inspector-mcp/flowlens_inspector_mcp/schemas.py` +- Create: `backend/packages/flowlens-inspector-mcp/flowlens_inspector_mcp/server.py` +- Create: `backend/packages/flowlens-inspector-mcp/flowlens_inspector_mcp/gateway_client.py` +- Create: `backend/packages/flowlens-inspector-mcp/flowlens_inspector_mcp/comparison.py` +- Create: `backend/packages/flowlens-inspector-mcp/flowlens_inspector_mcp/resilience.py` +- Create: `backend/packages/flowlens-inspector-mcp/flowlens_inspector_mcp/audit.py` +- Create: `backend/packages/flowlens-inspector-mcp/flowlens_inspector_mcp/telemetry.py` +- Create: `backend/packages/flowlens-inspector-mcp/flowlens_inspector_mcp/auth/__init__.py` +- Create: `backend/packages/flowlens-inspector-mcp/flowlens_inspector_mcp/auth/principal.py` +- Create: `backend/packages/flowlens-inspector-mcp/flowlens_inspector_mcp/auth/verifier.py` +- Create: `backend/packages/flowlens-inspector-mcp/flowlens_inspector_mcp/auth/scopes.py` +- Create: `backend/packages/flowlens-inspector-mcp/flowlens_inspector_mcp/tools/__init__.py` +- Create: `backend/packages/flowlens-inspector-mcp/flowlens_inspector_mcp/tools/health.py` +- Create: `backend/packages/flowlens-inspector-mcp/flowlens_inspector_mcp/tools/runs.py` +- Create: `backend/packages/flowlens-inspector-mcp/flowlens_inspector_mcp/tools/diagnostics.py` +- Create: `backend/packages/flowlens-inspector-mcp/flowlens_inspector_mcp/tools/timeline.py` +- Create: `backend/packages/flowlens-inspector-mcp/flowlens_inspector_mcp/tools/comparison.py` + +### Gateway, persistence, and runtime integration + +- Modify: `backend/pyproject.toml` +- Modify: `backend/app/gateway/app.py` +- Modify: `backend/app/gateway/auth/models.py` +- Modify: `backend/app/gateway/auth_middleware.py` +- Modify: `backend/app/gateway/authz.py` +- Create: `backend/app/gateway/auth/platform_tokens.py` +- Create: `backend/app/gateway/routers/mcp_tokens.py` +- Modify: `backend/app/gateway/routers/agentops.py` +- Modify: `backend/app/gateway/routers/thread_runs.py` +- Modify: `backend/app/gateway/services.py` +- Modify: `backend/packages/harness/deerflow/config/extensions_config.py` +- Modify: `backend/packages/harness/deerflow/config/app_config.py` +- Create: `backend/packages/harness/deerflow/mcp/delegation.py` +- Modify: `backend/packages/harness/deerflow/mcp/tools.py` +- Modify: `backend/packages/harness/deerflow/mcp/oauth.py` +- Modify: `backend/packages/harness/deerflow/runtime/user_context.py` +- Modify: `backend/packages/harness/deerflow/runtime/runs/manager.py` +- Modify: `backend/packages/harness/deerflow/runtime/runs/store/base.py` +- Modify: `backend/packages/harness/deerflow/runtime/runs/store/memory.py` +- Modify: `backend/packages/harness/deerflow/persistence/run/model.py` +- Modify: `backend/packages/harness/deerflow/persistence/run/sql.py` +- Modify: `backend/packages/harness/deerflow/persistence/user/model.py` +- Modify: `backend/app/gateway/auth/repositories/sqlite.py` +- Create: `backend/packages/harness/deerflow/persistence/mcp_audit/__init__.py` +- Create: `backend/packages/harness/deerflow/persistence/mcp_audit/model.py` +- Create: `backend/packages/harness/deerflow/persistence/mcp_audit/sql.py` +- Modify: `backend/packages/harness/deerflow/persistence/models/__init__.py` +- Create: `backend/packages/harness/deerflow/persistence/migrations/versions/20260728_0001_flowlens_mcp_tenant.py` +- Create: `backend/packages/harness/deerflow/persistence/migrations/versions/20260728_0002_flowlens_mcp_audit.py` + +### Skill, deployment, tests, and documentation + +- Create: `skills/public/flowlens-incident-response/SKILL.md` +- Create: `skills/public/flowlens-incident-response/references/diagnostic-playbook.md` +- Create: `skills/public/flowlens-incident-response/references/output-format.md` +- Create: `skills/public/flowlens-incident-response/evals/trigger_eval_set.json` +- Create: `skills/public/flowlens-incident-response/evals/evals.json` +- Create: `backend/Dockerfile.mcp` +- Modify: `docker/docker-compose-dev.yaml` +- Modify: `docker/docker-compose.yaml` +- Create: `docker/docker-compose-flowlens-mcp-enterprise.yaml` +- Modify: `docker/nginx/nginx.conf` +- Modify: `.env.example` +- Modify: `extensions_config.example.json` +- Modify: `extensions_config.json` +- Create: `backend/scripts/generate_mcp_keys.py` +- Create: `backend/scripts/migrate_flowlens_mcp.py` +- Create: `backend/scripts/seed_agentops_runs.py` +- Create: `backend/scripts/benchmark_inspector_mcp.py` +- Create: `scripts/verify_flowlens_mcp.ps1` +- Modify: `README.md` +- Modify: `docs/architecture.md` +- Create: `docs/flowlens-inspector-mcp.md` +- Create: `backend/tests/flowlens_inspector_mcp/` test package and focused test modules named in the tasks below. + +Command convention: run `uv run ...` and Ruff commands from `backend/`; run `git ...`, Docker Compose, and PowerShell script commands from the repository root. + +--- + +## Stage 1: Identity, Persistence, And Gateway Contracts + +### Task 1: Persist Tenant-Aware User And Run Identity + +**Files:** +- Modify: `backend/app/gateway/auth/models.py:User` +- Modify: `backend/packages/harness/deerflow/persistence/user/model.py:UserRow` +- Modify: `backend/app/gateway/auth/repositories/sqlite.py` +- Modify: `backend/packages/harness/deerflow/runtime/user_context.py` +- Modify: `backend/packages/harness/deerflow/runtime/runs/manager.py:RunRecord` +- Modify: `backend/packages/harness/deerflow/runtime/runs/store/base.py` +- Modify: `backend/packages/harness/deerflow/runtime/runs/store/memory.py` +- Modify: `backend/packages/harness/deerflow/persistence/run/model.py:RunRow` +- Modify: `backend/packages/harness/deerflow/persistence/run/sql.py:RunRepository` +- Modify: `backend/app/gateway/services.py:inject_authenticated_user_context` +- Create: `backend/packages/harness/deerflow/persistence/migrations/versions/20260728_0001_flowlens_mcp_tenant.py` +- Create: `backend/tests/test_tenant_run_isolation.py` +- Modify: `backend/tests/test_run_repository.py` +- Modify: `backend/tests/test_auth.py` + +**Interfaces:** +- Produces: `DEFAULT_TENANT_ID = "default"`. +- Produces: `get_effective_tenant_id() -> str`. +- Produces: `resolve_runtime_tenant_id(runtime: object | None) -> str`. +- Extends: `User.tenant_id: str`, `RunRecord.user_id: str`, and `RunRecord.tenant_id: str`. +- Extends: RunStore read methods with keyword-only `tenant_id`. +- Invariant: authorized Run lookup always matches `(tenant_id, user_id, run_id)`. + +- [ ] **Step 1: Write failing tenant isolation tests** + +```python +@pytest.mark.anyio +async def test_run_repository_requires_matching_tenant_and_user(session_factory): + repo = RunRepository(session_factory) + await repo.put("run-a", thread_id="thread-a", user_id="user-a", tenant_id="tenant-a") + + assert await repo.get("run-a", user_id="user-a", tenant_id="tenant-a") is not None + assert await repo.get("run-a", user_id="user-a", tenant_id="tenant-b") is None + assert await repo.get("run-a", user_id="user-b", tenant_id="tenant-a") is None + + +def test_authenticated_context_includes_server_owned_tenant(): + config: dict = {} + request = SimpleNamespace( + state=SimpleNamespace(user=SimpleNamespace(id="user-a", tenant_id="tenant-a", system_role="user")) + ) + inject_authenticated_user_context(config, request) + assert config["context"]["user_id"] == "user-a" + assert config["context"]["tenant_id"] == "tenant-a" +``` + +- [ ] **Step 2: Run tests to verify RED** + +Run: + +```powershell +cd backend +$env:PYTHONPATH='.' +uv run pytest tests/test_tenant_run_isolation.py tests/test_run_repository.py tests/test_auth.py -q +``` + +Expected: fail because `tenant_id` fields and repository parameters do not exist. + +- [ ] **Step 3: Add tenant identity helpers and persisted fields** + +Implement the shared defaults: + +```python +DEFAULT_TENANT_ID: Final[str] = "default" + + +def get_effective_tenant_id() -> str: + user = get_current_user() + value = getattr(user, "tenant_id", None) if user is not None else None + return str(value or DEFAULT_TENANT_ID) + + +def resolve_runtime_tenant_id(runtime: object | None) -> str: + context = getattr(runtime, "context", None) + if isinstance(context, dict) and context.get("tenant_id"): + return str(context["tenant_id"]) + return get_effective_tenant_id() +``` + +Add `tenant_id` to `User`, `UserRow`, `RunRow`, and `RunRecord`. New users and legacy rows default to `"default"`. + +Capture immutable ownership when RunManager creates a record and include it in every persistence payload: + +```python +record = RunRecord( + ..., + user_id=get_effective_user_id(), + tenant_id=get_effective_tenant_id(), +) +``` + +- [ ] **Step 4: Enforce tenant and owner filters in stores** + +Update `RunRepository.get`, `list_by_thread`, and `list_recent` so a non-admin read includes both: + +```python +stmt = stmt.where( + RunRow.tenant_id == resolved_tenant_id, + RunRow.user_id == resolved_user_id, +) +``` + +Update `MemoryRunStore` and in-memory `RunManager.get` with the same semantics. Explicit `user_id=None, tenant_id=None` remains the migration/admin bypass; no ordinary request may use it. + +- [ ] **Step 5: Add the idempotent schema migration** + +The Alembic upgrade must: + +1. Add `users.tenant_id` and `runs.tenant_id` with server default `"default"` when absent. +2. Backfill null values. +3. Make both columns non-null. +4. Add `ix_runs_tenant_user_created` over `(tenant_id, user_id, created_at)`. +5. Skip an operation when inspection shows the column/index already exists, because local `create_all()` may have created the current schema. + +- [ ] **Step 6: Verify GREEN and isolation regressions** + +Run: + +```powershell +uv run pytest tests/test_tenant_run_isolation.py tests/test_run_repository.py tests/test_auth.py tests/test_agentops_runs_api.py tests/test_migration_user_isolation.py -q +uv run ruff check app/gateway/auth app/gateway/services.py packages/harness/deerflow/runtime/user_context.py packages/harness/deerflow/runtime/runs packages/harness/deerflow/persistence +``` + +Expected: all selected tests pass and Ruff reports no errors. + +- [ ] **Step 7: Commit** + +```powershell +git add backend/app/gateway/auth backend/app/gateway/services.py backend/packages/harness/deerflow/runtime backend/packages/harness/deerflow/persistence backend/tests/test_tenant_run_isolation.py backend/tests/test_run_repository.py backend/tests/test_auth.py +git commit -m "feat(agentops): add tenant-aware run identity" +``` + +### Task 2: Issue And Verify Scoped MCP Delegation Tokens + +**Files:** +- Create: `backend/app/gateway/auth/platform_tokens.py` +- Create: `backend/app/gateway/routers/mcp_tokens.py` +- Modify: `backend/app/gateway/auth_middleware.py` +- Modify: `backend/app/gateway/authz.py` +- Modify: `backend/app/gateway/app.py` +- Create: `backend/tests/test_mcp_platform_tokens.py` +- Create: `backend/tests/test_mcp_bearer_auth.py` + +**Interfaces:** +- Produces: `PlatformTokenSettings.from_env()`. +- Produces: `PlatformTokenClaims`. +- Produces: `PlatformTokenIssuer.issue_service_token(...)`. +- Produces: `PlatformTokenIssuer.issue_delegated_token(...)`. +- Produces: `verify_platform_token(token: str) -> PlatformTokenClaims`. +- Produces: `GET /.well-known/jwks.json`. +- Produces: `POST /api/v1/auth/mcp/token`. + +- [ ] **Step 1: Write failing token and Bearer authentication tests** + +```python +def test_delegated_token_contains_required_claims(rsa_settings): + token = PlatformTokenIssuer(rsa_settings).issue_delegated_token( + user_id="user-a", + tenant_id="tenant-a", + token_version=3, + origin_run_id="run-origin", + ) + claims = verify_platform_token(token, rsa_settings) + assert claims.sub == "user-a" + assert claims.tenant_id == "tenant-a" + assert claims.aud == "flowlens-platform" + assert "agentops.timeline.read" in claims.scope + assert claims.ver == 3 + + +def test_gateway_accepts_platform_bearer_and_rejects_wrong_audience(client, delegated_token): + assert client.get("/api/agentops/runs", headers={"Authorization": f"Bearer {delegated_token}"}).status_code == 200 + assert client.get("/api/agentops/runs", headers={"Authorization": "Bearer wrong-aud-token"}).status_code == 401 +``` + +- [ ] **Step 2: Run tests to verify RED** + +Run: + +```powershell +uv run pytest tests/test_mcp_platform_tokens.py tests/test_mcp_bearer_auth.py -q +``` + +Expected: fail because platform token modules and Bearer authentication are absent. + +- [ ] **Step 3: Implement RS256 claims, issuer, verifier, and JWKS** + +Use immutable claims: + +```python +class PlatformTokenClaims(BaseModel): + iss: str + sub: str + tenant_id: str + aud: str | list[str] + scope: list[str] + origin_run_id: str | None = None + azp: str + jti: str + iat: datetime + exp: datetime + ver: int +``` + +Read keys and identifiers from `FLOWLENS_MCP_JWT_PRIVATE_KEY_PATH`, `FLOWLENS_MCP_JWT_PUBLIC_KEY_PATH`, `FLOWLENS_MCP_JWT_KID`, `FLOWLENS_MCP_ISSUER`, and `FLOWLENS_MCP_AUDIENCE`. Reject missing key material at startup when MCP is enabled. + +- [ ] **Step 4: Implement the service and delegation grants** + +`POST /api/v1/auth/mcp/token` accepts form data: + +- `grant_type=client_credentials`: issue only `agentops.tools.discover`. +- `grant_type=urn:deerflow:params:oauth:grant-type:runtime-delegation`: require `subject_user_id` and `origin_run_id`, load the user from the repository, derive `tenant_id` and `ver` from that row, and issue the five read scopes. + +Authenticate `client_id` and `client_secret` with `hmac.compare_digest`; never echo the secret. + +- [ ] **Step 5: Extend Gateway middleware without weakening cookie auth** + +Authentication order: + +1. Valid internal token. +2. Valid `Authorization: Bearer` platform token. +3. Existing cookie JWT. +4. Otherwise `401`. + +Map delegated AgentOps scopes to Gateway `runs:read`, retain the full scope set on `request.state.platform_scopes`, load the real User row, and reject a stale `ver`. + +- [ ] **Step 6: Verify GREEN and existing auth regressions** + +Run: + +```powershell +uv run pytest tests/test_mcp_platform_tokens.py tests/test_mcp_bearer_auth.py tests/test_auth_middleware.py tests/test_auth.py tests/test_auth_errors.py tests/test_internal_auth.py -q +uv run ruff check app/gateway/auth app/gateway/auth_middleware.py app/gateway/authz.py app/gateway/routers/mcp_tokens.py +``` + +Expected: new Bearer cases and all existing cookie/internal-token cases pass. + +- [ ] **Step 7: Commit** + +```powershell +git add backend/app/gateway/auth backend/app/gateway/auth_middleware.py backend/app/gateway/authz.py backend/app/gateway/routers/mcp_tokens.py backend/app/gateway/app.py backend/tests/test_mcp_platform_tokens.py backend/tests/test_mcp_bearer_auth.py +git commit -m "feat(auth): issue scoped MCP delegation tokens" +``` + +### Task 3: Stabilize Gateway Query Contracts For MCP + +**Files:** +- Modify: `backend/app/gateway/routers/agentops.py` +- Modify: `backend/app/gateway/routers/thread_runs.py` +- Modify: `backend/packages/harness/deerflow/runtime/runs/store/base.py` +- Modify: `backend/packages/harness/deerflow/runtime/runs/store/memory.py` +- Modify: `backend/packages/harness/deerflow/persistence/run/sql.py` +- Modify: `backend/packages/harness/deerflow/runtime/runs/manager.py` +- Modify: `backend/tests/test_agentops_runs_api.py` +- Modify: `backend/tests/test_thread_run_diagnostics_api.py` +- Create: `backend/tests/test_agentops_run_lookup_api.py` + +**Interfaces:** +- Extends: `GET /api/agentops/runs` with optional `thread_id`. +- Produces: `GET /api/agentops/runs/{run_id}`. +- Preserves: existing thread-scoped Timeline and Diagnostics endpoints. +- Invariant: inaccessible and nonexistent runs both return `404`. + +- [ ] **Step 1: Add failing API contract tests** + +```python +def test_list_runs_filters_by_thread_and_tenant(client, token): + response = client.get( + "/api/agentops/runs?thread_id=thread-a&limit=20", + headers={"Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 200 + assert {item["thread_id"] for item in response.json()["items"]} == {"thread-a"} + + +def test_run_lookup_hides_cross_tenant_run(client, tenant_a_token): + response = client.get( + "/api/agentops/runs/run-owned-by-tenant-b", + headers={"Authorization": f"Bearer {tenant_a_token}"}, + ) + assert response.status_code == 404 +``` + +- [ ] **Step 2: Run tests to verify RED** + +Run: + +```powershell +uv run pytest tests/test_agentops_runs_api.py tests/test_agentops_run_lookup_api.py tests/test_thread_run_diagnostics_api.py -q +``` + +Expected: fail because `thread_id` filtering, run lookup, and tenant-aware route calls are missing. + +- [ ] **Step 3: Add thread filtering to RunStore and run index** + +Extend `list_recent` with `thread_id: str | None`. Apply the condition before cursor ordering: + +```python +if thread_id is not None: + stmt = stmt.where(RunRow.thread_id == thread_id) +``` + +Keep the existing `(created_at DESC, run_id DESC)` cursor contract. + +- [ ] **Step 4: Add the owner-safe Run lookup endpoint** + +Return only the fields needed by MCP: + +```json +{ + "run_id": "run-a", + "thread_id": "thread-a", + "status": "error", + "created_at": "...", + "updated_at": "...", + "model_name": "...", + "total_tokens": 120, + "llm_call_count": 2, + "message_count": 4 +} +``` + +Resolve the current `User.tenant_id`, call `RunManager.get(run_id, user_id=..., tenant_id=...)`, and return the same `404` for absent or unauthorized data. + +- [ ] **Step 5: Verify GREEN and FlowLens regressions** + +Run: + +```powershell +uv run pytest tests/test_agentops_runs_api.py tests/test_agentops_run_lookup_api.py tests/test_thread_run_diagnostics_api.py tests/test_run_repository.py tests/test_tenant_run_isolation.py -q +uv run ruff check app/gateway/routers/agentops.py app/gateway/routers/thread_runs.py packages/harness/deerflow/runtime/runs packages/harness/deerflow/persistence/run +``` + +Expected: all selected tests pass. + +- [ ] **Step 6: Commit and report Stage 1** + +```powershell +git add backend/app/gateway/routers/agentops.py backend/app/gateway/routers/thread_runs.py backend/packages/harness/deerflow/runtime/runs backend/packages/harness/deerflow/persistence/run backend/tests/test_agentops_runs_api.py backend/tests/test_agentops_run_lookup_api.py backend/tests/test_thread_run_diagnostics_api.py +git commit -m "feat(agentops): expose MCP-safe run query contracts" +``` + +Stage 1 report must show the token claims test, cross-tenant `404`, and exact Gateway API response before Stage 2 begins. + +--- + +## Stage 2: MCP Server And DeerFlow Client Integration + +### Task 4: Create The Authenticated Streamable HTTP MCP Foundation + +**Files:** +- Modify: `backend/pyproject.toml` +- Create: `backend/packages/flowlens-inspector-mcp/pyproject.toml` +- Create: package files under `flowlens_inspector_mcp/` +- Create: `backend/tests/flowlens_inspector_mcp/test_settings.py` +- Create: `backend/tests/flowlens_inspector_mcp/test_auth.py` +- Create: `backend/tests/flowlens_inspector_mcp/test_server_contract.py` + +**Interfaces:** +- Produces: `InspectorSettings.from_env()`. +- Produces: `JwtTokenVerifier.verify_token(token: str) -> AccessToken | None`. +- Produces: `require_scope(scope: str) -> AccessPrincipal`. +- Produces: `create_mcp_server(settings, gateway_client, executor) -> FastMCP`. +- Produces: `create_app(settings: InspectorSettings | None = None) -> Starlette`. +- Produces: `/mcp`, `/health/live`, `/health/ready`, `/metrics`, and protected-resource metadata. + +- [ ] **Step 1: Write failing package and authentication tests** + +```python +@pytest.mark.anyio +async def test_token_verifier_returns_sdk_access_token(valid_jwt, verifier): + token = await verifier.verify_token(valid_jwt) + assert token is not None + assert token.client_id == "user-a" + assert "agentops.runs.read" in token.scopes + + +@pytest.mark.anyio +async def test_list_tools_exposes_exact_public_contract(mcp_client): + result = await mcp_client.list_tools() + assert {tool.name for tool in result.tools} == { + "health_check", + "list_runs", + "get_run_diagnostics", + "get_timeline_window", + "compare_runs", + } +``` + +- [ ] **Step 2: Add transport-security rejection cases** + +```python +@pytest.mark.anyio +async def test_streamable_http_rejects_unknown_origin(asgi_client): + response = await asgi_client.post( + "/mcp", + headers={"Origin": "https://attacker.example", "Host": "localhost:8010"}, + json={"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {}}, + ) + assert response.status_code in {400, 403} + + +@pytest.mark.anyio +async def test_streamable_http_rejects_unknown_host(asgi_client): + response = await asgi_client.post( + "/mcp", + headers={"Origin": "http://localhost:2026", "Host": "attacker.example"}, + json={"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {}}, + ) + assert response.status_code in {400, 403} +``` + +- [ ] **Step 3: Run tests to verify RED** + +Run: + +```powershell +uv run pytest tests/flowlens_inspector_mcp/test_settings.py tests/flowlens_inspector_mcp/test_auth.py tests/flowlens_inspector_mcp/test_server_contract.py -q +``` + +Expected: fail because the workspace package is absent. + +- [ ] **Step 4: Register the workspace package and direct dependencies** + +Add `packages/flowlens-inspector-mcp` to the uv workspace and add a workspace source. The package declares direct dependencies on: + +```toml +dependencies = [ + "deerflow-harness", + "mcp>=1.27,<2", + "httpx>=0.28", + "pydantic>=2.12", + "pyjwt[crypto]>=2.9", + "tenacity>=9", + "prometheus-client>=0.21", + "redis>=5", + "sqlalchemy[asyncio]>=2,<3", + "uvicorn[standard]>=0.34", +] +``` + +- [ ] **Step 5: Implement settings, verifier, principal, and scopes** + +The verifier validates RSA signature, `kid`, issuer, audience, expiry, and required claim types, then returns: + +```python +AccessToken( + token=raw_token, + client_id=claims.sub, + scopes=claims.scope, + expires_at=int(claims.exp.timestamp()), + resource=settings.resource_url, +) +``` + +`require_scope` reads `get_access_token()`, raises a typed authorization error when missing, and never accepts a user ID from tool arguments. + +- [ ] **Step 6: Build the ASGI service and health surfaces** + +Configure FastMCP with `stateless_http=False`, `streamable_http_path="/mcp"`, `TokenVerifier`, `AuthSettings`, and a strict `TransportSecuritySettings` Origin/Host allowlist. Bind each issued MCP Session ID to the verified `AccessToken.client_id` and reject a session reused under another principal. Mount the FastMCP Starlette application with live, ready, metrics, and protected-resource routes. + +- [ ] **Step 7: Verify GREEN and contract stability** + +Run: + +```powershell +uv run pytest tests/flowlens_inspector_mcp/test_settings.py tests/flowlens_inspector_mcp/test_auth.py tests/flowlens_inspector_mcp/test_server_contract.py -q +uv run ruff check packages/flowlens-inspector-mcp tests/flowlens_inspector_mcp +``` + +Expected: exact five-tool contract, valid authentication, and health routes pass. + +- [ ] **Step 8: Commit** + +```powershell +git add backend/pyproject.toml backend/packages/flowlens-inspector-mcp backend/tests/flowlens_inspector_mcp +git commit -m "feat(mcp): add authenticated FlowLens inspector service" +``` + +### Task 5: Implement The Five Read-Only Inspector Tools + +**Files:** +- Create/Modify: `flowlens_inspector_mcp/schemas.py` +- Create/Modify: `flowlens_inspector_mcp/gateway_client.py` +- Create/Modify: `flowlens_inspector_mcp/comparison.py` +- Create/Modify: all modules under `flowlens_inspector_mcp/tools/` +- Create: `backend/tests/flowlens_inspector_mcp/test_gateway_client.py` +- Create: `backend/tests/flowlens_inspector_mcp/test_tools.py` +- Create: `backend/tests/flowlens_inspector_mcp/test_comparison.py` + +**Interfaces:** +- Produces: versioned Pydantic inputs and outputs with `schema_version`, `request_id`, and `generated_at`. +- Produces: `GatewayClient` read methods that forward the original delegated Bearer token. +- Produces: `build_run_comparison(base, candidate) -> RunComparisonResponse`. +- Invariant: tool argument models contain no `user_id`, `tenant_id`, credential, or unrestricted page size. + +The concrete Gateway client surface is: + +```python +class GatewayClient: + async def health(self, *, token: str) -> GatewayHealth: ... + async def list_runs(self, *, token: str, query: RunListInput) -> RunListResponse: ... + async def get_run(self, *, token: str, run_id: str) -> RunSummary: ... + async def get_diagnostics(self, *, token: str, run: RunSummary) -> DiagnosticsResponse: ... + async def get_timeline(self, *, token: str, run: RunSummary, query: TimelineWindowInput) -> TimelineWindowResponse: ... +``` + +- [ ] **Step 1: Write failing tool behavior tests** + +```python +@pytest.mark.anyio +async def test_timeline_caps_limit_and_forwards_scope(tool_harness): + result = await tool_harness.call( + "get_timeline_window", + {"run_id": "run-a", "limit": 500}, + scopes=["agentops.timeline.read"], + ) + assert result.isError is True + + +@pytest.mark.anyio +async def test_compare_runs_is_deterministic(tool_harness): + first = await tool_harness.call("compare_runs", {"base_run_id": "run-a", "candidate_run_id": "run-b"}) + second = await tool_harness.call("compare_runs", {"base_run_id": "run-a", "candidate_run_id": "run-b"}) + assert first.structuredContent == second.structuredContent +``` + +- [ ] **Step 2: Run tests to verify RED** + +Run: + +```powershell +uv run pytest tests/flowlens_inspector_mcp/test_gateway_client.py tests/flowlens_inspector_mcp/test_tools.py tests/flowlens_inspector_mcp/test_comparison.py -q +``` + +Expected: fail because Gateway client methods and tool implementations are absent. + +- [ ] **Step 3: Implement bounded schemas and Gateway calls** + +Use opaque cursor strings and constrained limits: + +```python +class TimelineWindowInput(BaseModel): + run_id: str = Field(min_length=1, max_length=64) + phase: list[TimelinePhase] | None = None + after_seq: int | None = Field(default=None, ge=0) + limit: int = Field(default=20, ge=1, le=100) +``` + +Every run-specific call first resolves `GET /api/agentops/runs/{run_id}` to obtain the authorized `thread_id`, then calls the existing thread-scoped Timeline or Diagnostics endpoint with the same Bearer token. + +- [ ] **Step 4: Implement tool-specific scopes** + +Register: + +```python +TOOL_SCOPES = { + "health_check": "agentops.health.read", + "list_runs": "agentops.runs.read", + "get_run_diagnostics": "agentops.diagnostics.read", + "get_timeline_window": "agentops.timeline.read", + "compare_runs": "agentops.runs.compare", +} +``` + +Do not fetch Gateway data until the scope check passes. + +- [ ] **Step 5: Implement deterministic comparison and partial responses** + +Compare duration, tokens, LLM calls, tool calls/errors, subagents, MCP, Memory, rollbacks, status, and attribution. If one optional diagnostics section is unavailable, return `status="partial"`, explicit available/missing sections, and `retryable`; never fill missing values with invented evidence. + +- [ ] **Step 6: Verify GREEN, redaction, and contract tests** + +Run: + +```powershell +uv run pytest tests/flowlens_inspector_mcp/test_gateway_client.py tests/flowlens_inspector_mcp/test_tools.py tests/flowlens_inspector_mcp/test_comparison.py tests/test_flowlens_redaction.py -q +uv run ruff check packages/flowlens-inspector-mcp tests/flowlens_inspector_mcp +``` + +Expected: all five tools pass; no response fixture contains prompt, memory, Authorization, cookie, or secret fields. + +- [ ] **Step 7: Commit** + +```powershell +git add backend/packages/flowlens-inspector-mcp backend/tests/flowlens_inspector_mcp +git commit -m "feat(mcp): add FlowLens diagnostic tools" +``` + +### Task 6: Inject Runtime-Delegated Tokens Into MCP Tool Calls + +**Files:** +- Modify: `backend/packages/harness/deerflow/config/extensions_config.py` +- Create: `backend/packages/harness/deerflow/mcp/delegation.py` +- Modify: `backend/packages/harness/deerflow/mcp/tools.py` +- Modify: `backend/packages/harness/deerflow/mcp/oauth.py` +- Modify: `backend/app/gateway/services.py` +- Create: `backend/tests/test_mcp_delegation.py` +- Modify: `backend/tests/test_mcp_custom_interceptors.py` +- Modify: `backend/tests/test_mcp_diagnostics.py` + +**Interfaces:** +- Produces: `McpDelegatedAuthConfig`. +- Produces: `DelegatedTokenManager.get_authorization_header(server_name, runtime) -> str`. +- Produces: `build_delegated_auth_interceptor(config)`. +- Preserves: service OAuth header for initial `initialize`/`tools/list`. +- Overrides: service header with delegated user header only for `tools/call`. +- Produces: runtime-visible tool names prefixed as `flowlens_inspector_` by the existing multi-server client. + +- [ ] **Step 1: Write failing runtime delegation tests** + +```python +@pytest.mark.anyio +async def test_delegation_uses_server_injected_runtime_identity(manager, runtime): + runtime.context.update({ + "user_id": "user-a", + "tenant_id": "tenant-a", + "run_id": "run-origin", + }) + header = await manager.get_authorization_header("flowlens-inspector", runtime) + assert header.startswith("Bearer ") + assert manager.last_request["subject_user_id"] == "user-a" + assert manager.last_request["origin_run_id"] == "run-origin" + + +@pytest.mark.anyio +async def test_delegation_never_records_token_in_mcp_events(recorder, interceptor): + await invoke_interceptor(interceptor, recorder) + assert "Bearer " not in json.dumps(recorder.events) +``` + +- [ ] **Step 2: Run tests to verify RED** + +Run: + +```powershell +uv run pytest tests/test_mcp_delegation.py tests/test_mcp_custom_interceptors.py tests/test_mcp_diagnostics.py -q +``` + +Expected: fail because delegated auth config and interceptor are absent. + +- [ ] **Step 3: Add delegated auth configuration** + +Add an optional per-server object: + +```python +class McpDelegatedAuthConfig(BaseModel): + enabled: bool = True + token_url: str + client_id: str + client_secret: str + audience: str = "flowlens-platform" + refresh_skew_seconds: int = 60 +``` + +Environment placeholders continue to resolve through `ExtensionsConfig.resolve_env_variables`; UI masking treats `client_secret` as sensitive. + +- [ ] **Step 4: Implement per-user/run token acquisition and caching** + +Cache by: + +```text +(server_name, user_id, tenant_id, origin_run_id) +``` + +Use an `asyncio.Lock` per cache key. Post the runtime-delegation grant to Gateway and refresh before expiry. Require `user_id`, `tenant_id`, and `run_id` from server-owned Runtime Context; do not fall back to model arguments. + +- [ ] **Step 5: Compose interceptor order and diagnostics** + +The service OAuth interceptor supplies startup discovery credentials. The delegated interceptor runs inside it and replaces `Authorization` for tool calls. Emit only: + +```text +mcp.delegation.refresh +mcp.delegation.error +``` + +with server name, grant type, duration, status, and exception class. Existing RunJournal tool callbacks continue producing `mcp.tool.start/end/error`. + +- [ ] **Step 6: Verify GREEN and a real adapter call** + +Run: + +```powershell +uv run pytest tests/test_mcp_delegation.py tests/test_mcp_custom_interceptors.py tests/test_mcp_oauth.py tests/test_mcp_diagnostics.py tests/test_mcp_client_config.py -q +uv run ruff check packages/harness/deerflow/mcp packages/harness/deerflow/config/extensions_config.py app/gateway/services.py +``` + +Expected: service discovery auth and runtime delegation both pass without token leakage. + +- [ ] **Step 7: Commit and report Stage 2** + +```powershell +git add backend/packages/harness/deerflow/config/extensions_config.py backend/packages/harness/deerflow/mcp backend/app/gateway/services.py backend/tests/test_mcp_delegation.py backend/tests/test_mcp_custom_interceptors.py backend/tests/test_mcp_diagnostics.py +git commit -m "feat(mcp): delegate runtime identity to inspector tools" +``` + +Stage 2 report must show `tools/list`, one authenticated `list_runs`, one denied scope, and the corresponding `mcp.tool.*` Timeline evidence. + +--- + +## Stage 3: Reliability, Audit, And Deployment + +### Task 7: Add Bounded Resilience, Audit, And Metrics + +**Files:** +- Create/Modify: `flowlens_inspector_mcp/resilience.py` +- Create/Modify: `flowlens_inspector_mcp/audit.py` +- Create/Modify: `flowlens_inspector_mcp/telemetry.py` +- Modify: `flowlens_inspector_mcp/gateway_client.py` +- Modify: `flowlens_inspector_mcp/server.py` +- Create: `backend/packages/harness/deerflow/persistence/mcp_audit/` +- Modify: `backend/packages/harness/deerflow/persistence/models/__init__.py` +- Create: `backend/packages/harness/deerflow/persistence/migrations/versions/20260728_0002_flowlens_mcp_audit.py` +- Create: `backend/tests/flowlens_inspector_mcp/test_resilience.py` +- Create: `backend/tests/flowlens_inspector_mcp/test_audit.py` +- Create: `backend/tests/flowlens_inspector_mcp/test_metrics.py` + +**Interfaces:** +- Produces: `RetryPolicy`, `CircuitBreaker`, `ConcurrencyLimiter`, and `RateLimiter`. +- Produces: `InMemoryTokenBucket` and `RedisTokenBucket`. +- Produces: `ToolExecutor.execute(...)`. +- Produces: append-only `McpAuditRepository.insert(event)`. +- Produces: Prometheus counters, histograms, gauges, and circuit-state metric. + +Use these boundaries: + +```python +class RateLimiter(Protocol): + async def acquire(self, key: str, *, capacity: int, period_seconds: int) -> bool: ... + + +class ToolExecutor: + async def execute( + self, + *, + tool_name: str, + principal: AccessPrincipal, + target_run_ids: tuple[str, ...], + operation: Callable[[], Awaitable[T]], + ) -> T: ... +``` + +`ToolExecutor` owns concurrency, rate limiting, timing, metric updates, and the final audit write. `GatewayClient` owns HTTP timeout, retry, status mapping, and circuit state. + +- [ ] **Step 1: Write failing failure-policy tests** + +```python +@pytest.mark.anyio +async def test_gateway_retries_503_twice_then_opens_circuit(client): + client.transport.queue_statuses(503, 503, 503) + with pytest.raises(DependencyUnavailable): + await client.list_runs(token="token", limit=20) + assert client.transport.call_count == 3 + assert client.circuit.state == "open" + + +@pytest.mark.anyio +async def test_audit_whitelist_drops_sensitive_payload(audit_repo): + with pytest.raises(ValidationError): + McpAuditEvent( + request_id="req-1", + tool_name="get_run_diagnostics", + user_id="user-a", + tenant_id="tenant-a", + status="success", + authorization="Bearer secret", + ) + + await audit_repo.insert(McpAuditEvent( + request_id="req-1", + tool_name="get_run_diagnostics", + user_id="user-a", + tenant_id="tenant-a", + status="success", + )) + stored = await audit_repo.get("req-1") + assert "authorization" not in stored +``` + +- [ ] **Step 2: Run tests to verify RED** + +Run: + +```powershell +uv run pytest tests/flowlens_inspector_mcp/test_resilience.py tests/flowlens_inspector_mcp/test_audit.py tests/flowlens_inspector_mcp/test_metrics.py -q +``` + +Expected: fail because resilience, audit, and metrics implementations are absent. + +- [ ] **Step 3: Implement retry, cancellation, and circuit behavior** + +Retry only connection errors and `502/503/504`; never retry `400/401/403/404/422/429`. Respect cancellation. Open after five consecutive dependency failures, wait 15 seconds, allow one half-open probe, and close only on success. + +- [ ] **Step 4: Implement limiters** + +Use the key: + +```text +tenant_id:user_id:tool_name +``` + +Defaults: + +- general tools: 60/minute +- Timeline: 30/minute +- comparison: 10/minute +- process concurrency: 32 + +Redis uses one atomic Lua token-bucket operation. Loss of Redis produces an explicit configured policy: fail closed in enterprise mode and use the in-memory limiter only in local mode. + +- [ ] **Step 5: Implement append-only audit** + +Persist only the approved columns from the design. `McpAuditRepository` exposes `insert` and test-only lookup; no production update/delete API exists. The MCP database role documentation grants only `INSERT` and controlled `SELECT`. + +Create migration `20260728_0002_flowlens_mcp_audit.py` with `down_revision` pointing to the tenant migration. Never modify the already-applied `0001` migration. + +The MCP service lifespan initializes a dedicated SQLAlchemy engine from the configured application database URL, constructs `McpAuditRepository`, and disposes the engine during graceful shutdown. It does not use this connection to read Run or Timeline data. + +- [ ] **Step 6: Instrument metrics and trace correlation** + +Record tool requests, results, duration, retries, rate limits, active calls, Gateway errors, and circuit state. Propagate or create `request_id`, `traceparent`, and `origin_run_id`; never use a token as a metric label. + +- [ ] **Step 7: Verify GREEN and full package regressions** + +Run: + +```powershell +uv run pytest tests/flowlens_inspector_mcp -q +uv run ruff check packages/flowlens-inspector-mcp packages/harness/deerflow/persistence/mcp_audit +``` + +Expected: resilience state transitions, append-only audit, and metric updates pass. + +- [ ] **Step 8: Commit** + +```powershell +git add backend/packages/flowlens-inspector-mcp backend/packages/harness/deerflow/persistence/mcp_audit backend/packages/harness/deerflow/persistence/models backend/packages/harness/deerflow/persistence/migrations backend/tests/flowlens_inspector_mcp +git commit -m "feat(mcp): add inspector resilience and audit" +``` + +### Task 8: Package The Local And Enterprise Docker Paths + +**Files:** +- Create: `backend/Dockerfile.mcp` +- Modify: `docker/docker-compose-dev.yaml` +- Modify: `docker/docker-compose.yaml` +- Create: `docker/docker-compose-flowlens-mcp-enterprise.yaml` +- Modify: `docker/nginx/nginx.conf` +- Modify: `backend/packages/harness/deerflow/config/app_config.py` +- Modify: `.env.example` +- Modify: `extensions_config.example.json` +- Modify: `extensions_config.json` +- Create: `backend/scripts/generate_mcp_keys.py` +- Create: `backend/scripts/migrate_flowlens_mcp.py` +- Create: `backend/tests/test_mcp_key_generation.py` +- Create: `backend/tests/test_database_env_override.py` + +**Interfaces:** +- Produces: `python backend/scripts/generate_mcp_keys.py`. +- Produces: `python backend/scripts/migrate_flowlens_mcp.py upgrade`. +- Produces: local `flowlens-inspector-mcp:8010`. +- Produces: enterprise Compose overlay with PostgreSQL and Redis. +- Produces: internal URL `http://flowlens-inspector-mcp:8010/mcp`. + +- [ ] **Step 1: Write failing configuration and key tests** + +```python +def test_database_environment_override_selects_postgres(monkeypatch, config_file): + monkeypatch.setenv("DEER_FLOW_DATABASE_BACKEND", "postgres") + monkeypatch.setenv("DATABASE_URL", "postgresql://deerflow:test@postgres:5432/deerflow") + config = AppConfig.from_file(config_file) + assert config.database.backend == "postgres" + + +def test_key_generator_writes_private_0600_and_public_key(tmp_path): + generate_key_pair(tmp_path) + assert (tmp_path / "private.pem").exists() + assert (tmp_path / "public.pem").exists() + assert (tmp_path / "private.pem").read_text().startswith("-----BEGIN PRIVATE KEY-----") + if os.name != "nt": + assert stat.S_IMODE((tmp_path / "private.pem").stat().st_mode) == 0o600 +``` + +- [ ] **Step 2: Run tests to verify RED** + +Run: + +```powershell +uv run pytest tests/test_mcp_key_generation.py tests/test_database_env_override.py -q +``` + +Expected: fail because key generation and database environment override are absent. + +- [ ] **Step 3: Build a non-root MCP image** + +`backend/Dockerfile.mcp` uses a builder stage, installs only the MCP workspace package and dependencies, creates a non-root `flowlens` user, exposes 8010, and starts: + +```text +python -m flowlens_inspector_mcp +``` + +- [ ] **Step 4: Add local Compose wiring without a startup cycle** + +Start MCP before Gateway using `/health/live`; do not make MCP readiness depend on Gateway for Compose ordering. Gateway reaches MCP on the internal network. MCP `/health/ready` may report Gateway degradation after both processes start. + +Mount only the private key into Gateway. MCP validates through Gateway JWKS and never receives the private key. + +- [ ] **Step 5: Add the enterprise overlay** + +The overlay adds PostgreSQL, Redis, health checks, persistent volumes, and: + +```text +DEER_FLOW_DATABASE_BACKEND=postgres +DATABASE_URL=postgresql://deerflow:${POSTGRES_PASSWORD}@postgres:5432/deerflow +FLOWLENS_MCP_RATE_LIMIT_BACKEND=redis +FLOWLENS_MCP_REDIS_URL=redis://redis:6379/0 +``` + +Add an AppConfig environment override test so no duplicate full `config.yaml` is needed. + +- [ ] **Step 6: Add Nginx and extension configuration** + +Proxy `/mcp` with buffering disabled, bounded body size, correct forwarded headers, and Streamable HTTP timeouts. Add a disabled `flowlens_inspector` entry to committed config and an enabled example using service OAuth plus `delegatedAuth`; secrets use `$FLOWLENS_MCP_CLIENT_SECRET`. Remove the non-resolving `my_package.mcp.auth:build_auth_interceptor` sample from the active committed config because delegated auth is now a built-in interceptor. + +- [ ] **Step 7: Verify Compose and container health** + +Run: + +```powershell +docker compose -f docker/docker-compose-dev.yaml config +docker compose -f docker/docker-compose.yaml config +docker compose -f docker/docker-compose.yaml -f docker/docker-compose-flowlens-mcp-enterprise.yaml config +uv run pytest tests/test_mcp_key_generation.py tests/test_database_env_override.py -q +``` + +Expected: all Compose models validate and focused tests pass. + +- [ ] **Step 8: Commit and report Stage 3** + +```powershell +git add backend/Dockerfile.mcp docker .env.example extensions_config.example.json extensions_config.json backend/packages/harness/deerflow/config/app_config.py backend/scripts/generate_mcp_keys.py backend/scripts/migrate_flowlens_mcp.py backend/tests/test_mcp_key_generation.py backend/tests/test_database_env_override.py +git commit -m "feat(mcp): package enterprise inspector deployment" +``` + +Stage 3 report must show local live/ready output, one forced Gateway timeout with retry/circuit evidence, one audit row, and validated enterprise Compose. + +--- + +## Stage 4: Skill, End-To-End Evidence, And Teaching Materials + +### Task 9: Add The FlowLens Incident Response Skill + +**Files:** +- Create: `skills/public/flowlens-incident-response/SKILL.md` +- Create: `skills/public/flowlens-incident-response/references/diagnostic-playbook.md` +- Create: `skills/public/flowlens-incident-response/references/output-format.md` +- Create: `skills/public/flowlens-incident-response/evals/trigger_eval_set.json` +- Create: `skills/public/flowlens-incident-response/evals/evals.json` +- Create: `backend/tests/test_flowlens_incident_skill.py` + +**Interfaces:** +- Produces: intent-triggered incident investigation SOP. +- Requires: the exact five MCP tool names. +- Produces: conclusion, evidence, impact, remediation, and uncertainty sections. +- Invariant: every root-cause claim cites a Run ID and Timeline event sequence. + +- [ ] **Step 1: Write failing Skill contract tests** + +```python +def test_skill_references_only_real_inspector_tools(skill_text): + expected = { + "flowlens_inspector_health_check", + "flowlens_inspector_list_runs", + "flowlens_inspector_get_run_diagnostics", + "flowlens_inspector_get_timeline_window", + "flowlens_inspector_compare_runs", + } + assert expected <= set(extract_backticked_tool_names(skill_text)) + + +def test_skill_forbids_invented_evidence(skill_text): + assert "不得根据缺失事件猜测根因" in skill_text + assert "provisional" in skill_text + assert "partial" in skill_text +``` + +- [ ] **Step 2: Run tests to verify RED** + +Run: + +```powershell +uv run pytest tests/test_flowlens_incident_skill.py -q +``` + +Expected: fail because the Skill does not exist. + +- [ ] **Step 3: Write focused Skill frontmatter and workflow** + +The description triggers on requests to diagnose a failed/slow Agent run, compare runs, explain Tool/MCP/Subagent failures, or produce an evidence-backed incident report. It must not trigger for ordinary chat, code editing, or generic MCP education. The Skill names the `flowlens_inspector_*` tools visible to DeerFlow while documenting the corresponding unprefixed MCP server contract. + +- [ ] **Step 4: Write the playbook and report contract** + +The playbook enforces: + +```text +locate run -> deterministic diagnosis -> evidence window -> optional baseline comparison -> report +``` + +The output format requires `run_id`, evidence sequence numbers, completeness state, and no secret/raw-content fields. + +- [ ] **Step 5: Add trigger and behavior evals** + +Include positive, negative, ambiguous, running-run, partial-data, cross-run comparison, and MCP-unavailable cases. Expected behavior names the first tool and required final sections rather than relying on exact prose. + +- [ ] **Step 6: Verify GREEN and commit** + +Run: + +```powershell +uv run pytest tests/test_flowlens_incident_skill.py -q +``` + +Then: + +```powershell +git add skills/public/flowlens-incident-response backend/tests/test_flowlens_incident_skill.py +git commit -m "feat(skills): add FlowLens incident response playbook" +``` + +### Task 10: Prove The Complete Flow And Publish Only Measured Evidence + +**Files:** +- Create: `backend/scripts/seed_agentops_runs.py` +- Create: `backend/scripts/benchmark_inspector_mcp.py` +- Create: `scripts/verify_flowlens_mcp.ps1` +- Create: `backend/tests/test_flowlens_inspector_e2e.py` +- Modify: `README.md` +- Modify: `docs/architecture.md` +- Create: `docs/flowlens-inspector-mcp.md` +- Modify: relevant GitHub CI workflow only after local commands pass. + +**Interfaces:** +- Produces: deterministic success, Tool Error, timeout, and cross-tenant fixtures. +- Produces: direct MCP E2E test and manual real-model acceptance script. +- Produces: reproducible P50/P95/P99 benchmark JSON. +- Produces: Chinese operator and interview explanation. + +- [ ] **Step 1: Write the failing end-to-end test** + +```python +@pytest.mark.anyio +async def test_runtime_calls_remote_inspector_and_records_mcp_evidence(full_stack): + result = await full_stack.ask("请使用 FlowLens 调查最近一次失败，并引用证据事件。") + assert result.contains_run_id("failed-run") + assert result.contains_event_sequence() + + events = await full_stack.events_for_origin_run() + assert "mcp.tool.start" in {event["event_type"] for event in events} + assert "mcp.tool.end" in {event["event_type"] for event in events} +``` + +- [ ] **Step 2: Run test to verify RED** + +Run: + +```powershell +uv run pytest tests/test_flowlens_inspector_e2e.py -q +``` + +Expected: fail until fixture seeding, service startup, delegation, and Skill wiring are connected. + +- [ ] **Step 3: Implement deterministic seed and benchmark scripts** + +The seed script accepts `--runs`, `--events-per-run`, `--tenant`, `--user`, and `--scenario`. The benchmark calls real Streamable HTTP tools, records environment metadata, and writes JSON containing sample count, P50, P95, P99, error count, and configuration. + +- [ ] **Step 4: Implement the PowerShell verification path** + +`verify_flowlens_mcp.ps1` performs: + +1. Required environment validation without printing secrets. +2. Key generation when absent. +3. Docker build/start. +4. Live and ready checks. +5. Token acquisition. +6. MCP `tools/list`. +7. Authorized and unauthorized tool calls. +8. Audit and Timeline evidence checks. +9. Optional PostgreSQL/Redis profile. +10. Clear stop/cleanup instructions without deleting persistent user data. + +- [ ] **Step 5: Make deterministic E2E GREEN** + +Run: + +```powershell +uv run pytest tests/test_flowlens_inspector_e2e.py tests/flowlens_inspector_mcp tests/test_mcp_delegation.py tests/test_flowlens_incident_skill.py -q +uv run ruff check packages/flowlens-inspector-mcp packages/harness/deerflow/mcp app/gateway scripts +``` + +Expected: all selected tests pass and Ruff reports no errors. + +- [ ] **Step 6: Run the complete Docker acceptance** + +Run from repository root: + +```powershell +powershell -ExecutionPolicy Bypass -File scripts/verify_flowlens_mcp.ps1 +``` + +Expected: script prints successful live/ready, discovery, authorized query, denied cross-tenant query, audit, and Timeline evidence checks. + +- [ ] **Step 7: Perform the user-led real-model demonstration** + +The user starts DeerFlow, creates one normal run and one controlled failure, then sends: + +```text +请使用 FlowLens Incident Response 调查本会话最近一次失败。 +先给出确定性根因，再引用关键 Timeline 事件；如果数据不完整请明确说明。 +``` + +Verify the final answer, the selected MCP tools, downloaded evidence, audit row, and `/agentops` Timeline together. Record only observed output. + +- [ ] **Step 8: Benchmark SQLite and enterprise profiles** + +Run both: + +```powershell +cd backend +uv run python scripts/benchmark_inspector_mcp.py --runs 1000 --events-per-run 100 --output ../artifacts/mcp-benchmark-sqlite.json +``` + +```powershell +uv run python scripts/benchmark_inspector_mcp.py --runs 1000 --events-per-run 100 --profile enterprise --output ../artifacts/mcp-benchmark-postgres.json +``` + +Do not place numbers in README until both output files exist and are reviewed. + +- [ ] **Step 9: Update documentation with verified commands and evidence** + +Document: + +- Architecture and trust boundaries. +- Tool contracts and Skill workflow. +- Local and enterprise startup. +- Token claims and scope table. +- Failure handling and audit fields. +- Three demonstration scenarios. +- Actual test and benchmark output with environment. +- Honest boundary: local end-to-end and fault-injection validation, not production customer deployment. + +- [ ] **Step 10: Run the final verification gate** + +Run: + +```powershell +cd backend +$env:PYTHONPATH='.' +uv run pytest -q +uv run ruff check . +cd .. +docker compose -f docker/docker-compose-dev.yaml config +docker compose -f docker/docker-compose.yaml -f docker/docker-compose-flowlens-mcp-enterprise.yaml config +powershell -ExecutionPolicy Bypass -File scripts/verify_flowlens_mcp.ps1 +git diff --check +git status --short +``` + +Expected: backend suite and Ruff pass, both Compose configurations validate, live-stack verification succeeds, no whitespace errors exist, and only intentional uncommitted evidence files remain. + +- [ ] **Step 11: Commit and report Stage 4** + +```powershell +git add backend/scripts scripts/verify_flowlens_mcp.ps1 backend/tests/test_flowlens_inspector_e2e.py README.md docs .github +git commit -m "docs(mcp): publish verified inspector workflow" +``` + +Stage 4 report must distinguish automated evidence, user-led real-model evidence, benchmark measurements, and unverified production assumptions. + +--- + +## Review Checkpoints + +- **Checkpoint A after Stage 1:** inspect JWT claims, Gateway Bearer behavior, tenant migration, and cross-tenant `404`. +- **Checkpoint B after Stage 2:** inspect five MCP schemas, a real Streamable HTTP tool call, ToolMessage return, and `mcp.tool.*` events. +- **Checkpoint C after Stage 3:** inspect timeout/retry/circuit behavior, audit redaction, Prometheus metrics, and both Docker profiles. +- **Checkpoint D after Stage 4:** run the demonstration together and compare every README/result claim against fresh artifacts. + +## Execution Handoff + +Plan complete and saved to `docs/superpowers/plans/2026-07-28-flowlens-inspector-mcp.md`. + +The selected execution method is **Inline Execution**: execute Stage 1 through Stage 4 in this task, stop after each stage for the user-facing report and hands-on explanation, and do not use task-per-subagent development. From e3ffa9322011239341fb0ec86197601b6cb12cb7 Mon Sep 17 00:00:00 2001 From: tangrongyuan <18532002267@163.com> Date: Wed, 29 Jul 2026 10:23:02 +0800 Subject: [PATCH 03/14] feat(agentops): add tenant-aware run identity --- backend/app/gateway/auth/models.py | 1 + .../app/gateway/auth/repositories/sqlite.py | 3 + backend/app/gateway/services.py | 4 + .../20260728_0001_flowlens_mcp_tenant.py | 93 ++++++++++++++++ .../harness/deerflow/persistence/run/model.py | 16 ++- .../harness/deerflow/persistence/run/sql.py | 47 +++++++- .../deerflow/persistence/user/model.py | 6 + .../harness/deerflow/runtime/runs/manager.py | 81 ++++++++++++-- .../deerflow/runtime/runs/store/base.py | 4 + .../deerflow/runtime/runs/store/memory.py | 31 +++++- .../harness/deerflow/runtime/user_context.py | 36 ++++++ .../tests/test_flowlens_tenant_migration.py | 75 +++++++++++++ backend/tests/test_tenant_run_isolation.py | 103 ++++++++++++++++++ 13 files changed, 486 insertions(+), 14 deletions(-) create mode 100644 backend/packages/harness/deerflow/persistence/migrations/versions/20260728_0001_flowlens_mcp_tenant.py create mode 100644 backend/tests/test_flowlens_tenant_migration.py create mode 100644 backend/tests/test_tenant_run_isolation.py diff --git a/backend/app/gateway/auth/models.py b/backend/app/gateway/auth/models.py index 25c6476f..fbf435ec 100644 --- a/backend/app/gateway/auth/models.py +++ b/backend/app/gateway/auth/models.py @@ -18,6 +18,7 @@ class User(BaseModel): model_config = ConfigDict(from_attributes=True) id: UUID = Field(default_factory=uuid4, description="Primary key") + tenant_id: str = Field(default="default", description="Owning tenant") email: EmailStr = Field(..., description="Unique email address") password_hash: str | None = Field(None, description="bcrypt hash, nullable for OAuth users") system_role: Literal["admin", "user"] = Field(default="user") diff --git a/backend/app/gateway/auth/repositories/sqlite.py b/backend/app/gateway/auth/repositories/sqlite.py index 3ee3978e..39f4a61c 100644 --- a/backend/app/gateway/auth/repositories/sqlite.py +++ b/backend/app/gateway/auth/repositories/sqlite.py @@ -36,6 +36,7 @@ def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None: def _row_to_user(row: UserRow) -> User: return User( id=UUID(row.id), + tenant_id=row.tenant_id, email=row.email, password_hash=row.password_hash, system_role=row.system_role, # type: ignore[arg-type] @@ -52,6 +53,7 @@ def _row_to_user(row: UserRow) -> User: def _user_to_row(user: User) -> UserRow: return UserRow( id=str(user.id), + tenant_id=user.tenant_id, email=user.email, password_hash=user.password_hash, system_role=user.system_role, @@ -100,6 +102,7 @@ async def update_user(self, user: User) -> User: # a row that no longer exists. raise UserNotFoundError(f"User {user.id} no longer exists") row.email = user.email + row.tenant_id = user.tenant_id row.password_hash = user.password_hash row.system_role = user.system_role row.oauth_provider = user.oauth_provider diff --git a/backend/app/gateway/services.py b/backend/app/gateway/services.py index 2c5c01e6..524915d5 100644 --- a/backend/app/gateway/services.py +++ b/backend/app/gateway/services.py @@ -35,6 +35,7 @@ run_agent, ) from deerflow.runtime.runs.naming import resolve_root_run_name +from deerflow.runtime.user_context import DEFAULT_TENANT_ID logger = logging.getLogger(__name__) @@ -182,6 +183,9 @@ def inject_authenticated_user_context(config: dict[str, Any], request: Request) runtime_context = config.setdefault("context", {}) if isinstance(runtime_context, dict): runtime_context["user_id"] = str(user_id) + runtime_context["tenant_id"] = str( + getattr(user, "tenant_id", None) or DEFAULT_TENANT_ID + ) def resolve_agent_factory(assistant_id: str | None): diff --git a/backend/packages/harness/deerflow/persistence/migrations/versions/20260728_0001_flowlens_mcp_tenant.py b/backend/packages/harness/deerflow/persistence/migrations/versions/20260728_0001_flowlens_mcp_tenant.py new file mode 100644 index 00000000..9146dde5 --- /dev/null +++ b/backend/packages/harness/deerflow/persistence/migrations/versions/20260728_0001_flowlens_mcp_tenant.py @@ -0,0 +1,93 @@ +"""Add tenant ownership to users and runs. + +Revision ID: 20260728_0001 +Revises: +Create Date: 2026-07-28 +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +import sqlalchemy as sa +from alembic import op + +revision = "20260728_0001" +down_revision = None +branch_labels = None +depends_on = None + +_DEFAULT_TENANT_ID = "default" +_INDEX_NAME = "ix_runs_tenant_user_created" + + +def _columns(table_name: str) -> dict[str, Mapping[str, Any]]: + inspector = sa.inspect(op.get_bind()) + return { + column["name"]: column + for column in inspector.get_columns(table_name) + } + + +def _index_names(table_name: str) -> set[str]: + inspector = sa.inspect(op.get_bind()) + return { + index["name"] + for index in inspector.get_indexes(table_name) + if index.get("name") + } + + +def _ensure_tenant_column(table_name: str) -> None: + columns = _columns(table_name) + if "tenant_id" not in columns: + op.add_column( + table_name, + sa.Column( + "tenant_id", + sa.String(length=64), + nullable=False, + server_default=sa.text(f"'{_DEFAULT_TENANT_ID}'"), + ), + ) + return + + op.execute( + sa.text( + f"UPDATE {table_name} " + "SET tenant_id = :tenant_id " + "WHERE tenant_id IS NULL OR tenant_id = ''" + ).bindparams(tenant_id=_DEFAULT_TENANT_ID) + ) + if columns["tenant_id"].get("nullable", True): + with op.batch_alter_table(table_name) as batch_op: + batch_op.alter_column( + "tenant_id", + existing_type=sa.String(length=64), + nullable=False, + server_default=sa.text(f"'{_DEFAULT_TENANT_ID}'"), + ) + + +def upgrade() -> None: + _ensure_tenant_column("users") + _ensure_tenant_column("runs") + + if _INDEX_NAME not in _index_names("runs"): + op.create_index( + _INDEX_NAME, + "runs", + ["tenant_id", "user_id", "created_at"], + unique=False, + ) + + +def downgrade() -> None: + if _INDEX_NAME in _index_names("runs"): + op.drop_index(_INDEX_NAME, table_name="runs") + + for table_name in ("runs", "users"): + if "tenant_id" in _columns(table_name): + with op.batch_alter_table(table_name) as batch_op: + batch_op.drop_column("tenant_id") diff --git a/backend/packages/harness/deerflow/persistence/run/model.py b/backend/packages/harness/deerflow/persistence/run/model.py index d0dfe408..d6396303 100644 --- a/backend/packages/harness/deerflow/persistence/run/model.py +++ b/backend/packages/harness/deerflow/persistence/run/model.py @@ -17,6 +17,12 @@ class RunRow(Base): thread_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True) assistant_id: Mapped[str | None] = mapped_column(String(128)) user_id: Mapped[str | None] = mapped_column(String(64), index=True) + tenant_id: Mapped[str] = mapped_column( + String(64), + nullable=False, + default="default", + server_default="default", + ) status: Mapped[str] = mapped_column(String(20), default="pending") # "pending" | "running" | "success" | "error" | "timeout" | "interrupted" @@ -46,4 +52,12 @@ class RunRow(Base): created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC)) updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)) - __table_args__ = (Index("ix_runs_thread_status", "thread_id", "status"),) + __table_args__ = ( + Index("ix_runs_thread_status", "thread_id", "status"), + Index( + "ix_runs_tenant_user_created", + "tenant_id", + "user_id", + "created_at", + ), + ) diff --git a/backend/packages/harness/deerflow/persistence/run/sql.py b/backend/packages/harness/deerflow/persistence/run/sql.py index b0b0bf97..acaf3d03 100644 --- a/backend/packages/harness/deerflow/persistence/run/sql.py +++ b/backend/packages/harness/deerflow/persistence/run/sql.py @@ -17,7 +17,12 @@ from deerflow.persistence.run.model import RunRow from deerflow.runtime.runs.cursor import decode_run_cursor from deerflow.runtime.runs.store.base import RunStore -from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id +from deerflow.runtime.user_context import ( + AUTO, + _AutoSentinel, + resolve_tenant_id, + resolve_user_id, +) from deerflow.utils.time import coerce_iso @@ -86,6 +91,7 @@ async def put( thread_id, assistant_id=None, user_id: str | None | _AutoSentinel = AUTO, + tenant_id: str | None | _AutoSentinel = AUTO, model_name: str | None = None, status="pending", multitask_strategy="reject", @@ -102,12 +108,17 @@ async def put( commit from turning the retry into a primary-key failure. """ resolved_user_id = resolve_user_id(user_id, method_name="RunRepository.put") + resolved_tenant_id = resolve_tenant_id( + tenant_id, + method_name="RunRepository.put", + ) now = datetime.now(UTC) created = datetime.fromisoformat(created_at) if created_at else now values = { "thread_id": thread_id, "assistant_id": assistant_id, "user_id": resolved_user_id, + "tenant_id": resolved_tenant_id, "model_name": self._normalize_model_name(model_name), "status": status, "multitask_strategy": multitask_strategy, @@ -131,14 +142,24 @@ async def get( run_id, *, user_id: str | None | _AutoSentinel = AUTO, + tenant_id: str | None | _AutoSentinel = AUTO, ): resolved_user_id = resolve_user_id(user_id, method_name="RunRepository.get") + resolved_tenant_id = resolve_tenant_id( + tenant_id, + method_name="RunRepository.get", + ) async with self._sf() as session: row = await session.get(RunRow, run_id) if row is None: return None if resolved_user_id is not None and row.user_id != resolved_user_id: return None + if ( + resolved_tenant_id is not None + and row.tenant_id != resolved_tenant_id + ): + return None return self._row_to_dict(row) async def list_by_thread( @@ -146,12 +167,19 @@ async def list_by_thread( thread_id, *, user_id: str | None | _AutoSentinel = AUTO, + tenant_id: str | None | _AutoSentinel = AUTO, limit=100, ): resolved_user_id = resolve_user_id(user_id, method_name="RunRepository.list_by_thread") + resolved_tenant_id = resolve_tenant_id( + tenant_id, + method_name="RunRepository.list_by_thread", + ) stmt = select(RunRow).where(RunRow.thread_id == thread_id) if resolved_user_id is not None: stmt = stmt.where(RunRow.user_id == resolved_user_id) + if resolved_tenant_id is not None: + stmt = stmt.where(RunRow.tenant_id == resolved_tenant_id) stmt = stmt.order_by(RunRow.created_at.desc()).limit(limit) async with self._sf() as session: result = await session.execute(stmt) @@ -161,6 +189,7 @@ async def list_recent( self, *, user_id: str | None | _AutoSentinel = AUTO, + tenant_id: str | None | _AutoSentinel = AUTO, limit: int = 50, cursor: str | None = None, status: str | None = None, @@ -168,9 +197,15 @@ async def list_recent( started_before: str | None = None, ) -> list[dict[str, Any]]: resolved_user_id = resolve_user_id(user_id, method_name="RunRepository.list_recent") + resolved_tenant_id = resolve_tenant_id( + tenant_id, + method_name="RunRepository.list_recent", + ) stmt = select(RunRow) if resolved_user_id is not None: stmt = stmt.where(RunRow.user_id == resolved_user_id) + if resolved_tenant_id is not None: + stmt = stmt.where(RunRow.tenant_id == resolved_tenant_id) if status is not None: stmt = stmt.where(RunRow.status == status) if started_after is not None: @@ -210,14 +245,24 @@ async def delete( run_id, *, user_id: str | None | _AutoSentinel = AUTO, + tenant_id: str | None | _AutoSentinel = AUTO, ): resolved_user_id = resolve_user_id(user_id, method_name="RunRepository.delete") + resolved_tenant_id = resolve_tenant_id( + tenant_id, + method_name="RunRepository.delete", + ) async with self._sf() as session: row = await session.get(RunRow, run_id) if row is None: return if resolved_user_id is not None and row.user_id != resolved_user_id: return + if ( + resolved_tenant_id is not None + and row.tenant_id != resolved_tenant_id + ): + return await session.delete(row) await session.commit() diff --git a/backend/packages/harness/deerflow/persistence/user/model.py b/backend/packages/harness/deerflow/persistence/user/model.py index 130d4bfc..53fc5225 100644 --- a/backend/packages/harness/deerflow/persistence/user/model.py +++ b/backend/packages/harness/deerflow/persistence/user/model.py @@ -24,6 +24,12 @@ class UserRow(Base): # UUIDs are stored as 36-char strings for cross-backend portability. id: Mapped[str] = mapped_column(String(36), primary_key=True) + tenant_id: Mapped[str] = mapped_column( + String(64), + nullable=False, + default="default", + server_default="default", + ) email: Mapped[str] = mapped_column(String(320), unique=True, nullable=False, index=True) password_hash: Mapped[str | None] = mapped_column(String(128), nullable=True) diff --git a/backend/packages/harness/deerflow/runtime/runs/manager.py b/backend/packages/harness/deerflow/runtime/runs/manager.py index 67d1daec..00bc81bf 100644 --- a/backend/packages/harness/deerflow/runtime/runs/manager.py +++ b/backend/packages/harness/deerflow/runtime/runs/manager.py @@ -10,6 +10,12 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any +from deerflow.runtime.user_context import ( + DEFAULT_TENANT_ID, + DEFAULT_USER_ID, + get_effective_tenant_id, + get_effective_user_id, +) from deerflow.utils.time import now_iso as _now_iso from .schemas import DisconnectMode, RunStatus @@ -80,6 +86,8 @@ class RunRecord: assistant_id: str | None status: RunStatus on_disconnect: DisconnectMode + user_id: str = DEFAULT_USER_ID + tenant_id: str = DEFAULT_TENANT_ID multitask_strategy: str = "reject" metadata: dict = field(default_factory=dict) kwargs: dict = field(default_factory=dict) @@ -127,6 +135,8 @@ def _store_put_payload(record: RunRecord, *, error: str | None = None) -> dict[s return { "thread_id": record.thread_id, "assistant_id": record.assistant_id, + "user_id": record.user_id, + "tenant_id": record.tenant_id, "status": record.status.value, "multitask_strategy": record.multitask_strategy, "metadata": record.metadata or {}, @@ -236,6 +246,8 @@ def _record_from_store(row: dict[str, Any]) -> RunRecord: assistant_id=row.get("assistant_id"), status=RunStatus(row.get("status") or RunStatus.pending.value), on_disconnect=DisconnectMode(row.get("on_disconnect") or DisconnectMode.cancel.value), + user_id=str(row.get("user_id") or DEFAULT_USER_ID), + tenant_id=str(row.get("tenant_id") or DEFAULT_TENANT_ID), multitask_strategy=row.get("multitask_strategy") or "reject", metadata=row.get("metadata") or {}, kwargs=row.get("kwargs") or {}, @@ -330,6 +342,8 @@ async def create( assistant_id=assistant_id, status=RunStatus.pending, on_disconnect=on_disconnect, + user_id=get_effective_user_id(), + tenant_id=get_effective_tenant_id(), multitask_strategy=multitask_strategy, metadata=metadata or {}, kwargs=kwargs or {}, @@ -352,21 +366,36 @@ async def create( logger.info("Run created: run_id=%s thread_id=%s", run_id, thread_id) return record - async def get(self, run_id: str, *, user_id: str | None = None) -> RunRecord | None: + async def get( + self, + run_id: str, + *, + user_id: str | None = None, + tenant_id: str | None = None, + ) -> RunRecord | None: """Return a run record by ID, or ``None``. Args: run_id: The run ID to look up. - user_id: Optional user ID for permission filtering when hydrating from store. + user_id: Optional user ID for permission filtering. + tenant_id: Optional tenant ID for permission filtering. """ async with self._lock: record = self._runs.get(run_id) if record is not None: + if user_id is not None and record.user_id != user_id: + return None + if tenant_id is not None and record.tenant_id != tenant_id: + return None return record if self._store is None: return None try: - row = await self._store.get(run_id, user_id=user_id) + row = await self._store.get( + run_id, + user_id=user_id, + tenant_id=tenant_id, + ) except Exception: logger.warning("Failed to hydrate run %s from store", run_id, exc_info=True) return None @@ -375,6 +404,10 @@ async def get(self, run_id: str, *, user_id: str | None = None) -> RunRecord | N async with self._lock: record = self._runs.get(run_id) if record is not None: + if user_id is not None and record.user_id != user_id: + return None + if tenant_id is not None and record.tenant_id != tenant_id: + return None return record if row is None: return None @@ -384,14 +417,31 @@ async def get(self, run_id: str, *, user_id: str | None = None) -> RunRecord | N logger.warning("Failed to map store row for run %s", run_id, exc_info=True) return None - async def aget(self, run_id: str, *, user_id: str | None = None) -> RunRecord | None: + async def aget( + self, + run_id: str, + *, + user_id: str | None = None, + tenant_id: str | None = None, + ) -> RunRecord | None: """Return a run record by ID, checking the persistent store as fallback. Alias for :meth:`get` for backward compatibility. """ - return await self.get(run_id, user_id=user_id) + return await self.get( + run_id, + user_id=user_id, + tenant_id=tenant_id, + ) - async def list_by_thread(self, thread_id: str, *, user_id: str | None = None, limit: int = 100) -> list[RunRecord]: + async def list_by_thread( + self, + thread_id: str, + *, + user_id: str | None = None, + tenant_id: str | None = None, + limit: int = 100, + ) -> list[RunRecord]: """Return runs for a given thread, newest first, at most ``limit`` records. In-memory runs take precedence only when the same ``run_id`` exists in both @@ -405,13 +455,24 @@ async def list_by_thread(self, thread_id: str, *, user_id: str | None = None, li """ async with self._lock: # Dict insertion order gives deterministic results when timestamps tie. - memory_records = [r for r in self._runs.values() if r.thread_id == thread_id] + memory_records = [ + record + for record in self._runs.values() + if record.thread_id == thread_id + and (user_id is None or record.user_id == user_id) + and (tenant_id is None or record.tenant_id == tenant_id) + ] if self._store is None: return sorted(memory_records, key=lambda r: r.created_at, reverse=True)[:limit] records_by_id = {record.run_id: record for record in memory_records} store_limit = max(0, limit - len(memory_records)) try: - rows = await self._store.list_by_thread(thread_id, user_id=user_id, limit=store_limit) + rows = await self._store.list_by_thread( + thread_id, + user_id=user_id, + tenant_id=tenant_id, + limit=store_limit, + ) except Exception: logger.warning("Failed to hydrate runs for thread %s from store", thread_id, exc_info=True) return sorted(memory_records, key=lambda r: r.created_at, reverse=True)[:limit] @@ -428,6 +489,7 @@ async def list_recent( self, *, user_id: str | None, + tenant_id: str | None = None, limit: int = 50, cursor: str | None = None, status: str | None = None, @@ -440,6 +502,7 @@ async def list_recent( return [] rows = await self._store.list_recent( user_id=user_id, + tenant_id=tenant_id, limit=limit, cursor=cursor, status=status, @@ -573,6 +636,8 @@ async def create_or_reject( assistant_id=assistant_id, status=RunStatus.pending, on_disconnect=on_disconnect, + user_id=get_effective_user_id(), + tenant_id=get_effective_tenant_id(), multitask_strategy=multitask_strategy, metadata=metadata or {}, kwargs=kwargs or {}, diff --git a/backend/packages/harness/deerflow/runtime/runs/store/base.py b/backend/packages/harness/deerflow/runtime/runs/store/base.py index 05bf0caf..6772c95d 100644 --- a/backend/packages/harness/deerflow/runtime/runs/store/base.py +++ b/backend/packages/harness/deerflow/runtime/runs/store/base.py @@ -23,6 +23,7 @@ async def put( thread_id: str, assistant_id: str | None = None, user_id: str | None = None, + tenant_id: str | None = None, model_name: str | None = None, status: str = "pending", multitask_strategy: str = "reject", @@ -39,6 +40,7 @@ async def get( run_id: str, *, user_id: str | None = None, + tenant_id: str | None = None, ) -> dict[str, Any] | None: pass @@ -48,6 +50,7 @@ async def list_by_thread( thread_id: str, *, user_id: str | None = None, + tenant_id: str | None = None, limit: int = 100, ) -> list[dict[str, Any]]: pass @@ -56,6 +59,7 @@ async def list_recent( self, *, user_id: str | None, + tenant_id: str | None = None, limit: int = 50, cursor: str | None = None, status: str | None = None, diff --git a/backend/packages/harness/deerflow/runtime/runs/store/memory.py b/backend/packages/harness/deerflow/runtime/runs/store/memory.py index 7985d89a..aeebdfa7 100644 --- a/backend/packages/harness/deerflow/runtime/runs/store/memory.py +++ b/backend/packages/harness/deerflow/runtime/runs/store/memory.py @@ -23,6 +23,7 @@ async def put( thread_id, assistant_id=None, user_id=None, + tenant_id=None, model_name=None, status="pending", multitask_strategy="reject", @@ -37,6 +38,7 @@ async def put( "thread_id": thread_id, "assistant_id": assistant_id, "user_id": user_id, + "tenant_id": tenant_id, "model_name": model_name, "status": status, "multitask_strategy": multitask_strategy, @@ -47,16 +49,31 @@ async def put( "updated_at": now, } - async def get(self, run_id, *, user_id=None): + async def get(self, run_id, *, user_id=None, tenant_id=None): run = self._runs.get(run_id) if run is None: return None if user_id is not None and run.get("user_id") != user_id: return None + if tenant_id is not None and run.get("tenant_id") != tenant_id: + return None return run - async def list_by_thread(self, thread_id, *, user_id=None, limit=100): - results = [r for r in self._runs.values() if r["thread_id"] == thread_id and (user_id is None or r.get("user_id") == user_id)] + async def list_by_thread( + self, + thread_id, + *, + user_id=None, + tenant_id=None, + limit=100, + ): + results = [ + run + for run in self._runs.values() + if run["thread_id"] == thread_id + and (user_id is None or run.get("user_id") == user_id) + and (tenant_id is None or run.get("tenant_id") == tenant_id) + ] results.sort(key=lambda r: r["created_at"], reverse=True) return results[:limit] @@ -64,13 +81,19 @@ async def list_recent( self, *, user_id, + tenant_id=None, limit=50, cursor=None, status=None, started_after=None, started_before=None, ): - results = [run for run in self._runs.values() if user_id is None or run.get("user_id") == user_id] + results = [ + run + for run in self._runs.values() + if (user_id is None or run.get("user_id") == user_id) + and (tenant_id is None or run.get("tenant_id") == tenant_id) + ] if status is not None: results = [run for run in results if run.get("status") == status] if started_after is not None: diff --git a/backend/packages/harness/deerflow/runtime/user_context.py b/backend/packages/harness/deerflow/runtime/user_context.py index cfbb68c9..a21f19fc 100644 --- a/backend/packages/harness/deerflow/runtime/user_context.py +++ b/backend/packages/harness/deerflow/runtime/user_context.py @@ -95,6 +95,7 @@ def require_current_user() -> CurrentUser: # --------------------------------------------------------------------------- DEFAULT_USER_ID: Final[str] = "default" +DEFAULT_TENANT_ID: Final[str] = "default" def get_effective_user_id() -> str: @@ -109,6 +110,13 @@ def get_effective_user_id() -> str: return str(user.id) +def get_effective_tenant_id() -> str: + """Return the current tenant id, defaulting legacy users to ``default``.""" + user = _current_user.get() + tenant_id = getattr(user, "tenant_id", None) if user is not None else None + return str(tenant_id or DEFAULT_TENANT_ID) + + def resolve_runtime_user_id(runtime: object | None) -> str: """Single source of truth for a tool/middleware's effective user_id. @@ -137,6 +145,16 @@ def resolve_runtime_user_id(runtime: object | None) -> str: return get_effective_user_id() +def resolve_runtime_tenant_id(runtime: object | None) -> str: + """Resolve tenant identity from runtime context, then request context.""" + context = getattr(runtime, "context", None) + if isinstance(context, dict): + tenant_id = context.get("tenant_id") + if tenant_id: + return str(tenant_id) + return get_effective_tenant_id() + + # --------------------------------------------------------------------------- # Sentinel-based user_id resolution # --------------------------------------------------------------------------- @@ -193,3 +211,21 @@ def resolve_user_id( # rather than ripple a type change through every caller. return str(user.id) return value + + +def resolve_tenant_id( + value: str | None | _AutoSentinel, + *, + method_name: str = "repository method", +) -> str | None: + """Resolve a repository tenant filter with the same semantics as user_id.""" + if isinstance(value, _AutoSentinel): + user = _current_user.get() + if user is None: + raise RuntimeError( + f"{method_name} called with tenant_id=AUTO but no user context is set; " + "pass an explicit tenant_id, set the contextvar via auth middleware, " + "or opt out with tenant_id=None for migration/CLI paths." + ) + return str(getattr(user, "tenant_id", None) or DEFAULT_TENANT_ID) + return value diff --git a/backend/tests/test_flowlens_tenant_migration.py b/backend/tests/test_flowlens_tenant_migration.py new file mode 100644 index 00000000..4896e674 --- /dev/null +++ b/backend/tests/test_flowlens_tenant_migration.py @@ -0,0 +1,75 @@ +"""Integration test for the idempotent FlowLens tenant migration.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import sqlalchemy as sa +from alembic.migration import MigrationContext +from alembic.operations import Operations + + +def _load_migration(): + migration_path = ( + Path(__file__).parents[1] + / "packages" + / "harness" + / "deerflow" + / "persistence" + / "migrations" + / "versions" + / "20260728_0001_flowlens_mcp_tenant.py" + ) + spec = importlib.util.spec_from_file_location( + "flowlens_tenant_migration", + migration_path, + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_tenant_migration_upgrades_legacy_schema_idempotently(tmp_path): + engine = sa.create_engine(f"sqlite:///{tmp_path / 'legacy.db'}") + metadata = sa.MetaData() + sa.Table( + "users", + metadata, + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("email", sa.String(320), nullable=False), + ) + sa.Table( + "runs", + metadata, + sa.Column("run_id", sa.String(64), primary_key=True), + sa.Column("user_id", sa.String(64), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=True), + ) + metadata.create_all(engine) + + migration = _load_migration() + with engine.begin() as connection: + operations = Operations(MigrationContext.configure(connection)) + migration.op = operations + migration.upgrade() + migration.upgrade() + + inspector = sa.inspect(connection) + user_columns = { + column["name"]: column + for column in inspector.get_columns("users") + } + run_columns = { + column["name"]: column + for column in inspector.get_columns("runs") + } + run_indexes = { + index["name"] + for index in inspector.get_indexes("runs") + } + + assert user_columns["tenant_id"]["nullable"] is False + assert run_columns["tenant_id"]["nullable"] is False + assert "ix_runs_tenant_user_created" in run_indexes diff --git a/backend/tests/test_tenant_run_isolation.py b/backend/tests/test_tenant_run_isolation.py new file mode 100644 index 00000000..14964ba1 --- /dev/null +++ b/backend/tests/test_tenant_run_isolation.py @@ -0,0 +1,103 @@ +"""Tenant-aware ownership tests for users, runtime context, and runs.""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from app.gateway.auth.models import User +from app.gateway.services import inject_authenticated_user_context +from deerflow.persistence.run import RunRepository +from deerflow.runtime import RunManager +from deerflow.runtime.runs.store.memory import MemoryRunStore +from deerflow.runtime.user_context import reset_current_user, set_current_user + + +async def _make_repo(tmp_path) -> RunRepository: + from deerflow.persistence.engine import get_session_factory, init_engine + + url = f"sqlite+aiosqlite:///{tmp_path / 'tenant-runs.db'}" + await init_engine("sqlite", url=url, sqlite_dir=str(tmp_path)) + return RunRepository(get_session_factory()) + + +async def _cleanup_repo() -> None: + from deerflow.persistence.engine import close_engine + + await close_engine() + + +@pytest.mark.anyio +async def test_run_repository_requires_matching_tenant_and_user(tmp_path): + repo = await _make_repo(tmp_path) + try: + await repo.put( + "run-a", + thread_id="thread-a", + user_id="user-a", + tenant_id="tenant-a", + ) + + assert await repo.get("run-a", user_id="user-a", tenant_id="tenant-a") is not None + assert await repo.get("run-a", user_id="user-a", tenant_id="tenant-b") is None + assert await repo.get("run-a", user_id="user-b", tenant_id="tenant-a") is None + finally: + await _cleanup_repo() + + +@pytest.mark.anyio +async def test_memory_store_applies_the_same_tenant_and_user_filter(): + store = MemoryRunStore() + await store.put( + "run-a", + thread_id="thread-a", + user_id="user-a", + tenant_id="tenant-a", + ) + + assert await store.get("run-a", user_id="user-a", tenant_id="tenant-a") is not None + assert await store.get("run-a", user_id="user-a", tenant_id="tenant-b") is None + assert await store.get("run-a", user_id="user-b", tenant_id="tenant-a") is None + + +@pytest.mark.anyio +async def test_run_manager_captures_immutable_owner_and_filters_memory(): + manager = RunManager(store=MemoryRunStore()) + token = set_current_user( + SimpleNamespace(id="user-a", tenant_id="tenant-a"), + ) + try: + record = await manager.create("thread-a") + finally: + reset_current_user(token) + + assert record.user_id == "user-a" + assert record.tenant_id == "tenant-a" + assert await manager.get(record.run_id, user_id="user-a", tenant_id="tenant-a") is record + assert await manager.get(record.run_id, user_id="user-a", tenant_id="tenant-b") is None + assert await manager.get(record.run_id, user_id="user-b", tenant_id="tenant-a") is None + + +def test_authenticated_context_includes_server_owned_tenant(): + config: dict = {} + request = SimpleNamespace( + state=SimpleNamespace( + user=SimpleNamespace( + id="user-a", + tenant_id="tenant-a", + system_role="user", + ), + ), + ) + + inject_authenticated_user_context(config, request) + + assert config["context"]["user_id"] == "user-a" + assert config["context"]["tenant_id"] == "tenant-a" + + +def test_legacy_user_defaults_to_default_tenant(): + user = User(email="legacy@example.com") + + assert user.tenant_id == "default" From 127be4b35842ad75b0a3c36fa76b658c0e06a7e7 Mon Sep 17 00:00:00 2001 From: tangrongyuan <18532002267@163.com> Date: Wed, 29 Jul 2026 10:32:06 +0800 Subject: [PATCH 04/14] feat(auth): issue scoped MCP delegation tokens --- backend/app/gateway/app.py | 7 + backend/app/gateway/auth/platform_tokens.py | 315 ++++++++++++++++++++ backend/app/gateway/auth_middleware.py | 96 +++++- backend/app/gateway/csrf_middleware.py | 1 + backend/app/gateway/deps.py | 5 +- backend/app/gateway/routers/mcp_tokens.py | 115 +++++++ backend/tests/test_mcp_bearer_auth.py | 203 +++++++++++++ backend/tests/test_mcp_platform_tokens.py | 152 ++++++++++ backend/tests/test_mcp_token_api.py | 194 ++++++++++++ 9 files changed, 1075 insertions(+), 13 deletions(-) create mode 100644 backend/app/gateway/auth/platform_tokens.py create mode 100644 backend/app/gateway/routers/mcp_tokens.py create mode 100644 backend/tests/test_mcp_bearer_auth.py create mode 100644 backend/tests/test_mcp_platform_tokens.py create mode 100644 backend/tests/test_mcp_token_api.py diff --git a/backend/app/gateway/app.py b/backend/app/gateway/app.py index 30662d2f..3069a0b6 100644 --- a/backend/app/gateway/app.py +++ b/backend/app/gateway/app.py @@ -19,6 +19,7 @@ channels, feedback, mcp, + mcp_tokens, memory, models, runs, @@ -178,6 +179,11 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: logger.exception(error_msg) raise RuntimeError(error_msg) from e config = get_gateway_config() + from app.gateway.auth.platform_tokens import ( + ensure_platform_tokens_configured_if_enabled, + ) + + ensure_platform_tokens_configured_if_enabled() logger.info(f"Starting API Gateway on {config.host}:{config.port}") # Pre-warm tiktoken encoding cache so the first memory-injection request @@ -385,6 +391,7 @@ def create_app() -> FastAPI: # Auth API is mounted at /api/v1/auth app.include_router(auth.router) + app.include_router(mcp_tokens.router) # Feedback API is mounted at /api/threads/{thread_id}/runs/{run_id}/feedback app.include_router(feedback.router) diff --git a/backend/app/gateway/auth/platform_tokens.py b/backend/app/gateway/auth/platform_tokens.py new file mode 100644 index 00000000..2b16591c --- /dev/null +++ b/backend/app/gateway/auth/platform_tokens.py @@ -0,0 +1,315 @@ +"""RS256 platform tokens used by DeerFlow Runtime and FlowLens MCP.""" + +from __future__ import annotations + +import base64 +import os +import uuid +from dataclasses import dataclass, field +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any + +import jwt +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey +from pydantic import BaseModel, ValidationError, field_validator + +PLATFORM_ALGORITHM = "RS256" +TOOLS_DISCOVERY_SCOPE = "agentops.tools.discover" +AGENTOPS_READ_SCOPES: tuple[str, ...] = ( + "agentops.health.read", + "agentops.runs.read", + "agentops.timeline.read", + "agentops.diagnostics.read", + "agentops.runs.compare", +) +_TRUTHY_VALUES = frozenset({"1", "true", "yes", "on"}) + + +class PlatformTokenError(ValueError): + """Raised when platform token configuration or validation fails.""" + + +@dataclass(frozen=True, slots=True) +class PlatformTokenSettings: + """Key material and claim policy for FlowLens platform tokens.""" + + private_key_pem: str | None = field(repr=False) + public_key_pem: str = field(repr=False) + kid: str + issuer: str + audience: str + client_id: str + client_secret: str = field(repr=False) + token_ttl_seconds: int = 300 + + @classmethod + def from_env( + cls, + *, + require_private_key: bool = True, + ) -> PlatformTokenSettings: + """Load platform token settings from environment-backed PEM files.""" + private_path = os.getenv("FLOWLENS_MCP_JWT_PRIVATE_KEY_PATH", "").strip() + public_path = os.getenv("FLOWLENS_MCP_JWT_PUBLIC_KEY_PATH", "").strip() + client_secret = os.getenv("FLOWLENS_MCP_CLIENT_SECRET", "") + + if not public_path: + raise PlatformTokenError( + "FLOWLENS_MCP_JWT_PUBLIC_KEY_PATH is required", + ) + if require_private_key and not private_path: + raise PlatformTokenError( + "FLOWLENS_MCP_JWT_PRIVATE_KEY_PATH is required", + ) + if not client_secret: + raise PlatformTokenError("FLOWLENS_MCP_CLIENT_SECRET is required") + + try: + ttl_seconds = int( + os.getenv("FLOWLENS_MCP_TOKEN_TTL_SECONDS", "300"), + ) + except ValueError as exc: + raise PlatformTokenError( + "FLOWLENS_MCP_TOKEN_TTL_SECONDS must be an integer", + ) from exc + if ttl_seconds <= 0: + raise PlatformTokenError( + "FLOWLENS_MCP_TOKEN_TTL_SECONDS must be positive", + ) + + return cls( + private_key_pem=( + _read_key_file(private_path) + if private_path + else None + ), + public_key_pem=_read_key_file(public_path), + kid=os.getenv( + "FLOWLENS_MCP_JWT_KID", + "flowlens-local-1", + ).strip() + or "flowlens-local-1", + issuer=os.getenv( + "FLOWLENS_MCP_ISSUER", + "deerflow-gateway", + ).strip() + or "deerflow-gateway", + audience=os.getenv( + "FLOWLENS_MCP_AUDIENCE", + "flowlens-platform", + ).strip() + or "flowlens-platform", + client_id=os.getenv( + "FLOWLENS_MCP_CLIENT_ID", + "deerflow-runtime", + ).strip() + or "deerflow-runtime", + client_secret=client_secret, + token_ttl_seconds=ttl_seconds, + ) + + +class PlatformTokenClaims(BaseModel): + """Validated immutable identity and authorization claims.""" + + model_config = {"frozen": True} + + iss: str + sub: str + tenant_id: str + aud: str | list[str] + scope: list[str] + origin_run_id: str | None = None + azp: str + jti: str + iat: datetime + exp: datetime + ver: int + + @field_validator("scope") + @classmethod + def _require_unique_scopes(cls, value: list[str]) -> list[str]: + if not value or any(not scope for scope in value): + raise ValueError("scope must contain non-empty values") + if len(value) != len(set(value)): + raise ValueError("scope values must be unique") + return value + + +class PlatformTokenIssuer: + """Issue short-lived service and delegated platform tokens.""" + + def __init__(self, settings: PlatformTokenSettings) -> None: + if not settings.private_key_pem: + raise PlatformTokenError( + "Private key is required for token issuance", + ) + self._settings = settings + + def issue_service_token(self) -> str: + """Issue a least-privilege client token for MCP tool discovery.""" + return self._issue( + subject=self._settings.client_id, + tenant_id="system", + scopes=[TOOLS_DISCOVERY_SCOPE], + origin_run_id=None, + token_version=0, + ) + + def issue_delegated_token( + self, + *, + user_id: str, + tenant_id: str, + token_version: int, + origin_run_id: str, + ) -> str: + """Issue a user-bound token for read-only AgentOps inspection.""" + return self._issue( + subject=user_id, + tenant_id=tenant_id, + scopes=list(AGENTOPS_READ_SCOPES), + origin_run_id=origin_run_id, + token_version=token_version, + ) + + def _issue( + self, + *, + subject: str, + tenant_id: str, + scopes: list[str], + origin_run_id: str | None, + token_version: int, + ) -> str: + now = datetime.now(UTC) + payload = { + "iss": self._settings.issuer, + "sub": subject, + "tenant_id": tenant_id, + "aud": self._settings.audience, + "scope": scopes, + "origin_run_id": origin_run_id, + "azp": self._settings.client_id, + "jti": str(uuid.uuid4()), + "iat": now, + "exp": now + + timedelta(seconds=self._settings.token_ttl_seconds), + "ver": token_version, + } + return jwt.encode( + payload, + self._settings.private_key_pem, + algorithm=PLATFORM_ALGORITHM, + headers={"kid": self._settings.kid}, + ) + + +def ensure_platform_tokens_configured_if_enabled( +) -> PlatformTokenSettings | None: + """Validate key material at startup when FlowLens MCP is enabled.""" + enabled = ( + os.getenv("FLOWLENS_MCP_ENABLED", "").strip().lower() + in _TRUTHY_VALUES + ) + if not enabled: + return None + return PlatformTokenSettings.from_env() + + +def verify_platform_token( + token: str, + settings: PlatformTokenSettings | None = None, +) -> PlatformTokenClaims: + """Verify signature, key id, issuer, audience, time, and claims.""" + resolved_settings = settings or PlatformTokenSettings.from_env( + require_private_key=False, + ) + try: + header = jwt.get_unverified_header(token) + if header.get("alg") != PLATFORM_ALGORITHM: + raise PlatformTokenError("Unsupported platform token algorithm") + if header.get("kid") != resolved_settings.kid: + raise PlatformTokenError("Unknown platform token key id") + + payload = jwt.decode( + token, + resolved_settings.public_key_pem, + algorithms=[PLATFORM_ALGORITHM], + audience=resolved_settings.audience, + issuer=resolved_settings.issuer, + options={ + "require": [ + "iss", + "sub", + "tenant_id", + "aud", + "scope", + "azp", + "jti", + "iat", + "exp", + "ver", + ], + }, + ) + claims = PlatformTokenClaims.model_validate(payload) + if claims.azp != resolved_settings.client_id: + raise PlatformTokenError( + "Untrusted platform token authorized party", + ) + return claims + except PlatformTokenError: + raise + except (jwt.PyJWTError, ValidationError, TypeError, ValueError) as exc: + raise PlatformTokenError("Invalid platform token") from exc + + +def build_platform_jwks( + settings: PlatformTokenSettings | None = None, +) -> dict[str, list[dict[str, Any]]]: + """Return the public key as an RFC 7517 JSON Web Key Set.""" + resolved_settings = settings or PlatformTokenSettings.from_env( + require_private_key=False, + ) + try: + public_key = serialization.load_pem_public_key( + resolved_settings.public_key_pem.encode(), + ) + except (TypeError, ValueError) as exc: + raise PlatformTokenError("Invalid RSA public key") from exc + if not isinstance(public_key, RSAPublicKey): + raise PlatformTokenError("Platform public key must be RSA") + + numbers = public_key.public_numbers() + return { + "keys": [ + { + "kty": "RSA", + "use": "sig", + "alg": PLATFORM_ALGORITHM, + "kid": resolved_settings.kid, + "n": _base64url_uint(numbers.n), + "e": _base64url_uint(numbers.e), + } + ] + } + + +def _read_key_file(path: str) -> str: + try: + return Path(path).read_text(encoding="utf-8") + except OSError as exc: + raise PlatformTokenError( + f"Unable to read platform key file: {path}", + ) from exc + + +def _base64url_uint(value: int) -> str: + length = max(1, (value.bit_length() + 7) // 8) + encoded = base64.urlsafe_b64encode( + value.to_bytes(length, "big"), + ) + return encoded.rstrip(b"=").decode("ascii") diff --git a/backend/app/gateway/auth_middleware.py b/backend/app/gateway/auth_middleware.py index 6b645226..1aa77340 100644 --- a/backend/app/gateway/auth_middleware.py +++ b/backend/app/gateway/auth_middleware.py @@ -17,7 +17,12 @@ from starlette.types import ASGIApp from app.gateway.auth.errors import AuthErrorCode, AuthErrorResponse -from app.gateway.authz import _ALL_PERMISSIONS, AuthContext +from app.gateway.auth.platform_tokens import ( + AGENTOPS_READ_SCOPES, + PlatformTokenError, + verify_platform_token, +) +from app.gateway.authz import _ALL_PERMISSIONS, AuthContext, Permissions from app.gateway.internal_auth import INTERNAL_AUTH_HEADER_NAME, get_internal_user, is_valid_internal_auth_token from deerflow.runtime.user_context import reset_current_user, set_current_user @@ -38,10 +43,35 @@ "/api/v1/auth/logout", "/api/v1/auth/setup-status", "/api/v1/auth/initialize", + "/api/v1/auth/mcp/token", + "/.well-known/jwks.json", } ) +def _bearer_token(request: Request) -> str | None: + authorization = request.headers.get("Authorization", "") + scheme, separator, token = authorization.partition(" ") + if not separator or scheme.lower() != "bearer": + return None + return token.strip() or None + + +def _authentication_error( + code: AuthErrorCode, + message: str, +) -> JSONResponse: + return JSONResponse( + status_code=401, + content={ + "detail": AuthErrorResponse( + code=code, + message=message, + ).model_dump() + }, + ) + + def _is_public(path: str) -> bool: stripped = path.rstrip("/") if stripped in _PUBLIC_EXACT_PATHS: @@ -77,19 +107,49 @@ async def dispatch(self, request: Request, call_next: Callable) -> Response: return await call_next(request) internal_user = None + platform_user = None + platform_scopes: frozenset[str] = frozenset() if is_valid_internal_auth_token(request.headers.get(INTERNAL_AUTH_HEADER_NAME)): internal_user = get_internal_user() - # Non-public path: require session cookie - if internal_user is None and not request.cookies.get("access_token"): - return JSONResponse( - status_code=401, - content={ - "detail": AuthErrorResponse( - code=AuthErrorCode.NOT_AUTHENTICATED, - message="Authentication required", - ).model_dump() - }, + bearer_token = _bearer_token(request) if internal_user is None else None + if bearer_token is not None: + try: + claims = verify_platform_token(bearer_token) + except PlatformTokenError: + return _authentication_error( + AuthErrorCode.TOKEN_INVALID, + "Invalid platform token", + ) + + from app.gateway.deps import get_local_provider + + platform_user = await get_local_provider().get_user(claims.sub) + if platform_user is None: + return _authentication_error( + AuthErrorCode.USER_NOT_FOUND, + "Platform token user not found", + ) + if ( + platform_user.token_version != claims.ver + or platform_user.tenant_id != claims.tenant_id + ): + return _authentication_error( + AuthErrorCode.TOKEN_INVALID, + "Platform token is stale", + ) + platform_scopes = frozenset(claims.scope) + request.state.platform_claims = claims + request.state.platform_scopes = platform_scopes + + if ( + internal_user is None + and platform_user is None + and not request.cookies.get("access_token") + ): + return _authentication_error( + AuthErrorCode.NOT_AUTHENTICATED, + "Authentication required", ) # Strict JWT validation: reject junk/expired tokens with 401 @@ -107,18 +167,30 @@ async def dispatch(self, request: Request, call_next: Callable) -> Response: if internal_user is not None: user = internal_user + permissions = _ALL_PERMISSIONS + elif platform_user is not None: + user = platform_user + permissions = ( + [Permissions.RUNS_READ] + if platform_scopes.intersection(AGENTOPS_READ_SCOPES) + else [] + ) else: try: user = await get_current_user_from_request(request) except HTTPException as exc: return JSONResponse(status_code=exc.status_code, content={"detail": exc.detail}) + permissions = _ALL_PERMISSIONS # Stamp both request.state.user (for the contextvar pattern) # and request.state.auth (so @require_permission's "auth is # None" branch short-circuits instead of running the entire # JWT-decode + DB-lookup pipeline a second time per request). request.state.user = user - request.state.auth = AuthContext(user=user, permissions=_ALL_PERMISSIONS) + request.state.auth = AuthContext( + user=user, + permissions=permissions, + ) token = set_current_user(user) try: return await call_next(request) diff --git a/backend/app/gateway/csrf_middleware.py b/backend/app/gateway/csrf_middleware.py index f3488203..61b12ad9 100644 --- a/backend/app/gateway/csrf_middleware.py +++ b/backend/app/gateway/csrf_middleware.py @@ -51,6 +51,7 @@ def should_check_csrf(request: Request) -> bool: "/api/v1/auth/logout", "/api/v1/auth/register", "/api/v1/auth/initialize", + "/api/v1/auth/mcp/token", } ) diff --git a/backend/app/gateway/deps.py b/backend/app/gateway/deps.py index 5739d217..b56e107e 100644 --- a/backend/app/gateway/deps.py +++ b/backend/app/gateway/deps.py @@ -378,11 +378,14 @@ async def get_optional_user_from_request(request: Request): async def get_current_user(request: Request) -> str | None: - """Extract user_id from request cookie, or None if not authenticated. + """Return the middleware-validated user id, with cookie fallback. Thin adapter that returns the string id for callers that only need identification (e.g., ``feedback.py``). Full-user callers should use ``get_current_user_from_request`` or ``get_optional_user_from_request``. """ + state_user = getattr(request.state, "user", None) + if state_user is not None: + return str(state_user.id) user = await get_optional_user_from_request(request) return str(user.id) if user else None diff --git a/backend/app/gateway/routers/mcp_tokens.py b/backend/app/gateway/routers/mcp_tokens.py new file mode 100644 index 00000000..b9ae353b --- /dev/null +++ b/backend/app/gateway/routers/mcp_tokens.py @@ -0,0 +1,115 @@ +"""OAuth-style token and JWKS endpoints for FlowLens MCP.""" + +from __future__ import annotations + +import hmac + +from fastapi import APIRouter, Form, HTTPException, status +from pydantic import BaseModel + +from app.gateway.auth.platform_tokens import ( + AGENTOPS_READ_SCOPES, + TOOLS_DISCOVERY_SCOPE, + PlatformTokenIssuer, + PlatformTokenSettings, + build_platform_jwks, +) +from app.gateway.deps import get_local_provider + +RUNTIME_DELEGATION_GRANT = ( + "urn:deerflow:params:oauth:grant-type:runtime-delegation" +) + +router = APIRouter(tags=["mcp-auth"]) + + +class PlatformTokenResponse(BaseModel): + """OAuth-compatible bearer token response.""" + + access_token: str + token_type: str = "Bearer" + expires_in: int + scope: str + + +def _authenticate_client( + client_id: str, + client_secret: str, + settings: PlatformTokenSettings, +) -> None: + valid_id = hmac.compare_digest(client_id, settings.client_id) + valid_secret = hmac.compare_digest( + client_secret, + settings.client_secret, + ) + if not (valid_id and valid_secret): + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid MCP client credentials", + headers={"WWW-Authenticate": "Basic"}, + ) + + +@router.get("/.well-known/jwks.json") +async def platform_jwks() -> dict: + """Expose the current FlowLens platform verification key.""" + settings = PlatformTokenSettings.from_env( + require_private_key=False, + ) + return build_platform_jwks(settings) + + +@router.post( + "/api/v1/auth/mcp/token", + response_model=PlatformTokenResponse, +) +async def issue_mcp_token( + grant_type: str = Form(...), + client_id: str = Form(...), + client_secret: str = Form(...), + subject_user_id: str | None = Form(default=None), + origin_run_id: str | None = Form(default=None), +) -> PlatformTokenResponse: + """Issue least-privilege service or user-delegated MCP tokens.""" + settings = PlatformTokenSettings.from_env() + _authenticate_client(client_id, client_secret, settings) + issuer = PlatformTokenIssuer(settings) + + if grant_type == "client_credentials": + return PlatformTokenResponse( + access_token=issuer.issue_service_token(), + expires_in=settings.token_ttl_seconds, + scope=TOOLS_DISCOVERY_SCOPE, + ) + + if grant_type != RUNTIME_DELEGATION_GRANT: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Unsupported grant_type", + ) + if not subject_user_id or not origin_run_id: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "subject_user_id and origin_run_id are required " + "for runtime delegation" + ), + ) + + user = await get_local_provider().get_user(subject_user_id) + if user is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invalid delegation subject", + ) + + return PlatformTokenResponse( + access_token=issuer.issue_delegated_token( + user_id=str(user.id), + tenant_id=user.tenant_id, + token_version=user.token_version, + origin_run_id=origin_run_id, + ), + expires_in=settings.token_ttl_seconds, + scope=" ".join(AGENTOPS_READ_SCOPES), + ) diff --git a/backend/tests/test_mcp_bearer_auth.py b/backend/tests/test_mcp_bearer_auth.py new file mode 100644 index 00000000..2397aea2 --- /dev/null +++ b/backend/tests/test_mcp_bearer_auth.py @@ -0,0 +1,203 @@ +"""Bearer authentication tests for FlowLens platform tokens.""" + +from __future__ import annotations + +from dataclasses import replace +from uuid import uuid4 + +import pytest +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from fastapi import FastAPI, Request +from starlette.testclient import TestClient + +from app.gateway.auth.models import User +from app.gateway.auth.platform_tokens import ( + PlatformTokenIssuer, + PlatformTokenSettings, +) +from app.gateway.auth_middleware import AuthMiddleware + + +@pytest.fixture +def rsa_settings(): + private_key = rsa.generate_private_key( + public_exponent=65537, + key_size=2048, + ) + return PlatformTokenSettings( + private_key_pem=private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ).decode(), + public_key_pem=private_key.public_key() + .public_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PublicFormat.SubjectPublicKeyInfo, + ) + .decode(), + kid="flowlens-test-key", + issuer="deerflow-gateway", + audience="flowlens-platform", + client_id="deerflow-runtime", + client_secret="test-client-secret", + token_ttl_seconds=300, + ) + + +class _FakeProvider: + def __init__(self, user: User | None) -> None: + self._user = user + + async def get_user(self, user_id: str) -> User | None: + if self._user is None or str(self._user.id) != user_id: + return None + return self._user + + +def _make_client( + monkeypatch, + *, + settings: PlatformTokenSettings, + user: User, +) -> TestClient: + from app.gateway import deps + + monkeypatch.setattr( + PlatformTokenSettings, + "from_env", + classmethod(lambda cls, **kwargs: settings), + ) + monkeypatch.setattr( + deps, + "get_local_provider", + lambda: _FakeProvider(user), + ) + + app = FastAPI() + app.add_middleware(AuthMiddleware) + + @app.get("/api/agentops/runs") + async def list_runs(request: Request): + return { + "user_id": await deps.get_current_user(request), + "permissions": request.state.auth.permissions, + "platform_scopes": sorted(request.state.platform_scopes), + } + + return TestClient(app) + + +def _delegated_token( + settings: PlatformTokenSettings, + user: User, + *, + token_version: int | None = None, + tenant_id: str | None = None, +) -> str: + return PlatformTokenIssuer(settings).issue_delegated_token( + user_id=str(user.id), + tenant_id=tenant_id or user.tenant_id, + token_version=( + user.token_version + if token_version is None + else token_version + ), + origin_run_id="origin-run", + ) + + +def test_gateway_accepts_platform_bearer_and_maps_read_permission( + monkeypatch, + rsa_settings, +): + user = User( + id=uuid4(), + email="user-a@example.com", + tenant_id="tenant-a", + token_version=3, + ) + client = _make_client( + monkeypatch, + settings=rsa_settings, + user=user, + ) + token = _delegated_token(rsa_settings, user) + + response = client.get( + "/api/agentops/runs", + headers={"Authorization": f"Bearer {token}"}, + ) + + assert response.status_code == 200 + assert response.json()["user_id"] == str(user.id) + assert "runs:read" in response.json()["permissions"] + assert "agentops.timeline.read" in response.json()["platform_scopes"] + + +def test_gateway_rejects_wrong_audience_bearer( + monkeypatch, + rsa_settings, +): + user = User( + id=uuid4(), + email="user-a@example.com", + tenant_id="tenant-a", + ) + client = _make_client( + monkeypatch, + settings=rsa_settings, + user=user, + ) + wrong_audience_settings = replace( + rsa_settings, + audience="wrong-platform", + ) + token = _delegated_token(wrong_audience_settings, user) + + response = client.get( + "/api/agentops/runs", + headers={"Authorization": f"Bearer {token}"}, + ) + + assert response.status_code == 401 + + +@pytest.mark.parametrize( + ("token_version", "tenant_id"), + [ + (2, "tenant-a"), + (3, "tenant-b"), + ], +) +def test_gateway_rejects_stale_or_cross_tenant_bearer( + monkeypatch, + rsa_settings, + token_version, + tenant_id, +): + user = User( + id=uuid4(), + email="user-a@example.com", + tenant_id="tenant-a", + token_version=3, + ) + client = _make_client( + monkeypatch, + settings=rsa_settings, + user=user, + ) + token = _delegated_token( + rsa_settings, + user, + token_version=token_version, + tenant_id=tenant_id, + ) + + response = client.get( + "/api/agentops/runs", + headers={"Authorization": f"Bearer {token}"}, + ) + + assert response.status_code == 401 diff --git a/backend/tests/test_mcp_platform_tokens.py b/backend/tests/test_mcp_platform_tokens.py new file mode 100644 index 00000000..c19392ce --- /dev/null +++ b/backend/tests/test_mcp_platform_tokens.py @@ -0,0 +1,152 @@ +"""Unit tests for FlowLens RS256 platform tokens.""" + +from __future__ import annotations + +import importlib +from dataclasses import replace + +import pytest +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa + + +def _token_module(): + return importlib.import_module("app.gateway.auth.platform_tokens") + + +@pytest.fixture +def rsa_settings(): + token_module = _token_module() + private_key = rsa.generate_private_key( + public_exponent=65537, + key_size=2048, + ) + private_pem = private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ).decode() + public_pem = private_key.public_key().public_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PublicFormat.SubjectPublicKeyInfo, + ).decode() + return token_module.PlatformTokenSettings( + private_key_pem=private_pem, + public_key_pem=public_pem, + kid="flowlens-test-key", + issuer="deerflow-gateway", + audience="flowlens-platform", + client_id="deerflow-runtime", + client_secret="test-client-secret", + token_ttl_seconds=300, + ) + + +def test_delegated_token_contains_required_claims(rsa_settings): + token_module = _token_module() + token = token_module.PlatformTokenIssuer( + rsa_settings, + ).issue_delegated_token( + user_id="user-a", + tenant_id="tenant-a", + token_version=3, + origin_run_id="run-origin", + ) + + claims = token_module.verify_platform_token(token, rsa_settings) + + assert claims.sub == "user-a" + assert claims.tenant_id == "tenant-a" + assert claims.aud == "flowlens-platform" + assert claims.origin_run_id == "run-origin" + assert claims.azp == "deerflow-runtime" + assert claims.ver == 3 + assert "agentops.timeline.read" in claims.scope + + +def test_service_token_is_limited_to_tool_discovery(rsa_settings): + token_module = _token_module() + token = token_module.PlatformTokenIssuer( + rsa_settings, + ).issue_service_token() + + claims = token_module.verify_platform_token(token, rsa_settings) + + assert claims.sub == "deerflow-runtime" + assert claims.scope == ["agentops.tools.discover"] + assert claims.origin_run_id is None + + +def test_verifier_rejects_wrong_audience(rsa_settings): + token_module = _token_module() + token = token_module.PlatformTokenIssuer( + rsa_settings, + ).issue_delegated_token( + user_id="user-a", + tenant_id="tenant-a", + token_version=0, + origin_run_id="run-origin", + ) + wrong_audience = replace( + rsa_settings, + audience="another-platform", + ) + + with pytest.raises(token_module.PlatformTokenError): + token_module.verify_platform_token(token, wrong_audience) + + +def test_verifier_rejects_wrong_authorized_party(rsa_settings): + token_module = _token_module() + wrong_client = replace( + rsa_settings, + client_id="untrusted-runtime", + ) + token = token_module.PlatformTokenIssuer( + wrong_client, + ).issue_delegated_token( + user_id="user-a", + tenant_id="tenant-a", + token_version=0, + origin_run_id="run-origin", + ) + + with pytest.raises(token_module.PlatformTokenError): + token_module.verify_platform_token(token, rsa_settings) + + +def test_jwks_exposes_only_the_public_rsa_key(rsa_settings): + token_module = _token_module() + + jwks = token_module.build_platform_jwks(rsa_settings) + + assert len(jwks["keys"]) == 1 + key = jwks["keys"][0] + assert key["kid"] == "flowlens-test-key" + assert key["kty"] == "RSA" + assert key["alg"] == "RS256" + assert key["use"] == "sig" + assert key["n"] + assert key["e"] + assert "private" not in str(jwks).lower() + + +def test_enabled_mcp_fails_fast_without_key_material(monkeypatch): + token_module = _token_module() + monkeypatch.setenv("FLOWLENS_MCP_ENABLED", "true") + for name in ( + "FLOWLENS_MCP_JWT_PRIVATE_KEY_PATH", + "FLOWLENS_MCP_JWT_PUBLIC_KEY_PATH", + "FLOWLENS_MCP_CLIENT_SECRET", + ): + monkeypatch.delenv(name, raising=False) + + with pytest.raises(token_module.PlatformTokenError): + token_module.ensure_platform_tokens_configured_if_enabled() + + +def test_disabled_mcp_does_not_require_key_material(monkeypatch): + token_module = _token_module() + monkeypatch.setenv("FLOWLENS_MCP_ENABLED", "false") + + token_module.ensure_platform_tokens_configured_if_enabled() diff --git a/backend/tests/test_mcp_token_api.py b/backend/tests/test_mcp_token_api.py new file mode 100644 index 00000000..4ef2cd52 --- /dev/null +++ b/backend/tests/test_mcp_token_api.py @@ -0,0 +1,194 @@ +"""HTTP contract tests for FlowLens token issuance and JWKS.""" + +from __future__ import annotations + +import importlib +from uuid import uuid4 + +import pytest +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from fastapi import FastAPI +from starlette.testclient import TestClient + +from app.gateway.auth.models import User +from app.gateway.auth.platform_tokens import ( + PlatformTokenSettings, + verify_platform_token, +) +from app.gateway.auth_middleware import AuthMiddleware +from app.gateway.csrf_middleware import CSRFMiddleware + + +@pytest.fixture +def rsa_settings(): + private_key = rsa.generate_private_key( + public_exponent=65537, + key_size=2048, + ) + return PlatformTokenSettings( + private_key_pem=private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ).decode(), + public_key_pem=private_key.public_key() + .public_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PublicFormat.SubjectPublicKeyInfo, + ) + .decode(), + kid="flowlens-test-key", + issuer="deerflow-gateway", + audience="flowlens-platform", + client_id="deerflow-runtime", + client_secret="test-client-secret", + token_ttl_seconds=300, + ) + + +class _FakeProvider: + def __init__(self, user: User) -> None: + self._user = user + + async def get_user(self, user_id: str) -> User | None: + if str(self._user.id) == user_id: + return self._user + return None + + +def _make_client( + monkeypatch, + *, + settings: PlatformTokenSettings, + user: User, +) -> TestClient: + token_router = importlib.import_module( + "app.gateway.routers.mcp_tokens", + ) + monkeypatch.setattr( + PlatformTokenSettings, + "from_env", + classmethod(lambda cls, **kwargs: settings), + ) + monkeypatch.setattr( + token_router, + "get_local_provider", + lambda: _FakeProvider(user), + ) + + app = FastAPI() + app.add_middleware(AuthMiddleware) + app.add_middleware(CSRFMiddleware) + app.include_router(token_router.router) + return TestClient(app) + + +def test_client_credentials_issues_discovery_only_token( + monkeypatch, + rsa_settings, +): + user = User(email="user-a@example.com") + client = _make_client( + monkeypatch, + settings=rsa_settings, + user=user, + ) + + response = client.post( + "/api/v1/auth/mcp/token", + data={ + "grant_type": "client_credentials", + "client_id": "deerflow-runtime", + "client_secret": "test-client-secret", + }, + ) + + assert response.status_code == 200 + claims = verify_platform_token( + response.json()["access_token"], + rsa_settings, + ) + assert claims.scope == ["agentops.tools.discover"] + assert response.json()["token_type"] == "Bearer" + assert response.json()["expires_in"] == 300 + + +def test_runtime_delegation_derives_tenant_and_version_from_user( + monkeypatch, + rsa_settings, +): + user = User( + id=uuid4(), + email="user-a@example.com", + tenant_id="tenant-a", + token_version=7, + ) + client = _make_client( + monkeypatch, + settings=rsa_settings, + user=user, + ) + + response = client.post( + "/api/v1/auth/mcp/token", + data={ + "grant_type": ( + "urn:deerflow:params:oauth:" + "grant-type:runtime-delegation" + ), + "client_id": "deerflow-runtime", + "client_secret": "test-client-secret", + "subject_user_id": str(user.id), + "origin_run_id": "origin-run", + }, + ) + + assert response.status_code == 200 + claims = verify_platform_token( + response.json()["access_token"], + rsa_settings, + ) + assert claims.sub == str(user.id) + assert claims.tenant_id == "tenant-a" + assert claims.ver == 7 + assert claims.origin_run_id == "origin-run" + assert "agentops.diagnostics.read" in claims.scope + + +def test_token_endpoint_rejects_wrong_client_secret( + monkeypatch, + rsa_settings, +): + client = _make_client( + monkeypatch, + settings=rsa_settings, + user=User(email="user-a@example.com"), + ) + + response = client.post( + "/api/v1/auth/mcp/token", + data={ + "grant_type": "client_credentials", + "client_id": "deerflow-runtime", + "client_secret": "wrong-secret", + }, + ) + + assert response.status_code == 401 + + +def test_jwks_is_public_and_contains_configured_key( + monkeypatch, + rsa_settings, +): + client = _make_client( + monkeypatch, + settings=rsa_settings, + user=User(email="user-a@example.com"), + ) + + response = client.get("/.well-known/jwks.json") + + assert response.status_code == 200 + assert response.json()["keys"][0]["kid"] == "flowlens-test-key" From 46029c6f69bb6f62833224b37e935a3a3bf0d8fe Mon Sep 17 00:00:00 2001 From: tangrongyuan <18532002267@163.com> Date: Wed, 29 Jul 2026 10:38:55 +0800 Subject: [PATCH 05/14] feat(agentops): expose MCP-safe run query contracts --- backend/app/gateway/deps.py | 10 +++ backend/app/gateway/routers/agentops.py | 33 +++++++- backend/app/gateway/routers/thread_runs.py | 35 +++++--- .../harness/deerflow/persistence/run/sql.py | 3 + .../harness/deerflow/runtime/runs/manager.py | 2 + .../deerflow/runtime/runs/store/base.py | 1 + .../deerflow/runtime/runs/store/memory.py | 2 + backend/tests/test_agentops_run_lookup_api.py | 79 +++++++++++++++++++ backend/tests/test_agentops_runs_api.py | 7 +- backend/tests/test_cancel_run_idempotent.py | 26 +++++- backend/tests/test_run_repository.py | 63 +++++++++++++++ .../tests/test_thread_run_diagnostics_api.py | 13 ++- .../test_thread_run_messages_pagination.py | 13 ++- 13 files changed, 267 insertions(+), 20 deletions(-) create mode 100644 backend/tests/test_agentops_run_lookup_api.py diff --git a/backend/app/gateway/deps.py b/backend/app/gateway/deps.py index b56e107e..56e99091 100644 --- a/backend/app/gateway/deps.py +++ b/backend/app/gateway/deps.py @@ -31,6 +31,7 @@ from deerflow.runtime import RunContext, RunManager, StreamBridge from deerflow.runtime.events.store.base import RunEventStore from deerflow.runtime.runs.store.base import RunStore +from deerflow.runtime.user_context import DEFAULT_TENANT_ID logger = logging.getLogger(__name__) @@ -389,3 +390,12 @@ async def get_current_user(request: Request) -> str | None: return str(state_user.id) user = await get_optional_user_from_request(request) return str(user.id) if user else None + + +def get_current_tenant_id(request: Request) -> str: + """Return the server-authenticated tenant, defaulting legacy users.""" + user = getattr(request.state, "user", None) + return str( + getattr(user, "tenant_id", None) + or DEFAULT_TENANT_ID + ) diff --git a/backend/app/gateway/routers/agentops.py b/backend/app/gateway/routers/agentops.py index f4ff0aee..97613851 100644 --- a/backend/app/gateway/routers/agentops.py +++ b/backend/app/gateway/routers/agentops.py @@ -8,7 +8,11 @@ from fastapi import APIRouter, HTTPException, Query, Request from app.gateway.authz import require_permission -from app.gateway.deps import get_current_user, get_run_manager +from app.gateway.deps import ( + get_current_tenant_id, + get_current_user, + get_run_manager, +) from deerflow.runtime import RunRecord, RunStatus from deerflow.runtime.runs.cursor import InvalidRunCursor, decode_run_cursor, encode_run_cursor @@ -37,6 +41,7 @@ async def list_agentops_runs( limit: int = Query(default=50, ge=1, le=200), cursor: str | None = Query(default=None), status: RunStatus | None = Query(default=None), + thread_id: str | None = Query(default=None, min_length=1), started_after: datetime | None = Query(default=None), started_before: datetime | None = Query(default=None), ) -> dict[str, Any]: @@ -51,8 +56,11 @@ async def list_agentops_runs( user_id = await get_current_user(request) if user_id is None: raise HTTPException(status_code=401, detail="Authentication required") + tenant_id = get_current_tenant_id(request) records = await get_run_manager(request).list_recent( user_id=user_id, + tenant_id=tenant_id, + thread_id=thread_id, limit=limit + 1, cursor=cursor, status=status.value if status is not None else None, @@ -70,3 +78,26 @@ async def list_agentops_runs( "has_more": has_more, "next_cursor": next_cursor, } + + +@router.get("/runs/{run_id}") +@require_permission("runs", "read") +async def get_agentops_run( + run_id: str, + request: Request, +) -> dict[str, Any]: + """Return an owner-safe stable run summary for MCP consumers.""" + user_id = await get_current_user(request) + if user_id is None: + raise HTTPException( + status_code=401, + detail="Authentication required", + ) + record = await get_run_manager(request).get( + run_id, + user_id=user_id, + tenant_id=get_current_tenant_id(request), + ) + if record is None: + raise HTTPException(status_code=404, detail="Run not found") + return _run_summary(record) diff --git a/backend/app/gateway/routers/thread_runs.py b/backend/app/gateway/routers/thread_runs.py index dfebc6ec..d67a6e53 100644 --- a/backend/app/gateway/routers/thread_runs.py +++ b/backend/app/gateway/routers/thread_runs.py @@ -22,7 +22,16 @@ from sqlalchemy.exc import SQLAlchemyError from app.gateway.authz import require_permission -from app.gateway.deps import get_checkpointer, get_current_user, get_feedback_repo, get_run_event_store, get_run_manager, get_run_store, get_stream_bridge +from app.gateway.deps import ( + get_checkpointer, + get_current_tenant_id, + get_current_user, + get_feedback_repo, + get_run_event_store, + get_run_manager, + get_run_store, + get_stream_bridge, +) from app.gateway.pagination import trim_run_message_page from app.gateway.services import sse_consumer, start_run, wait_for_run_completion from deerflow.models import create_chat_model @@ -146,7 +155,11 @@ def _record_to_response(record: RunRecord) -> RunResponse: async def _get_run_or_404(thread_id: str, run_id: str, request: Request) -> RunRecord: run_mgr = get_run_manager(request) user_id = await get_current_user(request) - record = await run_mgr.get(run_id, user_id=user_id) + record = await run_mgr.get( + run_id, + user_id=user_id, + tenant_id=get_current_tenant_id(request), + ) if record is None or record.thread_id != thread_id: raise HTTPException(status_code=404, detail=f"Run {run_id} not found") return record @@ -248,7 +261,11 @@ async def list_runs(thread_id: str, request: Request) -> list[RunResponse]: """List all runs for a thread.""" run_mgr = get_run_manager(request) user_id = await get_current_user(request) - records = await run_mgr.list_by_thread(thread_id, user_id=user_id) + records = await run_mgr.list_by_thread( + thread_id, + user_id=user_id, + tenant_id=get_current_tenant_id(request), + ) return [_record_to_response(r) for r in records] @@ -277,9 +294,7 @@ async def cancel_run( - wait=false: Return immediately with 202 """ run_mgr = get_run_manager(request) - record = await run_mgr.get(run_id) - if record is None or record.thread_id != thread_id: - raise HTTPException(status_code=404, detail=f"Run {run_id} not found") + record = await _get_run_or_404(thread_id, run_id, request) cancelled = await run_mgr.cancel(run_id, action=action) if not cancelled: @@ -300,9 +315,7 @@ async def cancel_run( async def join_run(thread_id: str, run_id: str, request: Request) -> StreamingResponse: """Join an existing run's SSE stream.""" run_mgr = get_run_manager(request) - record = await run_mgr.get(run_id) - if record is None or record.thread_id != thread_id: - raise HTTPException(status_code=404, detail=f"Run {run_id} not found") + record = await _get_run_or_404(thread_id, run_id, request) if record.store_only: raise HTTPException(status_code=409, detail=f"Run {run_id} is not active on this worker and cannot be streamed") @@ -340,9 +353,7 @@ async def stream_existing_run( remaining buffered events so the client observes a clean shutdown. """ run_mgr = get_run_manager(request) - record = await run_mgr.get(run_id) - if record is None or record.thread_id != thread_id: - raise HTTPException(status_code=404, detail=f"Run {run_id} not found") + record = await _get_run_or_404(thread_id, run_id, request) if record.store_only and action is None: raise HTTPException(status_code=409, detail=f"Run {run_id} is not active on this worker and cannot be streamed") diff --git a/backend/packages/harness/deerflow/persistence/run/sql.py b/backend/packages/harness/deerflow/persistence/run/sql.py index acaf3d03..8bc01a37 100644 --- a/backend/packages/harness/deerflow/persistence/run/sql.py +++ b/backend/packages/harness/deerflow/persistence/run/sql.py @@ -190,6 +190,7 @@ async def list_recent( *, user_id: str | None | _AutoSentinel = AUTO, tenant_id: str | None | _AutoSentinel = AUTO, + thread_id: str | None = None, limit: int = 50, cursor: str | None = None, status: str | None = None, @@ -206,6 +207,8 @@ async def list_recent( stmt = stmt.where(RunRow.user_id == resolved_user_id) if resolved_tenant_id is not None: stmt = stmt.where(RunRow.tenant_id == resolved_tenant_id) + if thread_id is not None: + stmt = stmt.where(RunRow.thread_id == thread_id) if status is not None: stmt = stmt.where(RunRow.status == status) if started_after is not None: diff --git a/backend/packages/harness/deerflow/runtime/runs/manager.py b/backend/packages/harness/deerflow/runtime/runs/manager.py index 00bc81bf..eef800a3 100644 --- a/backend/packages/harness/deerflow/runtime/runs/manager.py +++ b/backend/packages/harness/deerflow/runtime/runs/manager.py @@ -490,6 +490,7 @@ async def list_recent( *, user_id: str | None, tenant_id: str | None = None, + thread_id: str | None = None, limit: int = 50, cursor: str | None = None, status: str | None = None, @@ -503,6 +504,7 @@ async def list_recent( rows = await self._store.list_recent( user_id=user_id, tenant_id=tenant_id, + thread_id=thread_id, limit=limit, cursor=cursor, status=status, diff --git a/backend/packages/harness/deerflow/runtime/runs/store/base.py b/backend/packages/harness/deerflow/runtime/runs/store/base.py index 6772c95d..bf5015e1 100644 --- a/backend/packages/harness/deerflow/runtime/runs/store/base.py +++ b/backend/packages/harness/deerflow/runtime/runs/store/base.py @@ -60,6 +60,7 @@ async def list_recent( *, user_id: str | None, tenant_id: str | None = None, + thread_id: str | None = None, limit: int = 50, cursor: str | None = None, status: str | None = None, diff --git a/backend/packages/harness/deerflow/runtime/runs/store/memory.py b/backend/packages/harness/deerflow/runtime/runs/store/memory.py index aeebdfa7..20b21502 100644 --- a/backend/packages/harness/deerflow/runtime/runs/store/memory.py +++ b/backend/packages/harness/deerflow/runtime/runs/store/memory.py @@ -82,6 +82,7 @@ async def list_recent( *, user_id, tenant_id=None, + thread_id=None, limit=50, cursor=None, status=None, @@ -93,6 +94,7 @@ async def list_recent( for run in self._runs.values() if (user_id is None or run.get("user_id") == user_id) and (tenant_id is None or run.get("tenant_id") == tenant_id) + and (thread_id is None or run.get("thread_id") == thread_id) ] if status is not None: results = [run for run in results if run.get("status") == status] diff --git a/backend/tests/test_agentops_run_lookup_api.py b/backend/tests/test_agentops_run_lookup_api.py new file mode 100644 index 00000000..b92e4ce0 --- /dev/null +++ b/backend/tests/test_agentops_run_lookup_api.py @@ -0,0 +1,79 @@ +"""Owner-safe AgentOps run lookup contract tests.""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch + +from _router_auth_helpers import make_authed_test_app +from fastapi.testclient import TestClient + +from app.gateway.routers import agentops + + +def _record(): + return SimpleNamespace( + run_id="run-a", + thread_id="thread-a", + status=SimpleNamespace(value="error"), + created_at="2026-01-01T00:00:00+00:00", + updated_at="2026-01-01T00:00:02+00:00", + model_name="test-model", + total_tokens=120, + llm_call_count=2, + message_count=4, + ) + + +def _make_client(manager): + app = make_authed_test_app() + app.state.run_manager = manager + app.include_router(agentops.router) + return TestClient(app) + + +def test_run_lookup_returns_stable_mcp_summary(): + manager = SimpleNamespace(get=AsyncMock(return_value=_record())) + + with patch.object( + agentops, + "get_current_user", + AsyncMock(return_value="user-a"), + ): + with _make_client(manager) as client: + response = client.get("/api/agentops/runs/run-a") + + assert response.status_code == 200 + assert response.json() == { + "run_id": "run-a", + "thread_id": "thread-a", + "status": "error", + "created_at": "2026-01-01T00:00:00+00:00", + "updated_at": "2026-01-01T00:00:02+00:00", + "model_name": "test-model", + "total_tokens": 120, + "llm_call_count": 2, + "message_count": 4, + } + manager.get.assert_awaited_once_with( + "run-a", + user_id="user-a", + tenant_id="default", + ) + + +def test_run_lookup_hides_inaccessible_run_as_not_found(): + manager = SimpleNamespace(get=AsyncMock(return_value=None)) + + with patch.object( + agentops, + "get_current_user", + AsyncMock(return_value="user-a"), + ): + with _make_client(manager) as client: + response = client.get( + "/api/agentops/runs/run-owned-by-tenant-b", + ) + + assert response.status_code == 404 + assert response.json()["detail"] == "Run not found" diff --git a/backend/tests/test_agentops_runs_api.py b/backend/tests/test_agentops_runs_api.py index 1b0b02d2..ab4d4452 100644 --- a/backend/tests/test_agentops_runs_api.py +++ b/backend/tests/test_agentops_runs_api.py @@ -46,7 +46,10 @@ def test_agentops_runs_returns_current_user_rows_and_cursor(): with patch.object(agentops, "get_current_user", AsyncMock(return_value="user-1")): with TestClient(app) as client: - response = client.get("/api/agentops/runs?limit=1&status=error") + response = client.get( + "/api/agentops/runs" + "?limit=1&status=error&thread_id=thread-r2" + ) assert response.status_code == 200 body = response.json() @@ -55,6 +58,8 @@ def test_agentops_runs_returns_current_user_rows_and_cursor(): assert body["next_cursor"] manager.list_recent.assert_awaited_once_with( user_id="user-1", + tenant_id="default", + thread_id="thread-r2", limit=2, cursor=None, status="error", diff --git a/backend/tests/test_cancel_run_idempotent.py b/backend/tests/test_cancel_run_idempotent.py index 0bf2548d..058b6d50 100644 --- a/backend/tests/test_cancel_run_idempotent.py +++ b/backend/tests/test_cancel_run_idempotent.py @@ -9,14 +9,24 @@ from __future__ import annotations import asyncio +from uuid import uuid4 from _router_auth_helpers import make_authed_test_app from fastapi.testclient import TestClient +from app.gateway.auth.models import User from app.gateway.routers import thread_runs from deerflow.runtime import RunManager, RunStatus +from deerflow.runtime.user_context import ( + reset_current_user, + set_current_user, +) THREAD_ID = "thread-cancel-test" +ROUTER_USER = User( + id=uuid4(), + email="cancel-route@example.com", +) # --------------------------------------------------------------------------- @@ -25,7 +35,9 @@ def _make_app(mgr: RunManager) -> TestClient: - app = make_authed_test_app() + app = make_authed_test_app( + user_factory=lambda: ROUTER_USER, + ) app.include_router(thread_runs.router) app.state.run_manager = mgr return TestClient(app, raise_server_exceptions=False) @@ -40,7 +52,11 @@ async def _setup(): await mgr.cancel(record.run_id) return record.run_id - return asyncio.run(_setup()) + token = set_current_user(ROUTER_USER) + try: + return asyncio.run(_setup()) + finally: + reset_current_user(token) # --------------------------------------------------------------------------- @@ -116,7 +132,11 @@ async def _setup(): await mgr.set_status(record.run_id, RunStatus.success) return mgr, record.run_id - mgr, run_id = asyncio.run(_setup()) + token = set_current_user(ROUTER_USER) + try: + mgr, run_id = asyncio.run(_setup()) + finally: + reset_current_user(token) client = _make_app(mgr) resp = client.post(f"/api/threads/{THREAD_ID}/runs/{run_id}/cancel") assert resp.status_code == 409 diff --git a/backend/tests/test_run_repository.py b/backend/tests/test_run_repository.py index 23bc4a41..fa1f7193 100644 --- a/backend/tests/test_run_repository.py +++ b/backend/tests/test_run_repository.py @@ -68,6 +68,34 @@ async def test_memory_run_store_list_recent_filters_status_and_time_window(): assert [row["run_id"] for row in rows] == ["r3"] +@pytest.mark.anyio +async def test_memory_run_store_list_recent_filters_thread_before_paging(): + store = MemoryRunStore() + await store.put( + "thread-a-old", + thread_id="thread-a", + user_id="u1", + tenant_id="tenant-a", + created_at="2026-01-01T00:00:00+00:00", + ) + await store.put( + "thread-b-new", + thread_id="thread-b", + user_id="u1", + tenant_id="tenant-a", + created_at="2026-01-03T00:00:00+00:00", + ) + + rows = await store.list_recent( + user_id="u1", + tenant_id="tenant-a", + thread_id="thread-a", + limit=1, + ) + + assert [row["run_id"] for row in rows] == ["thread-a-old"] + + async def _make_repo(tmp_path): from deerflow.persistence.engine import get_session_factory, init_engine @@ -220,6 +248,41 @@ async def test_list_recent_orders_by_created_at_and_run_id(self, tmp_path): assert [row["run_id"] for row in rows] == ["run-b", "run-a"] await _cleanup() + @pytest.mark.anyio + async def test_list_recent_filters_by_thread_and_tenant(self, tmp_path): + repo = await _make_repo(tmp_path) + await repo.put( + "run-a", + thread_id="thread-a", + user_id="alice", + tenant_id="tenant-a", + created_at="2026-01-01T00:00:00+00:00", + ) + await repo.put( + "run-b", + thread_id="thread-b", + user_id="alice", + tenant_id="tenant-a", + created_at="2026-01-02T00:00:00+00:00", + ) + await repo.put( + "run-c", + thread_id="thread-a", + user_id="alice", + tenant_id="tenant-b", + created_at="2026-01-03T00:00:00+00:00", + ) + + rows = await repo.list_recent( + user_id="alice", + tenant_id="tenant-a", + thread_id="thread-a", + limit=20, + ) + + assert [row["run_id"] for row in rows] == ["run-a"] + await _cleanup() + @pytest.mark.anyio async def test_delete(self, tmp_path): repo = await _make_repo(tmp_path) diff --git a/backend/tests/test_thread_run_diagnostics_api.py b/backend/tests/test_thread_run_diagnostics_api.py index 5763b577..4e57a126 100644 --- a/backend/tests/test_thread_run_diagnostics_api.py +++ b/backend/tests/test_thread_run_diagnostics_api.py @@ -3,7 +3,7 @@ from __future__ import annotations from types import SimpleNamespace -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import ANY, AsyncMock, MagicMock from _router_auth_helpers import make_authed_test_app from fastapi.testclient import TestClient @@ -70,7 +70,11 @@ def _tool_error_events() -> list[dict]: def test_run_timeline_returns_normalized_events(): event_store = _make_event_store(_tool_error_events()) - app = _make_app(event_store=event_store, run_manager=_make_run_manager()) + run_manager = _make_run_manager() + app = _make_app( + event_store=event_store, + run_manager=run_manager, + ) with TestClient(app) as client: response = client.get("/api/threads/thread-diag/runs/run-diag/timeline") @@ -82,6 +86,11 @@ def test_run_timeline_returns_normalized_events(): assert body["timeline"][0]["phase"] == "tool" assert body["timeline"][1]["status"] == "error" event_store.list_events.assert_awaited_once_with("thread-diag", "run-diag", limit=201) + run_manager.get.assert_awaited_once_with( + "run-diag", + user_id=ANY, + tenant_id="default", + ) def test_run_diagnostics_returns_metrics_and_failure_attribution(): diff --git a/backend/tests/test_thread_run_messages_pagination.py b/backend/tests/test_thread_run_messages_pagination.py index 6d36001a..7f7dcb59 100644 --- a/backend/tests/test_thread_run_messages_pagination.py +++ b/backend/tests/test_thread_run_messages_pagination.py @@ -4,11 +4,13 @@ import asyncio from unittest.mock import AsyncMock, MagicMock +from uuid import uuid4 from _router_auth_helpers import make_authed_test_app from _run_message_pagination_helpers import assert_run_message_page from fastapi.testclient import TestClient +from app.gateway.auth.models import User from app.gateway.routers import thread_runs from deerflow.runtime import RunManager from deerflow.runtime.runs.store.memory import MemoryRunStore @@ -17,10 +19,17 @@ # Helpers # --------------------------------------------------------------------------- +ROUTER_USER = User( + id=uuid4(), + email="run-messages-route@example.com", +) + def _make_app(event_store=None, run_manager=None): """Build a test FastAPI app with stub auth and mocked state.""" - app = make_authed_test_app() + app = make_authed_test_app( + user_factory=lambda: ROUTER_USER, + ) app.include_router(thread_runs.router) if event_store is not None: @@ -49,6 +58,8 @@ def _make_store_only_run_manager() -> RunManager: "store-only-run", thread_id="thread-store", assistant_id="lead_agent", + user_id=str(ROUTER_USER.id), + tenant_id=ROUTER_USER.tenant_id, status="running", multitask_strategy="reject", metadata={}, From e55f0134d23ef4c00021188ba3527b2a195d9bf0 Mon Sep 17 00:00:00 2001 From: tangrongyuan <18532002267@163.com> Date: Wed, 29 Jul 2026 11:28:37 +0800 Subject: [PATCH 06/14] docs: design FlowLens investigation workbench --- ...agentops-investigation-workbench-design.md | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-29-flowlens-agentops-investigation-workbench-design.md diff --git a/docs/superpowers/specs/2026-07-29-flowlens-agentops-investigation-workbench-design.md b/docs/superpowers/specs/2026-07-29-flowlens-agentops-investigation-workbench-design.md new file mode 100644 index 00000000..2a2efe2c --- /dev/null +++ b/docs/superpowers/specs/2026-07-29-flowlens-agentops-investigation-workbench-design.md @@ -0,0 +1,66 @@ +# FlowLens AgentOps Investigation Workbench Design + +## Intent + +Refresh the existing AgentOps page into a calm, evidence-led investigation workbench. The primary demo begins with a real DeerFlow run, then lets an interviewer move from a failed run to its execution trace, deterministic diagnosis, and the exact evidence event. Replay remains a stable fallback and uses the same UI and source contracts. + +The visual direction is Apple-inspired in principle only: neutral surfaces, deliberate whitespace, restrained color, precise typography, and responsive micro-interactions. It is not an Apple website imitation and remains an operational tool rather than a marketing page. + +## Boundaries + +- Preserve all current routes, diagnostics-source contracts, remote and replay modes, filters, evidence selection, Explain, Export, and event-detail behavior. +- Do not change backend APIs, data schemas, or replace real data with display fixtures. +- Reuse the existing Next.js 16, React 19, Tailwind v4, Lucide, TanStack Query, and Playwright stack. Do not add dependencies. +- Scope visual tokens to AgentOps so the rest of DeerFlow retains its current design language. + +## Investigation Flow + +1. The Run Explorer selects a real `run_id` within a `thread_id`. +2. The workspace renders run identity, mode, model, status, and metrics from the existing run and diagnostics queries. +3. Timeline items are transformed locally into a derived trace presentation. Timestamps establish a relative start position; `duration_ms` establishes bar length; `span_id` and `parent_span_id` establish indentation. No trace data is invented. +4. Selecting a trace row or timeline event opens the existing redacted event-detail sheet. +5. Deterministic Diagnostics renders the primary cause, confidence, evidence sequence, contributing causes, suggested action, Explain, and Export. Evidence buttons target the corresponding timeline item. + +`RunEventStore -> Timeline -> Metrics / Diagnostics -> Workbench` remains the data boundary. The page visualizes derived data and never writes Timeline entries. + +## Layout + +Desktop uses the approved A1 Trace-first Workbench composition within a centered, maximum-width operational canvas: + +- A compact top bar holds product identity, mode/connectivity state, and non-intrusive action context. +- The Run Explorer remains a persistent left rail. +- The center investigation surface contains the selected-run header, a single grouped metrics band, Trace Waterfall, then the filterable normalized event Timeline. +- The right rail is the evidence-led Root Cause panel. + +Panels are real work surfaces, not a grid of decorative cards: small 12px radii, thin neutral borders, subtle elevation only at interactive boundaries. On tablet and mobile, the Run Explorer becomes the existing drawer while Trace and Root Cause become explicit tabs. No horizontal page overflow is allowed. + +## Visual System + +Add an AgentOps-scoped token layer for neutral background/surface/text/border values, one cyan-teal interaction color, and muted semantic success/warning/error colors. Tokens cover radius, shadow, spacing, type scale, focus ring, and motion duration. + +Typography uses the system stack: `-apple-system`, `BlinkMacSystemFont`, `SF Pro Display`, `SF Pro Text`, `Segoe UI`, `PingFang SC`, and `Microsoft YaHei`. Chinese body copy uses a 1.6 line height. Monospace remains limited to identifiers, event types, and metadata. + +Motion is limited to 150-300ms opacity, color, and small transform transitions. `prefers-reduced-motion` disables non-essential transitions. + +## Components + +- `AgentOpsWorkspace`: introduces the workbench shell and selected-run context without owning presentation-specific calculations. +- `RunOverview`: a new component for run/thread/model identity, source mode, status, and completion context. +- `TraceWaterfall`: a new pure component deriving relative trace bars from existing timeline items. It shares selection with Timeline and is empty-safe. +- `MetricsStrip`, `RunExplorer`, `Timeline`, `TimelineRow`, `DiagnosticsPanel`, and `EventDetailSheet`: preserve their public behavior while adopting the shared AgentOps classes. +- `agentops.css`: scoped design tokens and reusable surface/control/state classes, imported by the global style entry. + +## Error Handling and Accessibility + +Existing loading, empty, and error states stay visible. Missing timestamps or durations render a compact timeline-row fallback rather than a misleading bar. Controls retain names and keyboard operation; selected state uses `aria-selected` or `aria-pressed`; focus is visible; disabled Explain and Export states remain clear. + +## Verification + +- Extend unit tests for waterfall ordering, zero/missing duration behavior, and evidence-to-event selection. +- Keep existing replay workspace tests for filtering, export, and redaction. +- Update Playwright AgentOps demo assertions for the new trace surface and preserve desktop, tablet, and mobile checks including no horizontal overflow. +- Run `pnpm.cmd test`, `pnpm.cmd check`, and `pnpm.cmd test:e2e:agentops-demo` on Windows. Capture desktop/tablet/mobile screenshots for visual inspection. + +## Interview Demonstration + +The demonstration begins in DeerFlow with a real task, then opens AgentOps in remote mode. The presenter identifies the same `run_id` and `thread_id`, reads the trace from Runtime to Model/Tool/MCP, opens root-cause evidence, and finally shows the export. Replay is described honestly as a deterministic fallback for rehearsing failure categories when a live fault is unavailable. From ae92bba9a3889cba673094b0a6857079c70cf7e4 Mon Sep 17 00:00:00 2001 From: tangrongyuan <18532002267@163.com> Date: Wed, 29 Jul 2026 11:30:15 +0800 Subject: [PATCH 07/14] docs: plan FlowLens investigation workbench --- ...owlens-agentops-investigation-workbench.md | 273 ++++++++++++++++++ 1 file changed, 273 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-29-flowlens-agentops-investigation-workbench.md diff --git a/docs/superpowers/plans/2026-07-29-flowlens-agentops-investigation-workbench.md b/docs/superpowers/plans/2026-07-29-flowlens-agentops-investigation-workbench.md new file mode 100644 index 00000000..e0ce7d08 --- /dev/null +++ b/docs/superpowers/plans/2026-07-29-flowlens-agentops-investigation-workbench.md @@ -0,0 +1,273 @@ +# FlowLens AgentOps Investigation Workbench Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use `superpowers:executing-plans` to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Transform AgentOps into a responsive, evidence-led Trace-first investigation workbench while preserving all current diagnostics behavior and API contracts. + +**Architecture:** Keep the existing diagnostics-source queries and replay/remote modes. Add a scoped AgentOps design-token stylesheet, a `RunOverview` presentational component, and a pure `TraceWaterfall` derived from normalized Timeline items. Compose them inside the existing workspace and restyle existing panes without changing source contracts. + +**Tech Stack:** Next.js 16, React 19, TypeScript 5.8, Tailwind CSS v4, Lucide React, TanStack Query, Vitest, Playwright. + +## Global Constraints + +- Preserve existing `/agentops` route, source contracts, remote/replay modes, filters, Explain, Export, evidence navigation, and event-detail redaction. +- Use only existing dependencies and real query data; do not modify backend schemas or APIs. +- Scope visual tokens to `.agentops-shell`; use the existing system font stack and `prefers-reduced-motion` behavior. +- No large gradients, neon effects, card-stack layout, excessive shadow, or horizontal page overflow. +- Validate with `pnpm.cmd test`, `pnpm.cmd check`, and `pnpm.cmd test:e2e:agentops-demo`. + +--- + +## File Structure + +- Create `frontend/src/styles/agentops.css`: AgentOps-only colors, spacing, typography, control, status, and motion tokens. +- Modify `frontend/src/styles/globals.css`: import the AgentOps token stylesheet. +- Create `frontend/src/components/agentops/run-overview.tsx`: selected-run identity and status header. +- Create `frontend/src/components/agentops/trace-waterfall.tsx`: pure trace-bar projection and selectable waterfall component. +- Modify `frontend/src/components/agentops/agentops-workspace.tsx`: compose workbench, overview, waterfall, existing Timeline, and responsive tabs/drawer. +- Modify `frontend/src/components/agentops/{metrics-strip,run-explorer,timeline,timeline-row,diagnostics-panel,event-detail-sheet}.tsx`: apply scoped surfaces, controls, status hierarchy, and responsive layout classes without changing public props. +- Create `frontend/tests/unit/components/trace-waterfall.test.tsx`: verify timestamp/duration projection and selection behavior. +- Modify `frontend/tests/unit/components/agentops-workspace.test.tsx`: verify overview, waterfall, and existing evidence navigation. +- Modify `frontend/tests/e2e/agentops-demo.spec.ts`: assert Trace Waterfall works at desktop/tablet/mobile and retain overflow check. + +## Task 1: Scope AgentOps Design Tokens and Run Identity Header + +**Files:** +- Create: `frontend/src/styles/agentops.css` +- Modify: `frontend/src/styles/globals.css` +- Create: `frontend/src/components/agentops/run-overview.tsx` +- Modify: `frontend/tests/unit/components/agentops-workspace.test.tsx` + +**Interfaces:** +- Consumes: `RunSummary` from `@/core/agentops/types` and `mode: "remote" | "replay"`. +- Produces: `RunOverview({ run, mode }: { run: RunSummary | null; mode: "remote" | "replay" })`. + +- [ ] **Step 1: Write the failing run overview test** + +```tsx +expect(await screen.findByText("Run context")).toBeVisible(); +expect(screen.getByText(/Thread:/)).toBeVisible(); +expect(screen.getByText(/Replay data source/)).toBeVisible(); +``` + +- [ ] **Step 2: Run the unit test to verify it fails** + +Run: `pnpm.cmd vitest run tests/unit/components/agentops-workspace.test.tsx` + +Expected: FAIL because `Run context` and the source label are not rendered. + +- [ ] **Step 3: Add scoped tokens and the header component** + +```tsx +export function RunOverview({ run, mode }: { run: RunSummary | null; mode: "remote" | "replay" }) { + return
...
; +} +``` + +Define `--agentops-surface`, `--agentops-canvas`, `--agentops-ink`, `--agentops-muted`, `--agentops-border`, `--agentops-accent`, semantic status variables, `--agentops-radius`, and `--agentops-motion` under `.agentops-shell`. Import the stylesheet after Tailwind imports. + +- [ ] **Step 4: Run the unit test to verify it passes** + +Run: `pnpm.cmd vitest run tests/unit/components/agentops-workspace.test.tsx` + +Expected: PASS with the existing replay interaction tests still green. + +- [ ] **Step 5: Commit the task** + +```powershell +git add frontend/src/styles/agentops.css frontend/src/styles/globals.css frontend/src/components/agentops/run-overview.tsx frontend/tests/unit/components/agentops-workspace.test.tsx +git commit -m "feat(agentops): add workbench design tokens" +``` + +## Task 2: Build the Real-data Trace Waterfall + +**Files:** +- Create: `frontend/src/components/agentops/trace-waterfall.tsx` +- Create: `frontend/tests/unit/components/trace-waterfall.test.tsx` + +**Interfaces:** +- Consumes: `items: TimelineItem[]`, `selectedSeq?: number`, and `onSelect?: (item: TimelineItem) => void`. +- Produces: `TraceWaterfall({ items, selectedSeq, onSelect })` and exported `buildTraceRows(items)` returning rows with `item`, `offsetPercent`, `widthPercent`, and `depth`. + +- [ ] **Step 1: Write failing projection and interaction tests** + +```tsx +expect(buildTraceRows(items)[1]).toMatchObject({ offsetPercent: 50, widthPercent: 50, depth: 1 }); +await user.click(screen.getByRole("button", { name: /trace event 2/i })); +expect(onSelect).toHaveBeenCalledWith(items[1]); +``` + +Use deterministic ISO timestamps with a 200ms total trace window and one item whose `parent_span_id` targets the first item. Add a zero-duration item and assert it renders a visible minimum-width marker. + +- [ ] **Step 2: Run the focused test to verify it fails** + +Run: `pnpm.cmd vitest run tests/unit/components/trace-waterfall.test.tsx` + +Expected: FAIL because `buildTraceRows` and `TraceWaterfall` do not exist. + +- [ ] **Step 3: Implement trace derivation and rendering** + +```tsx +const rows = buildTraceRows(items); +return

Trace Waterfall

{rows.map(renderTraceRow)}
; +``` + +Parse timestamps with `Date.parse`, use the earliest valid timestamp as the origin, and normalize the latest `timestamp + duration_ms` as the end. If timestamps are invalid or all durations are absent, render a compact chronological fallback list. Clamp each bar to 2%-100% so zero-duration events remain inspectable. Derive nesting with the existing cycle-safe parent-span traversal pattern. + +- [ ] **Step 4: Run focused tests to verify they pass** + +Run: `pnpm.cmd vitest run tests/unit/components/trace-waterfall.test.tsx` + +Expected: PASS for normal, zero-duration, invalid-timestamp, and selection cases. + +- [ ] **Step 5: Commit the task** + +```powershell +git add frontend/src/components/agentops/trace-waterfall.tsx frontend/tests/unit/components/trace-waterfall.test.tsx +git commit -m "feat(agentops): add timeline trace waterfall" +``` + +## Task 3: Compose the A1 Investigation Workbench + +**Files:** +- Modify: `frontend/src/components/agentops/agentops-workspace.tsx` +- Modify: `frontend/src/components/agentops/metrics-strip.tsx` +- Modify: `frontend/src/components/agentops/timeline.tsx` +- Modify: `frontend/tests/unit/components/agentops-workspace.test.tsx` + +**Interfaces:** +- Consumes: `RunOverview`, `TraceWaterfall`, existing `RunExplorer`, `MetricsStrip`, `Timeline`, and `DiagnosticsPanel`. +- Produces: a desktop three-surface workbench and existing tablet/mobile drawer plus tabs. + +- [ ] **Step 1: Add failing composition assertions** + +```tsx +expect(await screen.findByRole("region", { name: "Run context" })).toBeVisible(); +expect(screen.getByRole("region", { name: "Execution trace" })).toBeVisible(); +``` + +- [ ] **Step 2: Run the workspace test to verify it fails** + +Run: `pnpm.cmd vitest run tests/unit/components/agentops-workspace.test.tsx` + +Expected: FAIL because the new labelled regions are absent. + +- [ ] **Step 3: Integrate components and selection state** + +```tsx + + + +``` + +Place overview and metrics above the center surface. Preserve compact controls; add an `Trace` tab alongside Timeline and Root Cause below the desktop breakpoint. Use semantic `
` and `
+
+ +
+
+
+

+ Execution Timeline +

+ + {timelineQuery.data.items.length} events + +
+ +
+ +
+
+ + ) : null} + -
- {diagnosticsQuery.data && selected ? ( - { - setSelectedEvent( - diagnosticsQuery.data.timeline.find( - (item) => item.seq === seq, - ) ?? null, - ); - }} - /> - ) : ( - - )} -
+ {diagnosticsQuery.data && selected ? ( + { + setSelectedEvent( + diagnosticsQuery.data.timeline.find( + (item) => item.seq === seq, + ) ?? null, + ); + }} + /> + ) : ( + + )} + + + ) : null} void; + headerAction?: ReactNode; }) { const source = useDiagnosticsSource(); + const hasIncident = failure.primary_cause !== "NONE"; const [explanation, setExplanation] = useState( null, ); @@ -61,10 +65,13 @@ export function DiagnosticsPanel({ aria-hidden />

Root Cause

+ {headerAction}
- + Rule-based @@ -72,7 +79,9 @@ export function DiagnosticsPanel({
-

+

{failure.primary_cause}

diff --git a/frontend/src/components/agentops/event-detail-sheet.tsx b/frontend/src/components/agentops/event-detail-sheet.tsx index 1352fcb7..8d908dda 100644 --- a/frontend/src/components/agentops/event-detail-sheet.tsx +++ b/frontend/src/components/agentops/event-detail-sheet.tsx @@ -1,4 +1,5 @@ -import { X } from "lucide-react"; +import { ChevronDown, X } from "lucide-react"; +import { useEffect, useState } from "react"; import type { TimelineItem } from "@/core/agentops/types"; @@ -15,6 +16,12 @@ export function EventDetailSheet({ item: TimelineItem | null; onClose: () => void; }) { + const [metadataOpen, setMetadataOpen] = useState(false); + + useEffect(() => { + setMetadataOpen(false); + }, [item?.seq]); + if (!item) return null; return (

-

Safe metadata

-
- {Object.entries(item.metadata).map(([key, value]) => ( -
-
- {key} -
-
- {metadataValue(value)} -
-
- ))} -
+ + {metadataOpen ? ( +
+ {Object.entries(item.metadata).map(([key, value]) => ( +
+
+ {key} +
+
+ {metadataValue(value)} +
+
+ ))} +
+ ) : null}
diff --git a/frontend/src/components/agentops/run-explorer.tsx b/frontend/src/components/agentops/run-explorer.tsx index 989389f9..53f0966d 100644 --- a/frontend/src/components/agentops/run-explorer.tsx +++ b/frontend/src/components/agentops/run-explorer.tsx @@ -20,7 +20,28 @@ function runLabel(runId: string): string { .join(" "); } -function StatusIcon({ status }: { status: RunSummary["status"] }) { +function hasIncident(run: RunSummary): boolean { + return Boolean(run.diagnostic_category && run.diagnostic_category !== "NONE"); +} + +function isFailed(run: RunSummary): boolean { + return hasIncident(run) || ["error", "timeout", "interrupted"].includes(run.status); +} + +function statusLabel(run: RunSummary): string { + if (hasIncident(run)) { + return `success with incident: ${run.diagnostic_category}`; + } + return run.status; +} + +function StatusIcon({ run }: { run: RunSummary }) { + if (hasIncident(run)) { + return ( + + ); + } + const { status } = run; if (status === "success") { return ( { + if (statusFilter === "running" && run.status !== "running") return false; + if (statusFilter === "error" && !isFailed(run)) return false; if (!normalizedSearch) return true; return `${run.run_id} ${run.thread_id} ${run.model_name ?? ""}` .toLocaleLowerCase() @@ -143,13 +166,13 @@ export function RunExplorer({ ); diff --git a/frontend/src/components/agentops/run-overview.tsx b/frontend/src/components/agentops/run-overview.tsx index da1d845b..9ad24ba3 100644 --- a/frontend/src/components/agentops/run-overview.tsx +++ b/frontend/src/components/agentops/run-overview.tsx @@ -29,10 +29,15 @@ function sourceCopy(mode: "remote" | "replay") { export function RunOverview({ run, mode, + incidentCause, }: { run: RunSummary | null; mode: "remote" | "replay"; + incidentCause?: string; }) { + const hasIncident = run?.status === "success" && Boolean(incidentCause); + const displayStatus = hasIncident ? "Success with incident" : run?.status; + return (
- - {run.status} + {hasIncident ? : } + {displayStatus} + {hasIncident ? ( + + {incidentCause} + + ) : null} ) : null}
diff --git a/frontend/src/components/agentops/trace-waterfall.tsx b/frontend/src/components/agentops/trace-waterfall.tsx index 58a72ac0..01f9b4c0 100644 --- a/frontend/src/components/agentops/trace-waterfall.tsx +++ b/frontend/src/components/agentops/trace-waterfall.tsx @@ -125,10 +125,12 @@ export function TraceWaterfall({ items, selectedSeq, onSelect, + actions, }: { items: TimelineItem[]; selectedSeq?: number; onSelect?: (item: TimelineItem) => void; + actions?: React.ReactNode; }) { if (items.length === 0) { return ; @@ -150,11 +152,14 @@ export function TraceWaterfall({

Execution trace

Trace Waterfall

- - {rows.length - ? durationLabel(traceDuration) - : `${items.length} events`} - +
+ + {rows.length + ? durationLabel(traceDuration) + : `${items.length} events`} + + {actions} +
{rows.length === 0 ? ( diff --git a/frontend/src/core/agentops/schemas.ts b/frontend/src/core/agentops/schemas.ts index f2ecf2c3..39597be1 100644 --- a/frontend/src/core/agentops/schemas.ts +++ b/frontend/src/core/agentops/schemas.ts @@ -81,6 +81,7 @@ export const runSummarySchema = z total_tokens: z.number().int().nonnegative(), llm_call_count: z.number().int().nonnegative(), message_count: z.number().int().nonnegative(), + diagnostic_category: z.string().nullable().optional(), }) .strict(); diff --git a/frontend/src/styles/agentops.css b/frontend/src/styles/agentops.css index 412dd21a..639e8c73 100644 --- a/frontend/src/styles/agentops.css +++ b/frontend/src/styles/agentops.css @@ -90,6 +90,64 @@ border-color: var(--agentops-border); } +.agentops-icon-control { + border: 1px solid transparent; + border-radius: 7px; + color: var(--agentops-muted); + transition: + background-color var(--agentops-motion), + border-color var(--agentops-motion), + color var(--agentops-motion); +} + +.agentops-icon-control:hover, +.agentops-icon-control[aria-pressed="true"] { + border-color: var(--agentops-border); + background: var(--agentops-surface-muted); + color: var(--agentops-accent-ink); +} + +.agentops-resize-handle { + z-index: 1; + min-width: 10px; + min-height: 10px; + background: transparent; + transition: background-color var(--agentops-motion); +} + +.agentops-resize-handle::after { + border-radius: 999px; + background: transparent; + transition: background-color var(--agentops-motion); +} + +.agentops-resize-handle:hover::after, +.agentops-resize-handle:focus-visible::after { + background: var(--agentops-accent); +} + +.agentops-vertical-resize-handle { + min-height: 14px; + height: 14px !important; + width: 100% !important; + cursor: row-resize; + background: transparent; + pointer-events: auto; + touch-action: none; +} + +.agentops-vertical-resize-handle::after { + top: 50%; + right: 0; + bottom: auto; + left: 0; + width: auto; + height: 1px; + transform: translateY(-50%); + background: var(--agentops-border); + pointer-events: none; +} + .agentops-muted { color: var(--agentops-muted); } diff --git a/frontend/tests/e2e/agentops-demo.spec.ts b/frontend/tests/e2e/agentops-demo.spec.ts index 369b3610..9e69a5a9 100644 --- a/frontend/tests/e2e/agentops-demo.spec.ts +++ b/frontend/tests/e2e/agentops-demo.spec.ts @@ -27,6 +27,58 @@ test("static demo switches diagnostic scenarios", async ({ page }) => { expect(clientErrors).toEqual([]); }); +test("desktop lets users drag the full trace and timeline divider", async ({ + page, +}) => { + await page.setViewportSize({ width: 1440, height: 900 }); + await page.goto("/"); + await page.getByRole("button", { name: /tool error, error/i }).click(); + + const divider = page.getByRole("separator", { + name: "Resize trace and timeline panels", + }); + const dividerBox = await divider.boundingBox(); + expect(dividerBox).not.toBeNull(); + expect(dividerBox?.height).toBeGreaterThanOrEqual(14); + + const tracePanel = page + .locator("#agentops-vertical-layout > [data-panel]") + .first(); + const initialSize = await tracePanel.evaluate( + (panel) => panel.style.flexGrow, + ); + + const dragDividerAt = async (xOffset: number) => { + const currentDividerBox = await divider.boundingBox(); + expect(currentDividerBox).not.toBeNull(); + await page.mouse.move( + currentDividerBox!.x + xOffset, + currentDividerBox!.y + currentDividerBox!.height / 2, + ); + await page.mouse.down(); + await page.mouse.move( + currentDividerBox!.x + xOffset, + currentDividerBox!.y + currentDividerBox!.height / 2 + 35, + { steps: 4 }, + ); + await page.mouse.up(); + }; + + await dragDividerAt(8); + const leftDragSize = await tracePanel.evaluate( + (panel) => panel.style.flexGrow, + ); + expect(leftDragSize).not.toBe(initialSize); + + await dragDividerAt(dividerBox!.width - 8); + const rightDragSize = await tracePanel.evaluate( + (panel) => panel.style.flexGrow, + ); + expect(rightDragSize).not.toBe(leftDragSize); + + expect(rightDragSize).not.toBe(initialSize); +}); + for (const viewport of viewports) { test(`${viewport.name} layout keeps diagnostics usable`, async ({ page, @@ -76,7 +128,9 @@ for (const viewport of viewports) { await page.getByRole("tab", { name: "Root cause" }).click(); } - const rootCause = page.getByLabel("Root Cause"); + const rootCause = page.getByRole("complementary", { + name: "Root Cause", + }); await rootCause.scrollIntoViewIfNeeded(); await expect(rootCause).toBeVisible(); await expect(page.getByText("TOOL_ERROR")).toBeVisible(); diff --git a/frontend/tests/unit/components/agentops-workspace.test.tsx b/frontend/tests/unit/components/agentops-workspace.test.tsx index 079474c7..b4a7ebd3 100644 --- a/frontend/tests/unit/components/agentops-workspace.test.tsx +++ b/frontend/tests/unit/components/agentops-workspace.test.tsx @@ -7,7 +7,11 @@ import { AgentOpsWorkspace } from "@/components/agentops/agentops-workspace"; import { DiagnosticsSourceProvider } from "@/core/agentops/context"; import { createReplayDiagnosticsSource } from "@/core/agentops/replay-source"; -function renderWorkspace(initialScenarioId = "success") { +function renderWorkspace(initialScenarioId = "success", viewportWidth = 1024) { + Object.defineProperty(window, "innerWidth", { + configurable: true, + value: viewportWidth, + }); const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } }, }); @@ -85,9 +89,40 @@ describe("AgentOpsWorkspace", () => { expect( screen.getByRole("dialog", { name: /event 4 details/i }), ).toBeVisible(); + await user.click(screen.getByRole("button", { name: /safe metadata/i })); expect(screen.getByText("ToolException")).toBeVisible(); }); + test("keeps long execution traces in their own scroll region", async () => { + renderWorkspace("tool-error"); + + expect( + await screen.findByRole("region", { name: "Trace events" }), + ).toHaveClass("overflow-y-auto"); + }); + + test("exposes desktop workbench focus and layout reset controls", async () => { + renderWorkspace("tool-error", 1280); + + expect( + await screen.findByRole("button", { name: "Focus trace panel" }), + ).toBeVisible(); + expect( + screen.getByRole("button", { name: "Focus timeline panel" }), + ).toBeVisible(); + expect( + screen.getByRole("button", { name: "Focus root cause panel" }), + ).toBeVisible(); + expect( + screen.getByRole("button", { name: "Reset workbench layout" }), + ).toBeVisible(); + expect( + screen.getByRole("separator", { + name: "Resize trace and timeline panels", + }), + ).toHaveClass("agentops-vertical-resize-handle"); + }); + test("keeps selected run and phase filter states accessible", async () => { const user = userEvent.setup(); renderWorkspace("tool-error"); @@ -116,6 +151,7 @@ describe("AgentOpsWorkspace", () => { expect( screen.getByRole("dialog", { name: /event 4 details/i }), ).toBeVisible(); + await user.click(screen.getByRole("button", { name: /safe metadata/i })); expect(screen.getByText("ToolException")).toBeVisible(); expect(screen.queryByText(/secret/i)).not.toBeInTheDocument(); }); diff --git a/frontend/tests/unit/components/diagnostics-panel.test.tsx b/frontend/tests/unit/components/diagnostics-panel.test.tsx new file mode 100644 index 00000000..2cfa5795 --- /dev/null +++ b/frontend/tests/unit/components/diagnostics-panel.test.tsx @@ -0,0 +1,42 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, test, vi } from "vitest"; + +import { DiagnosticsPanel } from "@/components/agentops/diagnostics-panel"; +import { DiagnosticsSourceProvider } from "@/core/agentops/context"; +import type { DiagnosticsSource } from "@/core/agentops/source"; +import type { FailureAttribution } from "@/core/agentops/types"; + +const source = { mode: "remote" } as DiagnosticsSource; + +const failure: FailureAttribution = { + category: "MCP_TOOL_ERROR", + primary_cause: "MCP_TOOL_ERROR", + contributing_causes: [], + phase: "mcp", + confidence: 0.8, + root_cause: "MCP tool execution failed.", + evidence: [], + suggested_action: "Inspect MCP tool arguments.", +}; + +describe("DiagnosticsPanel", () => { + test("renders a diagnosed root cause with error styling", () => { + render( + + + , + ); + + expect(screen.getByText("MCP_TOOL_ERROR")).toHaveClass( + "text-[var(--agentops-danger)]", + ); + expect(screen.getByText("Rule-based")).toHaveClass( + "agentops-status-danger", + ); + }); +}); diff --git a/frontend/tests/unit/components/event-detail-sheet.test.tsx b/frontend/tests/unit/components/event-detail-sheet.test.tsx new file mode 100644 index 00000000..ad1bfe1b --- /dev/null +++ b/frontend/tests/unit/components/event-detail-sheet.test.tsx @@ -0,0 +1,37 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, test, vi } from "vitest"; + +import { EventDetailSheet } from "@/components/agentops/event-detail-sheet"; +import type { TimelineItem } from "@/core/agentops/types"; + +const item: TimelineItem = { + seq: 7, + timestamp: "2026-07-29T09:00:00.000Z", + phase: "tool", + component: "write_file", + event_type: "tool.end", + status: "success", + summary: "Tool completed", + duration_ms: 335, + span_id: "span-7", + parent_span_id: "span-3", + malformed: false, + metadata: { tool_name: "write_file", duration_ms: 335 }, +}; + +describe("EventDetailSheet", () => { + test("keeps safe metadata collapsed until requested", async () => { + const user = userEvent.setup(); + render(); + + const disclosure = screen.getByRole("button", { name: /safe metadata/i }); + expect(disclosure).toHaveAttribute("aria-expanded", "false"); + expect(screen.queryByText("tool_name")).not.toBeInTheDocument(); + + await user.click(disclosure); + + expect(disclosure).toHaveAttribute("aria-expanded", "true"); + expect(screen.getByText("tool_name")).toBeVisible(); + }); +}); diff --git a/frontend/tests/unit/components/run-explorer.test.tsx b/frontend/tests/unit/components/run-explorer.test.tsx new file mode 100644 index 00000000..e641b27d --- /dev/null +++ b/frontend/tests/unit/components/run-explorer.test.tsx @@ -0,0 +1,42 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, test, vi } from "vitest"; + +import { RunExplorer } from "@/components/agentops/run-explorer"; +import type { RunSummary } from "@/core/agentops/types"; + +const incidentRun = { + run_id: "run-incident", + thread_id: "thread-incident", + status: "success", + created_at: "2026-01-02T00:00:00.000Z", + updated_at: "2026-01-02T00:00:05.000Z", + model_name: "test-model", + total_tokens: 12, + llm_call_count: 1, + message_count: 2, + diagnostic_category: "MCP_TOOL_ERROR", +} as RunSummary; + +describe("RunExplorer", () => { + test("marks a successful run with an unrecovered incident as failed", () => { + render( + , + ); + + expect( + screen.getByRole("button", { + name: /run incident, success with incident: mcp_tool_error/i, + }), + ).toBeVisible(); + }); +}); diff --git a/frontend/tests/unit/components/run-overview.test.tsx b/frontend/tests/unit/components/run-overview.test.tsx new file mode 100644 index 00000000..160faaaf --- /dev/null +++ b/frontend/tests/unit/components/run-overview.test.tsx @@ -0,0 +1,32 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, test } from "vitest"; + +import { RunOverview } from "@/components/agentops/run-overview"; +import type { RunSummary } from "@/core/agentops/types"; + +const run: RunSummary = { + run_id: "run-mcp-incident", + thread_id: "thread-mcp-incident", + status: "success", + created_at: "2026-07-29T12:00:00.000Z", + updated_at: "2026-07-29T12:00:05.000Z", + model_name: "deepseek-chat", + total_tokens: 100, + llm_call_count: 1, + message_count: 2, +}; + +describe("RunOverview", () => { + test("surfaces a successful run with a diagnosed MCP incident", () => { + render( + , + ); + + expect(screen.getByText("Success with incident")).toBeVisible(); + expect(screen.getByText("MCP_TOOL_ERROR")).toBeVisible(); + }); +}); diff --git a/frontend/tests/unit/setup.ts b/frontend/tests/unit/setup.ts index 004761e5..174c3ae8 100644 --- a/frontend/tests/unit/setup.ts +++ b/frontend/tests/unit/setup.ts @@ -3,6 +3,22 @@ import "@testing-library/jest-dom/vitest"; import { cleanup } from "@testing-library/react"; import { afterEach } from "vitest"; +class ResizeObserverMock { + observe() { + return undefined; + } + unobserve() { + return undefined; + } + disconnect() { + return undefined; + } +} + +if (typeof window !== "undefined" && !window.ResizeObserver) { + window.ResizeObserver = ResizeObserverMock as typeof ResizeObserver; +} + afterEach(() => { cleanup(); }); diff --git a/now.md b/now.md new file mode 100644 index 00000000..5c48fad5 --- /dev/null +++ b/now.md @@ -0,0 +1,272 @@ +# FlowLens AgentOps 交接说明 + +> 最后整理：2026-07-31 +> +> 工作分支：`codex/runlens-agentops` +> 远程仓库：`flowlens` -> `https://github.com/try1004/FlowLens-AgentOps.git` + +## 1. 项目定位与当前目标 + +FlowLens AgentOps 是在 DeerFlow Agent Runtime 上完成的可观测性与确定性故障归因扩展。它不改变 Agent 的决策，而是把一次 Agent Run 中 Runtime、模型、Tool、Subagent、Middleware、MCP、Memory 等行为记录为事件流，再派生为可查看、可统计、可归因的诊断结果。 + +当前目标不是宣称生产级 AgentOps 平台，而是交付一个可开源、可本地验证、面试可讲清楚的完整最小闭环： + +```text +Agent Runtime + -> RunJournal / append-only RunEventStore + -> Timeline normalization + Metrics + -> deterministic Diagnostics + -> FastAPI AgentOps API + -> FlowLens Dashboard +``` + +本轮还补齐了真实 MCP 与 Skill 的教学/验证链路：SQLite Analytics MCP 负责只读本地演示数据分析，FlowLens Inspector MCP 负责只读诊断查询。两者通过 MCP delegated token 实现发现阶段与执行阶段的权限分离。 + +## 2. 已完成进度 + +### 2.1 运行链路与事件采集 + +- Runtime 在一次用户任务开始时创建 `run_id`，并与 DeerFlow 会话的 `thread_id` 关联。 +- RunJournal 将生命周期与组件埋点写入 append-only `RunEventStore`；事件不覆盖旧记录，便于回放、审计和后续重新归因。 +- Timeline 将原始事件归一化为统一的阶段、组件、状态、摘要、耗时、span 与 parent span。 +- Metrics 从 RunManager 与 Timeline 派生时长、Token、工具调用、错误、Subagent、MCP、Memory 等指标，并按 `span_id` 去重。 + +### 2.2 Diagnostics 与事件状态 + +- Diagnostics 使用确定性规则而非 LLM 归因；主要原因遵循“可解释、可复现、可测试”的优先级。 +- 外层 Run 的生命周期状态与 FlowLens 的诊断状态分离：Run 可以 `Success`，但如果包含未恢复的 MCP/Tool 等 incident，仍会显示诊断分类。 +- 有真实成功重试或结构化 `report_fallback` 且证明任务约束已满足时，错误可以视为恢复；否则事件仍保留为证据。 +- Run Explorer 对 `diagnostic_category != NONE` 的运行显示红色 incident 标识，并将其纳入 Failed 筛选；避免“绿色成功”掩盖内部错误。 +- Root Cause 面板中的非 `NONE` 类别使用红色错误语义；`Success with incident` 表示 Agent 图结束了，但需要开发者关注过程中的异常。 + +### 2.3 真实 MCP 与权限设计 + +- 新增 `backend/packages/sqlite-analytics-mcp/`： + - `sqlite_analytics_schema_overview` + - `sqlite_analytics_describe_table` + - `sqlite_analytics_query_readonly` +- 新增 `backend/packages/flowlens-inspector-mcp/`：面向诊断数据的只读查询能力。 +- MCP discovery 使用受限服务令牌；真正执行工具时根据 `user_id + run_id` 申请短时 delegated token，模型只看到 Tool schema 与结果，不直接接触令牌。 +- 修复同步 MCP Tool wrapper 丢失 LangGraph `ToolRuntime` 注解的问题：`@functools.wraps` 保留原协程签名，因此 delegated token 注入可在真实 `ToolNode` 中工作。 +- 在容器内已做过真实链路探针：`ToolNode -> OAuth delegation -> SQLite MCP schema_overview` 返回 `agent_runs` 与 `tool_calls` schema。不要把本地用户标识或令牌写入日志、文档或 Git。 + +### 2.4 Skill、UI 与文档 + +- 新增 `skills/public/sqlite-analytics/SKILL.md`：要求 Agent 使用已注册的 SQLite MCP，不询问数据库路径、不建议用 Python 临时建库、只读 SQL。 +- AgentOps Dashboard 已改为可调节工作台：左右区和 Trace Waterfall / Execution Timeline 可拖拽调整，事件详情抽屉展示 span 链路；Safe Metadata 默认折叠。 +- Waterfall 与 Timeline 不重复承担同一任务：Waterfall 展示并发、重叠与关键路径；Timeline 作为可筛选的事件账本和审计入口。 +- 已新增 MCP 双链路验证文档、MCP 验证指南、架构说明、局部单元/接口/E2E 测试。 + +## 3. 核心数据模型与讲解口径 + +| 概念 | 含义 | 面试时的简述 | +| --- | --- | --- | +| `thread_id` | DeerFlow 的对话/会话标识，可跨多次任务 | “它把同一段用户对话串起来。” | +| `run_id` | 一次 Agent Loop 的执行标识，由 Runtime 在任务启动时创建 | “它是这一轮执行的唯一追踪主键。” | +| `span_id` | 一个具体工作单元的执行区间，例如一次 Tool 调用 | “它像一次函数调用的计时区间。” | +| `parent_span_id` | 当前 span 的父工作单元 | “它把工具、模型、子任务组织成可追溯的调用树。” | +| RunEventStore | append-only 原始事件存储 | “像录像的原始素材，不被后续展示逻辑覆盖。” | +| Timeline | 面向人阅读的归一化执行账本 | “回答先后顺序、组件做了什么、最后状态是什么。” | +| Waterfall | 面向性能/并发分析的时间轴几何视图 | “回答谁在什么时候运行、是否重叠、瓶颈在哪里。” | +| Safe Metadata | 脱敏后的扩展字段 | “保留调试上下文，但不直接展示令牌、原始敏感输入或连接串。” | + +## 4. 本地使用与验证 + +### 4.1 前置条件 + +- Windows + PowerShell。 +- Docker Desktop 已启动，并且 Docker Engine 正常运行。 +- 根目录的 `.env` 已配置模型提供方所需的本地密钥。不要提交 `.env`。 +- 在本工作树根目录执行：`D:\vscode\项目\deer-flow\.worktrees\runlens-agentops`。 + +### 4.2 启动完整本地环境 + +```powershell +Set-Location 'D:\vscode\项目\deer-flow\.worktrees\runlens-agentops' + +$env:DEER_FLOW_ROOT=(Get-Location).Path +$env:HOME=$env:USERPROFILE +$env:DEER_FLOW_CONTAINER_PREFIX='flowlens-agentops' +$env:DEER_FLOW_HTTP_PORT='2027' + +docker compose -p flowlens-agentops ` + -f docker/docker-compose-dev.yaml ` + -f docker/docker-compose-agentops-isolated.yaml ` + up -d --remove-orphans frontend gateway nginx + +docker compose -p flowlens-agentops ` + -f docker/docker-compose-dev.yaml ` + -f docker/docker-compose-agentops-isolated.yaml ` + ps + +Invoke-RestMethod http://localhost:2027/health +``` + +预期：`frontend`、`gateway`、`nginx`、SQLite Analytics MCP、FlowLens Inspector MCP 都为运行状态；健康检查返回 `healthy`。浏览器打开： + +- DeerFlow： +- FlowLens AgentOps： + +停止环境： + +```powershell +docker compose -p flowlens-agentops ` + -f docker/docker-compose-dev.yaml ` + -f docker/docker-compose-agentops-isolated.yaml ` + down +``` + +### 4.3 成功路径：SQLite Analytics MCP + +在 DeerFlow 新建对话，发送： + +```text +使用 SQLite Analytics MCP 完成本地演示数据分析：先查看 schema，再描述 agent_runs 表，最后按 status 统计运行数量。只执行只读 SQL，写出使用的 SQL，并明确标记结果为“本地演示数据”。 +``` + +预期行为（模型可能将文案略作调整，但工具意图应保持一致）： + +1. 调用 `sqlite_analytics_schema_overview`。 +2. 调用 `sqlite_analytics_describe_table(table_name="agent_runs")`。 +3. 调用 `sqlite_analytics_query_readonly`，执行类似： + +```sql +SELECT status, COUNT(*) AS run_count +FROM agent_runs +GROUP BY status +ORDER BY run_count DESC; +``` + +4. 在 AgentOps 刷新后选择最新 Run：顶部 `MCP > 0`；Timeline 的 MCP 筛选可看到 `mcp.tool.start` / `mcp.tool.end`；无异常时 Root Cause 为 `NONE`，Run 为 `Success`。 + +若模型回答“工具未注册”或 Timeline 出现 `mcp.tool.error`，不要先改模型提示词。先检查容器状态、gateway 日志和 MCP 发现流程： + +```powershell +docker compose -p flowlens-agentops ` + -f docker/docker-compose-dev.yaml ` + -f docker/docker-compose-agentops-isolated.yaml ` + logs --tail=200 gateway sqlite-analytics +``` + +### 4.4 失败路径：只读策略 + +在新的 DeerFlow 对话中发送： + +```text +使用 SQLite Analytics MCP 执行 DELETE FROM agent_runs，并告诉我删除了多少数据。 +``` + +预期：系统拒绝变更型 SQL，演示数据不被删除。若 MCP 尝试或策略拒绝形成未恢复错误，AgentOps 应显示 MCP/Tool 事件证据并给出相应诊断；如果 Agent 主动拒绝且没有真正发起工具调用，则不会凭空制造 MCP 错误。这是正确行为。 + +### 4.5 Dashboard 演示顺序 + +1. 在 DeerFlow 完成一轮任务，再打开/刷新 `/agentops`。 +2. Run Explorer：绿色圆点是无 incident 的成功 Run；红色 `X` 表示生命周期失败或诊断发现未恢复 incident。 +3. Run Context：说明 `thread_id`、`run_id`、模型与 Run 生命周期状态。 +4. Metrics：看耗时、Token、Tools/Errors、MCP、Memory、Subagents。 +5. Trace Waterfall：解释时间重叠与关键路径；上下拖拽分隔条可调整视图高度。 +6. Execution Timeline：按 MCP/Tool/Model 等筛选，点击事件打开详情。 +7. Event detail：先看 Phase、Component、Status、Duration、Span、Parent span；按需展开 Safe Metadata。 +8. Root Cause：查看规则分类、证据事件与建议操作；`Explain` 展示补充解释，`Export` 下载当前 Run 的 JSON。 + +## 5. 测试与验证命令 + +提交前应至少运行： + +```powershell +# FlowLens 后端、前端、构建与 Replay Demo gate +make flowlens-check + +# MCP/委托令牌/同步 wrapper/诊断关键回归 +Set-Location backend +uv run pytest ` + tests/test_mcp_sync_wrapper.py ` + tests/test_mcp_oauth.py ` + tests/test_mcp_diagnostics.py ` + tests/test_agentops_runs_api.py ` + tests/test_agentops_run_lookup_api.py ` + tests/test_flowlens_attribution.py -q +``` + +若 Windows 未安装 `make`，使用与 `flowlens-check` 等价的 PowerShell 验证： + +```powershell +Set-Location backend +$flowLensTests = Get-ChildItem tests/test_flowlens_*.py | ForEach-Object { $_.FullName } +uv run pytest @flowLensTests tests/test_agentops_runs_api.py tests/test_thread_run_diagnostics_api.py -q +uv run ruff check packages/harness/deerflow/runtime/diagnostics app/gateway/routers/agentops.py app/gateway/routers/thread_runs.py + +Set-Location ../frontend +pnpm.cmd check +pnpm.cmd test +pnpm.cmd build +pnpm.cmd agentops:demo:build +``` + +补充的人工验收为：Docker 健康检查、一次自然语言 SQLite MCP 成功路径、一次只读拒绝路径、AgentOps UI 的 Run/Timeline/详情/Root Cause 检查。外部模型具有非确定性，浏览器自然语言链路不应被伪装成稳定的自动化测试。 + +## 6. 已知边界与后续优化方案 + +### P0：先完成可演示闭环 + +- 用新对话完整人工验证 SQLite 的 `schema -> describe -> query_readonly` 三步，保存 DeerFlow 与 AgentOps 截图。 +- 验证只读拒绝路径，明确“模型拒绝”和“已实际调用但 MCP 拒绝”的事件差异。 +- 回归 Run Explorer：只要 `diagnostic_category != NONE`，即使原始 Run 生命周期为 `success`，也必须显示红色 incident。 + +### P1：让诊断更接近任务真实结果 + +- 新增任务结果契约/Verifier：区分“LangGraph 图正常结束”和“用户目标被满足”。当前 FlowLens 已能记录 success-with-incident，但不能完全判断自然语言任务是否语义完成。 +- 将 incident 摘要写入独立索引或持久化字段。当前 runs 列表为每个 Run 最多读取 5000 条事件、并发 8 路重新归因，适合本地演示，不适合高并发历史查询。 +- 对关键 MCP 失败增加受控降级：只在替代路径有可验证结果、满足原任务约束时，调用结构化 `report_fallback` 清除 incident；不能因为模型“说已完成”而清除。 + +### P2：扩展为工程化平台 + +- 事件存储迁移/补充 PostgreSQL、分区与保留策略；Timeline 使用虚拟列表、预聚合 Metrics、按需拉取详情。 +- 增加 OpenTelemetry trace/span 对接，保留 FlowLens 的 Agent 领域事件归一化与确定性归因优势。 +- MCP 接入企业 IdP/SSO、令牌撤销、审计日志、RBAC、工具级 policy 与限流。 +- 增加评估集：成功率、任务验证率、工具失败恢复率、诊断命中率、P95 耗时、Token 成本；数据必须来自真实测试，不编造业务指标。 + +## 7. 关键文件索引 + +| 领域 | 主要位置 | +| --- | --- | +| Diagnostics | `backend/packages/harness/deerflow/runtime/diagnostics/attribution.py` | +| Run 列表诊断富化 API | `backend/app/gateway/routers/agentops.py` | +| MCP delegated token | `backend/packages/harness/deerflow/mcp/oauth.py`、`backend/app/gateway/routers/mcp_tokens.py` | +| ToolRuntime 注入修复 | `backend/packages/harness/deerflow/tools/sync.py` | +| Fallback 证据 | `backend/packages/harness/deerflow/tools/builtins/report_fallback_tool.py` | +| SQLite Analytics MCP | `backend/packages/sqlite-analytics-mcp/` | +| FlowLens Inspector MCP | `backend/packages/flowlens-inspector-mcp/` | +| SQLite Skill | `skills/public/sqlite-analytics/SKILL.md` | +| Dashboard 工作台 | `frontend/src/components/agentops/agentops-workspace.tsx` | +| Run incident UI | `frontend/src/components/agentops/run-explorer.tsx` | +| Root Cause UI | `frontend/src/components/agentops/diagnostics-panel.tsx` | +| 视觉样式 | `frontend/src/styles/agentops.css` | +| Docker 隔离编排 | `docker/docker-compose-agentops-isolated.yaml` | +| MCP 双链路手册 | `docs/flowlens-dual-mcp-test-runbook.zh-CN.md` | + +## 8. 给下一位 Codex 的接手提示词 + +```text +请先完整阅读仓库根目录 now.md，并把它作为 FlowLens AgentOps 的当前交接契约。 + +工作目录应为 deer-flow/.worktrees/runlens-agentops，当前开发分支为 codex/runlens-agentops。不要 reset、checkout 覆盖、删除或回退任何已有变更；先运行 git status、阅读相关实现与测试，再决定修改范围。绝不提交 .env、令牌、日志、SQLite 本地数据或用户标识。 + +项目目标是可本地验证的 DeerFlow Agent Runtime 可观测性和确定性故障归因，不应夸大为已生产落地的平台。区分 Run 生命周期成功、任务语义完成与 FlowLens incident 三个概念。 + +若处理 Bug 或新增行为： +1. 先定位到现有测试和真实数据路径； +2. 先写/补充会失败的最小测试； +3. 仅使用 apply_patch 修改代码； +4. 运行相关测试、lint、typecheck/build； +5. 对 Docker/MCP 改动再跑最小真实链路； +6. 汇报已验证事实、已知限制和未验证项。 + +优先级：先完成 now.md 的 P0 人工闭环，再实现 P1 的任务结果契约与 incident 索引；不要为了漂亮 UI 删除诊断证据，也不要将模型自然语言成功当作已验证的 fallback。 +``` + +## 9. Git 协作约定 + +- 当前代码应推送到 `flowlens` 远程，而非上游 `origin`（ByteDance DeerFlow）。 +- 推送前执行本文件第 5 节的验证，检查 `git diff --check`，并确认暂存区没有 `.env`、日志、数据库或 token。 +- 先推送 `codex/runlens-agentops` 分支；是否合并到默认分支由仓库所有者决定，避免未经确认改写主分支历史。 diff --git a/scripts/verify_flowlens-dual-mcp.ps1 b/scripts/verify_flowlens-dual-mcp.ps1 new file mode 100644 index 00000000..b833b8f8 --- /dev/null +++ b/scripts/verify_flowlens-dual-mcp.ps1 @@ -0,0 +1,38 @@ +[CmdletBinding()] +param( + [string]$ProjectName = "flowlens-agentops", + [int]$Port = 2027 +) + +$ErrorActionPreference = "Stop" + +$requiredServices = @("gateway", "nginx", "frontend", "sqlite-analytics", "flowlens-inspector") +$composeFiles = @( + "-p", $ProjectName, + "-f", "docker/docker-compose-dev.yaml", + "-f", "docker/docker-compose-agentops-isolated.yaml" +) + +$rows = @(& docker compose @composeFiles ps --format json | ForEach-Object { + if ($_.Trim()) { + $_ | ConvertFrom-Json + } +}) + +foreach ($service in $requiredServices) { + $row = $rows | Where-Object { $_.Service -eq $service } + if ($null -eq $row -or $row.State -ne "running") { + throw "FlowLens service '$service' is not running." + } + if ($service -in @("sqlite-analytics", "flowlens-inspector") -and $row.Health -ne "healthy") { + throw "FlowLens MCP service '$service' is not healthy." + } +} + +$health = Invoke-RestMethod "http://localhost:$Port/health" -TimeoutSec 10 +if ($health.status -ne "healthy") { + throw "Gateway health response was not healthy." +} + +Write-Host "FlowLens dual-MCP stack is healthy on http://localhost:$Port" -ForegroundColor Green +Write-Host "MCP services: sqlite-analytics, flowlens-inspector" -ForegroundColor Green diff --git a/skills/public/sqlite-analytics/SKILL.md b/skills/public/sqlite-analytics/SKILL.md new file mode 100644 index 00000000..df89842c --- /dev/null +++ b/skills/public/sqlite-analytics/SKILL.md @@ -0,0 +1,40 @@ +--- +name: sqlite-analytics +description: Use this skill when a user asks to inspect the local FlowLens demonstration analytics database, compare aggregate run outcomes, or write bounded read-only SQL. Start with the schema tools and return the SQL, assumptions, and evidence used. +--- + +# SQLite Analytics + +Use the `sqlite_analytics` MCP tools only for the local demonstration analytics dataset. They are read-only and intentionally bounded. The runtime already provides the local demonstration database: do not ask the user for a database path and do not propose creating a database with Python. + +## Method + +1. Call `sqlite_analytics_schema_overview` first. +2. Call `sqlite_analytics_describe_table` before querying a table for the first time. +3. Call `sqlite_analytics_query_readonly` with one `SELECT`, `WITH`, or `EXPLAIN QUERY PLAN` statement. +4. Keep the `limit` at 50 or below unless the user asks for a larger bounded result. +5. State the SQL and the returned row count in the answer. Label the data as local demonstration data, not production telemetry. + +If the user explicitly asks for SQLite Analytics MCP, execute the named MCP tools above. Do not replace them with sandbox file tools, shell commands, or a generic explanation of unavailable tooling. + +## Query Patterns + +```sql +SELECT status, COUNT(*) AS run_count +FROM agent_runs +GROUP BY status +ORDER BY run_count DESC; +``` + +```sql +SELECT tool_name, COUNT(*) AS call_count, AVG(duration_ms) AS avg_duration_ms +FROM tool_calls +GROUP BY tool_name +ORDER BY call_count DESC; +``` + +## Guardrails + +- Do not attempt `INSERT`, `UPDATE`, `DELETE`, DDL, `ATTACH`, `PRAGMA`, or multiple statements. +- Do not claim this data reflects real users, production load, or benchmarking results. +- If the question requires run-level root-cause evidence, use FlowLens Inspector MCP instead of inferring it from aggregate rows. From 10b45874db855b36b4febf04e798dcdaa53d97ac Mon Sep 17 00:00:00 2001 From: tangrongyuan <18532002267@163.com> Date: Fri, 31 Jul 2026 09:47:29 +0800 Subject: [PATCH 14/14] docs: complete GitHub README and MCP runtime displays --- README.md | 61 +++++++++++++++--- .../agentops/event-detail-sheet.tsx | 35 +++++++++- .../src/components/agentops/metrics-strip.tsx | 53 ++++++++++----- .../components/event-detail-sheet.test.tsx | 24 +++++++ .../unit/components/metrics-strip.test.tsx | 64 +++++++++++++++++++ 5 files changed, 213 insertions(+), 24 deletions(-) create mode 100644 frontend/tests/unit/components/metrics-strip.test.tsx diff --git a/README.md b/README.md index b331d461..9edfa58a 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ English | [中文](./README_zh.md) ![FlowLens AgentOps dashboard](./docs/assets/flowlens-agentops-dashboard.png) -[Pages Demo](https://try1004.github.io/FlowLens-AgentOps/) | [Quick Start](#quick-start) | [Architecture](./docs/architecture.md) | [Dual MCP Validation](./docs/mcp-validation-guide.md) +[Pages Demo](https://try1004.github.io/FlowLens-AgentOps/) | [Quick Start](#quick-start) | [Architecture](./docs/architecture.md) | [Dual MCP Validation](./docs/mcp-validation-guide.md) | [Maintainer Handoff](./now.md) > Built on [DeerFlow 2.0](https://github.com/bytedance/deer-flow) under the MIT License. DeerFlow provides the Agent runtime, tools, subagents, memory, MCP, sandbox, and application foundation. FlowLens adds the diagnostics and AgentOps product layer described below. @@ -45,6 +45,14 @@ The diagnostics path is read-only: it does not mutate checkpoints, prompts, RunR ## Quick Start +### Choose a mode + +| Mode | What it proves | Requirements | URL | +| --- | --- | --- | --- | +| Replay | Dashboard interactions and deterministic synthetic failure scenarios | Node.js or Docker; no model key | `http://localhost:5173` or `http://localhost:4173` | +| Connected Runtime | Real DeerFlow runs emitted into the AgentOps event pipeline | Normal DeerFlow configuration, account, and model provider | `http://localhost:2026/agentops` | +| Connected + dual MCP | Runtime delegation, read-only SQLite Analytics MCP, and FlowLens Inspector MCP | Docker Desktop plus the Connected Runtime requirements | `http://localhost:2027/agentops` | + ### No-key Replay with pnpm ```bash @@ -67,6 +75,14 @@ make flowlens-demo Open `http://localhost:4173`, then stop it with `make flowlens-stop`. +On Windows without `make`, run the Compose command directly: + +```powershell +docker compose -f docker/docker-compose-flowlens-demo.yaml up --build +``` + +Press `Ctrl+C` to stop the foreground Replay container. + ### Connected DeerFlow runtime ```bash @@ -78,11 +94,36 @@ Open `http://localhost:2026/agentops`. Connected mode requires the normal DeerFl ### Connected runtime with dual MCP services -The development compose stack can run two internal Streamable HTTP MCP services. -They use a discovery-only service token for schema loading and a short user/run-bound -delegated token for each tool call. The full isolated Docker command and validation -prompts are in the [Dual MCP Validation Guide](./docs/mcp-validation-guide.md) -and the [Chinese end-to-end test runbook](./docs/flowlens-dual-mcp-test-runbook.zh-CN.md). +The development stack runs two internal-only Streamable HTTP MCP services: + +- **SQLite Analytics MCP** exposes schema inspection and `SELECT`-only local demonstration-data queries. +- **FlowLens Inspector MCP** queries owner-checked diagnostic data through the Gateway. + +Schemas are discovered with a restricted service token. A real tool invocation receives a short-lived token bound to the current `user_id` and `run_id`; the model never receives that token. + +From the repository root in PowerShell: + +```powershell +$env:DEER_FLOW_ROOT = (Get-Location).Path +$env:HOME = $env:USERPROFILE +$env:DEER_FLOW_CONTAINER_PREFIX = "flowlens-agentops" +$env:DEER_FLOW_HTTP_PORT = "2027" + +docker compose -p flowlens-agentops ` + -f docker/docker-compose-dev.yaml ` + -f docker/docker-compose-agentops-isolated.yaml ` + up -d --build --remove-orphans frontend gateway nginx sqlite-analytics flowlens-inspector + +Invoke-RestMethod http://localhost:2027/health +``` + +Open `http://localhost:2027/`, sign in, then open `http://localhost:2027/agentops`. To exercise the real SQLite path, start a new chat and send: + +```text +Use the SQLite Analytics MCP to inspect the available schema, describe the agent_runs table, then count runs by status. Execute read-only SQL only, state the SQL used, and label the result as local demonstration data. +``` + +The selected Run should show MCP events in Timeline and an `MCP > 0` metric. The [Dual MCP Validation Guide](./docs/mcp-validation-guide.md) and [Chinese end-to-end runbook](./docs/flowlens-dual-mcp-test-runbook.zh-CN.md) cover expected evidence, policy-rejection cases, and shutdown. ## Diagnostic APIs @@ -134,8 +175,10 @@ FlowLens does not claim authorship of the DeerFlow runtime modules. See [NOTICE] | Gate | Result | | ---------------------------------- | ------------------------------------------------------------------ | -| Focused backend FlowLens/API suite | 53 tests passed | -| Frontend Vitest suite | 219 tests passed | +| Focused backend FlowLens/API suite | 58 tests passed on 2026-07-31 | +| Frontend Vitest suite | 232 tests passed on 2026-07-31 | +| Frontend quality gate | ESLint, TypeScript, Next production build, and Vite Replay build passed on 2026-07-31 | +| Real MCP integration probe | `ToolNode -> delegated OAuth token -> SQLite schema tool` completed locally | | Next and static Replay Playwright | 5 tests passed across route, desktop, tablet, and mobile workflows | | Secret scan | No high-confidence credentials in Git-tracked files | | In-process diagnostics benchmark | 2,000 events, 20 iterations, median 24.7578 ms, P95 36.5606 ms | @@ -165,6 +208,8 @@ The machine-readable result is committed at [flowlens-benchmark.json](./docs/ass - [Failure attribution rules](./docs/failure-attribution.md) - [Replay and Connected demo guide](./docs/demo-guide.md) - [Dual MCP validation guide](./docs/mcp-validation-guide.md) +- [Chinese dual MCP end-to-end runbook](./docs/flowlens-dual-mcp-test-runbook.zh-CN.md) +- [Maintainer handoff and next steps](./now.md) - [Contributing](./CONTRIBUTING.md) - [Changelog](./CHANGELOG.md) diff --git a/frontend/src/components/agentops/event-detail-sheet.tsx b/frontend/src/components/agentops/event-detail-sheet.tsx index 8d908dda..d9c4b28b 100644 --- a/frontend/src/components/agentops/event-detail-sheet.tsx +++ b/frontend/src/components/agentops/event-detail-sheet.tsx @@ -9,6 +9,17 @@ function metadataValue(value: unknown): string { return JSON.stringify(value); } +function metadataString( + metadata: TimelineItem["metadata"], + keys: string[], +): string | null { + for (const key of keys) { + const value = metadata[key]; + if (typeof value === "string" && value.trim()) return value; + } + return null; +} + export function EventDetailSheet({ item, onClose, @@ -23,6 +34,12 @@ export function EventDetailSheet({ }, [item?.seq]); if (!item) return null; + const isMcp = item.phase === "mcp"; + const mcpServer = isMcp + ? (metadataString(item.metadata, ["server_name", "server", "mcp_server"]) ?? + "Not reported") + : null; + return (
Phase
{item.phase}
-
Component
+
+ {isMcp ? "MCP tool" : "Component"} +
{item.component}
Status
{item.status}
@@ -73,6 +92,20 @@ export function EventDetailSheet({ + {isMcp ? ( +
+

MCP context

+
+
Server
+
{mcpServer}
+
Tool lifecycle
+
+ {item.event_type} +
+
+
+ ) : null} +

Summary

{item.summary}

diff --git a/frontend/src/components/agentops/metrics-strip.tsx b/frontend/src/components/agentops/metrics-strip.tsx index a05a6cff..61e6b1c4 100644 --- a/frontend/src/components/agentops/metrics-strip.tsx +++ b/frontend/src/components/agentops/metrics-strip.tsx @@ -1,5 +1,12 @@ import type { RunMetrics, TimelineItem } from "@/core/agentops/types"; +type MetricCell = { + label: string; + value: string; + detail?: string; + detailClassName?: string; +}; + function durationLabel(durationMs: number | null): string { if (durationMs === null) return "--"; if (durationMs < 1000) return `${durationMs} ms`; @@ -14,28 +21,37 @@ export function MetricsStrip({ timeline: TimelineItem[]; }) { const mcpEvents = timeline.filter((item) => item.phase === "mcp").length; + const mcpIncidents = timeline.filter( + (item) => item.phase === "mcp" && item.status === "error", + ).length; const memoryEvents = timeline.filter( (item) => item.phase === "memory", ).length; - const cells = [ - ["Duration", durationLabel(metrics.duration_ms)], - ["Tokens", metrics.total_tokens.toLocaleString()], - [ - "Tools / Errors", - `${metrics.tool_call_count} / ${metrics.tool_error_count}`, - ], - [ - "Subagents", - `${metrics.subagent_count} / ${metrics.subagent_error_count}`, - ], - ["MCP", String(mcpEvents)], - ["Memory", String(memoryEvents)], - ["Status", metrics.final_status], + const cells: MetricCell[] = [ + { label: "Duration", value: durationLabel(metrics.duration_ms) }, + { label: "Tokens", value: metrics.total_tokens.toLocaleString() }, + { + label: "Tools / Errors", + value: `${metrics.tool_call_count} / ${metrics.tool_error_count}`, + }, + { + label: "Subagents", + value: `${metrics.subagent_count} / ${metrics.subagent_error_count}`, + }, + { + label: "MCP", + value: mcpEvents === 1 ? "1 event" : `${mcpEvents} events`, + detail: mcpIncidents > 0 ? `${mcpIncidents} incident` : undefined, + detailClassName: + mcpIncidents > 0 ? "agentops-metric-danger" : undefined, + }, + { label: "Memory", value: String(memoryEvents) }, + { label: "Status", value: metrics.final_status }, ]; return (
- {cells.map(([label, value]) => ( + {cells.map(({ label, value, detail, detailClassName }) => (
{value} + {detail ? ( +

+ {detail} +

+ ) : null}
))}
diff --git a/frontend/tests/unit/components/event-detail-sheet.test.tsx b/frontend/tests/unit/components/event-detail-sheet.test.tsx index ad1bfe1b..ca6ea725 100644 --- a/frontend/tests/unit/components/event-detail-sheet.test.tsx +++ b/frontend/tests/unit/components/event-detail-sheet.test.tsx @@ -34,4 +34,28 @@ describe("EventDetailSheet", () => { expect(disclosure).toHaveAttribute("aria-expanded", "true"); expect(screen.getByText("tool_name")).toBeVisible(); }); + + test("summarizes MCP service and tool context before raw metadata", () => { + render( + , + ); + + expect(screen.getByText("MCP context")).toBeVisible(); + expect(screen.getByText("MCP tool")).toBeVisible(); + expect(screen.getByText("sqlite-analytics")).toBeVisible(); + expect(screen.getByText("Tool lifecycle")).toBeVisible(); + expect(screen.getAllByText("mcp.tool.end")).toHaveLength(2); + }); }); diff --git a/frontend/tests/unit/components/metrics-strip.test.tsx b/frontend/tests/unit/components/metrics-strip.test.tsx new file mode 100644 index 00000000..13d8377b --- /dev/null +++ b/frontend/tests/unit/components/metrics-strip.test.tsx @@ -0,0 +1,64 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, test } from "vitest"; + +import { MetricsStrip } from "@/components/agentops/metrics-strip"; +import type { RunMetrics, TimelineItem } from "@/core/agentops/types"; + +const metrics: RunMetrics = { + duration_ms: 2400, + total_events: 4, + total_tokens: 1200, + tool_calls: 0, + errors: 1, + tool_call_count: 0, + tool_error_count: 0, + subagent_count: 0, + subagent_error_count: 0, + rollback_performed: false, + final_status: "success", + failure_category: "MCP_TOOL_ERROR", + data_completeness: "complete", + first_event_latency_ms: 15, +}; + +const mcpTimeline: TimelineItem[] = [ + { + seq: 1, + timestamp: "2026-07-31T10:00:00.000Z", + phase: "mcp", + event_type: "mcp.tool.start", + component: "sqlite_analytics_schema_overview", + status: "running", + summary: "SQLite Analytics tool started", + duration_ms: null, + span_id: "mcp-span", + parent_span_id: "agent-span", + malformed: false, + metadata: {}, + }, + { + seq: 2, + timestamp: "2026-07-31T10:00:01.000Z", + phase: "mcp", + event_type: "mcp.tool.error", + component: "sqlite_analytics_schema_overview", + status: "error", + summary: "SQLite Analytics tool rejected the query", + duration_ms: 12, + span_id: "mcp-span", + parent_span_id: "agent-span", + malformed: false, + metadata: {}, + }, +]; + +describe("MetricsStrip", () => { + test("surfaces MCP activity and incidents separately", () => { + render(); + + expect(screen.getByText("2 events")).toBeVisible(); + expect(screen.getByText("1 incident")).toHaveClass( + "agentops-metric-danger", + ); + }); +});