From 7aab2a4fc5e221074cd4fbe42a0d2a79303f2a29 Mon Sep 17 00:00:00 2001 From: Xianquan Date: Wed, 15 Jul 2026 17:55:14 +0800 Subject: [PATCH 1/5] fix: make mongo index sync non-destructive --- .../common/mongo-index-sync-strategy.md | 219 ++++++++++++++++++ .../mongo-index-sync-customer-index-loss.md | 83 +++++++ packages/global/common/system/constants.ts | 3 + packages/service/common/mongo/index.ts | 23 +- packages/service/common/mongo/indexManager.ts | 107 +++++++++ packages/service/env.ts | 7 +- .../test/common/mongo/indexManager.test.ts | 86 +++++++ packages/service/test/env.test.ts | 45 ++++ .../dataClean/cleanupDuplicateChats.test.ts | 2 +- projects/marketplace/src/env.ts | 3 +- .../marketplace/src/service/mongo/index.ts | 25 +- projects/marketplace/test/env.test.ts | 28 ++- 12 files changed, 600 insertions(+), 31 deletions(-) create mode 100644 .agents/design/common/mongo-index-sync-strategy.md create mode 100644 .agents/issue/mongo-index-sync-customer-index-loss.md create mode 100644 packages/service/common/mongo/indexManager.ts create mode 100644 packages/service/test/common/mongo/indexManager.test.ts diff --git a/.agents/design/common/mongo-index-sync-strategy.md b/.agents/design/common/mongo-index-sync-strategy.md new file mode 100644 index 000000000000..957376e8fcaa --- /dev/null +++ b/.agents/design/common/mongo-index-sync-strategy.md @@ -0,0 +1,219 @@ +# MongoDB 索引同步安全化方案 + +## 背景 + +私有化部署客户可能会通过 `mongosh`、运维脚本或数据库管理平台为 MongoDB 集合添加自定义索引。当前 FastGPT 在模型加载时调用 Mongoose `model.syncIndexes()`，该 API 的语义是让数据库索引与 schema 中声明的索引保持一致：创建缺失索引，同时删除 schema 中不存在的索引。 + +这会导致服务每次重启时都可能删除客户自建索引。现有 `SYNC_INDEX=false` 能临时关闭启动同步，但它把“补建 FastGPT 必需索引”和“删除旧索引/未知索引”一起关闭了，升级时容易漏建新索引或遗留旧唯一索引。 + +## 改造前代码现状 + +- 主应用入口：`packages/service/common/mongo/index.ts` + - `getMongoModel()` / `getMongoLogModel()` 编译 model 后调用 `syncMongoIndex(model)`。 + - `syncMongoIndex()` 在非测试、非构建、`SYNC_INDEX=true` 且存在 `MONGO_URL` 时执行 `await model.syncIndexes({ background: true })`。 +- Marketplace 入口：`projects/marketplace/src/service/mongo/index.ts` + - 同样在 model 编译后调用 `model.syncIndexes({ background: true })`。 + - 当前没有 `await`，`try/catch` 捕不到异步失败。 +- 环境变量： + - `packages/service/env.ts` 和 `projects/marketplace/src/env.ts` 均定义 `SYNC_INDEX: BoolSchema.default(true)`。 +- 部署文档默认仍配置 `SYNC_INDEX: true`。 + +## 已确认决策 + +1. 私有化默认行为从破坏性全量同步改为 safe create：只补建 FastGPT 缺失索引，不删除 schema 外索引。 +2. 不保留 `SYNC_INDEX` 兼容逻辑，直接移除旧布尔变量，统一改用 `MONGO_INDEX_SYNC_MODE`。 +3. 当前启动索引处理不提供删除索引能力，历史旧索引清理留给后续独立迁移工具。 +4. `llm_request_records.requestId_1` 已确认不再需要，但不在本次启动索引同步中清理。 + +## 根因判断 + +`syncIndexes()` 不是单纯“创建索引”，而是完整 reconcile： + +1. 计算 schema 与数据库已有索引差异。 +2. 删除 schema 中不存在的索引。 +3. 创建 schema 中缺失的索引。 + +因此客户自建索引被删除是当前 API 选择带来的必然行为，不是异常分支。 + +## 设计目标 + +1. 默认保护客户自建索引：启动时不删除 schema 未声明的索引。 +2. 仍能自动补建 FastGPT 新版本需要的缺失索引。 +3. 提供 dry-run 差异检查，帮助管理员识别 schema 外索引和缺失索引。 +4. 当前方案不再保留启动期强制全量同步能力，避免换一种方式重新引入删除客户索引的风险。 +5. 主应用与 Marketplace 使用一致的索引策略和错误处理。 + +## 非目标 + +- 不尝试自动判断所有未知索引是否由客户创建。历史上未带所有权标记，完全自动判断不可靠。 +- 不在普通服务重启时执行危险删除。 +- 不把 `SYNC_INDEX=false` 作为长期推荐方案，因为它也会阻止新索引创建。 + +## 推荐方案 + +### 1. 引入索引同步模式 + +新增枚举型环境变量，例如 `MONGO_INDEX_SYNC_MODE`： + +| 模式 | 启动行为 | 是否删除未知索引 | 使用场景 | +| --- | --- | --- | --- | +| `off` | 不处理索引 | 否 | 极端保守环境，客户自行维护索引 | +| `create` | 只创建 schema 声明但数据库缺失的索引 | 否 | 私有化默认推荐 | +| `dryRun` | 只扫描并记录差异 | 否 | 升级前检查 | + +移除旧变量策略： + +- 移除 `packages/service/env.ts` 和 `projects/marketplace/src/env.ts` 中的 `SYNC_INDEX`。 +- 测试统一改成 `MONGO_INDEX_SYNC_MODE`；`.mdx` 文档和 `.yml` 部署文件先不手动修改，等代码改完后再统一处理。 +- 未配置 `MONGO_INDEX_SYNC_MODE` 时默认 `create`。 +- 旧配置 `SYNC_INDEX=false` 的用户如需继续完全跳过索引处理，升级时改为 `MONGO_INDEX_SYNC_MODE=off`。 +- 旧配置 `SYNC_INDEX=true` 的用户无需迁移值；未配置新变量时默认等价于 `MONGO_INDEX_SYNC_MODE=create`，只补建缺失索引。 + +模式值通过共享的 `mongoIndexSyncModeList` 约束，主应用和 Marketplace 使用同一组枚举，避免两个入口分叉。这里故意不读取 `SYNC_INDEX`。这属于配置破坏性变更，但可以避免长期维护两个入口造成语义混乱。 + +### 2. 默认从 `syncIndexes()` 改为安全创建 + +`create` 模式下使用 Mongoose 的 `createIndexes()` 或基于 `diffIndexes()` 的 `toCreate` 结果只创建缺失索引，不执行 `cleanIndexes()`。 + +建议同时记录差异日志： + +- `toCreate`：即将创建的 FastGPT schema 索引。 +- `toDrop`：数据库存在但 schema 不声明的索引，只告警，不删除。 +- 冲突：同名或同 key 但 options 不一致的索引，交由 `createIndexes()` 抛错并记录日志，提示需要迁移或人工处理。 + +不建议只简单替换成 `model.createIndexes()` 后结束，原因是： + +1. `createIndexes()` 对已有索引 options 不做修正。TTL、unique、partialFilterExpression 变化时，旧索引仍会保留旧行为。 +2. 它缺少差异报告，管理员无法知道哪些旧索引需要人工处理或迁移。 +3. 它不处理旧索引 options 变化或历史旧索引清理，这些应由独立迁移工具处理。 + +因此可以内部用 `createIndexes()` 完成安全创建，但对外的抽象应是 `mongoIndexManager`，保留 inspect 和未来管理入口的扩展点。 + +### 3. 历史旧索引处理边界 + +本次不在服务启动索引同步里删除任何索引，包括 FastGPT 历史旧索引。原因： + +1. 当前核心目标是保护客户自建索引，启动路径不应包含 drop index 行为。 +2. 历史旧索引识别和删除需要更明确的运维确认，不应混入普通重启流程。 +3. 如后续确实需要清理旧索引，应做独立 migration 脚本或 Root 管理工具，并提供 dry-run、确认参数和审计日志。 + +### 4. 提供可审计的管理入口 + +建议新增 Root 管理员 API 或脚本： + +- `POST /api/admin/mongoIndexes/inspect` + - 默认 dry-run。 + - 输出每个集合的 `toCreate`、`unknownIndexes`、`conflicts`、`fastgptObsoleteIndexes`。 +- `POST /api/admin/mongoIndexes/apply` + - 入参明确指定模式：`create`。 + - 不提供旧式全量同步能力。 + +本次先通过环境变量完成启动期 `off/create/dryRun`，Root 管理员 API 或脚本作为后续增强。 + +### 5. 未来索引命名规范 + +新建 FastGPT schema 索引尽量显式命名，例如 `fg__`，降低未来识别成本。 + +注意：不能一次性给所有旧索引改名，因为 MongoDB 不支持直接重命名索引，改名通常等价于新建再删除，容易触发冲突和重建成本。旧索引应通过迁移清单逐步治理。 + +## 推荐落地架构 + +### 1. 模块划分 + +建议在 `packages/service/common/mongo/` 下新增索引管理模块： + +- `indexManager.ts` + - `runMongoIndexSyncForModel()` + - `inspectMongoModelIndexes()` + +`getMongoModel()` / `getMongoLogModel()` 只负责把 model 注册给 manager，不直接调用 Mongoose 的 destructive API。这样后续可以统一做并发控制、日志聚合和管理 API 复用。 + +### 2. 启动流程 + +启动时允许三类行为： + +- `off`：跳过。 +- `dryRun`：只记录差异。 +- `create`：创建缺失索引，并记录未知索引/冲突。 + +不在常规启动模式中暴露 Mongoose `syncIndexes()`。如未来确实需要旧式全量同步，应作为单独危险运维工具重新评审，而不是并入默认索引同步路径。 + +### 3. 并发与失败处理 + +当前 model 加载时即触发索引同步。后续实现至少要保证： + +1. 同一进程内同一 model 只执行一次索引任务。 +2. 多实例并发启动时，重复创建索引错误可识别并降噪；真正的冲突错误必须记录。 +3. 索引任务失败不应让 model 注册失败，但要有明确日志，必要时在健康检查或管理 API 暴露状态。 +4. Marketplace 已修正异步错误捕获，后续如果引入管理入口，应继续复用同一 manager。 + +### 4. 本次交付边界 + +- 新增 `MONGO_INDEX_SYNC_MODE`。 +- 将默认启动行为从 `syncIndexes()` 改为安全 `create`。 +- 支持 `off/create/dryRun`。 +- 加日志说明 `toCreate/toDrop`，但 `toDrop` 不删除；索引冲突由 `createIndexes()` 错误日志暴露。 +- 主应用与 Marketplace 行为一致。 +- 补回归测试：客户自建索引在 `create/dryRun` 下不会被删除。 +- 代码确认后再更新 `.mdx` 文档，并由生成流程统一产出 `.yml` 部署文件。 + +Root 管理员 inspect/apply API 或等价脚本作为后续增强，不阻塞本次修复。 + +## 已知历史旧索引记录 + +### `llm_request_records.requestId_1` + +- 引入提交：`76d6234de V4.14.7 features (#6406)`，提交时间 `2026-02-12 16:37:50 +0800`。 +- 引入方式：`packages/service/core/ai/record/schema.ts` 初次新增 LLM 请求追踪记录时，在 `requestId` path 上声明 `unique: true`。MongoDB 自动生成索引名 `requestId_1`。 +- 变更提交：`f008ea971 feat: teamId in reacord llm`，提交时间 `2026-06-25 12:27:57 +0800`；后续合并提交 `60c62b7af Fix test (#7179)` 包含同样变更。 +- 变更原因：LLM 请求追踪记录新增 `teamId` 隔离，查询从 `{ requestId }` 调整为 `{ requestId, teamId }`，唯一索引从单字段 `requestId` 调整为复合 `{ teamId: 1, requestId: 1 }`。 +- 当前判断：旧 `requestId_1` 对当前 schema 已无必要，且可能继续施加跨团队全局唯一约束。但 `requestId` 由 `getNanoid(12)` 生成，实际碰撞概率很低；请求追踪记录保存失败只记录错误，不阻塞主流程。 +- 处理边界：本次默认 `create` 不删除它。后续如需清理，应通过独立 migration 脚本或 Root 管理工具处理。 + +## 边界与风险 + +1. `createIndexes()` 不会更新已存在索引的 options。TTL 秒数、unique、partialFilterExpression 等发生变化时，需要独立 migration 或人工处理。 +2. 旧唯一索引如果不删除，可能继续影响业务。例如 `llm_request_records` 从 `{ requestId }` 改到 `{ teamId, requestId }` 后，旧 `requestId_1` 若残留，会继续限制跨团队相同 requestId。 +3. 旧式全量同步仍可能有排障价值，但不作为常规启动模式暴露，避免重新引入删除客户索引的风险。 +4. 多实例同时启动会并发创建索引。MongoDB 创建已存在索引通常是幂等的，但冲突错误需要聚合成清晰日志，避免噪声刷屏。 +5. Marketplace 已统一接入 `mongoIndexManager`，异步错误会被捕获并记录。 + +## 测试策略 + +至少需要覆盖： + +1. `MONGO_INDEX_SYNC_MODE` 环境变量解析： + - 未设置新变量时为 `create`。 + - 合法模式按原值返回。 + - 非法模式在 env 解析阶段失败。 +2. `create` 模式： + - 会创建 schema 中缺失索引。 + - 不删除手动创建的客户索引。 + - 遇到 schema 外索引只记录为 `toDrop/unknownIndexes`。 +3. `dryRun` 模式： + - 不创建索引。 + - 不删除索引。 + - 能返回差异结果。 +4. Marketplace： + - 解析环境变量与主应用一致。 + - 异步错误可以被捕获并记录。 + +仓库已有 `mongodb-memory-server` 测试环境，可以写真实 MongoDB 索引回归测试，而不是只 mock Mongoose 方法。 + +## 后续增强 + +1. 是否继续补 Root 管理员 inspect/apply API，作为比环境变量重启更友好的运维入口。 +2. 是否需要独立 MongoDB 索引迁移脚本，用于清理确认无用的 FastGPT 历史旧索引。 +3. 对客户自己改动 FastGPT 既有索引 options 的情况，当前只告警；如未来需要自动修正，应以明确迁移项实现。 + +## TODO + +- [x] 补充需求确认，确定环境变量命名和默认行为。 +- [x] 移除 `SYNC_INDEX`，新增 `MONGO_INDEX_SYNC_MODE`。 +- [x] 设计并实现 `mongoIndexManager`：统一主应用和 Marketplace 的索引处理。 +- [x] 实现 `off/create/dryRun` 基础模式。 +- [x] 接入主应用 `getMongoModel()` / `getMongoLogModel()`。 +- [x] 接入 Marketplace `getMongoModel()`，修正异步错误捕获。 +- [ ] 后续增强：新增 Root 管理员 inspect/apply API 或等价脚本。 +- [ ] 后续补充 `.mdx` 文档，并通过生成流程产出 `.yml` 部署文件。 +- [x] 增加测试，覆盖 env 解析、`create/dryRun` 行为、客户自建索引不会被删除。 diff --git a/.agents/issue/mongo-index-sync-customer-index-loss.md b/.agents/issue/mongo-index-sync-customer-index-loss.md new file mode 100644 index 000000000000..07592ff7a41a --- /dev/null +++ b/.agents/issue/mongo-index-sync-customer-index-loss.md @@ -0,0 +1,83 @@ +# MongoDB 启动索引同步删除客户自建索引问题分析 + +## 问题描述 + +私有化部署客户可能会直接在 MongoDB 中添加自定义索引，用于客户自己的查询、报表、审计或集成任务。FastGPT 服务重启时会加载 Mongoose model，并执行索引同步。当前同步使用 Mongoose `model.syncIndexes()`，会删除 schema 中未声明的索引，导致客户自建索引被清理。 + +`SYNC_INDEX=false` 可以阻止启动同步，但它同时阻止 FastGPT 新版本缺失索引的创建，升级时可能留下性能问题或唯一约束缺失。因此它只能算临时开关，不是长期索引治理方案。当前决策是不再保留该布尔变量，改为 `MONGO_INDEX_SYNC_MODE` 表达更明确的索引处理模式。 + +## 证据 + +当前主应用代码： + +- `packages/service/common/mongo/index.ts` + - `getMongoModel()` / `getMongoLogModel()` 创建 model 后调用 `syncMongoIndex(model)`。 + - `syncMongoIndex()` 在 `SYNC_INDEX=true` 时执行 `await model.syncIndexes({ background: true })`。 + +当前 Marketplace 代码： + +- `projects/marketplace/src/service/mongo/index.ts` + - model 创建后也调用 `model.syncIndexes({ background: true })`。 + - 该调用未 `await`，异步错误无法被当前 `try/catch` 捕获。 + +Mongoose `syncIndexes()` 的语义是： + +1. 对比 schema 索引与数据库已有索引。 +2. 删除 schema 中不存在的索引。 +3. 创建 schema 中缺失的索引。 + +所以删除客户自建索引是当前 API 语义导致的确定性风险。 + +## 影响范围 + +- 私有化部署环境：风险最高，因为客户可能对数据库有直接运维权。 +- 多实例部署：多个服务重启时会重复触发索引同步，日志与冲突更难排查。 +- 版本升级：如果关闭 `SYNC_INDEX`，新索引可能不会创建；如果打开 `SYNC_INDEX`，客户自建索引可能被删。 +- 日志库：`getMongoLogModel()` 也会同步索引，日志集合里的客户索引同样受影响。 +- Marketplace：存在同类 destructive sync，并且异步错误处理不完整。 + +## 方案取舍 + +### 不推荐：继续依赖 `SYNC_INDEX` + +优点是改动小。缺点是它只有开/关两种状态，无法表达“只创建 FastGPT 缺失索引，但保留客户索引”。 + +### 不推荐：直接全量改成 `createIndexes()` + +能保留客户索引，但无法发现和治理 FastGPT 历史旧索引。比如唯一索引或 TTL 索引的 options 发生变化时，旧索引可能继续影响业务。 + +当前最终方案仍以 `createIndexes()` 作为安全创建动作，但通过统一的 index manager 包装差异检查、日志和启动入口，历史旧索引清理不放在服务启动路径中。 + +### 推荐：引入索引同步模式 + +将索引同步拆成不同意图： + +- `off`：完全跳过。 +- `create`：只创建缺失索引，不删除未知索引。 +- `dryRun`：只检查差异。 +常规启动不再暴露旧 `syncIndexes()` 全量同步行为，避免重新引入删除客户索引的风险。 + +私有化默认推荐 `create`。 + +## 已确认方向 + +1. 默认行为从破坏性全量同步改为 safe create。 +2. 直接移除 `SYNC_INDEX`，不做长期兼容映射。 +3. 启动索引处理不提供删除索引能力，旧索引清理后续通过独立迁移脚本或 Root 管理工具处理。 +4. `llm_request_records.requestId_1` 已确认不再需要，但不在本次启动索引同步中清理。 + +## 已知旧索引样例 + +`llm_request_records.requestId_1` 来源： + +- `76d6234de V4.14.7 features (#6406)` 在 `2026-02-12 16:37:50 +0800` 引入 LLM 请求追踪记录，`requestId` path 声明 `unique: true`，MongoDB 因此创建 `requestId_1`。 +- `f008ea971 feat: teamId in reacord llm` 在 `2026-06-25 12:27:57 +0800` 将 schema 改为 `teamId + requestId` 复合唯一索引。 + +当前判断：旧 `requestId_1` 对现在的团队隔离查询已无必要，且可能继续施加跨团队全局唯一约束。但 `requestId` 由 `getNanoid(12)` 生成，实际碰撞概率很低；请求追踪记录保存失败只记录错误，不阻塞主流程。因此本次不在启动索引同步中清理它。 + +## 已确认边界 + +1. 常规启动默认只执行 `create`，不会删除未知索引。 +2. `dryRun` 只检查差异，不创建、不删除。 +3. 不保留旧式全量同步入口；Root 管理员 inspect/apply API 或等价脚本作为后续增强，不阻塞本次修复。 +4. 历史旧索引清理不进入启动路径，后续如需要再做独立迁移工具。 diff --git a/packages/global/common/system/constants.ts b/packages/global/common/system/constants.ts index 0bec97774e2f..a10cc8a49ae7 100644 --- a/packages/global/common/system/constants.ts +++ b/packages/global/common/system/constants.ts @@ -12,3 +12,6 @@ export const isTestEnv = process.env.NODE_ENV === 'test'; export const isPhaseProductionBuild = process.env.NEXT_PHASE === 'phase-production-build'; export const FASTGPT_PRO_TOKEN_HEADER = 'x-fastgpt-pro-token'; + +export const mongoIndexSyncModeList = ['off', 'create', 'dryRun'] as const; +export type MongoIndexSyncMode = (typeof mongoIndexSyncModeList)[number]; diff --git a/packages/service/common/mongo/index.ts b/packages/service/common/mongo/index.ts index 2491df300a40..273c1566245d 100644 --- a/packages/service/common/mongo/index.ts +++ b/packages/service/common/mongo/index.ts @@ -9,6 +9,7 @@ import type { } from 'mongoose'; import mongoose, { Mongoose } from 'mongoose'; import { serviceEnv } from '../../env'; +import { runMongoIndexSyncForModel } from './indexManager'; const logger = getLogger(LogCategories.INFRA.MONGO); @@ -136,7 +137,6 @@ export const getMongoModel = (name: string, schema: mongoose.Schema): Model; - // Sync index syncMongoIndex(model); return model; @@ -148,27 +148,32 @@ export const getMongoLogModel = (name: string, schema: mongoose.Schema): Mode const model = connectionLogMongo.model(name, schema) as Model; - // Sync index syncMongoIndex(model); return model; }; -const syncMongoIndex = async (model: Model) => { +const syncMongoIndex = (model: Model) => { if ( process.env.NODE_ENV === 'test' || process.env.NEXT_PHASE === 'phase-production-build' || - !serviceEnv.SYNC_INDEX || !MONGO_URL ) { return; } - try { - await model.syncIndexes({ background: true }); - } catch (error) { - logger.error('Failed to sync MongoDB indexes', { modelName: model.modelName, error }); - } + void runMongoIndexSyncForModel({ + model, + mode: serviceEnv.MONGO_INDEX_SYNC_MODE, + logger + }).catch((error) => { + logger.error('Failed to ensure MongoDB indexes', { + modelName: model.modelName, + collectionName: model.collection.collectionName, + mode: serviceEnv.MONGO_INDEX_SYNC_MODE, + error + }); + }); }; export const ReadPreference = connectionMongo.mongo.ReadPreference; diff --git a/packages/service/common/mongo/indexManager.ts b/packages/service/common/mongo/indexManager.ts new file mode 100644 index 000000000000..8f32e6598693 --- /dev/null +++ b/packages/service/common/mongo/indexManager.ts @@ -0,0 +1,107 @@ +import type { MongoIndexSyncMode } from '@fastgpt/global/common/system/constants'; +import { getLogger, LogCategories } from '../logger'; +import type { Model } from 'mongoose'; + +const defaultLogger = getLogger(LogCategories.INFRA.MONGO); + +type MongoIndexLogger = { + debug: (message: string, data?: Record) => void; + info: (message: string, data?: Record) => void; + warn: (message: string, data?: Record) => void; + error: (message: string, data?: Record) => void; +}; + +type MongooseDiffIndexesResult = { + toDrop: string[]; + toCreate: unknown[]; +}; + +export type MongoIndexSyncResult = { + mode: MongoIndexSyncMode; + modelName: string; + collectionName: string; + toDrop: string[]; + toCreate: unknown[]; +}; + +const getCollectionName = (model: Model) => model.collection.collectionName; + +/** + * 只计算当前 schema 和数据库索引的差异，不创建也不删除任何索引。 + * + * `toDrop` 只表示 Mongoose 认为 schema 外存在的索引。默认安全模式下这些索引 + * 只会被记录为未知索引，不会删除，以保护客户自建索引。 + */ +export const inspectMongoModelIndexes = async ( + model: Model +): Promise> => { + const diff = (await model.diffIndexes({ + indexOptionsToCreate: true + })) as MongooseDiffIndexesResult; + + return { + modelName: model.modelName, + collectionName: getCollectionName(model), + toDrop: diff.toDrop, + toCreate: diff.toCreate + }; +}; + +/** + * 按配置模式处理单个 Mongoose model 的索引。 + * + * 默认 `create` 只创建 schema 中缺失的索引，并保留所有 schema 外索引。 + */ +export const runMongoIndexSyncForModel = async ({ + model, + mode, + logger = defaultLogger +}: { + model: Model; + mode: MongoIndexSyncMode; + logger?: MongoIndexLogger; +}): Promise => { + const baseResult = { + mode, + modelName: model.modelName, + collectionName: getCollectionName(model), + toDrop: [], + toCreate: [] + } satisfies MongoIndexSyncResult; + + if (mode === 'off') { + return baseResult; + } + + const inspection = await inspectMongoModelIndexes(model); + const result: MongoIndexSyncResult = { + ...baseResult, + toDrop: inspection.toDrop, + toCreate: inspection.toCreate + }; + + if (result.toDrop.length > 0) { + logger.warn('Detected MongoDB indexes not declared by FastGPT schema', { + mode, + modelName: model.modelName, + collectionName: getCollectionName(model), + indexes: result.toDrop + }); + } + + if (mode === 'dryRun') { + logger.info('MongoDB index dry-run completed', result); + return result; + } + + await model.createIndexes({ background: true }); + + logger.info('MongoDB indexes ensured', { + mode, + modelName: model.modelName, + collectionName: getCollectionName(model), + toCreateCount: result.toCreate.length + }); + + return result; +}; diff --git a/packages/service/env.ts b/packages/service/env.ts index 8e9a85230223..a4dfca32ab78 100644 --- a/packages/service/env.ts +++ b/packages/service/env.ts @@ -1,6 +1,9 @@ import { createEnv } from '@t3-oss/env-core'; import z from 'zod'; -import { isPhaseProductionBuild } from '@fastgpt/global/common/system/constants'; +import { + isPhaseProductionBuild, + mongoIndexSyncModeList +} from '@fastgpt/global/common/system/constants'; import { DEFAULT_MAX_FOLDER_DEPTH } from '@fastgpt/global/common/parentFolder/depth'; import { BoolSchema, IntSchema, NumSchema, UrlSchema } from '@fastgpt/global/common/zod'; import { agentSandboxProviderList } from '@fastgpt/global/core/ai/sandbox/constants'; @@ -29,7 +32,7 @@ export const serviceEnv = createEnv({ server: { // ==================== 基础配置 ==================== DB_MAX_LINK: IntSchema.min(1).default(5), - SYNC_INDEX: BoolSchema.default(true), + MONGO_INDEX_SYNC_MODE: z.enum(mongoIndexSyncModeList).default('create'), // ==================== 密钥 ==================== ROOT_KEY: z diff --git a/packages/service/test/common/mongo/indexManager.test.ts b/packages/service/test/common/mongo/indexManager.test.ts new file mode 100644 index 000000000000..8c721bf44135 --- /dev/null +++ b/packages/service/test/common/mongo/indexManager.test.ts @@ -0,0 +1,86 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { connectionMongo, Schema } from '@fastgpt/service/common/mongo'; +import { runMongoIndexSyncForModel } from '@fastgpt/service/common/mongo/indexManager'; + +const logger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn() +}; + +const getIndexNames = async (collectionName: string) => { + const indexes = await connectionMongo.connection.db?.collection(collectionName).indexes(); + return new Set(indexes?.map((index: { name?: string }) => index.name)); +}; + +const dropCollection = async (collectionName: string) => { + try { + await connectionMongo.connection.db?.collection(collectionName).drop(); + } catch (error: any) { + if (error?.codeName !== 'NamespaceNotFound') { + throw error; + } + } +}; + +describe('runMongoIndexSyncForModel', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('creates missing schema indexes without deleting customer indexes in create mode', async () => { + const collectionName = `mongo_index_create_${Date.now()}`; + await dropCollection(collectionName); + + const schema = new Schema( + { + name: String, + customerField: String + }, + { autoIndex: false } + ); + schema.index({ name: 1 }, { name: 'schema_name_1' }); + + const model = connectionMongo.model(`MongoIndexCreate${Date.now()}`, schema, collectionName); + await model.collection.createIndex({ customerField: 1 }, { name: 'customer_custom_1' }); + + await runMongoIndexSyncForModel({ + model, + mode: 'create', + logger + }); + + const indexNames = await getIndexNames(collectionName); + expect(indexNames.has('schema_name_1')).toBe(true); + expect(indexNames.has('customer_custom_1')).toBe(true); + }); + + it('does not create or delete indexes in dryRun mode', async () => { + const collectionName = `mongo_index_dry_run_${Date.now()}`; + await dropCollection(collectionName); + + const schema = new Schema( + { + name: String, + customerField: String + }, + { autoIndex: false } + ); + schema.index({ name: 1 }, { name: 'schema_name_1' }); + + const model = connectionMongo.model(`MongoIndexDryRun${Date.now()}`, schema, collectionName); + await model.collection.createIndex({ customerField: 1 }, { name: 'customer_custom_1' }); + + const result = await runMongoIndexSyncForModel({ + model, + mode: 'dryRun', + logger + }); + + const indexNames = await getIndexNames(collectionName); + expect(result.toCreate.length).toBeGreaterThan(0); + expect(indexNames.has('schema_name_1')).toBe(false); + expect(indexNames.has('customer_custom_1')).toBe(true); + }); +}); diff --git a/packages/service/test/env.test.ts b/packages/service/test/env.test.ts index e4f056fdc42a..916a3bcb1893 100644 --- a/packages/service/test/env.test.ts +++ b/packages/service/test/env.test.ts @@ -14,6 +14,7 @@ const originalEnv = { PRO_TOKEN: process.env.PRO_TOKEN, VITEST: process.env.VITEST, NODE_ENV: process.env.NODE_ENV, + MONGO_INDEX_SYNC_MODE: process.env.MONGO_INDEX_SYNC_MODE, AGENT_SANDBOX_PROVIDER: process.env.AGENT_SANDBOX_PROVIDER, AGENT_SANDBOX_SEALOS_BASEURL: process.env.AGENT_SANDBOX_SEALOS_BASEURL, AGENT_SANDBOX_SEALOS_TOKEN: process.env.AGENT_SANDBOX_SEALOS_TOKEN, @@ -42,6 +43,7 @@ describe('serviceEnv', () => { vi.stubEnv('PRO_TOKEN', originalEnv.PRO_TOKEN); vi.stubEnv('VITEST', originalEnv.VITEST); vi.stubEnv('NODE_ENV', originalEnv.NODE_ENV); + vi.stubEnv('MONGO_INDEX_SYNC_MODE', originalEnv.MONGO_INDEX_SYNC_MODE); vi.stubEnv('AGENT_SANDBOX_PROVIDER', originalEnv.AGENT_SANDBOX_PROVIDER); vi.stubEnv('AGENT_SANDBOX_SEALOS_BASEURL', originalEnv.AGENT_SANDBOX_SEALOS_BASEURL); vi.stubEnv('AGENT_SANDBOX_SEALOS_TOKEN', originalEnv.AGENT_SANDBOX_SEALOS_TOKEN); @@ -73,6 +75,49 @@ describe('serviceEnv', () => { }); }); + it('parses MongoDB index sync mode during service env init', async () => { + vi.stubEnv('FILE_TOKEN_KEY', 'filetokenkey'); + vi.stubEnv('AES256_SECRET_KEY', 'fastgptsecret'); + vi.stubEnv('INVOKE_TOKEN_SECRET', validInvokeTokenSecret); + + vi.stubEnv('MONGO_INDEX_SYNC_MODE', undefined); + await expect(importServiceEnv()).resolves.toMatchObject({ + serviceEnv: { + MONGO_INDEX_SYNC_MODE: 'create' + } + }); + + vi.stubEnv('MONGO_INDEX_SYNC_MODE', ''); + await expect(importServiceEnv()).resolves.toMatchObject({ + serviceEnv: { + MONGO_INDEX_SYNC_MODE: 'create' + } + }); + + vi.stubEnv('MONGO_INDEX_SYNC_MODE', 'off'); + await expect(importServiceEnv()).resolves.toMatchObject({ + serviceEnv: { + MONGO_INDEX_SYNC_MODE: 'off' + } + }); + + vi.stubEnv('MONGO_INDEX_SYNC_MODE', 'dryRun'); + await expect(importServiceEnv()).resolves.toMatchObject({ + serviceEnv: { + MONGO_INDEX_SYNC_MODE: 'dryRun' + } + }); + }); + + it('rejects invalid MongoDB index sync mode during service env init', async () => { + vi.stubEnv('FILE_TOKEN_KEY', 'filetokenkey'); + vi.stubEnv('AES256_SECRET_KEY', 'fastgptsecret'); + vi.stubEnv('INVOKE_TOKEN_SECRET', validInvokeTokenSecret); + vi.stubEnv('MONGO_INDEX_SYNC_MODE', 'migrate'); + + await expect(importServiceEnv()).rejects.toThrow('Invalid environment variables'); + }); + it('rejects invalid SYSTEM_MAX_STRING_LENGTH_M during service env init', async () => { vi.stubEnv('FILE_TOKEN_KEY', 'filetokenkey'); vi.stubEnv('AES256_SECRET_KEY', 'fastgptsecret'); diff --git a/projects/app/test/pages/api/admin/dataClean/cleanupDuplicateChats.test.ts b/projects/app/test/pages/api/admin/dataClean/cleanupDuplicateChats.test.ts index de91f1d8156d..1f0876603571 100644 --- a/projects/app/test/pages/api/admin/dataClean/cleanupDuplicateChats.test.ts +++ b/projects/app/test/pages/api/admin/dataClean/cleanupDuplicateChats.test.ts @@ -37,7 +37,7 @@ const ensureLegacyDuplicateWritableCollection = async () => { }; const restoreChatSchemaIndexes = async () => { - await MongoChat.syncIndexes(); + await MongoChat.createIndexes(); }; const createChatHeader = ({ diff --git a/projects/marketplace/src/env.ts b/projects/marketplace/src/env.ts index df8134f4a763..348ebcd4bfbd 100644 --- a/projects/marketplace/src/env.ts +++ b/projects/marketplace/src/env.ts @@ -1,6 +1,7 @@ import { createEnv } from '@t3-oss/env-core'; import z from 'zod'; import { BoolSchema, IntSchema } from '@fastgpt/global/common/zod'; +import { mongoIndexSyncModeList } from '@fastgpt/global/common/system/constants'; const LogLevelSchema = z.enum(['trace', 'debug', 'info', 'warning', 'error', 'fatal']); const StorageVendorSchema = z.enum(['minio', 'aws-s3', 'cos', 'oss']); @@ -15,7 +16,7 @@ export const marketplaceEnv = createEnv({ COMMUNITY_AUTH_TOKEN: z.string().optional(), MONGODB_URI: z.string().optional().default(''), DB_MAX_LINK: IntSchema.default(20), - SYNC_INDEX: BoolSchema.default(true), + MONGO_INDEX_SYNC_MODE: z.enum(mongoIndexSyncModeList).default('create'), // 对象存储。保持与主项目 packages/service/env.ts 同名,并兼容 marketplace 旧 S3_* 变量。 STORAGE_VENDOR: StorageVendorSchema.default('minio'), diff --git a/projects/marketplace/src/service/mongo/index.ts b/projects/marketplace/src/service/mongo/index.ts index 1eba0ec327b3..417606e0a0f6 100644 --- a/projects/marketplace/src/service/mongo/index.ts +++ b/projects/marketplace/src/service/mongo/index.ts @@ -2,12 +2,12 @@ import type { Model, Schema } from 'mongoose'; import { Mongoose } from 'mongoose'; import { getLogger, LogCategories } from '../logger'; import { marketplaceEnv } from '../../env'; +import { runMongoIndexSyncForModel } from '@fastgpt/service/common/mongo/indexManager'; export const MONGO_URL = marketplaceEnv.MONGODB_URI; const maxConnecting = Math.max(30, marketplaceEnv.DB_MAX_LINK); const logger = getLogger(LogCategories.INFRA.MONGO); -const syncIndex = marketplaceEnv.SYNC_INDEX; declare global { var mongodb: Mongoose | undefined; @@ -32,12 +32,23 @@ export const getMongoModel = (name: string, schema: T) => { }; const syncMongoIndex = async (model: Model) => { - if (syncIndex && process.env.NODE_ENV !== 'test') { - try { - model.syncIndexes({ background: true }); - } catch (error: any) { - logger.error('Create index error', { error }); - } + if (process.env.NODE_ENV === 'test' || !MONGO_URL) { + return; + } + + try { + await runMongoIndexSyncForModel({ + model, + mode: marketplaceEnv.MONGO_INDEX_SYNC_MODE, + logger + }); + } catch (error: any) { + logger.error('Failed to ensure MongoDB indexes', { + modelName: model.modelName, + collectionName: model.collection.collectionName, + mode: marketplaceEnv.MONGO_INDEX_SYNC_MODE, + error + }); } }; diff --git a/projects/marketplace/test/env.test.ts b/projects/marketplace/test/env.test.ts index 96feeb4c8981..a0f09a4de076 100644 --- a/projects/marketplace/test/env.test.ts +++ b/projects/marketplace/test/env.test.ts @@ -1,7 +1,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; -const originalSyncIndex = process.env.SYNC_INDEX; const originalCommunityAuthToken = process.env.COMMUNITY_AUTH_TOKEN; +const originalMongoIndexSyncMode = process.env.MONGO_INDEX_SYNC_MODE; const importEnv = async () => { vi.resetModules(); @@ -10,32 +10,38 @@ const importEnv = async () => { describe('marketplace env', () => { afterEach(() => { - vi.stubEnv('SYNC_INDEX', originalSyncIndex); vi.stubEnv('COMMUNITY_AUTH_TOKEN', originalCommunityAuthToken); + vi.stubEnv('MONGO_INDEX_SYNC_MODE', originalMongoIndexSyncMode); }); - it('defaults SYNC_INDEX to true when it is not configured', async () => { - vi.stubEnv('SYNC_INDEX', undefined); + it('defaults MONGO_INDEX_SYNC_MODE to create when it is not configured', async () => { + vi.stubEnv('MONGO_INDEX_SYNC_MODE', undefined); const { marketplaceEnv } = await importEnv(); - expect(marketplaceEnv.SYNC_INDEX).toBe(true); + expect(marketplaceEnv.MONGO_INDEX_SYNC_MODE).toBe('create'); }); - it('defaults SYNC_INDEX to true when it is empty', async () => { - vi.stubEnv('SYNC_INDEX', ''); + it('defaults MONGO_INDEX_SYNC_MODE to create when it is empty', async () => { + vi.stubEnv('MONGO_INDEX_SYNC_MODE', ''); const { marketplaceEnv } = await importEnv(); - expect(marketplaceEnv.SYNC_INDEX).toBe(true); + expect(marketplaceEnv.MONGO_INDEX_SYNC_MODE).toBe('create'); }); - it('parses explicit false-like SYNC_INDEX values', async () => { - vi.stubEnv('SYNC_INDEX', 'false'); + it('parses explicit MONGO_INDEX_SYNC_MODE values', async () => { + vi.stubEnv('MONGO_INDEX_SYNC_MODE', 'off'); const { marketplaceEnv } = await importEnv(); - expect(marketplaceEnv.SYNC_INDEX).toBe(false); + expect(marketplaceEnv.MONGO_INDEX_SYNC_MODE).toBe('off'); + }); + + it('rejects invalid MONGO_INDEX_SYNC_MODE values', async () => { + vi.stubEnv('MONGO_INDEX_SYNC_MODE', 'full'); + + await expect(importEnv()).rejects.toThrow('Invalid marketplace environment variables'); }); it('parses optional community auth token', async () => { From 7d60a1a4a205192ff966f454b0b39c6a390364f8 Mon Sep 17 00:00:00 2001 From: Xianquan Date: Fri, 17 Jul 2026 17:06:59 +0800 Subject: [PATCH 2/5] feat: add safe mongo index maintenance --- .../common/mongo-index-sync-strategy.md | 45 +- .../mongo-index-sync-customer-index-loss.md | 8 +- AGENTS.md | 6 + packages/global/common/system/constants.ts | 3 - .../service/common/mongo/deprecatedIndexes.ts | 111 +++ packages/service/common/mongo/index.ts | 13 +- packages/service/common/mongo/indexManager.ts | 653 ++++++++++++++++-- packages/service/common/mongo/init.ts | 19 +- packages/service/env.ts | 7 +- .../test/common/mongo/indexManager.test.ts | 426 +++++++++++- packages/service/test/env.test.ts | 7 + projects/app/src/instrumentation-node.ts | 1 + projects/marketplace/src/env.ts | 3 +- .../marketplace/src/service/mongo/index.ts | 20 +- projects/marketplace/test/env.test.ts | 8 + 15 files changed, 1228 insertions(+), 102 deletions(-) create mode 100644 packages/service/common/mongo/deprecatedIndexes.ts diff --git a/.agents/design/common/mongo-index-sync-strategy.md b/.agents/design/common/mongo-index-sync-strategy.md index 957376e8fcaa..ec9688562342 100644 --- a/.agents/design/common/mongo-index-sync-strategy.md +++ b/.agents/design/common/mongo-index-sync-strategy.md @@ -22,7 +22,7 @@ 1. 私有化默认行为从破坏性全量同步改为 safe create：只补建 FastGPT 缺失索引，不删除 schema 外索引。 2. 不保留 `SYNC_INDEX` 兼容逻辑，直接移除旧布尔变量，统一改用 `MONGO_INDEX_SYNC_MODE`。 -3. 当前启动索引处理不提供删除索引能力，历史旧索引清理留给后续独立迁移工具。 +3. 默认启动索引处理不删除索引；如显式配置 `MONGO_INDEX_SYNC_MODE=sync`，也只执行 manager 管理的 `create + deprecated cleanup`，不会使用 Mongoose destructive sync 语义。 4. `llm_request_records.requestId_1` 已确认不再需要，但不在本次启动索引同步中清理。 ## 根因判断 @@ -58,6 +58,7 @@ | 模式 | 启动行为 | 是否删除未知索引 | 使用场景 | | --- | --- | --- | --- | | `off` | 不处理索引 | 否 | 极端保守环境，客户自行维护索引 | +| `sync` | 创建 schema 缺失索引，并清理 registry 中登记的 FastGPT 废弃索引 | 只删除登记过且精确匹配的废弃索引 | 升级时需要同步 FastGPT 新索引并清理已确认废弃索引 | | `create` | 只创建 schema 声明但数据库缺失的索引 | 否 | 私有化默认推荐 | | `dryRun` | 只扫描并记录差异 | 否 | 升级前检查 | @@ -91,11 +92,13 @@ ### 3. 历史旧索引处理边界 -本次不在服务启动索引同步里删除任何索引，包括 FastGPT 历史旧索引。原因： +默认 `create` 不在服务启动索引同步里删除任何索引，包括 FastGPT 历史旧索引。显式 `sync` 只允许清理 `deprecatedIndexes` registry 中登记的 FastGPT 废弃索引，并且删除前必须精确匹配 collection、index name、key 和关键 options。 -1. 当前核心目标是保护客户自建索引，启动路径不应包含 drop index 行为。 -2. 历史旧索引识别和删除需要更明确的运维确认，不应混入普通重启流程。 -3. 如后续确实需要清理旧索引，应做独立 migration 脚本或 Root 管理工具，并提供 dry-run、确认参数和审计日志。 +该边界的原因： + +1. 当前核心目标是保护客户自建索引，不能按 schema 外索引一概删除。 +2. 历史旧索引只能通过明确维护的 registry 识别，不能自动猜测。 +3. 更复杂的批量清理仍应复用 `cleanupDeprecatedIndexes()`，通过升级脚本或 Root 管理工具触发，并提供 dry-run、确认参数和审计日志。 ### 4. 提供可审计的管理入口 @@ -108,7 +111,7 @@ - 入参明确指定模式：`create`。 - 不提供旧式全量同步能力。 -本次先通过环境变量完成启动期 `off/create/dryRun`，Root 管理员 API 或脚本作为后续增强。 +本次先通过环境变量完成启动期 `off/sync/create/dryRun`，Root 管理员 API 或脚本作为后续增强。 ### 5. 未来索引命名规范 @@ -123,38 +126,48 @@ 建议在 `packages/service/common/mongo/` 下新增索引管理模块： - `indexManager.ts` - - `runMongoIndexSyncForModel()` - - `inspectMongoModelIndexes()` + - `MongoIndexManager.runModelIndexMode()` + - `MongoIndexManager.inspectModelIndexes()` + - `MongoIndexManager.cleanupDeprecatedIndexes()` + - `MongoIndexManager.formatCleanupReport()` `getMongoModel()` / `getMongoLogModel()` 只负责把 model 注册给 manager，不直接调用 Mongoose 的 destructive API。这样后续可以统一做并发控制、日志聚合和管理 API 复用。 ### 2. 启动流程 -启动时允许三类行为： +启动时允许四类行为： - `off`：跳过。 +- `sync`：各 model 只创建 schema 缺失索引；主 MongoDB 连接等待所有已登记的 model 索引任务结束后，全局执行一次 registry 废弃索引清理。不会删除客户自建或其他未知索引。 - `dryRun`：只记录差异。 - `create`：创建缺失索引，并记录未知索引/冲突。 -不在常规启动模式中暴露 Mongoose `syncIndexes()`。如未来确实需要旧式全量同步，应作为单独危险运维工具重新评审，而不是并入默认索引同步路径。 +常规默认模式仍是 `create`。`sync` 是显式维护模式，用于升级时清理 FastGPT 自己已废弃的旧索引，但不等同于 Mongoose `syncIndexes()`。 ### 3. 并发与失败处理 当前 model 加载时即触发索引同步。后续实现至少要保证： -1. 同一进程内同一 model 只执行一次索引任务。 +1. 同一进程内同一 model 的并发索引任务会复用；废弃索引 cleanup 按 MongoDB 连接全局只执行一次，不会随每个 schema 重复扫描。 2. 多实例并发启动时，重复创建索引错误可识别并降噪；真正的冲突错误必须记录。 3. 索引任务失败不应让 model 注册失败，但要有明确日志，必要时在健康检查或管理 API 暴露状态。 4. Marketplace 已修正异步错误捕获，后续如果引入管理入口，应继续复用同一 manager。 -### 4. 本次交付边界 +### 4. 日志分级 + +- `debug`：每个 model 输出完整 diff，例如 `toCreate`、schema 外索引名称；连接级 cleanup 输出 registry 扫描明细和重复调用跳过原因。 +- `info`：每个 model 输出索引创建阶段和结果摘要；连接级 cleanup 仅输出一次清理汇总，并记录实际 drop 的废弃索引。 +- `warn`：输出需要运维关注但不阻塞启动的问题，例如检测到 schema 外索引、废弃索引定义不匹配、replacement index 缺失。 +- `error`：输出索引创建或废弃索引清理失败，保留 model、collection、mode 和 error 信息。 + +### 5. 本次交付边界 - 新增 `MONGO_INDEX_SYNC_MODE`。 - 将默认启动行为从 `syncIndexes()` 改为安全 `create`。 -- 支持 `off/create/dryRun`。 +- 支持 `off/sync/create/dryRun`，其中 `sync` 为 manager 语义：`createIndexes()` + `cleanupDeprecatedIndexes(apply=true)`。 - 加日志说明 `toCreate/toDrop`，但 `toDrop` 不删除；索引冲突由 `createIndexes()` 错误日志暴露。 - 主应用与 Marketplace 行为一致。 -- 补回归测试：客户自建索引在 `create/dryRun` 下不会被删除。 +- 补回归测试：客户自建索引在 `create/dryRun/sync` 下不会被删除，`sync` 只删除 registry 中登记的废弃索引。 - 代码确认后再更新 `.mdx` 文档，并由生成流程统一产出 `.yml` 部署文件。 Root 管理员 inspect/apply API 或等价脚本作为后续增强，不阻塞本次修复。 @@ -174,7 +187,7 @@ Root 管理员 inspect/apply API 或等价脚本作为后续增强，不阻塞 1. `createIndexes()` 不会更新已存在索引的 options。TTL 秒数、unique、partialFilterExpression 等发生变化时，需要独立 migration 或人工处理。 2. 旧唯一索引如果不删除，可能继续影响业务。例如 `llm_request_records` 从 `{ requestId }` 改到 `{ teamId, requestId }` 后，旧 `requestId_1` 若残留，会继续限制跨团队相同 requestId。 -3. 旧式全量同步仍可能有排障价值，但不作为常规启动模式暴露，避免重新引入删除客户索引的风险。 +3. 旧式 Mongoose 全量同步不作为启动模式暴露，避免重新引入删除客户索引的风险。 4. 多实例同时启动会并发创建索引。MongoDB 创建已存在索引通常是幂等的，但冲突错误需要聚合成清晰日志，避免噪声刷屏。 5. Marketplace 已统一接入 `mongoIndexManager`，异步错误会被捕获并记录。 @@ -211,7 +224,7 @@ Root 管理员 inspect/apply API 或等价脚本作为后续增强，不阻塞 - [x] 补充需求确认，确定环境变量命名和默认行为。 - [x] 移除 `SYNC_INDEX`，新增 `MONGO_INDEX_SYNC_MODE`。 - [x] 设计并实现 `mongoIndexManager`：统一主应用和 Marketplace 的索引处理。 -- [x] 实现 `off/create/dryRun` 基础模式。 +- [x] 实现 `off/sync/create/dryRun` 基础模式。 - [x] 接入主应用 `getMongoModel()` / `getMongoLogModel()`。 - [x] 接入 Marketplace `getMongoModel()`，修正异步错误捕获。 - [ ] 后续增强：新增 Root 管理员 inspect/apply API 或等价脚本。 diff --git a/.agents/issue/mongo-index-sync-customer-index-loss.md b/.agents/issue/mongo-index-sync-customer-index-loss.md index 07592ff7a41a..f8833e4c562e 100644 --- a/.agents/issue/mongo-index-sync-customer-index-loss.md +++ b/.agents/issue/mongo-index-sync-customer-index-loss.md @@ -46,7 +46,7 @@ Mongoose `syncIndexes()` 的语义是： 能保留客户索引，但无法发现和治理 FastGPT 历史旧索引。比如唯一索引或 TTL 索引的 options 发生变化时，旧索引可能继续影响业务。 -当前最终方案仍以 `createIndexes()` 作为安全创建动作，但通过统一的 index manager 包装差异检查、日志和启动入口，历史旧索引清理不放在服务启动路径中。 +当前最终方案仍以 `createIndexes()` 作为安全创建动作，但通过统一的 index manager 包装差异检查、日志和启动入口。默认 `create` 不做任何删除；显式 `sync` 只清理 registry 中登记且精确匹配的 FastGPT 废弃索引。 ### 推荐：引入索引同步模式 @@ -55,6 +55,8 @@ Mongoose `syncIndexes()` 的语义是： - `off`：完全跳过。 - `create`：只创建缺失索引，不删除未知索引。 - `dryRun`：只检查差异。 +- `sync`：创建缺失索引，并删除 `deprecatedIndexes` registry 中登记且精确匹配的 FastGPT 废弃索引。 + 常规启动不再暴露旧 `syncIndexes()` 全量同步行为，避免重新引入删除客户索引的风险。 私有化默认推荐 `create`。 @@ -63,7 +65,7 @@ Mongoose `syncIndexes()` 的语义是： 1. 默认行为从破坏性全量同步改为 safe create。 2. 直接移除 `SYNC_INDEX`，不做长期兼容映射。 -3. 启动索引处理不提供删除索引能力，旧索引清理后续通过独立迁移脚本或 Root 管理工具处理。 +3. 启动索引处理不提供按 schema 外索引批量删除的能力；显式 `sync` 只允许删除 registry 中登记且精确匹配的 FastGPT 废弃索引。 4. `llm_request_records.requestId_1` 已确认不再需要，但不在本次启动索引同步中清理。 ## 已知旧索引样例 @@ -80,4 +82,4 @@ Mongoose `syncIndexes()` 的语义是： 1. 常规启动默认只执行 `create`，不会删除未知索引。 2. `dryRun` 只检查差异，不创建、不删除。 3. 不保留旧式全量同步入口；Root 管理员 inspect/apply API 或等价脚本作为后续增强，不阻塞本次修复。 -4. 历史旧索引清理不进入启动路径，后续如需要再做独立迁移工具。 +4. 历史旧索引清理必须走 `deprecatedIndexes` registry；客户自建或未知索引不会被 `sync` 删除。 diff --git a/AGENTS.md b/AGENTS.md index 2352b63d9ff9..f2e40de46dde 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -89,6 +89,12 @@ FastGPT 是一个 AI Agent 构建平台,通过 Flow 提供开箱即用的数据 - 所有代码编写、修改、重构和测试调整都必须遵守 [FastGPT 代码规范](./.agents/code/syntax.md)。开始改动前先查看相关规范；如果规范与当前实现习惯冲突，优先按规范执行，并只在有明确业务或兼容性理由时说明例外。 +### MongoDB Schema 与索引维护 + +- 每当新增、修改、删除或重命名 MongoDB/Mongoose Schema 字段、索引定义、唯一约束、TTL、partialFilterExpression、collation 等索引相关配置时，必须同步检查 `packages/service/common/mongo/deprecatedIndexes.ts` 是否需要更新 FastGPT 历史废弃索引清单。 +- 如果当前变更会让 FastGPT 旧版本自己创建过的索引不再被新 schema 使用，且该旧索引可能继续影响写入约束、查询计划或存储成本，应在 `deprecatedIndexes.ts` 中登记对应的废弃索引，并补充/调整 `packages/service/test/common/mongo/indexManager.test.ts` 的清理行为覆盖。 +- 不要把客户自建索引、无法确认来源的索引、或仅凭当前 schema 未声明就推断为废弃的索引加入 `deprecatedIndexes.ts`。该文件只维护 FastGPT 明确创建过、明确废弃、可由维护/升级脚本安全清理的历史索引。 + ### API 入参校验 - 编写或修改 NextJS API 路由时，如果需要校验接口入参（`req.body`、`req.query`、`req.params`），必须使用 `parseApiInput`，不要直接写 `SomeSchema.parse(req.body)`、`SomeSchema.parse(req.query)` 或 `SomeSchema.parse(req.params)`。 diff --git a/packages/global/common/system/constants.ts b/packages/global/common/system/constants.ts index a10cc8a49ae7..0bec97774e2f 100644 --- a/packages/global/common/system/constants.ts +++ b/packages/global/common/system/constants.ts @@ -12,6 +12,3 @@ export const isTestEnv = process.env.NODE_ENV === 'test'; export const isPhaseProductionBuild = process.env.NEXT_PHASE === 'phase-production-build'; export const FASTGPT_PRO_TOKEN_HEADER = 'x-fastgpt-pro-token'; - -export const mongoIndexSyncModeList = ['off', 'create', 'dryRun'] as const; -export type MongoIndexSyncMode = (typeof mongoIndexSyncModeList)[number]; diff --git a/packages/service/common/mongo/deprecatedIndexes.ts b/packages/service/common/mongo/deprecatedIndexes.ts new file mode 100644 index 000000000000..dc51731ca08f --- /dev/null +++ b/packages/service/common/mongo/deprecatedIndexes.ts @@ -0,0 +1,111 @@ +type DeprecatedMongoIndexOptions = { + unique?: boolean; + sparse?: boolean; + expireAfterSeconds?: number; + partialFilterExpression?: unknown; + collation?: unknown; +}; + +export type DeprecatedMongoIndexDefinition = { + collectionName: string; + indexName: string; + key: Record; + options?: DeprecatedMongoIndexOptions; + deprecatedVersion: string; + reason: string; + replacementIndexNames?: string[]; +}; + +const deprecatedMongoIndexProperties = { + version: { + v4150Beta6: '4.15.0-beta6' + }, + collection: { + chat: 'chats', + sandboxInstance: 'agent_sandbox_instances' + }, + replacementIndex: { + chatSourceIdentity: 'sourceType_1_appId_1_chatId_1', + sandboxProviderIdentity: 'provider_1_sandboxId_1', + sandboxSourceLookup: 'sourceType_1_sourceId_1_chatId_1' + } +} as const; + +const deprecatedChatIndexes: DeprecatedMongoIndexDefinition[] = [ + { + collectionName: deprecatedMongoIndexProperties.collection.chat, + indexName: 'appId_1_chatId_1', + key: { appId: 1, chatId: 1 }, + options: { unique: true }, + deprecatedVersion: deprecatedMongoIndexProperties.version.v4150Beta6, + reason: 'Chat identity index was expanded to include sourceType for source-aware chats.', + replacementIndexNames: [deprecatedMongoIndexProperties.replacementIndex.chatSourceIdentity] + } +]; + +const deprecatedSandboxIndexes: DeprecatedMongoIndexDefinition[] = [ + { + collectionName: deprecatedMongoIndexProperties.collection.sandboxInstance, + indexName: 'provider_1_appId_1_userId_1_chatId_1', + key: { provider: 1, appId: 1, userId: 1, chatId: 1 }, + options: { + unique: true, + partialFilterExpression: { + appId: { $exists: true }, + userId: { $exists: true }, + chatId: { $exists: true } + } + }, + deprecatedVersion: deprecatedMongoIndexProperties.version.v4150Beta6, + reason: 'Sandbox ownership moved from appId/userId/type fields to sourceType/sourceId.', + replacementIndexNames: [ + deprecatedMongoIndexProperties.replacementIndex.sandboxProviderIdentity, + deprecatedMongoIndexProperties.replacementIndex.sandboxSourceLookup + ] + }, + { + collectionName: deprecatedMongoIndexProperties.collection.sandboxInstance, + indexName: 'appId_1_chatId_1', + key: { appId: 1, chatId: 1 }, + options: { + unique: true, + partialFilterExpression: { + appId: { $exists: true }, + chatId: { $exists: true }, + type: { $exists: true } + } + }, + deprecatedVersion: deprecatedMongoIndexProperties.version.v4150Beta6, + reason: 'Sandbox appId/type lookup was replaced by sourceType/sourceId lookup.', + replacementIndexNames: [deprecatedMongoIndexProperties.replacementIndex.sandboxSourceLookup] + }, + { + collectionName: deprecatedMongoIndexProperties.collection.sandboxInstance, + indexName: 'metadata.skillId_1', + key: { 'metadata.skillId': 1 }, + deprecatedVersion: deprecatedMongoIndexProperties.version.v4150Beta6, + reason: 'Skill edit sandbox ownership no longer uses metadata.skillId.', + replacementIndexNames: [deprecatedMongoIndexProperties.replacementIndex.sandboxSourceLookup] + }, + { + collectionName: deprecatedMongoIndexProperties.collection.sandboxInstance, + indexName: 'type_1_chatId_1', + key: { type: 1, chatId: 1 }, + deprecatedVersion: deprecatedMongoIndexProperties.version.v4150Beta6, + reason: 'Sandbox type/chat lookup was replaced by sourceType/sourceId lookup.', + replacementIndexNames: [deprecatedMongoIndexProperties.replacementIndex.sandboxSourceLookup] + } +]; + +/** + * FastGPT 历史版本明确废弃、允许通过维护脚本清理的 MongoDB 索引清单。 + * + * 维护规则： + * - 只记录 FastGPT 自己历史创建过的索引。 + * - 删除前必须精确匹配 collection、index name、key 和关键 options。 + * - 客户自建索引不要加入这里，即使它不在当前 schema 中。 + */ +export const deprecatedMongoIndexes: DeprecatedMongoIndexDefinition[] = [ + ...deprecatedChatIndexes, + ...deprecatedSandboxIndexes +]; diff --git a/packages/service/common/mongo/index.ts b/packages/service/common/mongo/index.ts index 273c1566245d..9b7b834593e8 100644 --- a/packages/service/common/mongo/index.ts +++ b/packages/service/common/mongo/index.ts @@ -9,7 +9,7 @@ import type { } from 'mongoose'; import mongoose, { Mongoose } from 'mongoose'; import { serviceEnv } from '../../env'; -import { runMongoIndexSyncForModel } from './indexManager'; +import { MongoIndexManager } from './indexManager'; const logger = getLogger(LogCategories.INFRA.MONGO); @@ -162,7 +162,7 @@ const syncMongoIndex = (model: Model) => { return; } - void runMongoIndexSyncForModel({ + void MongoIndexManager.runModelIndexMode({ model, mode: serviceEnv.MONGO_INDEX_SYNC_MODE, logger @@ -177,3 +177,12 @@ const syncMongoIndex = (model: Model) => { }; export const ReadPreference = connectionMongo.mongo.ReadPreference; + +export { MongoIndexManager } from './indexManager'; +export type { + MongoIndexCleanupAction, + MongoIndexCleanupReport, + MongoIndexCleanupReportItem, + MongoIndexCleanupSummary, + MongoIndexSyncResult +} from './indexManager'; diff --git a/packages/service/common/mongo/indexManager.ts b/packages/service/common/mongo/indexManager.ts index 8f32e6598693..0e3fb5379bc6 100644 --- a/packages/service/common/mongo/indexManager.ts +++ b/packages/service/common/mongo/indexManager.ts @@ -1,9 +1,15 @@ -import type { MongoIndexSyncMode } from '@fastgpt/global/common/system/constants'; import { getLogger, LogCategories } from '../logger'; -import type { Model } from 'mongoose'; +import type { Connection, Model } from 'mongoose'; +import { + deprecatedMongoIndexes as defaultDeprecatedMongoIndexes, + type DeprecatedMongoIndexDefinition +} from './deprecatedIndexes'; +import type { serviceEnv } from '../../env'; const defaultLogger = getLogger(LogCategories.INFRA.MONGO); +type MongoIndexSyncMode = typeof serviceEnv.MONGO_INDEX_SYNC_MODE; + type MongoIndexLogger = { debug: (message: string, data?: Record) => void; info: (message: string, data?: Record) => void; @@ -24,84 +30,603 @@ export type MongoIndexSyncResult = { toCreate: unknown[]; }; -const getCollectionName = (model: Model) => model.collection.collectionName; +export type MongoIndexDescription = { + name?: string; + key?: Record; + unique?: boolean; + sparse?: boolean; + expireAfterSeconds?: number; + partialFilterExpression?: unknown; + collation?: unknown; +}; -/** - * 只计算当前 schema 和数据库索引的差异，不创建也不删除任何索引。 - * - * `toDrop` 只表示 Mongoose 认为 schema 外存在的索引。默认安全模式下这些索引 - * 只会被记录为未知索引，不会删除，以保护客户自建索引。 - */ -export const inspectMongoModelIndexes = async ( - model: Model -): Promise> => { - const diff = (await model.diffIndexes({ - indexOptionsToCreate: true - })) as MongooseDiffIndexesResult; - - return { - modelName: model.modelName, - collectionName: getCollectionName(model), - toDrop: diff.toDrop, - toCreate: diff.toCreate - }; +type MongoIndexCollection = { + indexes: () => Promise; + dropIndex: (indexName: string) => Promise; }; -/** - * 按配置模式处理单个 Mongoose model 的索引。 - * - * 默认 `create` 只创建 schema 中缺失的索引，并保留所有 schema 外索引。 - */ -export const runMongoIndexSyncForModel = async ({ - model, - mode, - logger = defaultLogger -}: { +type MongoIndexDb = { + collection: (collectionName: string) => MongoIndexCollection; +}; + +export type MongoIndexCleanupAction = + | 'drop' + | 'skip_missing' + | 'skip_mismatch' + | 'skip_missing_replacement' + | 'error'; + +export type MongoIndexCleanupReportItem = { + collectionName: string; + indexName: string; + action: MongoIndexCleanupAction; + applied: boolean; + reason: string; + deprecatedVersion: string; + replacementIndexNames?: string[]; + missingReplacementIndexNames?: string[]; + error?: string; +}; + +export type MongoIndexCleanupReport = { + apply: boolean; + items: MongoIndexCleanupReportItem[]; +}; + +export type MongoIndexCleanupSummary = { + total: number; + dropped: number; + droppable: number; + skippedMissing: number; + skippedMismatch: number; + skippedMissingReplacement: number; + errors: number; +}; + +type RunModelIndexModeParams = { model: Model; mode: MongoIndexSyncMode; logger?: MongoIndexLogger; -}): Promise => { - const baseResult = { - mode, - modelName: model.modelName, - collectionName: getCollectionName(model), - toDrop: [], - toCreate: [] - } satisfies MongoIndexSyncResult; +}; + +type RunDeprecatedIndexCleanupOnceParams = { + db: MongoIndexDb; + cleanupKey: string; + apply: boolean; + indexes?: DeprecatedMongoIndexDefinition[]; + logger?: MongoIndexLogger; +}; + +const optionKeys = [ + 'unique', + 'sparse', + 'expireAfterSeconds', + 'partialFilterExpression', + 'collation' +] as const; + +/** + * MongoDB 索引管理入口。 + * + * 默认启动期只负责非破坏性补建 schema 索引；只有显式 `sync` 模式才会额外清理 + * `deprecatedIndexes` 中登记的 FastGPT 历史旧索引，避免普通服务重启误删客户自建索引。 + */ +export class MongoIndexManager { + private static modelIndexTasks = new Map, Promise>(); + private static deprecatedCleanupTasks = new Map>(); - if (mode === 'off') { - return baseResult; + private static getCollectionName(model: Model) { + return model.collection.collectionName; } - const inspection = await inspectMongoModelIndexes(model); - const result: MongoIndexSyncResult = { - ...baseResult, - toDrop: inspection.toDrop, - toCreate: inspection.toCreate - }; + /** + * 只计算当前 schema 和数据库索引的差异，不创建也不删除任何索引。 + * + * `toDrop` 只表示 Mongoose 认为 schema 外存在的索引。默认安全模式下这些索引 + * 只会被记录为未知索引，不会删除，以保护客户自建索引。 + */ + static async inspectModelIndexes( + model: Model + ): Promise> { + const diff = (await model.diffIndexes({ + indexOptionsToCreate: true + })) as MongooseDiffIndexesResult; - if (result.toDrop.length > 0) { - logger.warn('Detected MongoDB indexes not declared by FastGPT schema', { - mode, + return { modelName: model.modelName, - collectionName: getCollectionName(model), - indexes: result.toDrop + collectionName: MongoIndexManager.getCollectionName(model), + toDrop: diff.toDrop, + toCreate: diff.toCreate + }; + } + + /** + * 按配置模式处理单个 Mongoose model 的索引。 + * + * 默认 `create` 只创建 schema 中缺失的索引，并保留所有 schema 外索引。 + * `sync` 在 model 级别仍只创建缺失索引；废弃索引由连接初始化流程全局清理一次。 + */ + static async runModelIndexMode(params: RunModelIndexModeParams): Promise { + const existingTask = MongoIndexManager.modelIndexTasks.get(params.model); + if (existingTask) { + return existingTask; + } + + const task = MongoIndexManager.runModelIndexModeInner(params); + + MongoIndexManager.modelIndexTasks.set(params.model, task); + + try { + return await task; + } finally { + if (MongoIndexManager.modelIndexTasks.get(params.model) === task) { + MongoIndexManager.modelIndexTasks.delete(params.model); + } + } + } + + /** + * 等待指定 Mongoose 连接上已经登记的 model 索引任务结束。 + * + * 全局清理废弃索引前必须先等待替代索引创建完成，否则可能因 replacement 尚不存在而跳过清理。 + */ + static async waitForModelIndexTasks(connection: Connection): Promise { + while (true) { + const tasks = [...MongoIndexManager.modelIndexTasks.entries()] + .filter(([model]) => model.db === connection) + .map(([, task]) => task); + + if (tasks.length === 0) { + return; + } + + await Promise.allSettled(tasks); + } + } + + /** + * 对同一个 MongoDB 连接全局执行一次废弃索引清理。 + * + * 清理范围只来自 deprecated registry；重复调用会复用首次任务，不会再次扫描或删除。 + */ + static async runDeprecatedIndexCleanupOnce({ + db, + cleanupKey, + apply, + indexes = defaultDeprecatedMongoIndexes, + logger = defaultLogger + }: RunDeprecatedIndexCleanupOnceParams): Promise { + const existingTask = MongoIndexManager.deprecatedCleanupTasks.get(cleanupKey); + if (existingTask) { + logger.debug('MongoDB deprecated index cleanup skipped', { + cleanupKey, + reason: 'already_started_or_completed' + }); + return existingTask; + } + + const task = MongoIndexManager.cleanupDeprecatedIndexes({ + db, + apply, + indexes, + logger }); + MongoIndexManager.deprecatedCleanupTasks.set(cleanupKey, task); + + const report = await task; + logger.info('MongoDB deprecated index cleanup finished', { + cleanupKey, + apply, + summary: MongoIndexManager.summarizeCleanupReport(report) + }); + return report; + } + + /** 生成不包含凭据的连接级 cleanup 去重键。 */ + static getConnectionCleanupKey(connection: Connection): string { + return [connection.host, connection.port, connection.name].filter(Boolean).join(':'); } - if (mode === 'dryRun') { - logger.info('MongoDB index dry-run completed', result); + private static async runModelIndexModeInner({ + model, + mode, + logger = defaultLogger + }: RunModelIndexModeParams): Promise { + const collectionName = MongoIndexManager.getCollectionName(model); + const baseResult = { + mode, + modelName: model.modelName, + collectionName, + toDrop: [], + toCreate: [] + } satisfies MongoIndexSyncResult; + + if (mode === 'off') { + logger.debug('MongoDB index management skipped', { + ...MongoIndexManager.buildIndexModeLogData(baseResult), + reason: 'mode_off' + }); + return baseResult; + } + + const inspection = await MongoIndexManager.inspectModelIndexes(model); + const result: MongoIndexSyncResult = { + ...baseResult, + toDrop: inspection.toDrop, + toCreate: inspection.toCreate + }; + + logger.debug('MongoDB index diff inspected', { + ...MongoIndexManager.buildIndexModeLogData(result), + schemaExternalIndexNames: result.toDrop, + toCreate: result.toCreate + }); + + if (result.toDrop.length > 0) { + logger.warn('Detected MongoDB indexes not declared by FastGPT schema', { + ...MongoIndexManager.buildIndexModeLogData(result), + schemaExternalIndexNames: result.toDrop, + cleanupPolicy: 'only_registered_deprecated_indexes_can_be_dropped' + }); + } + + if (mode === 'dryRun') { + logger.info( + 'MongoDB index dry-run completed', + MongoIndexManager.buildIndexModeLogData(result) + ); + return result; + } + + if (mode === 'sync') { + logger.info('MongoDB managed index sync started', { + ...MongoIndexManager.buildIndexModeLogData(result), + cleanupPolicy: 'deprecated_indexes_are_cleaned_once_per_connection' + }); + logger.debug('MongoDB managed index sync detail', { + ...MongoIndexManager.buildIndexModeLogData(result), + schemaExternalIndexNames: result.toDrop, + toCreate: result.toCreate, + cleanupPolicy: 'deprecated_indexes_are_cleaned_once_per_connection' + }); + + await model.createIndexes({ background: true }); + + logger.info('MongoDB managed index sync completed', { + ...MongoIndexManager.buildIndexModeLogData(result), + cleanupPolicy: 'deprecated_indexes_are_cleaned_once_per_connection' + }); + + return result; + } + + await model.createIndexes({ background: true }); + + logger.info('MongoDB schema indexes ensured', MongoIndexManager.buildIndexModeLogData(result)); + logger.debug('MongoDB schema index ensure detail', { + ...MongoIndexManager.buildIndexModeLogData(result), + schemaExternalIndexNames: result.toDrop, + toCreate: result.toCreate + }); + return result; } - await model.createIndexes({ background: true }); + /** + * 清理 FastGPT 明确登记的废弃 MongoDB 索引。 + * + * 默认 dry-run 只输出计划；只有 `apply=true` 且 registry 与数据库索引精确匹配时才会删除。 + * schema 外未知索引不在本函数处理范围内，避免误删客户自建索引。 + */ + static async cleanupDeprecatedIndexes({ + db, + apply, + indexes = defaultDeprecatedMongoIndexes, + logger + }: { + db: MongoIndexDb; + apply: boolean; + indexes?: DeprecatedMongoIndexDefinition[]; + logger?: MongoIndexLogger; + }): Promise { + const items: MongoIndexCleanupReportItem[] = []; - logger.info('MongoDB indexes ensured', { - mode, - modelName: model.modelName, - collectionName: getCollectionName(model), - toCreateCount: result.toCreate.length - }); + if (indexes.length === 0) { + return { + apply, + items + }; + } - return result; -}; + logger?.debug('MongoDB deprecated index cleanup started', { + apply, + deprecatedIndexCount: indexes.length + }); + + for (const definition of indexes) { + try { + const collection = db.collection(definition.collectionName); + const currentIndexes = await collection.indexes().catch((error) => { + if (MongoIndexManager.isNamespaceNotFoundError(error)) { + return []; + } + throw error; + }); + const targetIndex = currentIndexes.find((index) => index.name === definition.indexName); + + if (!targetIndex) { + const item = MongoIndexManager.buildCleanupItem({ + definition, + action: 'skip_missing', + reason: 'Deprecated index does not exist' + }); + logger?.debug('Deprecated MongoDB index does not exist', item); + items.push(item); + continue; + } + + if (!MongoIndexManager.isDeprecatedIndexMatched({ definition, index: targetIndex })) { + const item = MongoIndexManager.buildCleanupItem({ + definition, + action: 'skip_mismatch', + reason: 'Index definition does not match deprecated registry entry' + }); + logger?.warn('Skip deprecated MongoDB index cleanup because definition mismatched', { + ...item, + expectedKey: definition.key, + actualKey: targetIndex.key, + expectedOptions: MongoIndexManager.getExpectedOptions(definition), + actualOptions: MongoIndexManager.getActualOptions(targetIndex) + }); + items.push(item); + continue; + } + + const replacementIndexNames = definition.replacementIndexNames ?? []; + const missingReplacementIndexNames = replacementIndexNames.filter( + (indexName) => !currentIndexes.some((index) => index.name === indexName) + ); + + if (missingReplacementIndexNames.length > 0) { + const item = MongoIndexManager.buildCleanupItem({ + definition, + action: 'skip_missing_replacement', + reason: 'Replacement index does not exist', + missingReplacementIndexNames + }); + logger?.warn( + 'Skip deprecated MongoDB index cleanup because replacement is missing', + item + ); + items.push(item); + continue; + } + + if (apply) { + await collection.dropIndex(definition.indexName); + } + + const item = MongoIndexManager.buildCleanupItem({ + definition, + action: 'drop', + applied: apply, + reason: apply ? 'Deprecated index dropped' : 'Deprecated index can be dropped' + }); + if (apply) { + logger?.info('Dropped deprecated MongoDB index', item); + } else { + logger?.debug('Deprecated MongoDB index can be dropped', item); + } + items.push(item); + } catch (error) { + const item = MongoIndexManager.buildCleanupItem({ + definition, + action: 'error', + reason: 'Failed to inspect or cleanup deprecated index', + error: MongoIndexManager.getErrorMessage(error) + }); + logger?.error('Failed to cleanup deprecated MongoDB index', item); + items.push(item); + } + } + + logger?.debug('MongoDB deprecated index cleanup completed', { + apply, + summary: MongoIndexManager.summarizeCleanupReport({ apply, items }) + }); + + return { + apply, + items + }; + } + + static summarizeCleanupReport(report: MongoIndexCleanupReport): MongoIndexCleanupSummary { + return report.items.reduce( + (summary, item) => { + summary.total += 1; + + if (item.action === 'drop' && item.applied) { + summary.dropped += 1; + } else if (item.action === 'drop') { + summary.droppable += 1; + } else if (item.action === 'skip_missing') { + summary.skippedMissing += 1; + } else if (item.action === 'skip_mismatch') { + summary.skippedMismatch += 1; + } else if (item.action === 'skip_missing_replacement') { + summary.skippedMissingReplacement += 1; + } else if (item.action === 'error') { + summary.errors += 1; + } + + return summary; + }, + { + total: 0, + dropped: 0, + droppable: 0, + skippedMissing: 0, + skippedMismatch: 0, + skippedMissingReplacement: 0, + errors: 0 + } + ); + } + + static formatCleanupReport(report: MongoIndexCleanupReport) { + const lines = [ + `MongoDB deprecated index cleanup ${report.apply ? 'apply' : 'dry-run'} report`, + `Total: ${report.items.length}` + ]; + + for (const item of report.items) { + lines.push( + [ + `- [${item.action}]`, + item.applied ? 'applied' : 'not-applied', + `${item.collectionName}.${item.indexName}`, + `version=${item.deprecatedVersion}`, + `reason=${item.reason}`, + item.replacementIndexNames?.length + ? `replacement=${item.replacementIndexNames.join(',')}` + : undefined, + item.missingReplacementIndexNames?.length + ? `missingReplacement=${item.missingReplacementIndexNames.join(',')}` + : undefined, + item.error ? `error=${item.error}` : undefined + ] + .filter(Boolean) + .join(' ') + ); + } + + return lines.join('\n'); + } + + private static buildIndexModeLogData(result: MongoIndexSyncResult) { + return { + mode: result.mode, + modelName: result.modelName, + collectionName: result.collectionName, + toCreateCount: result.toCreate.length, + schemaExternalIndexCount: result.toDrop.length + }; + } + + private static normalizeForCompare( + value: unknown, + { sortObjectKeys }: { sortObjectKeys: boolean } + ): unknown { + if (Array.isArray(value)) { + return value.map((item) => MongoIndexManager.normalizeForCompare(item, { sortObjectKeys })); + } + + if (value && typeof value === 'object') { + const keys = Object.keys(value as Record); + const orderedKeys = sortObjectKeys ? keys.sort() : keys; + + return orderedKeys.reduce>((result, key) => { + result[key] = MongoIndexManager.normalizeForCompare( + (value as Record)[key], + { sortObjectKeys } + ); + return result; + }, {}); + } + + return value; + } + + private static isSameValue( + left: unknown, + right: unknown, + { sortObjectKeys = true }: { sortObjectKeys?: boolean } = {} + ) { + return ( + JSON.stringify(MongoIndexManager.normalizeForCompare(left, { sortObjectKeys })) === + JSON.stringify(MongoIndexManager.normalizeForCompare(right, { sortObjectKeys })) + ); + } + + private static getExpectedOptions(definition: DeprecatedMongoIndexDefinition) { + return optionKeys.reduce>((result, key) => { + const value = definition.options?.[key]; + if (value !== undefined) { + result[key] = value; + } + return result; + }, {}); + } + + private static getActualOptions(index: MongoIndexDescription) { + return optionKeys.reduce>((result, key) => { + const value = index[key]; + if (value !== undefined) { + result[key] = value; + } + return result; + }, {}); + } + + private static isDeprecatedIndexMatched({ + definition, + index + }: { + definition: DeprecatedMongoIndexDefinition; + index: MongoIndexDescription; + }) { + if (!MongoIndexManager.isSameValue(index.key, definition.key, { sortObjectKeys: false })) { + return false; + } + + return MongoIndexManager.isSameValue( + MongoIndexManager.getActualOptions(index), + MongoIndexManager.getExpectedOptions(definition) + ); + } + + private static buildCleanupItem({ + definition, + action, + applied = false, + reason, + missingReplacementIndexNames, + error + }: { + definition: DeprecatedMongoIndexDefinition; + action: MongoIndexCleanupAction; + applied?: boolean; + reason: string; + missingReplacementIndexNames?: string[]; + error?: string; + }): MongoIndexCleanupReportItem { + return { + collectionName: definition.collectionName, + indexName: definition.indexName, + action, + applied, + reason, + deprecatedVersion: definition.deprecatedVersion, + replacementIndexNames: definition.replacementIndexNames, + missingReplacementIndexNames, + error + }; + } + + private static getErrorMessage(error: unknown) { + if (error instanceof Error) { + return error.message; + } + return String(error); + } + + private static isNamespaceNotFoundError(error: unknown) { + if (typeof error !== 'object' || error === null) { + return false; + } + + const { codeName, message } = error as { codeName?: string; message?: string }; + return codeName === 'NamespaceNotFound' || message?.includes('ns does not exist') === true; + } +} diff --git a/packages/service/common/mongo/init.ts b/packages/service/common/mongo/init.ts index ced13c03482b..07ac2cf1f1a5 100644 --- a/packages/service/common/mongo/init.ts +++ b/packages/service/common/mongo/init.ts @@ -2,6 +2,7 @@ import { delay } from '@fastgpt/global/common/system/utils'; import { getLogger, LogCategories } from '../logger'; import type { Mongoose } from 'mongoose'; import { serviceEnv } from '../../env'; +import { MongoIndexManager } from './indexManager'; const logger = getLogger(LogCategories.INFRA.MONGO); @@ -14,8 +15,9 @@ export async function connectMongo(props: { db: Mongoose; url: string; connectedCb?: () => void; + cleanupDeprecatedIndexes?: boolean; }): Promise { - const { db, url, connectedCb } = props; + const { db, url, connectedCb, cleanupDeprecatedIndexes = false } = props; /* Connecting, connected will return */ if (db.connection.readyState !== 0) { @@ -63,6 +65,21 @@ export async function connectMongo(props: { connectedCb?.(); + if (cleanupDeprecatedIndexes && serviceEnv.MONGO_INDEX_SYNC_MODE === 'sync') { + const mongoDb = db.connection.db; + if (!mongoDb) { + throw new Error('MongoDB connection has no database instance after connecting'); + } + + await MongoIndexManager.waitForModelIndexTasks(db.connection); + await MongoIndexManager.runDeprecatedIndexCleanupOnce({ + db: mongoDb, + cleanupKey: MongoIndexManager.getConnectionCleanupKey(db.connection), + apply: true, + logger + }); + } + return db; } catch (error) { logger.error('MongoDB connection failed, will retry', { error }); diff --git a/packages/service/env.ts b/packages/service/env.ts index a4dfca32ab78..b90831290d99 100644 --- a/packages/service/env.ts +++ b/packages/service/env.ts @@ -1,9 +1,6 @@ import { createEnv } from '@t3-oss/env-core'; import z from 'zod'; -import { - isPhaseProductionBuild, - mongoIndexSyncModeList -} from '@fastgpt/global/common/system/constants'; +import { isPhaseProductionBuild } from '@fastgpt/global/common/system/constants'; import { DEFAULT_MAX_FOLDER_DEPTH } from '@fastgpt/global/common/parentFolder/depth'; import { BoolSchema, IntSchema, NumSchema, UrlSchema } from '@fastgpt/global/common/zod'; import { agentSandboxProviderList } from '@fastgpt/global/core/ai/sandbox/constants'; @@ -32,7 +29,7 @@ export const serviceEnv = createEnv({ server: { // ==================== 基础配置 ==================== DB_MAX_LINK: IntSchema.min(1).default(5), - MONGO_INDEX_SYNC_MODE: z.enum(mongoIndexSyncModeList).default('create'), + MONGO_INDEX_SYNC_MODE: z.enum(['off', 'sync', 'create', 'dryRun']).default('create'), // ==================== 密钥 ==================== ROOT_KEY: z diff --git a/packages/service/test/common/mongo/indexManager.test.ts b/packages/service/test/common/mongo/indexManager.test.ts index 8c721bf44135..089d185b0b9e 100644 --- a/packages/service/test/common/mongo/indexManager.test.ts +++ b/packages/service/test/common/mongo/indexManager.test.ts @@ -1,6 +1,8 @@ +import { randomUUID } from 'node:crypto'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import { connectionMongo, Schema } from '@fastgpt/service/common/mongo'; -import { runMongoIndexSyncForModel } from '@fastgpt/service/common/mongo/indexManager'; +import { MongoIndexManager } from '@fastgpt/service/common/mongo/indexManager'; +import type { DeprecatedMongoIndexDefinition } from '@fastgpt/service/common/mongo/deprecatedIndexes'; const logger = { debug: vi.fn(), @@ -14,6 +16,8 @@ const getIndexNames = async (collectionName: string) => { return new Set(indexes?.map((index: { name?: string }) => index.name)); }; +const getCleanupCollectionName = () => `mongo_index_cleanup_${randomUUID().replace(/-/g, '')}`; + const dropCollection = async (collectionName: string) => { try { await connectionMongo.connection.db?.collection(collectionName).drop(); @@ -24,7 +28,15 @@ const dropCollection = async (collectionName: string) => { } }; -describe('runMongoIndexSyncForModel', () => { +const getDb = () => { + const db = connectionMongo.connection.db; + if (!db) { + throw new Error('Mongo test db is not connected'); + } + return db; +}; + +describe('MongoIndexManager.runModelIndexMode', () => { beforeEach(() => { vi.clearAllMocks(); }); @@ -45,7 +57,7 @@ describe('runMongoIndexSyncForModel', () => { const model = connectionMongo.model(`MongoIndexCreate${Date.now()}`, schema, collectionName); await model.collection.createIndex({ customerField: 1 }, { name: 'customer_custom_1' }); - await runMongoIndexSyncForModel({ + await MongoIndexManager.runModelIndexMode({ model, mode: 'create', logger @@ -54,6 +66,24 @@ describe('runMongoIndexSyncForModel', () => { const indexNames = await getIndexNames(collectionName); expect(indexNames.has('schema_name_1')).toBe(true); expect(indexNames.has('customer_custom_1')).toBe(true); + expect(logger.info).toHaveBeenCalledWith( + 'MongoDB schema indexes ensured', + expect.objectContaining({ + mode: 'create', + collectionName, + toCreateCount: expect.any(Number), + schemaExternalIndexCount: expect.any(Number) + }) + ); + expect(logger.debug).toHaveBeenCalledWith( + 'MongoDB schema index ensure detail', + expect.objectContaining({ + mode: 'create', + collectionName, + schemaExternalIndexNames: expect.arrayContaining(['customer_custom_1']), + toCreate: expect.any(Array) + }) + ); }); it('does not create or delete indexes in dryRun mode', async () => { @@ -72,7 +102,7 @@ describe('runMongoIndexSyncForModel', () => { const model = connectionMongo.model(`MongoIndexDryRun${Date.now()}`, schema, collectionName); await model.collection.createIndex({ customerField: 1 }, { name: 'customer_custom_1' }); - const result = await runMongoIndexSyncForModel({ + const result = await MongoIndexManager.runModelIndexMode({ model, mode: 'dryRun', logger @@ -82,5 +112,393 @@ describe('runMongoIndexSyncForModel', () => { expect(result.toCreate.length).toBeGreaterThan(0); expect(indexNames.has('schema_name_1')).toBe(false); expect(indexNames.has('customer_custom_1')).toBe(true); + expect(logger.info).toHaveBeenCalledWith( + 'MongoDB index dry-run completed', + expect.objectContaining({ + mode: 'dryRun', + collectionName, + toCreateCount: result.toCreate.length, + schemaExternalIndexCount: expect.any(Number) + }) + ); + expect(logger.debug).toHaveBeenCalledWith( + 'MongoDB index diff inspected', + expect.objectContaining({ + mode: 'dryRun', + collectionName, + schemaExternalIndexNames: expect.arrayContaining(['customer_custom_1']), + toCreate: result.toCreate + }) + ); + }); + + it('creates schema indexes without deleting indexes in per-model sync mode', async () => { + const collectionName = `mongo_index_sync_${Date.now()}`; + await dropCollection(collectionName); + + const schema = new Schema( + { + name: String, + customerField: String, + legacyField: String + }, + { autoIndex: false } + ); + schema.index({ name: 1 }, { name: 'schema_name_1' }); + + const model = connectionMongo.model(`MongoIndexSync${Date.now()}`, schema, collectionName); + await model.collection.createIndex({ customerField: 1 }, { name: 'customer_custom_1' }); + await model.collection.createIndex({ legacyField: 1 }, { name: 'legacy_field_1' }); + + const result = await MongoIndexManager.runModelIndexMode({ + model, + mode: 'sync', + logger + }); + + const indexNames = await getIndexNames(collectionName); + expect(result.toDrop).toContain('customer_custom_1'); + expect(result.toDrop).toContain('legacy_field_1'); + expect(indexNames.has('schema_name_1')).toBe(true); + expect(indexNames.has('customer_custom_1')).toBe(true); + expect(indexNames.has('legacy_field_1')).toBe(true); + expect(logger.info).toHaveBeenCalledWith( + 'MongoDB managed index sync started', + expect.objectContaining({ + mode: 'sync', + collectionName, + cleanupPolicy: 'deprecated_indexes_are_cleaned_once_per_connection' + }) + ); + expect(logger.info).toHaveBeenCalledWith( + 'MongoDB managed index sync completed', + expect.objectContaining({ + mode: 'sync', + collectionName, + cleanupPolicy: 'deprecated_indexes_are_cleaned_once_per_connection' + }) + ); + expect(logger.debug).toHaveBeenCalledWith( + 'MongoDB managed index sync detail', + expect.objectContaining({ + mode: 'sync', + collectionName, + schemaExternalIndexNames: expect.arrayContaining(['customer_custom_1', 'legacy_field_1']), + toCreate: expect.any(Array), + cleanupPolicy: 'deprecated_indexes_are_cleaned_once_per_connection' + }) + ); + expect(logger.warn).toHaveBeenCalledWith( + 'Detected MongoDB indexes not declared by FastGPT schema', + expect.objectContaining({ + mode: 'sync', + collectionName, + schemaExternalIndexNames: expect.arrayContaining(['customer_custom_1', 'legacy_field_1']), + cleanupPolicy: 'only_registered_deprecated_indexes_can_be_dropped' + }) + ); + }); +}); + +describe('MongoIndexManager.cleanupDeprecatedIndexes', () => { + it('runs deprecated index cleanup only once for the same connection key', async () => { + vi.clearAllMocks(); + const collectionName = getCleanupCollectionName(); + const cleanupKey = `test:${randomUUID()}`; + await dropCollection(collectionName); + const collection = getDb().collection(collectionName); + await collection.createIndex({ legacyField: 1 }, { name: 'legacy_field_1' }); + + const indexes: DeprecatedMongoIndexDefinition[] = [ + { + collectionName, + indexName: 'legacy_field_1', + key: { legacyField: 1 }, + deprecatedVersion: '4.15.0', + reason: 'test legacy index' + } + ]; + + const firstReport = await MongoIndexManager.runDeprecatedIndexCleanupOnce({ + db: getDb(), + cleanupKey, + apply: true, + indexes, + logger + }); + await collection.createIndex({ legacyField: 1 }, { name: 'legacy_field_1' }); + const secondReport = await MongoIndexManager.runDeprecatedIndexCleanupOnce({ + db: getDb(), + cleanupKey, + apply: true, + indexes, + logger + }); + + const indexNames = await getIndexNames(collectionName); + expect(firstReport).toBe(secondReport); + expect(indexNames.has('legacy_field_1')).toBe(true); + expect(logger.info).toHaveBeenCalledTimes(2); + expect(logger.debug).toHaveBeenCalledWith( + 'MongoDB deprecated index cleanup skipped', + expect.objectContaining({ + cleanupKey, + reason: 'already_started_or_completed' + }) + ); + }); + + it('dry-runs matched deprecated indexes without deleting them', async () => { + const collectionName = getCleanupCollectionName(); + await dropCollection(collectionName); + const collection = getDb().collection(collectionName); + await collection.createIndex({ legacyField: 1 }, { name: 'legacy_field_1' }); + + const indexes: DeprecatedMongoIndexDefinition[] = [ + { + collectionName, + indexName: 'legacy_field_1', + key: { legacyField: 1 }, + deprecatedVersion: '4.15.0', + reason: 'test legacy index' + } + ]; + + const report = await MongoIndexManager.cleanupDeprecatedIndexes({ + db: getDb(), + apply: false, + indexes + }); + + const indexNames = await getIndexNames(collectionName); + expect(report.items).toEqual([ + expect.objectContaining({ + action: 'drop', + applied: false, + collectionName, + indexName: 'legacy_field_1' + }) + ]); + expect(indexNames.has('legacy_field_1')).toBe(true); + }); + + it('drops only matched deprecated indexes when apply is enabled', async () => { + const collectionName = getCleanupCollectionName(); + await dropCollection(collectionName); + const collection = getDb().collection(collectionName); + await collection.createIndex({ legacyField: 1 }, { name: 'legacy_field_1' }); + await collection.createIndex({ currentField: 1 }, { name: 'current_field_1' }); + + const indexes: DeprecatedMongoIndexDefinition[] = [ + { + collectionName, + indexName: 'legacy_field_1', + key: { legacyField: 1 }, + deprecatedVersion: '4.15.0', + replacementIndexNames: ['current_field_1'], + reason: 'test legacy index' + } + ]; + + const report = await MongoIndexManager.cleanupDeprecatedIndexes({ + db: getDb(), + apply: true, + indexes + }); + + const indexNames = await getIndexNames(collectionName); + expect(report.items).toEqual([ + expect.objectContaining({ + action: 'drop', + applied: true, + collectionName, + indexName: 'legacy_field_1' + }) + ]); + expect(indexNames.has('legacy_field_1')).toBe(false); + expect(indexNames.has('current_field_1')).toBe(true); + }); + + it('skips same-name indexes when key or options do not match registry', async () => { + const collectionName = getCleanupCollectionName(); + await dropCollection(collectionName); + const collection = getDb().collection(collectionName); + await collection.createIndex({ customerField: 1 }, { name: 'legacy_field_1' }); + + const indexes: DeprecatedMongoIndexDefinition[] = [ + { + collectionName, + indexName: 'legacy_field_1', + key: { legacyField: 1 }, + options: { unique: true }, + deprecatedVersion: '4.15.0', + reason: 'test legacy index' + } + ]; + + const report = await MongoIndexManager.cleanupDeprecatedIndexes({ + db: getDb(), + apply: true, + indexes + }); + + const indexNames = await getIndexNames(collectionName); + expect(report.items).toEqual([ + expect.objectContaining({ + action: 'skip_mismatch', + applied: false, + collectionName, + indexName: 'legacy_field_1' + }) + ]); + expect(indexNames.has('legacy_field_1')).toBe(true); + }); + + it('skips same-name compound indexes when key order does not match registry', async () => { + const collectionName = getCleanupCollectionName(); + await dropCollection(collectionName); + const collection = getDb().collection(collectionName); + await collection.createIndex( + { customerField: 1, legacyField: 1 }, + { name: 'legacy_compound_1' } + ); + + const indexes: DeprecatedMongoIndexDefinition[] = [ + { + collectionName, + indexName: 'legacy_compound_1', + key: { legacyField: 1, customerField: 1 }, + deprecatedVersion: '4.15.0', + reason: 'test legacy index' + } + ]; + + const report = await MongoIndexManager.cleanupDeprecatedIndexes({ + db: getDb(), + apply: true, + indexes + }); + + const indexNames = await getIndexNames(collectionName); + expect(report.items).toEqual([ + expect.objectContaining({ + action: 'skip_mismatch', + applied: false, + collectionName, + indexName: 'legacy_compound_1' + }) + ]); + expect(indexNames.has('legacy_compound_1')).toBe(true); + }); + + it('skips cleanup when replacement index is required but missing', async () => { + const collectionName = getCleanupCollectionName(); + await dropCollection(collectionName); + const collection = getDb().collection(collectionName); + await collection.createIndex({ legacyField: 1 }, { name: 'legacy_field_1' }); + + const indexes: DeprecatedMongoIndexDefinition[] = [ + { + collectionName, + indexName: 'legacy_field_1', + key: { legacyField: 1 }, + deprecatedVersion: '4.15.0', + replacementIndexNames: ['current_field_1'], + reason: 'test legacy index' + } + ]; + + const report = await MongoIndexManager.cleanupDeprecatedIndexes({ + db: getDb(), + apply: true, + indexes + }); + + const indexNames = await getIndexNames(collectionName); + expect(report.items).toEqual([ + expect.objectContaining({ + action: 'skip_missing_replacement', + applied: false, + collectionName, + indexName: 'legacy_field_1' + }) + ]); + expect(indexNames.has('legacy_field_1')).toBe(true); + }); + + it('skips cleanup until every replacement index exists', async () => { + const collectionName = getCleanupCollectionName(); + await dropCollection(collectionName); + const collection = getDb().collection(collectionName); + await collection.createIndex({ legacyField: 1 }, { name: 'legacy_field_1' }); + await collection.createIndex({ currentField: 1 }, { name: 'current_field_1' }); + + const indexes: DeprecatedMongoIndexDefinition[] = [ + { + collectionName, + indexName: 'legacy_field_1', + key: { legacyField: 1 }, + deprecatedVersion: '4.15.0', + replacementIndexNames: ['current_field_1', 'current_field_2'], + reason: 'test legacy index' + } + ]; + + const report = await MongoIndexManager.cleanupDeprecatedIndexes({ + db: getDb(), + apply: true, + indexes + }); + + const indexNames = await getIndexNames(collectionName); + expect(report.items).toEqual([ + expect.objectContaining({ + action: 'skip_missing_replacement', + applied: false, + collectionName, + indexName: 'legacy_field_1', + missingReplacementIndexNames: ['current_field_2'] + }) + ]); + expect(indexNames.has('legacy_field_1')).toBe(true); + }); + + it('reports missing deprecated indexes and formats the report', async () => { + const collectionName = getCleanupCollectionName(); + await dropCollection(collectionName); + + const report = await MongoIndexManager.cleanupDeprecatedIndexes({ + db: getDb(), + apply: false, + indexes: [ + { + collectionName, + indexName: 'missing_1', + key: { missing: 1 }, + deprecatedVersion: '4.15.0', + reason: 'test missing index' + } + ] + }); + + expect(report.items).toEqual([ + expect.objectContaining({ + action: 'skip_missing', + collectionName, + indexName: 'missing_1' + }) + ]); + expect(MongoIndexManager.summarizeCleanupReport(report)).toEqual({ + total: 1, + dropped: 0, + droppable: 0, + skippedMissing: 1, + skippedMismatch: 0, + skippedMissingReplacement: 0, + errors: 0 + }); + expect(MongoIndexManager.formatCleanupReport(report)).toContain( + `${collectionName}.missing_1 version=4.15.0` + ); }); }); diff --git a/packages/service/test/env.test.ts b/packages/service/test/env.test.ts index 916a3bcb1893..c4139fca4187 100644 --- a/packages/service/test/env.test.ts +++ b/packages/service/test/env.test.ts @@ -101,6 +101,13 @@ describe('serviceEnv', () => { } }); + vi.stubEnv('MONGO_INDEX_SYNC_MODE', 'sync'); + await expect(importServiceEnv()).resolves.toMatchObject({ + serviceEnv: { + MONGO_INDEX_SYNC_MODE: 'sync' + } + }); + vi.stubEnv('MONGO_INDEX_SYNC_MODE', 'dryRun'); await expect(importServiceEnv()).resolves.toMatchObject({ serviceEnv: { diff --git a/projects/app/src/instrumentation-node.ts b/projects/app/src/instrumentation-node.ts index 2a001c5e386a..2dca7db4906e 100644 --- a/projects/app/src/instrumentation-node.ts +++ b/projects/app/src/instrumentation-node.ts @@ -99,6 +99,7 @@ export async function registerNodeInstrumentation() { connectMongo({ db: connectionMongo, url: MONGO_URL, + cleanupDeprecatedIndexes: true, connectedCb: () => startMongoWatch() }), logger, diff --git a/projects/marketplace/src/env.ts b/projects/marketplace/src/env.ts index 348ebcd4bfbd..44f31abf6e54 100644 --- a/projects/marketplace/src/env.ts +++ b/projects/marketplace/src/env.ts @@ -1,7 +1,6 @@ import { createEnv } from '@t3-oss/env-core'; import z from 'zod'; import { BoolSchema, IntSchema } from '@fastgpt/global/common/zod'; -import { mongoIndexSyncModeList } from '@fastgpt/global/common/system/constants'; const LogLevelSchema = z.enum(['trace', 'debug', 'info', 'warning', 'error', 'fatal']); const StorageVendorSchema = z.enum(['minio', 'aws-s3', 'cos', 'oss']); @@ -16,7 +15,7 @@ export const marketplaceEnv = createEnv({ COMMUNITY_AUTH_TOKEN: z.string().optional(), MONGODB_URI: z.string().optional().default(''), DB_MAX_LINK: IntSchema.default(20), - MONGO_INDEX_SYNC_MODE: z.enum(mongoIndexSyncModeList).default('create'), + MONGO_INDEX_SYNC_MODE: z.enum(['off', 'sync', 'create', 'dryRun']).default('create'), // 对象存储。保持与主项目 packages/service/env.ts 同名,并兼容 marketplace 旧 S3_* 变量。 STORAGE_VENDOR: StorageVendorSchema.default('minio'), diff --git a/projects/marketplace/src/service/mongo/index.ts b/projects/marketplace/src/service/mongo/index.ts index 417606e0a0f6..26dbfdec0117 100644 --- a/projects/marketplace/src/service/mongo/index.ts +++ b/projects/marketplace/src/service/mongo/index.ts @@ -2,7 +2,7 @@ import type { Model, Schema } from 'mongoose'; import { Mongoose } from 'mongoose'; import { getLogger, LogCategories } from '../logger'; import { marketplaceEnv } from '../../env'; -import { runMongoIndexSyncForModel } from '@fastgpt/service/common/mongo/indexManager'; +import { MongoIndexManager } from '@fastgpt/service/common/mongo/indexManager'; export const MONGO_URL = marketplaceEnv.MONGODB_URI; const maxConnecting = Math.max(30, marketplaceEnv.DB_MAX_LINK); @@ -37,7 +37,7 @@ const syncMongoIndex = async (model: Model) => { } try { - await runMongoIndexSyncForModel({ + await MongoIndexManager.runModelIndexMode({ model, mode: marketplaceEnv.MONGO_INDEX_SYNC_MODE, logger @@ -92,6 +92,22 @@ export async function connectMongo(db: Mongoose, url: string): Promise serverSelectionTimeoutMS: 10000, // 服务器选择超时: 10秒,防止副本集故障时长时间阻塞 heartbeatFrequencyMS: 5000 // 5s 进行一次健康检查 }); + + if (marketplaceEnv.MONGO_INDEX_SYNC_MODE === 'sync') { + const mongoDb = db.connection.db; + if (!mongoDb) { + throw new Error('MongoDB connection has no database instance after connecting'); + } + + await MongoIndexManager.waitForModelIndexTasks(db.connection); + await MongoIndexManager.runDeprecatedIndexCleanupOnce({ + db: mongoDb, + cleanupKey: MongoIndexManager.getConnectionCleanupKey(db.connection), + apply: true, + logger + }); + } + return db; } catch (error) { logger.error('Mongo connect error', { error }); diff --git a/projects/marketplace/test/env.test.ts b/projects/marketplace/test/env.test.ts index a0f09a4de076..dda4b2fa3620 100644 --- a/projects/marketplace/test/env.test.ts +++ b/projects/marketplace/test/env.test.ts @@ -38,6 +38,14 @@ describe('marketplace env', () => { expect(marketplaceEnv.MONGO_INDEX_SYNC_MODE).toBe('off'); }); + it('parses destructive MongoDB index sync mode', async () => { + vi.stubEnv('MONGO_INDEX_SYNC_MODE', 'sync'); + + const { marketplaceEnv } = await importEnv(); + + expect(marketplaceEnv.MONGO_INDEX_SYNC_MODE).toBe('sync'); + }); + it('rejects invalid MONGO_INDEX_SYNC_MODE values', async () => { vi.stubEnv('MONGO_INDEX_SYNC_MODE', 'full'); From 7ff1c848398c2b3847df8249ab1a4ba238564e4e Mon Sep 17 00:00:00 2001 From: Xianquan Date: Fri, 17 Jul 2026 17:41:38 +0800 Subject: [PATCH 3/5] docs: document mongo index sync modes --- .../self-host/upgrading/4-15/4152.en.mdx | 18 +++++++++++++++++ .../content/self-host/upgrading/4-15/4152.mdx | 20 ++++++++++++++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/document/content/self-host/upgrading/4-15/4152.en.mdx b/document/content/self-host/upgrading/4-15/4152.en.mdx index b4401c4d0821..a729b99ef2ca 100644 --- a/document/content/self-host/upgrading/4-15/4152.en.mdx +++ b/document/content/self-host/upgrading/4-15/4152.en.mdx @@ -26,6 +26,24 @@ Starting with V4.15.2, `AGENT_ENGINE` uses new enum values. Update the environme The previous values are no longer supported. Using `default` or `pi` will fail environment variable validation and prevent FastGPT from starting. If `AGENT_ENGINE` is not set, FastGPT uses `fastAgent` by default. +### Update the MongoDB Index Sync Environment Variable + +Starting with V4.15.2, the `SYNC_INDEX` environment variable has been removed and replaced by `MONGO_INDEX_SYNC_MODE`. The old variable is no longer used. Update your deployment configuration before upgrading: + +| Previous configuration | New configuration | +| ---------------------- | ------------------------------ | +| `SYNC_INDEX=true` | `MONGO_INDEX_SYNC_MODE=create` | +| `SYNC_INDEX=false` | `MONGO_INDEX_SYNC_MODE=off` | + +`MONGO_INDEX_SYNC_MODE` supports the following modes: + +- `create`: The default. Creates indexes missing from the FastGPT schema without deleting other indexes. +- `off`: Disables automatic MongoDB index management. +- `dryRun`: Inspects and logs index differences without creating or deleting indexes. +- `sync`: Creates missing indexes and removes only historical FastGPT indexes that are explicitly registered and whose definitions match exactly. It does not remove custom created or other unknown indexes. + +To automatically remove deprecated FastGPT indexes during an upgrade, temporarily set `MONGO_INDEX_SYNC_MODE=sync` and restart the service. After confirming that cleanup has completed, switch back to the default `create` mode. + ### Configure the File Download URL Mode V4.15.2 introduces the `STORAGE_DOWNLOAD_URL_MODE` environment variable, which defaults to `short-proxy`. diff --git a/document/content/self-host/upgrading/4-15/4152.mdx b/document/content/self-host/upgrading/4-15/4152.mdx index c465708b42e7..f196ec0c285f 100644 --- a/document/content/self-host/upgrading/4-15/4152.mdx +++ b/document/content/self-host/upgrading/4-15/4152.mdx @@ -26,7 +26,25 @@ V4.15.2 起，`AGENT_ENGINE` 使用新的枚举值。升级前，请按下表修 旧值不再兼容。继续使用 `default` 或 `pi` 会导致环境变量校验失败，FastGPT 无法启动。未配置 `AGENT_ENGINE` 时，可正常启动，系统默认使用 `fastAgent`。 -### 3. 修改文件下载模式变量 +### MongoDB 索引同步环境变量调整 + +V4.15.2 起，原 `SYNC_INDEX` 环境变量已移除，统一改用 `MONGO_INDEX_SYNC_MODE`。旧变量将不再生效，升级前请按下表调整： + +| 原配置 | 新配置 | +| ------------------ | ------------------------------ | +| `SYNC_INDEX=true` | `MONGO_INDEX_SYNC_MODE=create` | +| `SYNC_INDEX=false` | `MONGO_INDEX_SYNC_MODE=off` | + +`MONGO_INDEX_SYNC_MODE` 支持以下模式： + +- `create`：默认值，只创建 FastGPT Schema 中缺失的索引，不删除其他索引。 +- `off`：不自动处理 MongoDB 索引。 +- `dryRun`：只检查并记录索引差异，不创建或删除索引。 +- `sync`：创建缺失索引，并清理 FastGPT 明确登记且定义完全匹配的历史废弃索引；不会删除自建索引或其他未知索引。 + +版本升级时，如需自动清理 FastGPT 的历史废弃索引，可临时配置 `MONGO_INDEX_SYNC_MODE=sync` 并重启服务。确认清理完成后，建议改回默认的 `create` 模式。 + +### 修改文件下载模式变量 V4.15.2 新增 `STORAGE_DOWNLOAD_URL_MODE` 环境变量，默认值为 `short-proxy`。 From 6e083fe0437e2c2111ed00c260c84562cff91963 Mon Sep 17 00:00:00 2001 From: Xianquan Date: Mon, 20 Jul 2026 15:11:00 +0800 Subject: [PATCH 4/5] refactor: simplify mongo index synchronization --- .../common/mongo-index-sync-strategy.md | 172 ++--- .../mongo-index-sync-customer-index-loss.md | 30 +- AGENTS.md | 6 +- .../self-host/upgrading/4-15/4152.en.mdx | 20 +- .../content/self-host/upgrading/4-15/4152.mdx | 20 +- document/data/doc-last-modified.json | 16 +- .../service/common/mongo/deprecatedIndexes.ts | 111 ---- packages/service/common/mongo/index.ts | 9 +- packages/service/common/mongo/indexManager.ts | 357 ++++------- packages/service/common/mongo/init.ts | 19 +- .../service/common/mongo/schemaIndexes.ts | 44 ++ .../sandbox/infrastructure/instance/schema.ts | 41 +- packages/service/core/chat/chatSchema.ts | 10 +- packages/service/env.ts | 1 - .../test/common/mongo/indexManager.test.ts | 602 +++++++----------- .../test/common/mongo/schemaIndexes.test.ts | 46 ++ .../infrastructure/instance/schema.test.ts | 12 + .../service/test/core/chat/schema.test.ts | 11 + packages/service/test/env.test.ts | 52 -- projects/app/src/instrumentation-node.ts | 1 - projects/marketplace/src/env.ts | 1 - .../marketplace/src/service/mongo/index.ts | 21 +- projects/marketplace/test/env.test.ts | 40 -- 23 files changed, 629 insertions(+), 1013 deletions(-) delete mode 100644 packages/service/common/mongo/deprecatedIndexes.ts create mode 100644 packages/service/common/mongo/schemaIndexes.ts create mode 100644 packages/service/test/common/mongo/schemaIndexes.test.ts diff --git a/.agents/design/common/mongo-index-sync-strategy.md b/.agents/design/common/mongo-index-sync-strategy.md index ec9688562342..86151edffcab 100644 --- a/.agents/design/common/mongo-index-sync-strategy.md +++ b/.agents/design/common/mongo-index-sync-strategy.md @@ -20,10 +20,12 @@ ## 已确认决策 -1. 私有化默认行为从破坏性全量同步改为 safe create：只补建 FastGPT 缺失索引，不删除 schema 外索引。 -2. 不保留 `SYNC_INDEX` 兼容逻辑，直接移除旧布尔变量，统一改用 `MONGO_INDEX_SYNC_MODE`。 -3. 默认启动索引处理不删除索引；如显式配置 `MONGO_INDEX_SYNC_MODE=sync`，也只执行 manager 管理的 `create + deprecated cleanup`，不会使用 Mongoose destructive sync 语义。 -4. `llm_request_records.requestId_1` 已确认不再需要，但不在本次启动索引同步中清理。 +1. 启动时默认执行 manager 管理的主动安全同步，不再通过环境变量切换索引处理模式。 +2. 移除 `MONGO_INDEX_SYNC_MODE`；已经移除的 `SYNC_INDEX` 也不恢复兼容逻辑。 +3. 主动安全同步固定执行“创建当前 schema 缺失索引 + 删除该 schema 显式声明的废弃索引”，不会调用 Mongoose 具有破坏性语义的 `syncIndexes()`。 +4. 废弃索引不再集中维护 registry，而是与当前索引一起维护在所属 Schema 文件中；collection 从 model 推导，声明本身不再重复填写 collection name。 +5. schema 外且未被显式声明为废弃的索引一律保留，只记录差异和告警，避免删除客户自建索引。 +6. `llm_request_records.requestId_1` 已确认不再需要，但此前已明确不纳入本次清理；本次仅迁移现有中心清单中的 chat 和 sandbox 废弃索引，不顺带扩大删除范围。 ## 根因判断 @@ -39,42 +41,23 @@ 1. 默认保护客户自建索引：启动时不删除 schema 未声明的索引。 2. 仍能自动补建 FastGPT 新版本需要的缺失索引。 -3. 提供 dry-run 差异检查，帮助管理员识别 schema 外索引和缺失索引。 -4. 当前方案不再保留启动期强制全量同步能力，避免换一种方式重新引入删除客户索引的风险。 +3. 保留可复用的差异检查能力，帮助日志或后续管理入口识别 schema 外索引和缺失索引。 +4. 不再保留启动期模式选择或强制全量同步能力，避免配置分支导致不同部署的索引状态长期分叉。 5. 主应用与 Marketplace 使用一致的索引策略和错误处理。 ## 非目标 - 不尝试自动判断所有未知索引是否由客户创建。历史上未带所有权标记，完全自动判断不可靠。 -- 不在普通服务重启时执行危险删除。 -- 不把 `SYNC_INDEX=false` 作为长期推荐方案，因为它也会阻止新索引创建。 +- 不删除 Schema 未明确登记为废弃的索引。 +- 不提供环境变量关闭索引创建或废弃索引清理。 ## 推荐方案 -### 1. 引入索引同步模式 +### 1. 固定执行主动安全同步 -新增枚举型环境变量，例如 `MONGO_INDEX_SYNC_MODE`： +移除 `packages/service/env.ts` 和 `projects/marketplace/src/env.ts` 中的 `MONGO_INDEX_SYNC_MODE`。主应用和 Marketplace 在满足现有运行条件时固定执行同一条索引同步链路，不再支持 `off/create/dryRun/sync` 启动模式。 -| 模式 | 启动行为 | 是否删除未知索引 | 使用场景 | -| --- | --- | --- | --- | -| `off` | 不处理索引 | 否 | 极端保守环境，客户自行维护索引 | -| `sync` | 创建 schema 缺失索引，并清理 registry 中登记的 FastGPT 废弃索引 | 只删除登记过且精确匹配的废弃索引 | 升级时需要同步 FastGPT 新索引并清理已确认废弃索引 | -| `create` | 只创建 schema 声明但数据库缺失的索引 | 否 | 私有化默认推荐 | -| `dryRun` | 只扫描并记录差异 | 否 | 升级前检查 | - -移除旧变量策略： - -- 移除 `packages/service/env.ts` 和 `projects/marketplace/src/env.ts` 中的 `SYNC_INDEX`。 -- 测试统一改成 `MONGO_INDEX_SYNC_MODE`；`.mdx` 文档和 `.yml` 部署文件先不手动修改，等代码改完后再统一处理。 -- 未配置 `MONGO_INDEX_SYNC_MODE` 时默认 `create`。 -- 旧配置 `SYNC_INDEX=false` 的用户如需继续完全跳过索引处理，升级时改为 `MONGO_INDEX_SYNC_MODE=off`。 -- 旧配置 `SYNC_INDEX=true` 的用户无需迁移值；未配置新变量时默认等价于 `MONGO_INDEX_SYNC_MODE=create`，只补建缺失索引。 - -模式值通过共享的 `mongoIndexSyncModeList` 约束，主应用和 Marketplace 使用同一组枚举，避免两个入口分叉。这里故意不读取 `SYNC_INDEX`。这属于配置破坏性变更，但可以避免长期维护两个入口造成语义混乱。 - -### 2. 默认从 `syncIndexes()` 改为安全创建 - -`create` 模式下使用 Mongoose 的 `createIndexes()` 或基于 `diffIndexes()` 的 `toCreate` 结果只创建缺失索引，不执行 `cleanIndexes()`。 +同步内部使用 Mongoose 的 `createIndexes()` 创建当前 Schema 索引，不执行 `syncIndexes()` 或 `cleanIndexes()`。创建完成后，再处理当前 Schema 显式声明的废弃索引。 建议同时记录差异日志： @@ -86,19 +69,48 @@ 1. `createIndexes()` 对已有索引 options 不做修正。TTL、unique、partialFilterExpression 变化时，旧索引仍会保留旧行为。 2. 它缺少差异报告，管理员无法知道哪些旧索引需要人工处理或迁移。 -3. 它不处理旧索引 options 变化或历史旧索引清理，这些应由独立迁移工具处理。 +3. 它不处理旧索引 options 变化或历史旧索引清理，因此需要由 schema-local 废弃声明补足。 因此可以内部用 `createIndexes()` 完成安全创建，但对外的抽象应是 `mongoIndexManager`，保留 inspect 和未来管理入口的扩展点。 +### 2. Schema 内声明废弃索引 + +在 Mongo 公共模块提供 schema-level 注册 helper。Schema 文件继续使用 `schema.index(...)` 声明当前索引，并在相邻位置调用 helper 声明该 Schema 需要删除的历史索引。helper 只保存元数据，不把自定义 options 传给 Mongoose 或 MongoDB。 + +建议 API 形态： + +```ts +registerDeprecatedMongoIndexes(ChatSchema, [ + { + indexName: 'appId_1_chatId_1', + key: { appId: 1, chatId: 1 }, + options: { unique: true } + } +]); +``` + +元数据应挂在 Schema 实例自身，并使用 Mongo 公共模块私有的 symbol key 读写；不使用中心数组，也不使用可能在热加载或重复模块实例间丢失关联的进程级 `WeakMap`。manager 从 `model.schema` 读取这份元数据。 + +废弃索引定义只包含 `indexName`、`key` 和需要参与精确匹配的可选关键 options。 + +定义不包含 `collectionName`、版本或说明字段。manager 从 model 获取 collection，并且只处理 model.schema 上登记的废弃项。这能让当前索引和历史删除声明在同一个业务 Schema 文件中完成 review，也避免中心清单与 Schema 演进脱节。 + +现有中心清单迁移位置： + +- chat 废弃索引迁移到 `packages/service/core/chat/chatSchema.ts`。 +- sandbox instance 废弃索引迁移到 `packages/service/core/ai/sandbox/infrastructure/instance/schema.ts`。 +- 迁移完成后删除 `packages/service/common/mongo/deprecatedIndexes.ts`。 + ### 3. 历史旧索引处理边界 -默认 `create` 不在服务启动索引同步里删除任何索引，包括 FastGPT 历史旧索引。显式 `sync` 只允许清理 `deprecatedIndexes` registry 中登记的 FastGPT 废弃索引，并且删除前必须精确匹配 collection、index name、key 和关键 options。 +主动同步只允许清理当前 Schema 显式登记的 FastGPT 废弃索引，并且删除前必须精确匹配 index name、key 和关键 options。 该边界的原因： 1. 当前核心目标是保护客户自建索引，不能按 schema 外索引一概删除。 -2. 历史旧索引只能通过明确维护的 registry 识别，不能自动猜测。 -3. 更复杂的批量清理仍应复用 `cleanupDeprecatedIndexes()`，通过升级脚本或 Root 管理工具触发，并提供 dry-run、确认参数和审计日志。 +2. 历史旧索引只能通过所属 Schema 的明确声明识别，不能自动猜测。 +3. 必须先成功创建当前索引，再执行该 model 的废弃索引清理；创建失败时不继续删除，避免替代索引缺失。 +4. 多实例重复执行必须保持幂等：索引已不存在时视为跳过，定义不匹配时保留并告警。 ### 4. 提供可审计的管理入口 @@ -108,10 +120,10 @@ - 默认 dry-run。 - 输出每个集合的 `toCreate`、`unknownIndexes`、`conflicts`、`fastgptObsoleteIndexes`。 - `POST /api/admin/mongoIndexes/apply` - - 入参明确指定模式：`create`。 + - 只应用 manager 已定义的安全同步动作。 - 不提供旧式全量同步能力。 -本次先通过环境变量完成启动期 `off/sync/create/dryRun`，Root 管理员 API 或脚本作为后续增强。 +本次先完成固定的启动同步行为，Root 管理员 API 或脚本作为后续增强。 ### 5. 未来索引命名规范 @@ -126,48 +138,51 @@ 建议在 `packages/service/common/mongo/` 下新增索引管理模块： - `indexManager.ts` - - `MongoIndexManager.runModelIndexMode()` + - `MongoIndexManager.syncModelIndexes()` - `MongoIndexManager.inspectModelIndexes()` - - `MongoIndexManager.cleanupDeprecatedIndexes()` + - `MongoIndexManager.cleanupModelDeprecatedIndexes()` - `MongoIndexManager.formatCleanupReport()` +- schema metadata helper + - 注册和读取某个 Schema 的废弃索引定义。 + - 类型不包含 collection name，避免业务 Schema 重复维护集合信息。 `getMongoModel()` / `getMongoLogModel()` 只负责把 model 注册给 manager，不直接调用 Mongoose 的 destructive API。这样后续可以统一做并发控制、日志聚合和管理 API 复用。 ### 2. 启动流程 -启动时允许四类行为： +每个 model 固定执行： -- `off`：跳过。 -- `sync`：各 model 只创建 schema 缺失索引；主 MongoDB 连接等待所有已登记的 model 索引任务结束后，全局执行一次 registry 废弃索引清理。不会删除客户自建或其他未知索引。 -- `dryRun`：只记录差异。 -- `create`：创建缺失索引，并记录未知索引/冲突。 +1. 用 `diffIndexes()` 检查 `toCreate` 和 schema 外索引，仅用于日志与结果报告。 +2. 用 `createIndexes()` 创建当前 Schema 声明的索引。 +3. 读取 model.schema 上登记的废弃索引定义。 +4. 在当前 model 对应 collection 中逐项精确匹配并删除；未知索引不参与删除。 -常规默认模式仍是 `create`。`sync` 是显式维护模式，用于升级时清理 FastGPT 自己已废弃的旧索引，但不等同于 Mongoose `syncIndexes()`。 +废弃清理从连接级全局阶段下沉到 model 任务后，不再需要等待全连接所有 model 的索引任务，也不再需要 `connectMongo({ cleanupDeprecatedIndexes })` 这类入口开关。 ### 3. 并发与失败处理 当前 model 加载时即触发索引同步。后续实现至少要保证： -1. 同一进程内同一 model 的并发索引任务会复用；废弃索引 cleanup 按 MongoDB 连接全局只执行一次，不会随每个 schema 重复扫描。 +1. 同一进程内同一 model 的并发索引任务会复用；任务结束后允许后续调用重新检查，以支持热加载和重连。 2. 多实例并发启动时，重复创建索引错误可识别并降噪；真正的冲突错误必须记录。 3. 索引任务失败不应让 model 注册失败，但要有明确日志，必要时在健康检查或管理 API 暴露状态。 4. Marketplace 已修正异步错误捕获，后续如果引入管理入口，应继续复用同一 manager。 ### 4. 日志分级 -- `debug`：每个 model 输出完整 diff，例如 `toCreate`、schema 外索引名称；连接级 cleanup 输出 registry 扫描明细和重复调用跳过原因。 -- `info`：每个 model 输出索引创建阶段和结果摘要；连接级 cleanup 仅输出一次清理汇总，并记录实际 drop 的废弃索引。 -- `warn`：输出需要运维关注但不阻塞启动的问题，例如检测到 schema 外索引、废弃索引定义不匹配、replacement index 缺失。 -- `error`：输出索引创建或废弃索引清理失败，保留 model、collection、mode 和 error 信息。 +- `debug`：每个 model 输出完整 diff、schema 外索引名称和 schema-local 废弃项扫描明细。 +- `info`：每个 model 输出索引同步结果摘要，并记录实际 drop 的废弃索引。 +- `warn`：输出需要运维关注但不阻塞启动的问题，例如检测到 schema 外索引或废弃索引定义不匹配。 +- `error`：输出索引创建或废弃索引清理失败，保留 model、collection 和 error 信息。 ### 5. 本次交付边界 -- 新增 `MONGO_INDEX_SYNC_MODE`。 -- 将默认启动行为从 `syncIndexes()` 改为安全 `create`。 -- 支持 `off/sync/create/dryRun`，其中 `sync` 为 manager 语义：`createIndexes()` + `cleanupDeprecatedIndexes(apply=true)`。 +- 移除 `MONGO_INDEX_SYNC_MODE`，启动时固定执行主动安全同步。 +- 固定同步语义为 `createIndexes()` + 当前 Schema 显式声明的 deprecated cleanup。 +- 将中心废弃索引清单迁移到 chat 和 sandbox instance Schema，随后删除中心 registry。 - 加日志说明 `toCreate/toDrop`，但 `toDrop` 不删除；索引冲突由 `createIndexes()` 错误日志暴露。 - 主应用与 Marketplace 行为一致。 -- 补回归测试：客户自建索引在 `create/dryRun/sync` 下不会被删除，`sync` 只删除 registry 中登记的废弃索引。 +- 补回归测试：客户自建索引不会被删除，只删除当前 Schema 登记且精确匹配的废弃索引。 - 代码确认后再更新 `.mdx` 文档，并由生成流程统一产出 `.yml` 部署文件。 Root 管理员 inspect/apply API 或等价脚本作为后续增强，不阻塞本次修复。 @@ -181,52 +196,49 @@ Root 管理员 inspect/apply API 或等价脚本作为后续增强，不阻塞 - 变更提交：`f008ea971 feat: teamId in reacord llm`，提交时间 `2026-06-25 12:27:57 +0800`；后续合并提交 `60c62b7af Fix test (#7179)` 包含同样变更。 - 变更原因：LLM 请求追踪记录新增 `teamId` 隔离，查询从 `{ requestId }` 调整为 `{ requestId, teamId }`，唯一索引从单字段 `requestId` 调整为复合 `{ teamId: 1, requestId: 1 }`。 - 当前判断：旧 `requestId_1` 对当前 schema 已无必要，且可能继续施加跨团队全局唯一约束。但 `requestId` 由 `getNanoid(12)` 生成，实际碰撞概率很低；请求追踪记录保存失败只记录错误，不阻塞主流程。 -- 处理边界：本次默认 `create` 不删除它。后续如需清理，应通过独立 migration 脚本或 Root 管理工具处理。 +- 处理边界：本次不为它新增 schema-local 废弃声明。后续如需清理，应单独确认后在 `packages/service/core/ai/record/schema.ts` 声明并补回归测试。 ## 边界与风险 1. `createIndexes()` 不会更新已存在索引的 options。TTL 秒数、unique、partialFilterExpression 等发生变化时，需要独立 migration 或人工处理。 2. 旧唯一索引如果不删除，可能继续影响业务。例如 `llm_request_records` 从 `{ requestId }` 改到 `{ teamId, requestId }` 后，旧 `requestId_1` 若残留，会继续限制跨团队相同 requestId。 -3. 旧式 Mongoose 全量同步不作为启动模式暴露，避免重新引入删除客户索引的风险。 +3. 旧式 Mongoose 全量同步不作为启动行为暴露，避免重新引入删除客户索引的风险。 4. 多实例同时启动会并发创建索引。MongoDB 创建已存在索引通常是幂等的，但冲突错误需要聚合成清晰日志，避免噪声刷屏。 5. Marketplace 已统一接入 `mongoIndexManager`，异步错误会被捕获并记录。 +6. 主动清理意味着错误的 schema-local 声明会在服务启动时生效，因此定义 review 和精确匹配是必须保留的防线。 ## 测试策略 至少需要覆盖： -1. `MONGO_INDEX_SYNC_MODE` 环境变量解析： - - 未设置新变量时为 `create`。 - - 合法模式按原值返回。 - - 非法模式在 env 解析阶段失败。 -2. `create` 模式： - - 会创建 schema 中缺失索引。 - - 不删除手动创建的客户索引。 - - 遇到 schema 外索引只记录为 `toDrop/unknownIndexes`。 -3. `dryRun` 模式： - - 不创建索引。 - - 不删除索引。 - - 能返回差异结果。 -4. Marketplace： - - 解析环境变量与主应用一致。 - - 异步错误可以被捕获并记录。 +1. 固定同步会创建 Schema 中缺失索引，并保留未登记的 schema 外索引。 +2. Schema 未登记废弃项时不执行任何删除。 +3. Schema 登记的废弃索引只有在 name、key 和关键 options 精确匹配时才删除。 +4. 当前索引创建失败时不删除废弃索引。 +5. 相同名称但定义不同的客户索引保留并告警。 +6. 已不存在的废弃索引、多实例并发重复清理保持幂等。 +7. chat 和 sandbox 的现有中心清单迁移后行为不变。 +8. 主应用与 Marketplace 均固定调用同一 manager，异步错误可以被捕获并记录。 仓库已有 `mongodb-memory-server` 测试环境，可以写真实 MongoDB 索引回归测试，而不是只 mock Mongoose 方法。 ## 后续增强 -1. 是否继续补 Root 管理员 inspect/apply API，作为比环境变量重启更友好的运维入口。 -2. 是否需要独立 MongoDB 索引迁移脚本，用于清理确认无用的 FastGPT 历史旧索引。 +1. 是否继续补 Root 管理员 inspect/apply API，提供启动日志之外的索引诊断入口。 +2. 是否将 `llm_request_records.requestId_1` 纳入对应 Schema 的废弃索引声明。 3. 对客户自己改动 FastGPT 既有索引 options 的情况，当前只告警；如未来需要自动修正，应以明确迁移项实现。 ## TODO -- [x] 补充需求确认，确定环境变量命名和默认行为。 -- [x] 移除 `SYNC_INDEX`，新增 `MONGO_INDEX_SYNC_MODE`。 -- [x] 设计并实现 `mongoIndexManager`：统一主应用和 Marketplace 的索引处理。 -- [x] 实现 `off/sync/create/dryRun` 基础模式。 -- [x] 接入主应用 `getMongoModel()` / `getMongoLogModel()`。 -- [x] 接入 Marketplace `getMongoModel()`，修正异步错误捕获。 +- [x] 确认修订方案：移除模式变量，固定执行主动安全同步，废弃索引下沉到所属 Schema。 +- [x] 新增 schema-level 废弃索引注册与读取能力；定义仅包含 name、key 和用于精确匹配的可选关键 options。 +- [x] 重构 `MongoIndexManager`：移除 mode 分支，按 model 依次执行 inspect、`createIndexes()` 和 schema-local deprecated cleanup。 +- [x] 将 chat 与 sandbox instance 的现有废弃索引定义迁移到对应 Schema 文件，并删除中心 `deprecatedIndexes.ts`。 +- [x] 移除连接级 cleanup 任务、等待逻辑、`connectMongo.cleanupDeprecatedIndexes` 参数及 instrumentation 调用。 +- [x] 从主应用和 Marketplace env 中移除 `MONGO_INDEX_SYNC_MODE`，删除 manager 中的 mode 类型及对应 env 测试。 +- [x] 简化主应用 `getMongoModel()` / `getMongoLogModel()` 与 Marketplace `getMongoModel()` 的 manager 调用，保持异步错误日志完整。 +- [x] 更新 `AGENTS.md` 的 MongoDB Schema 与索引维护约束，要求在所属 Schema 中声明废弃索引，不再引用中心清单。 +- [x] 调整 manager 单元测试和真实 MongoDB 回归测试，覆盖安全创建、schema-local 精确删除、未知索引保留和并发幂等。 +- [x] 运行索引管理相关局部测试与类型检查；实现全部完成后再运行全量测试（全量并发运行出现 9 个无关超时失败，失败文件在单 worker 下全部复跑通过）。 +- [x] 检查并移除 `.mdx`、部署配置和其他文档中的 `MONGO_INDEX_SYNC_MODE` 引用；需要生成的 `.yml` 通过既有生成流程更新。 - [ ] 后续增强：新增 Root 管理员 inspect/apply API 或等价脚本。 -- [ ] 后续补充 `.mdx` 文档，并通过生成流程产出 `.yml` 部署文件。 -- [x] 增加测试，覆盖 env 解析、`create/dryRun` 行为、客户自建索引不会被删除。 diff --git a/.agents/issue/mongo-index-sync-customer-index-loss.md b/.agents/issue/mongo-index-sync-customer-index-loss.md index f8833e4c562e..7e55c82c9ce9 100644 --- a/.agents/issue/mongo-index-sync-customer-index-loss.md +++ b/.agents/issue/mongo-index-sync-customer-index-loss.md @@ -4,7 +4,7 @@ 私有化部署客户可能会直接在 MongoDB 中添加自定义索引，用于客户自己的查询、报表、审计或集成任务。FastGPT 服务重启时会加载 Mongoose model，并执行索引同步。当前同步使用 Mongoose `model.syncIndexes()`，会删除 schema 中未声明的索引，导致客户自建索引被清理。 -`SYNC_INDEX=false` 可以阻止启动同步，但它同时阻止 FastGPT 新版本缺失索引的创建，升级时可能留下性能问题或唯一约束缺失。因此它只能算临时开关，不是长期索引治理方案。当前决策是不再保留该布尔变量，改为 `MONGO_INDEX_SYNC_MODE` 表达更明确的索引处理模式。 +`SYNC_INDEX=false` 可以阻止启动同步，但它同时阻止 FastGPT 新版本缺失索引的创建，升级时可能留下性能问题或唯一约束缺失。因此它只能算临时开关，不是长期索引治理方案。最终决策是不再保留索引同步环境变量，启动时固定执行 manager 管理的主动安全同步。 ## 证据 @@ -46,26 +46,24 @@ Mongoose `syncIndexes()` 的语义是： 能保留客户索引，但无法发现和治理 FastGPT 历史旧索引。比如唯一索引或 TTL 索引的 options 发生变化时，旧索引可能继续影响业务。 -当前最终方案仍以 `createIndexes()` 作为安全创建动作，但通过统一的 index manager 包装差异检查、日志和启动入口。默认 `create` 不做任何删除；显式 `sync` 只清理 registry 中登记且精确匹配的 FastGPT 废弃索引。 +当前最终方案仍以 `createIndexes()` 作为安全创建动作，但通过统一的 index manager 包装差异检查、日志和启动入口。创建当前索引后，只清理所属 Schema 显式登记且精确匹配的 FastGPT 废弃索引。 -### 推荐：引入索引同步模式 +### 推荐：固定执行主动安全同步 -将索引同步拆成不同意图： +每个 model 固定执行： -- `off`：完全跳过。 -- `create`：只创建缺失索引，不删除未知索引。 -- `dryRun`：只检查差异。 -- `sync`：创建缺失索引，并删除 `deprecatedIndexes` registry 中登记且精确匹配的 FastGPT 废弃索引。 +- 检查并记录当前 Schema 与数据库索引的差异。 +- 创建当前 Schema 缺失的索引。 +- 删除当前 Schema 通过 `registerDeprecatedMongoIndexes` 显式登记、定义精确匹配且替代索引已存在的历史索引。 +- 保留客户自建索引及其他未登记索引。 -常规启动不再暴露旧 `syncIndexes()` 全量同步行为，避免重新引入删除客户索引的风险。 - -私有化默认推荐 `create`。 +启动过程不再暴露模式选项或旧 `syncIndexes()` 全量同步行为，避免不同部署的索引状态因配置分支长期分叉。 ## 已确认方向 1. 默认行为从破坏性全量同步改为 safe create。 -2. 直接移除 `SYNC_INDEX`，不做长期兼容映射。 -3. 启动索引处理不提供按 schema 外索引批量删除的能力；显式 `sync` 只允许删除 registry 中登记且精确匹配的 FastGPT 废弃索引。 +2. 直接移除 `SYNC_INDEX` 和 `MONGO_INDEX_SYNC_MODE`，不做长期兼容映射。 +3. 启动索引处理不提供按 schema 外索引批量删除的能力；只允许删除所属 Schema 登记且精确匹配的 FastGPT 废弃索引。 4. `llm_request_records.requestId_1` 已确认不再需要，但不在本次启动索引同步中清理。 ## 已知旧索引样例 @@ -79,7 +77,7 @@ Mongoose `syncIndexes()` 的语义是： ## 已确认边界 -1. 常规启动默认只执行 `create`，不会删除未知索引。 -2. `dryRun` 只检查差异，不创建、不删除。 +1. 常规启动固定补建当前索引，并清理所属 Schema 明确声明的废弃索引。 +2. 未登记的 schema 外索引只记录差异，不会删除。 3. 不保留旧式全量同步入口；Root 管理员 inspect/apply API 或等价脚本作为后续增强，不阻塞本次修复。 -4. 历史旧索引清理必须走 `deprecatedIndexes` registry；客户自建或未知索引不会被 `sync` 删除。 +4. 历史旧索引必须在所属 Schema 中登记；客户自建或未知索引不会被主动同步删除。 diff --git a/AGENTS.md b/AGENTS.md index f2e40de46dde..6d995f2ed109 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -91,9 +91,9 @@ FastGPT 是一个 AI Agent 构建平台,通过 Flow 提供开箱即用的数据 ### MongoDB Schema 与索引维护 -- 每当新增、修改、删除或重命名 MongoDB/Mongoose Schema 字段、索引定义、唯一约束、TTL、partialFilterExpression、collation 等索引相关配置时，必须同步检查 `packages/service/common/mongo/deprecatedIndexes.ts` 是否需要更新 FastGPT 历史废弃索引清单。 -- 如果当前变更会让 FastGPT 旧版本自己创建过的索引不再被新 schema 使用，且该旧索引可能继续影响写入约束、查询计划或存储成本，应在 `deprecatedIndexes.ts` 中登记对应的废弃索引，并补充/调整 `packages/service/test/common/mongo/indexManager.test.ts` 的清理行为覆盖。 -- 不要把客户自建索引、无法确认来源的索引、或仅凭当前 schema 未声明就推断为废弃的索引加入 `deprecatedIndexes.ts`。该文件只维护 FastGPT 明确创建过、明确废弃、可由维护/升级脚本安全清理的历史索引。 +- 每当新增、修改、删除或重命名 MongoDB/Mongoose Schema 字段、索引定义、唯一约束、TTL、partialFilterExpression、collation 等索引相关配置时，必须同步检查是否有 FastGPT 旧版本创建的索引不再被当前 Schema 使用。 +- 如果历史索引可能继续影响写入约束、查询计划或存储成本，应在所属 Schema 文件中通过 `registerDeprecatedMongoIndexes` 紧邻当前索引声明登记删除定义，并补充/调整 `packages/service/test/common/mongo/indexManager.test.ts` 的清理行为覆盖。定义只包含旧索引的 name、key，以及需要参与精确匹配的关键 options。 +- 不要登记客户自建索引、无法确认来源的索引，或仅凭当前 Schema 未声明就推断为废弃的索引。主动同步只允许删除 FastGPT 明确创建过、明确废弃且与 Schema 本地声明精确匹配的历史索引。 ### API 入参校验 diff --git a/document/content/self-host/upgrading/4-15/4152.en.mdx b/document/content/self-host/upgrading/4-15/4152.en.mdx index a729b99ef2ca..9200d4618811 100644 --- a/document/content/self-host/upgrading/4-15/4152.en.mdx +++ b/document/content/self-host/upgrading/4-15/4152.en.mdx @@ -26,23 +26,17 @@ Starting with V4.15.2, `AGENT_ENGINE` uses new enum values. Update the environme The previous values are no longer supported. Using `default` or `pi` will fail environment variable validation and prevent FastGPT from starting. If `AGENT_ENGINE` is not set, FastGPT uses `fastAgent` by default. -### Update the MongoDB Index Sync Environment Variable +### MongoDB Index Synchronization Changes -Starting with V4.15.2, the `SYNC_INDEX` environment variable has been removed and replaced by `MONGO_INDEX_SYNC_MODE`. The old variable is no longer used. Update your deployment configuration before upgrading: +Starting with V4.15.2, FastGPT no longer uses the `SYNC_INDEX` or `MONGO_INDEX_SYNC_MODE` environment variables. You can remove both variables before upgrading; no MongoDB index synchronization mode needs to be configured. -| Previous configuration | New configuration | -| ---------------------- | ------------------------------ | -| `SYNC_INDEX=true` | `MONGO_INDEX_SYNC_MODE=create` | -| `SYNC_INDEX=false` | `MONGO_INDEX_SYNC_MODE=off` | +FastGPT now performs a safe synchronization automatically at startup: -`MONGO_INDEX_SYNC_MODE` supports the following modes: +- Creates indexes that are missing from the current FastGPT schemas. +- Removes only historical FastGPT indexes explicitly declared by the corresponding schema and whose name, key, and relevant options match exactly. +- Preserves custom indexes and any other indexes that are not explicitly declared as deprecated. -- `create`: The default. Creates indexes missing from the FastGPT schema without deleting other indexes. -- `off`: Disables automatic MongoDB index management. -- `dryRun`: Inspects and logs index differences without creating or deleting indexes. -- `sync`: Creates missing indexes and removes only historical FastGPT indexes that are explicitly registered and whose definitions match exactly. It does not remove custom created or other unknown indexes. - -To automatically remove deprecated FastGPT indexes during an upgrade, temporarily set `MONGO_INDEX_SYNC_MODE=sync` and restart the service. After confirming that cleanup has completed, switch back to the default `create` mode. +This process does not call Mongoose's full `syncIndexes()` operation, so indexes are never removed simply because they are absent from a FastGPT schema. ### Configure the File Download URL Mode diff --git a/document/content/self-host/upgrading/4-15/4152.mdx b/document/content/self-host/upgrading/4-15/4152.mdx index f196ec0c285f..237c847dce67 100644 --- a/document/content/self-host/upgrading/4-15/4152.mdx +++ b/document/content/self-host/upgrading/4-15/4152.mdx @@ -26,23 +26,17 @@ V4.15.2 起，`AGENT_ENGINE` 使用新的枚举值。升级前，请按下表修 旧值不再兼容。继续使用 `default` 或 `pi` 会导致环境变量校验失败，FastGPT 无法启动。未配置 `AGENT_ENGINE` 时，可正常启动，系统默认使用 `fastAgent`。 -### MongoDB 索引同步环境变量调整 +### MongoDB 索引同步调整 -V4.15.2 起，原 `SYNC_INDEX` 环境变量已移除，统一改用 `MONGO_INDEX_SYNC_MODE`。旧变量将不再生效，升级前请按下表调整： +V4.15.2 起，不再使用 `SYNC_INDEX` 和 `MONGO_INDEX_SYNC_MODE` 环境变量，无需配置 MongoDB 索引同步模式。升级前可直接删除这两个变量。 -| 原配置 | 新配置 | -| ------------------ | ------------------------------ | -| `SYNC_INDEX=true` | `MONGO_INDEX_SYNC_MODE=create` | -| `SYNC_INDEX=false` | `MONGO_INDEX_SYNC_MODE=off` | +FastGPT 启动时会自动执行安全的主动同步： -`MONGO_INDEX_SYNC_MODE` 支持以下模式： +- 创建当前 FastGPT Schema 中缺失的索引。 +- 仅删除对应 Schema 明确声明、且 name、key 和关键 options 完全匹配的 FastGPT 历史废弃索引。 +- 保留客户自建索引及其他未声明的索引。 -- `create`：默认值，只创建 FastGPT Schema 中缺失的索引，不删除其他索引。 -- `off`：不自动处理 MongoDB 索引。 -- `dryRun`：只检查并记录索引差异，不创建或删除索引。 -- `sync`：创建缺失索引，并清理 FastGPT 明确登记且定义完全匹配的历史废弃索引；不会删除自建索引或其他未知索引。 - -版本升级时，如需自动清理 FastGPT 的历史废弃索引，可临时配置 `MONGO_INDEX_SYNC_MODE=sync` 并重启服务。确认清理完成后，建议改回默认的 `create` 模式。 +该同步不会调用 Mongoose 的全量 `syncIndexes()`，因此不会按“未在 Schema 中声明”这一条件批量删除索引。 ### 修改文件下载模式变量 diff --git a/document/data/doc-last-modified.json b/document/data/doc-last-modified.json index 11ebdde32e32..4000d4e3fe7b 100644 --- a/document/data/doc-last-modified.json +++ b/document/data/doc-last-modified.json @@ -167,8 +167,8 @@ "content/plugin/model-presets.mdx": "2026-06-04T16:10:15+08:00", "content/plugin/system-tool-development.en.mdx": "2026-07-02T11:54:55+08:00", "content/plugin/system-tool-development.mdx": "2026-07-02T11:54:55+08:00", - "content/self-host/config/env.en.mdx": "2026-07-17T19:17:14+08:00", - "content/self-host/config/env.mdx": "2026-07-17T19:17:14+08:00", + "content/self-host/config/env.en.mdx": "2026-07-18T12:51:59+08:00", + "content/self-host/config/env.mdx": "2026-07-18T12:51:59+08:00", "content/self-host/config/model/intro.en.mdx": "2026-06-04T16:10:15+08:00", "content/self-host/config/model/intro.mdx": "2026-06-04T16:10:15+08:00", "content/self-host/config/model/minimax.en.mdx": "2026-06-03T10:40:17+08:00", @@ -320,10 +320,10 @@ "content/self-host/upgrading/4-15/41507.mdx": "2026-06-30T17:31:43+08:00", "content/self-host/upgrading/4-15/4151.en.mdx": "2026-07-07T21:14:28+08:00", "content/self-host/upgrading/4-15/4151.mdx": "2026-07-07T21:14:28+08:00", - "content/self-host/upgrading/4-15/4152.en.mdx": "2026-07-17T19:17:14+08:00", - "content/self-host/upgrading/4-15/4152.mdx": "2026-07-17T19:34:15+08:00", - "content/self-host/upgrading/4-15/4153.en.mdx": "2026-07-18T11:22:45+08:00", - "content/self-host/upgrading/4-15/4153.mdx": "2026-07-18T11:22:45+08:00", + "content/self-host/upgrading/4-15/4152.en.mdx": "2026-07-17T17:41:38+08:00", + "content/self-host/upgrading/4-15/4152.mdx": "2026-07-17T17:41:38+08:00", + "content/self-host/upgrading/4-15/4153.en.mdx": "2026-07-18T12:51:59+08:00", + "content/self-host/upgrading/4-15/4153.mdx": "2026-07-18T12:51:59+08:00", "content/self-host/upgrading/outdated/40.en.mdx": "2026-04-26T21:08:47+08:00", "content/self-host/upgrading/outdated/40.mdx": "2026-04-26T21:08:47+08:00", "content/self-host/upgrading/outdated/41.en.mdx": "2026-04-26T21:08:47+08:00", @@ -464,6 +464,6 @@ "content/self-host/upgrading/outdated/499.mdx": "2026-05-07T15:06:40+08:00", "content/self-host/upgrading/upgrade-intruction.en.mdx": "2026-04-26T21:08:47+08:00", "content/self-host/upgrading/upgrade-intruction.mdx": "2026-04-26T21:08:47+08:00", - "content/toc.en.mdx": "2026-07-18T11:22:45+08:00", - "content/toc.mdx": "2026-07-18T11:22:45+08:00" + "content/toc.en.mdx": "2026-07-18T12:51:59+08:00", + "content/toc.mdx": "2026-07-18T12:51:59+08:00" } \ No newline at end of file diff --git a/packages/service/common/mongo/deprecatedIndexes.ts b/packages/service/common/mongo/deprecatedIndexes.ts deleted file mode 100644 index dc51731ca08f..000000000000 --- a/packages/service/common/mongo/deprecatedIndexes.ts +++ /dev/null @@ -1,111 +0,0 @@ -type DeprecatedMongoIndexOptions = { - unique?: boolean; - sparse?: boolean; - expireAfterSeconds?: number; - partialFilterExpression?: unknown; - collation?: unknown; -}; - -export type DeprecatedMongoIndexDefinition = { - collectionName: string; - indexName: string; - key: Record; - options?: DeprecatedMongoIndexOptions; - deprecatedVersion: string; - reason: string; - replacementIndexNames?: string[]; -}; - -const deprecatedMongoIndexProperties = { - version: { - v4150Beta6: '4.15.0-beta6' - }, - collection: { - chat: 'chats', - sandboxInstance: 'agent_sandbox_instances' - }, - replacementIndex: { - chatSourceIdentity: 'sourceType_1_appId_1_chatId_1', - sandboxProviderIdentity: 'provider_1_sandboxId_1', - sandboxSourceLookup: 'sourceType_1_sourceId_1_chatId_1' - } -} as const; - -const deprecatedChatIndexes: DeprecatedMongoIndexDefinition[] = [ - { - collectionName: deprecatedMongoIndexProperties.collection.chat, - indexName: 'appId_1_chatId_1', - key: { appId: 1, chatId: 1 }, - options: { unique: true }, - deprecatedVersion: deprecatedMongoIndexProperties.version.v4150Beta6, - reason: 'Chat identity index was expanded to include sourceType for source-aware chats.', - replacementIndexNames: [deprecatedMongoIndexProperties.replacementIndex.chatSourceIdentity] - } -]; - -const deprecatedSandboxIndexes: DeprecatedMongoIndexDefinition[] = [ - { - collectionName: deprecatedMongoIndexProperties.collection.sandboxInstance, - indexName: 'provider_1_appId_1_userId_1_chatId_1', - key: { provider: 1, appId: 1, userId: 1, chatId: 1 }, - options: { - unique: true, - partialFilterExpression: { - appId: { $exists: true }, - userId: { $exists: true }, - chatId: { $exists: true } - } - }, - deprecatedVersion: deprecatedMongoIndexProperties.version.v4150Beta6, - reason: 'Sandbox ownership moved from appId/userId/type fields to sourceType/sourceId.', - replacementIndexNames: [ - deprecatedMongoIndexProperties.replacementIndex.sandboxProviderIdentity, - deprecatedMongoIndexProperties.replacementIndex.sandboxSourceLookup - ] - }, - { - collectionName: deprecatedMongoIndexProperties.collection.sandboxInstance, - indexName: 'appId_1_chatId_1', - key: { appId: 1, chatId: 1 }, - options: { - unique: true, - partialFilterExpression: { - appId: { $exists: true }, - chatId: { $exists: true }, - type: { $exists: true } - } - }, - deprecatedVersion: deprecatedMongoIndexProperties.version.v4150Beta6, - reason: 'Sandbox appId/type lookup was replaced by sourceType/sourceId lookup.', - replacementIndexNames: [deprecatedMongoIndexProperties.replacementIndex.sandboxSourceLookup] - }, - { - collectionName: deprecatedMongoIndexProperties.collection.sandboxInstance, - indexName: 'metadata.skillId_1', - key: { 'metadata.skillId': 1 }, - deprecatedVersion: deprecatedMongoIndexProperties.version.v4150Beta6, - reason: 'Skill edit sandbox ownership no longer uses metadata.skillId.', - replacementIndexNames: [deprecatedMongoIndexProperties.replacementIndex.sandboxSourceLookup] - }, - { - collectionName: deprecatedMongoIndexProperties.collection.sandboxInstance, - indexName: 'type_1_chatId_1', - key: { type: 1, chatId: 1 }, - deprecatedVersion: deprecatedMongoIndexProperties.version.v4150Beta6, - reason: 'Sandbox type/chat lookup was replaced by sourceType/sourceId lookup.', - replacementIndexNames: [deprecatedMongoIndexProperties.replacementIndex.sandboxSourceLookup] - } -]; - -/** - * FastGPT 历史版本明确废弃、允许通过维护脚本清理的 MongoDB 索引清单。 - * - * 维护规则： - * - 只记录 FastGPT 自己历史创建过的索引。 - * - 删除前必须精确匹配 collection、index name、key 和关键 options。 - * - 客户自建索引不要加入这里，即使它不在当前 schema 中。 - */ -export const deprecatedMongoIndexes: DeprecatedMongoIndexDefinition[] = [ - ...deprecatedChatIndexes, - ...deprecatedSandboxIndexes -]; diff --git a/packages/service/common/mongo/index.ts b/packages/service/common/mongo/index.ts index 9b7b834593e8..82c07ab53772 100644 --- a/packages/service/common/mongo/index.ts +++ b/packages/service/common/mongo/index.ts @@ -162,15 +162,13 @@ const syncMongoIndex = (model: Model) => { return; } - void MongoIndexManager.runModelIndexMode({ + void MongoIndexManager.syncModelIndexes({ model, - mode: serviceEnv.MONGO_INDEX_SYNC_MODE, logger }).catch((error) => { logger.error('Failed to ensure MongoDB indexes', { modelName: model.modelName, collectionName: model.collection.collectionName, - mode: serviceEnv.MONGO_INDEX_SYNC_MODE, error }); }); @@ -179,6 +177,10 @@ const syncMongoIndex = (model: Model) => { export const ReadPreference = connectionMongo.mongo.ReadPreference; export { MongoIndexManager } from './indexManager'; +export { + getDeprecatedIndexes as getSchemaDeprecatedMongoIndexes, + defineDeprecatedIndexes as defineDeprecatedIndexes +} from './schemaIndexes'; export type { MongoIndexCleanupAction, MongoIndexCleanupReport, @@ -186,3 +188,4 @@ export type { MongoIndexCleanupSummary, MongoIndexSyncResult } from './indexManager'; +export type { DeprecatedMongoIndexDefinition, DeprecatedMongoIndexOptions } from './schemaIndexes'; diff --git a/packages/service/common/mongo/indexManager.ts b/packages/service/common/mongo/indexManager.ts index 0e3fb5379bc6..0f516fe63d18 100644 --- a/packages/service/common/mongo/indexManager.ts +++ b/packages/service/common/mongo/indexManager.ts @@ -1,15 +1,9 @@ import { getLogger, LogCategories } from '../logger'; -import type { Connection, Model } from 'mongoose'; -import { - deprecatedMongoIndexes as defaultDeprecatedMongoIndexes, - type DeprecatedMongoIndexDefinition -} from './deprecatedIndexes'; -import type { serviceEnv } from '../../env'; +import type { Model } from 'mongoose'; +import { getDeprecatedIndexes, type DeprecatedMongoIndexDefinition } from './schemaIndexes'; const defaultLogger = getLogger(LogCategories.INFRA.MONGO); -type MongoIndexSyncMode = typeof serviceEnv.MONGO_INDEX_SYNC_MODE; - type MongoIndexLogger = { debug: (message: string, data?: Record) => void; info: (message: string, data?: Record) => void; @@ -23,11 +17,11 @@ type MongooseDiffIndexesResult = { }; export type MongoIndexSyncResult = { - mode: MongoIndexSyncMode; modelName: string; collectionName: string; toDrop: string[]; toCreate: unknown[]; + cleanupReport: MongoIndexCleanupReport; }; export type MongoIndexDescription = { @@ -40,21 +34,7 @@ export type MongoIndexDescription = { collation?: unknown; }; -type MongoIndexCollection = { - indexes: () => Promise; - dropIndex: (indexName: string) => Promise; -}; - -type MongoIndexDb = { - collection: (collectionName: string) => MongoIndexCollection; -}; - -export type MongoIndexCleanupAction = - | 'drop' - | 'skip_missing' - | 'skip_mismatch' - | 'skip_missing_replacement' - | 'error'; +export type MongoIndexCleanupAction = 'drop' | 'skip_missing' | 'skip_mismatch' | 'error'; export type MongoIndexCleanupReportItem = { collectionName: string; @@ -62,9 +42,6 @@ export type MongoIndexCleanupReportItem = { action: MongoIndexCleanupAction; applied: boolean; reason: string; - deprecatedVersion: string; - replacementIndexNames?: string[]; - missingReplacementIndexNames?: string[]; error?: string; }; @@ -79,21 +56,11 @@ export type MongoIndexCleanupSummary = { droppable: number; skippedMissing: number; skippedMismatch: number; - skippedMissingReplacement: number; errors: number; }; -type RunModelIndexModeParams = { +type SyncModelIndexesParams = { model: Model; - mode: MongoIndexSyncMode; - logger?: MongoIndexLogger; -}; - -type RunDeprecatedIndexCleanupOnceParams = { - db: MongoIndexDb; - cleanupKey: string; - apply: boolean; - indexes?: DeprecatedMongoIndexDefinition[]; logger?: MongoIndexLogger; }; @@ -108,22 +75,20 @@ const optionKeys = [ /** * MongoDB 索引管理入口。 * - * 默认启动期只负责非破坏性补建 schema 索引；只有显式 `sync` 模式才会额外清理 - * `deprecatedIndexes` 中登记的 FastGPT 历史旧索引，避免普通服务重启误删客户自建索引。 + * 每个 model 固定执行安全同步：补建当前 Schema 索引，再删除该 Schema 明确登记且 + * 精确匹配的历史索引。Schema 外未知索引只记录不删除，以保护客户自建索引。 */ export class MongoIndexManager { private static modelIndexTasks = new Map, Promise>(); - private static deprecatedCleanupTasks = new Map>(); private static getCollectionName(model: Model) { return model.collection.collectionName; } /** - * 只计算当前 schema 和数据库索引的差异，不创建也不删除任何索引。 + * 只计算当前 Schema 和数据库索引的差异，不创建也不删除任何索引。 * - * `toDrop` 只表示 Mongoose 认为 schema 外存在的索引。默认安全模式下这些索引 - * 只会被记录为未知索引，不会删除，以保护客户自建索引。 + * `toDrop` 仅表示 Mongoose 认为 Schema 外存在的索引，不能直接作为删除清单。 */ static async inspectModelIndexes( model: Model @@ -141,19 +106,18 @@ export class MongoIndexManager { } /** - * 按配置模式处理单个 Mongoose model 的索引。 + * 主动同步单个 Model 的索引。 * - * 默认 `create` 只创建 schema 中缺失的索引，并保留所有 schema 外索引。 - * `sync` 在 model 级别仍只创建缺失索引；废弃索引由连接初始化流程全局清理一次。 + * 当前索引必须先创建成功，之后才会清理 Schema 本地登记的废弃索引。同一进程内 + * 针对同一个 Model 的并发调用复用进行中的任务，完成后允许重连或热加载再次检查。 */ - static async runModelIndexMode(params: RunModelIndexModeParams): Promise { + static async syncModelIndexes(params: SyncModelIndexesParams): Promise { const existingTask = MongoIndexManager.modelIndexTasks.get(params.model); if (existingTask) { return existingTask; } - const task = MongoIndexManager.runModelIndexModeInner(params); - + const task = MongoIndexManager.syncModelIndexesInner(params); MongoIndexManager.modelIndexTasks.set(params.model, task); try { @@ -165,197 +129,96 @@ export class MongoIndexManager { } } - /** - * 等待指定 Mongoose 连接上已经登记的 model 索引任务结束。 - * - * 全局清理废弃索引前必须先等待替代索引创建完成，否则可能因 replacement 尚不存在而跳过清理。 - */ - static async waitForModelIndexTasks(connection: Connection): Promise { - while (true) { - const tasks = [...MongoIndexManager.modelIndexTasks.entries()] - .filter(([model]) => model.db === connection) - .map(([, task]) => task); - - if (tasks.length === 0) { - return; - } - - await Promise.allSettled(tasks); - } - } - - /** - * 对同一个 MongoDB 连接全局执行一次废弃索引清理。 - * - * 清理范围只来自 deprecated registry；重复调用会复用首次任务，不会再次扫描或删除。 - */ - static async runDeprecatedIndexCleanupOnce({ - db, - cleanupKey, - apply, - indexes = defaultDeprecatedMongoIndexes, - logger = defaultLogger - }: RunDeprecatedIndexCleanupOnceParams): Promise { - const existingTask = MongoIndexManager.deprecatedCleanupTasks.get(cleanupKey); - if (existingTask) { - logger.debug('MongoDB deprecated index cleanup skipped', { - cleanupKey, - reason: 'already_started_or_completed' - }); - return existingTask; - } - - const task = MongoIndexManager.cleanupDeprecatedIndexes({ - db, - apply, - indexes, - logger - }); - MongoIndexManager.deprecatedCleanupTasks.set(cleanupKey, task); - - const report = await task; - logger.info('MongoDB deprecated index cleanup finished', { - cleanupKey, - apply, - summary: MongoIndexManager.summarizeCleanupReport(report) - }); - return report; - } - - /** 生成不包含凭据的连接级 cleanup 去重键。 */ - static getConnectionCleanupKey(connection: Connection): string { - return [connection.host, connection.port, connection.name].filter(Boolean).join(':'); - } - - private static async runModelIndexModeInner({ + private static async syncModelIndexesInner({ model, - mode, logger = defaultLogger - }: RunModelIndexModeParams): Promise { - const collectionName = MongoIndexManager.getCollectionName(model); - const baseResult = { - mode, - modelName: model.modelName, - collectionName, - toDrop: [], - toCreate: [] - } satisfies MongoIndexSyncResult; - - if (mode === 'off') { - logger.debug('MongoDB index management skipped', { - ...MongoIndexManager.buildIndexModeLogData(baseResult), - reason: 'mode_off' - }); - return baseResult; - } - + }: SyncModelIndexesParams): Promise { const inspection = await MongoIndexManager.inspectModelIndexes(model); - const result: MongoIndexSyncResult = { - ...baseResult, - toDrop: inspection.toDrop, - toCreate: inspection.toCreate - }; + const logData = MongoIndexManager.buildIndexSyncLogData(inspection); logger.debug('MongoDB index diff inspected', { - ...MongoIndexManager.buildIndexModeLogData(result), - schemaExternalIndexNames: result.toDrop, - toCreate: result.toCreate + ...logData, + schemaExternalIndexNames: inspection.toDrop, + toCreate: inspection.toCreate }); - if (result.toDrop.length > 0) { + if (inspection.toDrop.length > 0) { logger.warn('Detected MongoDB indexes not declared by FastGPT schema', { - ...MongoIndexManager.buildIndexModeLogData(result), - schemaExternalIndexNames: result.toDrop, - cleanupPolicy: 'only_registered_deprecated_indexes_can_be_dropped' + ...logData, + schemaExternalIndexNames: inspection.toDrop, + cleanupPolicy: 'only_schema_registered_deprecated_indexes_can_be_dropped' }); } - if (mode === 'dryRun') { - logger.info( - 'MongoDB index dry-run completed', - MongoIndexManager.buildIndexModeLogData(result) - ); - return result; - } - - if (mode === 'sync') { - logger.info('MongoDB managed index sync started', { - ...MongoIndexManager.buildIndexModeLogData(result), - cleanupPolicy: 'deprecated_indexes_are_cleaned_once_per_connection' - }); - logger.debug('MongoDB managed index sync detail', { - ...MongoIndexManager.buildIndexModeLogData(result), - schemaExternalIndexNames: result.toDrop, - toCreate: result.toCreate, - cleanupPolicy: 'deprecated_indexes_are_cleaned_once_per_connection' - }); - - await model.createIndexes({ background: true }); - - logger.info('MongoDB managed index sync completed', { - ...MongoIndexManager.buildIndexModeLogData(result), - cleanupPolicy: 'deprecated_indexes_are_cleaned_once_per_connection' - }); - - return result; - } - await model.createIndexes({ background: true }); - logger.info('MongoDB schema indexes ensured', MongoIndexManager.buildIndexModeLogData(result)); - logger.debug('MongoDB schema index ensure detail', { - ...MongoIndexManager.buildIndexModeLogData(result), - schemaExternalIndexNames: result.toDrop, - toCreate: result.toCreate + const cleanupReport = await MongoIndexManager.cleanupModelDeprecatedIndexes({ + model, + apply: true, + logger + }); + const result: MongoIndexSyncResult = { + ...inspection, + cleanupReport + }; + + logger.info('MongoDB indexes synchronized', { + ...logData, + cleanupSummary: MongoIndexManager.summarizeCleanupReport(cleanupReport) + }); + logger.debug('MongoDB index synchronization detail', { + ...logData, + schemaExternalIndexNames: inspection.toDrop, + toCreate: inspection.toCreate, + cleanupReport }); return result; } /** - * 清理 FastGPT 明确登记的废弃 MongoDB 索引。 + * 清理当前 Model 所属 Schema 明确登记的废弃索引。 * - * 默认 dry-run 只输出计划；只有 `apply=true` 且 registry 与数据库索引精确匹配时才会删除。 - * schema 外未知索引不在本函数处理范围内，避免误删客户自建索引。 + * 只有 name、key 和关键 options 精确匹配时才允许删除；定义不匹配或未知索引均 + * 保留。`apply=false` 仅供诊断入口复用，启动同步固定传 true。 */ - static async cleanupDeprecatedIndexes({ - db, + static async cleanupModelDeprecatedIndexes({ + model, apply, - indexes = defaultDeprecatedMongoIndexes, logger }: { - db: MongoIndexDb; + model: Model; apply: boolean; - indexes?: DeprecatedMongoIndexDefinition[]; logger?: MongoIndexLogger; }): Promise { + const collectionName = MongoIndexManager.getCollectionName(model); + const definitions = getDeprecatedIndexes(model.schema); const items: MongoIndexCleanupReportItem[] = []; - if (indexes.length === 0) { - return { - apply, - items - }; + if (definitions.length === 0) { + return { apply, items }; } logger?.debug('MongoDB deprecated index cleanup started', { + modelName: model.modelName, + collectionName, apply, - deprecatedIndexCount: indexes.length + deprecatedIndexCount: definitions.length }); - for (const definition of indexes) { + for (const definition of definitions) { try { - const collection = db.collection(definition.collectionName); - const currentIndexes = await collection.indexes().catch((error) => { + const currentIndexes = (await model.collection.indexes().catch((error) => { if (MongoIndexManager.isNamespaceNotFoundError(error)) { return []; } throw error; - }); + })) as MongoIndexDescription[]; const targetIndex = currentIndexes.find((index) => index.name === definition.indexName); if (!targetIndex) { const item = MongoIndexManager.buildCleanupItem({ + collectionName, definition, action: 'skip_missing', reason: 'Deprecated index does not exist' @@ -367,9 +230,10 @@ export class MongoIndexManager { if (!MongoIndexManager.isDeprecatedIndexMatched({ definition, index: targetIndex })) { const item = MongoIndexManager.buildCleanupItem({ + collectionName, definition, action: 'skip_mismatch', - reason: 'Index definition does not match deprecated registry entry' + reason: 'Index definition does not match Schema declaration' }); logger?.warn('Skip deprecated MongoDB index cleanup because definition mismatched', { ...item, @@ -382,31 +246,27 @@ export class MongoIndexManager { continue; } - const replacementIndexNames = definition.replacementIndexNames ?? []; - const missingReplacementIndexNames = replacementIndexNames.filter( - (indexName) => !currentIndexes.some((index) => index.name === indexName) - ); - - if (missingReplacementIndexNames.length > 0) { - const item = MongoIndexManager.buildCleanupItem({ - definition, - action: 'skip_missing_replacement', - reason: 'Replacement index does not exist', - missingReplacementIndexNames - }); - logger?.warn( - 'Skip deprecated MongoDB index cleanup because replacement is missing', - item - ); - items.push(item); - continue; - } - if (apply) { - await collection.dropIndex(definition.indexName); + try { + await model.collection.dropIndex(definition.indexName); + } catch (error) { + if (MongoIndexManager.isIndexNotFoundError(error)) { + const item = MongoIndexManager.buildCleanupItem({ + collectionName, + definition, + action: 'skip_missing', + reason: 'Deprecated index was already removed' + }); + logger?.debug('Deprecated MongoDB index was already removed', item); + items.push(item); + continue; + } + throw error; + } } const item = MongoIndexManager.buildCleanupItem({ + collectionName, definition, action: 'drop', applied: apply, @@ -420,6 +280,7 @@ export class MongoIndexManager { items.push(item); } catch (error) { const item = MongoIndexManager.buildCleanupItem({ + collectionName, definition, action: 'error', reason: 'Failed to inspect or cleanup deprecated index', @@ -431,14 +292,13 @@ export class MongoIndexManager { } logger?.debug('MongoDB deprecated index cleanup completed', { + modelName: model.modelName, + collectionName, apply, summary: MongoIndexManager.summarizeCleanupReport({ apply, items }) }); - return { - apply, - items - }; + return { apply, items }; } static summarizeCleanupReport(report: MongoIndexCleanupReport): MongoIndexCleanupSummary { @@ -454,8 +314,6 @@ export class MongoIndexManager { summary.skippedMissing += 1; } else if (item.action === 'skip_mismatch') { summary.skippedMismatch += 1; - } else if (item.action === 'skip_missing_replacement') { - summary.skippedMissingReplacement += 1; } else if (item.action === 'error') { summary.errors += 1; } @@ -468,7 +326,6 @@ export class MongoIndexManager { droppable: 0, skippedMissing: 0, skippedMismatch: 0, - skippedMissingReplacement: 0, errors: 0 } ); @@ -486,14 +343,7 @@ export class MongoIndexManager { `- [${item.action}]`, item.applied ? 'applied' : 'not-applied', `${item.collectionName}.${item.indexName}`, - `version=${item.deprecatedVersion}`, `reason=${item.reason}`, - item.replacementIndexNames?.length - ? `replacement=${item.replacementIndexNames.join(',')}` - : undefined, - item.missingReplacementIndexNames?.length - ? `missingReplacement=${item.missingReplacementIndexNames.join(',')}` - : undefined, item.error ? `error=${item.error}` : undefined ] .filter(Boolean) @@ -504,9 +354,10 @@ export class MongoIndexManager { return lines.join('\n'); } - private static buildIndexModeLogData(result: MongoIndexSyncResult) { + private static buildIndexSyncLogData( + result: Pick + ) { return { - mode: result.mode, modelName: result.modelName, collectionName: result.collectionName, toCreateCount: result.toCreate.length, @@ -523,14 +374,13 @@ export class MongoIndexManager { } if (value && typeof value === 'object') { - const keys = Object.keys(value as Record); + const keys = Object.keys(value); const orderedKeys = sortObjectKeys ? keys.sort() : keys; return orderedKeys.reduce>((result, key) => { - result[key] = MongoIndexManager.normalizeForCompare( - (value as Record)[key], - { sortObjectKeys } - ); + result[key] = MongoIndexManager.normalizeForCompare(Reflect.get(value, key), { + sortObjectKeys + }); return result; }, {}); } @@ -587,29 +437,26 @@ export class MongoIndexManager { } private static buildCleanupItem({ + collectionName, definition, action, applied = false, reason, - missingReplacementIndexNames, error }: { + collectionName: string; definition: DeprecatedMongoIndexDefinition; action: MongoIndexCleanupAction; applied?: boolean; reason: string; - missingReplacementIndexNames?: string[]; error?: string; }): MongoIndexCleanupReportItem { return { - collectionName: definition.collectionName, + collectionName, indexName: definition.indexName, action, applied, reason, - deprecatedVersion: definition.deprecatedVersion, - replacementIndexNames: definition.replacementIndexNames, - missingReplacementIndexNames, error }; } @@ -626,7 +473,21 @@ export class MongoIndexManager { return false; } - const { codeName, message } = error as { codeName?: string; message?: string }; - return codeName === 'NamespaceNotFound' || message?.includes('ns does not exist') === true; + const codeName = Reflect.get(error, 'codeName'); + const message = Reflect.get(error, 'message'); + return ( + codeName === 'NamespaceNotFound' || + (typeof message === 'string' && message.includes('ns does not exist')) + ); + } + + private static isIndexNotFoundError(error: unknown) { + if (typeof error !== 'object' || error === null) { + return false; + } + + const code = Reflect.get(error, 'code'); + const codeName = Reflect.get(error, 'codeName'); + return code === 27 || codeName === 'IndexNotFound'; } } diff --git a/packages/service/common/mongo/init.ts b/packages/service/common/mongo/init.ts index 07ac2cf1f1a5..ced13c03482b 100644 --- a/packages/service/common/mongo/init.ts +++ b/packages/service/common/mongo/init.ts @@ -2,7 +2,6 @@ import { delay } from '@fastgpt/global/common/system/utils'; import { getLogger, LogCategories } from '../logger'; import type { Mongoose } from 'mongoose'; import { serviceEnv } from '../../env'; -import { MongoIndexManager } from './indexManager'; const logger = getLogger(LogCategories.INFRA.MONGO); @@ -15,9 +14,8 @@ export async function connectMongo(props: { db: Mongoose; url: string; connectedCb?: () => void; - cleanupDeprecatedIndexes?: boolean; }): Promise { - const { db, url, connectedCb, cleanupDeprecatedIndexes = false } = props; + const { db, url, connectedCb } = props; /* Connecting, connected will return */ if (db.connection.readyState !== 0) { @@ -65,21 +63,6 @@ export async function connectMongo(props: { connectedCb?.(); - if (cleanupDeprecatedIndexes && serviceEnv.MONGO_INDEX_SYNC_MODE === 'sync') { - const mongoDb = db.connection.db; - if (!mongoDb) { - throw new Error('MongoDB connection has no database instance after connecting'); - } - - await MongoIndexManager.waitForModelIndexTasks(db.connection); - await MongoIndexManager.runDeprecatedIndexCleanupOnce({ - db: mongoDb, - cleanupKey: MongoIndexManager.getConnectionCleanupKey(db.connection), - apply: true, - logger - }); - } - return db; } catch (error) { logger.error('MongoDB connection failed, will retry', { error }); diff --git a/packages/service/common/mongo/schemaIndexes.ts b/packages/service/common/mongo/schemaIndexes.ts new file mode 100644 index 000000000000..a3e6051242cf --- /dev/null +++ b/packages/service/common/mongo/schemaIndexes.ts @@ -0,0 +1,44 @@ +import type { IndexDefinition, IndexOptions, Schema } from 'mongoose'; + +const deprecatedMongoIndexesKey = Symbol.for('fastgpt.mongo.deprecatedIndexes'); + +export type DeprecatedMongoIndexOptions = Pick< + IndexOptions, + 'unique' | 'sparse' | 'expireAfterSeconds' | 'partialFilterExpression' | 'collation' +>; + +export type DeprecatedMongoIndexDefinition = { + indexName: string; + key: IndexDefinition; + options?: DeprecatedMongoIndexOptions; +}; + +/** + * 将 FastGPT 明确废弃的历史索引登记到所属 Schema。 + * + * collection 由 model 推导，因此定义只描述索引本身。重复名称会在启动前直接报错， + * 避免同一个 Schema 对删除目标给出互相冲突的定义。 + */ +export const defineDeprecatedIndexes = ( + schema: Schema, + indexes: DeprecatedMongoIndexDefinition[] +) => { + const registeredIndexes = getDeprecatedIndexes(schema); + const nextIndexes = [...registeredIndexes, ...indexes]; + const duplicateIndexName = nextIndexes.find( + (index, position) => + nextIndexes.findIndex((candidate) => candidate.indexName === index.indexName) !== position + )?.indexName; + + if (duplicateIndexName) { + throw new Error(`Duplicate deprecated MongoDB index declaration: ${duplicateIndexName}`); + } + + Reflect.set(schema, deprecatedMongoIndexesKey, nextIndexes); +}; + +/** 读取某个 Schema 自身登记的废弃索引，不聚合其他集合或全局清单。 */ +export const getDeprecatedIndexes = (schema: Schema): readonly DeprecatedMongoIndexDefinition[] => { + const indexes: unknown = Reflect.get(schema, deprecatedMongoIndexesKey); + return Array.isArray(indexes) ? indexes : []; +}; diff --git a/packages/service/core/ai/sandbox/infrastructure/instance/schema.ts b/packages/service/core/ai/sandbox/infrastructure/instance/schema.ts index 1e7fcbdaeb94..bac3307a43c1 100644 --- a/packages/service/core/ai/sandbox/infrastructure/instance/schema.ts +++ b/packages/service/core/ai/sandbox/infrastructure/instance/schema.ts @@ -3,7 +3,11 @@ * * 只描述本地实例记录结构，不编排 provider、归档或运行态流程。 */ -import { connectionMongo, getMongoModel } from '../../../../../common/mongo'; +import { + connectionMongo, + getMongoModel, + defineDeprecatedIndexes +} from '../../../../../common/mongo'; const { Schema } = connectionMongo; import type { SandboxInstanceSchemaType } from '../../type'; import { SandboxStatusEnum, SandboxTypeEnum } from '@fastgpt/global/core/ai/sandbox/constants'; @@ -82,6 +86,41 @@ SandboxInstanceSchema.index({ status: 1, lastActiveAt: 1, 'metadata.archive.stat SandboxInstanceSchema.index({ 'metadata.archive.state': 1, 'metadata.archive.startedAt': 1 }); SandboxInstanceSchema.index({ 'metadata.archive.state': 1, 'metadata.archive.deleteStartedAt': 1 }); +defineDeprecatedIndexes(SandboxInstanceSchema, [ + { + indexName: 'provider_1_appId_1_userId_1_chatId_1', + key: { provider: 1, appId: 1, userId: 1, chatId: 1 }, + options: { + unique: true, + partialFilterExpression: { + appId: { $exists: true }, + userId: { $exists: true }, + chatId: { $exists: true } + } + } + }, + { + indexName: 'appId_1_chatId_1', + key: { appId: 1, chatId: 1 }, + options: { + unique: true, + partialFilterExpression: { + appId: { $exists: true }, + chatId: { $exists: true }, + type: { $exists: true } + } + } + }, + { + indexName: 'metadata.skillId_1', + key: { 'metadata.skillId': 1 } + }, + { + indexName: 'type_1_chatId_1', + key: { type: 1, chatId: 1 } + } +]); + /** * sandbox 实例 Mongo model。 * diff --git a/packages/service/core/chat/chatSchema.ts b/packages/service/core/chat/chatSchema.ts index 6e7dfd2eb146..166187b14d15 100644 --- a/packages/service/core/chat/chatSchema.ts +++ b/packages/service/core/chat/chatSchema.ts @@ -1,4 +1,4 @@ -import { connectionMongo, getMongoModel } from '../../common/mongo'; +import { connectionMongo, getMongoModel, defineDeprecatedIndexes } from '../../common/mongo'; const { Schema } = connectionMongo; import { type ChatSchemaType } from '@fastgpt/global/core/chat/type'; import { @@ -249,4 +249,12 @@ ChatSchema.index( } ); +defineDeprecatedIndexes(ChatSchema, [ + { + indexName: 'appId_1_chatId_1', + key: { appId: 1, chatId: 1 }, + options: { unique: true } + } +]); + export const MongoChat = getMongoModel(chatCollectionName, ChatSchema); diff --git a/packages/service/env.ts b/packages/service/env.ts index b90831290d99..a28dd73e42bf 100644 --- a/packages/service/env.ts +++ b/packages/service/env.ts @@ -29,7 +29,6 @@ export const serviceEnv = createEnv({ server: { // ==================== 基础配置 ==================== DB_MAX_LINK: IntSchema.min(1).default(5), - MONGO_INDEX_SYNC_MODE: z.enum(['off', 'sync', 'create', 'dryRun']).default('create'), // ==================== 密钥 ==================== ROOT_KEY: z diff --git a/packages/service/test/common/mongo/indexManager.test.ts b/packages/service/test/common/mongo/indexManager.test.ts index 089d185b0b9e..3fb706b48ee2 100644 --- a/packages/service/test/common/mongo/indexManager.test.ts +++ b/packages/service/test/common/mongo/indexManager.test.ts @@ -1,8 +1,7 @@ import { randomUUID } from 'node:crypto'; import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { connectionMongo, Schema } from '@fastgpt/service/common/mongo'; +import { connectionMongo, defineDeprecatedIndexes, Schema } from '@fastgpt/service/common/mongo'; import { MongoIndexManager } from '@fastgpt/service/common/mongo/indexManager'; -import type { DeprecatedMongoIndexDefinition } from '@fastgpt/service/common/mongo/deprecatedIndexes'; const logger = { debug: vi.fn(), @@ -11,494 +10,329 @@ const logger = { error: vi.fn() }; -const getIndexNames = async (collectionName: string) => { - const indexes = await connectionMongo.connection.db?.collection(collectionName).indexes(); - return new Set(indexes?.map((index: { name?: string }) => index.name)); +const createModel = ({ + schema, + prefix = 'MongoIndexManager' +}: { + schema: InstanceType; + prefix?: string; +}) => { + const suffix = randomUUID().replaceAll('-', ''); + return connectionMongo.model(`${prefix}${suffix}`, schema, `${prefix.toLowerCase()}_${suffix}`); }; -const getCleanupCollectionName = () => `mongo_index_cleanup_${randomUUID().replace(/-/g, '')}`; +const getIndexNames = async (model: ReturnType) => + new Set((await model.collection.indexes()).map((index) => index.name)); -const dropCollection = async (collectionName: string) => { - try { - await connectionMongo.connection.db?.collection(collectionName).drop(); - } catch (error: any) { - if (error?.codeName !== 'NamespaceNotFound') { - throw error; - } - } -}; - -const getDb = () => { - const db = connectionMongo.connection.db; - if (!db) { - throw new Error('Mongo test db is not connected'); - } - return db; -}; +const legacyDefinition = { + indexName: 'legacy_field_1', + key: { legacyField: 1 } +} as const; -describe('MongoIndexManager.runModelIndexMode', () => { +describe('MongoIndexManager.syncModelIndexes', () => { beforeEach(() => { vi.clearAllMocks(); }); - it('creates missing schema indexes without deleting customer indexes in create mode', async () => { - const collectionName = `mongo_index_create_${Date.now()}`; - await dropCollection(collectionName); - + it('creates current indexes, removes declared legacy indexes, and preserves customer indexes', async () => { const schema = new Schema( { - name: String, + currentField: String, + legacyField: String, customerField: String }, { autoIndex: false } ); - schema.index({ name: 1 }, { name: 'schema_name_1' }); - - const model = connectionMongo.model(`MongoIndexCreate${Date.now()}`, schema, collectionName); + schema.index({ currentField: 1 }, { name: 'current_field_1' }); + defineDeprecatedIndexes(schema, [legacyDefinition]); + const model = createModel({ schema }); + await model.collection.createIndex({ legacyField: 1 }, { name: 'legacy_field_1' }); await model.collection.createIndex({ customerField: 1 }, { name: 'customer_custom_1' }); - await MongoIndexManager.runModelIndexMode({ - model, - mode: 'create', - logger - }); + const result = await MongoIndexManager.syncModelIndexes({ model, logger }); - const indexNames = await getIndexNames(collectionName); - expect(indexNames.has('schema_name_1')).toBe(true); - expect(indexNames.has('customer_custom_1')).toBe(true); - expect(logger.info).toHaveBeenCalledWith( - 'MongoDB schema indexes ensured', + const indexNames = await getIndexNames(model); + expect(indexNames).toContain('current_field_1'); + expect(indexNames).toContain('customer_custom_1'); + expect(indexNames).not.toContain('legacy_field_1'); + expect(result.cleanupReport.items).toEqual([ expect.objectContaining({ - mode: 'create', - collectionName, - toCreateCount: expect.any(Number), - schemaExternalIndexCount: expect.any(Number) + action: 'drop', + applied: true, + collectionName: model.collection.collectionName, + indexName: 'legacy_field_1' }) - ); - expect(logger.debug).toHaveBeenCalledWith( - 'MongoDB schema index ensure detail', + ]); + expect(logger.warn).toHaveBeenCalledWith( + 'Detected MongoDB indexes not declared by FastGPT schema', expect.objectContaining({ - mode: 'create', - collectionName, - schemaExternalIndexNames: expect.arrayContaining(['customer_custom_1']), - toCreate: expect.any(Array) + schemaExternalIndexNames: expect.arrayContaining(['legacy_field_1', 'customer_custom_1']) }) ); }); - it('does not create or delete indexes in dryRun mode', async () => { - const collectionName = `mongo_index_dry_run_${Date.now()}`; - await dropCollection(collectionName); - + it('does not delete schema-external indexes when the Schema has no deprecated declarations', async () => { const schema = new Schema( - { - name: String, - customerField: String - }, + { currentField: String, customerField: String }, { autoIndex: false } ); - schema.index({ name: 1 }, { name: 'schema_name_1' }); - - const model = connectionMongo.model(`MongoIndexDryRun${Date.now()}`, schema, collectionName); + schema.index({ currentField: 1 }, { name: 'current_field_1' }); + const model = createModel({ schema }); await model.collection.createIndex({ customerField: 1 }, { name: 'customer_custom_1' }); - const result = await MongoIndexManager.runModelIndexMode({ - model, - mode: 'dryRun', - logger - }); + const result = await MongoIndexManager.syncModelIndexes({ model }); - const indexNames = await getIndexNames(collectionName); - expect(result.toCreate.length).toBeGreaterThan(0); - expect(indexNames.has('schema_name_1')).toBe(false); - expect(indexNames.has('customer_custom_1')).toBe(true); - expect(logger.info).toHaveBeenCalledWith( - 'MongoDB index dry-run completed', - expect.objectContaining({ - mode: 'dryRun', - collectionName, - toCreateCount: result.toCreate.length, - schemaExternalIndexCount: expect.any(Number) - }) - ); - expect(logger.debug).toHaveBeenCalledWith( - 'MongoDB index diff inspected', - expect.objectContaining({ - mode: 'dryRun', - collectionName, - schemaExternalIndexNames: expect.arrayContaining(['customer_custom_1']), - toCreate: result.toCreate - }) - ); + expect(await getIndexNames(model)).toContain('customer_custom_1'); + expect(result.cleanupReport.items).toEqual([]); }); - it('creates schema indexes without deleting indexes in per-model sync mode', async () => { - const collectionName = `mongo_index_sync_${Date.now()}`; - await dropCollection(collectionName); + it('reuses an in-flight task for concurrent calls on the same Model', async () => { + const schema = new Schema({ currentField: String }, { autoIndex: false }); + schema.index({ currentField: 1 }, { name: 'current_field_1' }); + const model = createModel({ schema }); + const createIndexes = vi.spyOn(model, 'createIndexes'); + const [firstResult, secondResult] = await Promise.all([ + MongoIndexManager.syncModelIndexes({ model }), + MongoIndexManager.syncModelIndexes({ model }) + ]); + + expect(firstResult).toBe(secondResult); + expect(createIndexes).toHaveBeenCalledTimes(1); + }); + + it('does not clean deprecated indexes when creating current indexes fails', async () => { const schema = new Schema( - { - name: String, - customerField: String, - legacyField: String - }, + { currentField: String, conflictingField: String, legacyField: String }, { autoIndex: false } ); - schema.index({ name: 1 }, { name: 'schema_name_1' }); - - const model = connectionMongo.model(`MongoIndexSync${Date.now()}`, schema, collectionName); - await model.collection.createIndex({ customerField: 1 }, { name: 'customer_custom_1' }); + schema.index({ currentField: 1 }, { name: 'current_field_1' }); + defineDeprecatedIndexes(schema, [legacyDefinition]); + const model = createModel({ schema }); + await model.collection.createIndex({ conflictingField: 1 }, { name: 'current_field_1' }); await model.collection.createIndex({ legacyField: 1 }, { name: 'legacy_field_1' }); - const result = await MongoIndexManager.runModelIndexMode({ - model, - mode: 'sync', - logger - }); + await expect(MongoIndexManager.syncModelIndexes({ model })).rejects.toThrow(); - const indexNames = await getIndexNames(collectionName); - expect(result.toDrop).toContain('customer_custom_1'); - expect(result.toDrop).toContain('legacy_field_1'); - expect(indexNames.has('schema_name_1')).toBe(true); - expect(indexNames.has('customer_custom_1')).toBe(true); - expect(indexNames.has('legacy_field_1')).toBe(true); - expect(logger.info).toHaveBeenCalledWith( - 'MongoDB managed index sync started', - expect.objectContaining({ - mode: 'sync', - collectionName, - cleanupPolicy: 'deprecated_indexes_are_cleaned_once_per_connection' - }) - ); - expect(logger.info).toHaveBeenCalledWith( - 'MongoDB managed index sync completed', - expect.objectContaining({ - mode: 'sync', - collectionName, - cleanupPolicy: 'deprecated_indexes_are_cleaned_once_per_connection' - }) - ); - expect(logger.debug).toHaveBeenCalledWith( - 'MongoDB managed index sync detail', - expect.objectContaining({ - mode: 'sync', - collectionName, - schemaExternalIndexNames: expect.arrayContaining(['customer_custom_1', 'legacy_field_1']), - toCreate: expect.any(Array), - cleanupPolicy: 'deprecated_indexes_are_cleaned_once_per_connection' - }) - ); - expect(logger.warn).toHaveBeenCalledWith( - 'Detected MongoDB indexes not declared by FastGPT schema', - expect.objectContaining({ - mode: 'sync', - collectionName, - schemaExternalIndexNames: expect.arrayContaining(['customer_custom_1', 'legacy_field_1']), - cleanupPolicy: 'only_registered_deprecated_indexes_can_be_dropped' - }) - ); + expect(await getIndexNames(model)).toContain('legacy_field_1'); }); }); -describe('MongoIndexManager.cleanupDeprecatedIndexes', () => { - it('runs deprecated index cleanup only once for the same connection key', async () => { +describe('MongoIndexManager.cleanupModelDeprecatedIndexes', () => { + beforeEach(() => { vi.clearAllMocks(); - const collectionName = getCleanupCollectionName(); - const cleanupKey = `test:${randomUUID()}`; - await dropCollection(collectionName); - const collection = getDb().collection(collectionName); - await collection.createIndex({ legacyField: 1 }, { name: 'legacy_field_1' }); + }); - const indexes: DeprecatedMongoIndexDefinition[] = [ + it('supports dry-run without deleting a matched index', async () => { + const schema = new Schema({ legacyField: String }, { autoIndex: false }); + defineDeprecatedIndexes(schema, [ { - collectionName, - indexName: 'legacy_field_1', - key: { legacyField: 1 }, - deprecatedVersion: '4.15.0', - reason: 'test legacy index' + ...legacyDefinition, + options: { unique: true } } - ]; + ]); + const model = createModel({ schema }); + await model.collection.createIndex( + { legacyField: 1 }, + { name: 'legacy_field_1', unique: true } + ); - const firstReport = await MongoIndexManager.runDeprecatedIndexCleanupOnce({ - db: getDb(), - cleanupKey, - apply: true, - indexes, - logger - }); - await collection.createIndex({ legacyField: 1 }, { name: 'legacy_field_1' }); - const secondReport = await MongoIndexManager.runDeprecatedIndexCleanupOnce({ - db: getDb(), - cleanupKey, - apply: true, - indexes, + const report = await MongoIndexManager.cleanupModelDeprecatedIndexes({ + model, + apply: false, logger }); - const indexNames = await getIndexNames(collectionName); - expect(firstReport).toBe(secondReport); - expect(indexNames.has('legacy_field_1')).toBe(true); - expect(logger.info).toHaveBeenCalledTimes(2); + expect(report.items).toEqual([ + expect.objectContaining({ action: 'drop', applied: false, indexName: 'legacy_field_1' }) + ]); + expect(await getIndexNames(model)).toContain('legacy_field_1'); expect(logger.debug).toHaveBeenCalledWith( - 'MongoDB deprecated index cleanup skipped', - expect.objectContaining({ - cleanupKey, - reason: 'already_started_or_completed' - }) + 'Deprecated MongoDB index can be dropped', + expect.objectContaining({ indexName: 'legacy_field_1' }) ); }); - it('dry-runs matched deprecated indexes without deleting them', async () => { - const collectionName = getCleanupCollectionName(); - await dropCollection(collectionName); - const collection = getDb().collection(collectionName); - await collection.createIndex({ legacyField: 1 }, { name: 'legacy_field_1' }); - - const indexes: DeprecatedMongoIndexDefinition[] = [ + it('preserves same-name indexes when key, key order, or options do not match', async () => { + const schema = new Schema( + { customerField: String, legacyField: String, otherField: String }, + { autoIndex: false } + ); + defineDeprecatedIndexes(schema, [ { - collectionName, - indexName: 'legacy_field_1', - key: { legacyField: 1 }, - deprecatedVersion: '4.15.0', - reason: 'test legacy index' + ...legacyDefinition, + options: { unique: true } + }, + { + indexName: 'legacy_compound_1', + key: { legacyField: 1, otherField: 1 } } - ]; + ]); + const model = createModel({ schema }); + await model.collection.createIndex({ customerField: 1 }, { name: 'legacy_field_1' }); + await model.collection.createIndex( + { otherField: 1, legacyField: 1 }, + { name: 'legacy_compound_1' } + ); - const report = await MongoIndexManager.cleanupDeprecatedIndexes({ - db: getDb(), - apply: false, - indexes + const report = await MongoIndexManager.cleanupModelDeprecatedIndexes({ + model, + apply: true, + logger }); - const indexNames = await getIndexNames(collectionName); expect(report.items).toEqual([ - expect.objectContaining({ - action: 'drop', - applied: false, - collectionName, - indexName: 'legacy_field_1' - }) + expect.objectContaining({ action: 'skip_mismatch', indexName: 'legacy_field_1' }), + expect.objectContaining({ action: 'skip_mismatch', indexName: 'legacy_compound_1' }) ]); - expect(indexNames.has('legacy_field_1')).toBe(true); + expect(await getIndexNames(model)).toEqual( + expect.objectContaining(new Set(['_id_', 'legacy_field_1', 'legacy_compound_1'])) + ); }); - it('drops only matched deprecated indexes when apply is enabled', async () => { - const collectionName = getCleanupCollectionName(); - await dropCollection(collectionName); - const collection = getDb().collection(collectionName); - await collection.createIndex({ legacyField: 1 }, { name: 'legacy_field_1' }); - await collection.createIndex({ currentField: 1 }, { name: 'current_field_1' }); + it('reports a missing deprecated index and formats its report', async () => { + const schema = new Schema({ legacyField: String }, { autoIndex: false }); + defineDeprecatedIndexes(schema, [legacyDefinition]); + const model = createModel({ schema }); - const indexes: DeprecatedMongoIndexDefinition[] = [ - { - collectionName, - indexName: 'legacy_field_1', - key: { legacyField: 1 }, - deprecatedVersion: '4.15.0', - replacementIndexNames: ['current_field_1'], - reason: 'test legacy index' - } - ]; - - const report = await MongoIndexManager.cleanupDeprecatedIndexes({ - db: getDb(), - apply: true, - indexes + const report = await MongoIndexManager.cleanupModelDeprecatedIndexes({ + model, + apply: true }); - const indexNames = await getIndexNames(collectionName); expect(report.items).toEqual([ - expect.objectContaining({ - action: 'drop', - applied: true, - collectionName, - indexName: 'legacy_field_1' - }) + expect.objectContaining({ action: 'skip_missing', indexName: 'legacy_field_1' }) ]); - expect(indexNames.has('legacy_field_1')).toBe(false); - expect(indexNames.has('current_field_1')).toBe(true); + expect(MongoIndexManager.summarizeCleanupReport(report)).toEqual({ + total: 1, + dropped: 0, + droppable: 0, + skippedMissing: 1, + skippedMismatch: 0, + errors: 0 + }); + expect(MongoIndexManager.formatCleanupReport(report)).toContain( + `${model.collection.collectionName}.legacy_field_1 reason=Deprecated index does not exist` + ); }); - it('skips same-name indexes when key or options do not match registry', async () => { - const collectionName = getCleanupCollectionName(); - await dropCollection(collectionName); - const collection = getDb().collection(collectionName); - await collection.createIndex({ customerField: 1 }, { name: 'legacy_field_1' }); - - const indexes: DeprecatedMongoIndexDefinition[] = [ - { - collectionName, - indexName: 'legacy_field_1', - key: { legacyField: 1 }, - options: { unique: true }, - deprecatedVersion: '4.15.0', - reason: 'test legacy index' - } - ]; + it('treats a concurrent IndexNotFound response as an idempotent skip', async () => { + const schema = new Schema({ legacyField: String }, { autoIndex: false }); + defineDeprecatedIndexes(schema, [legacyDefinition]); + const model = createModel({ schema }); + await model.collection.createIndex({ legacyField: 1 }, { name: 'legacy_field_1' }); + vi.spyOn(model.collection, 'dropIndex').mockRejectedValueOnce({ + codeName: 'IndexNotFound' + }); - const report = await MongoIndexManager.cleanupDeprecatedIndexes({ - db: getDb(), - apply: true, - indexes + const report = await MongoIndexManager.cleanupModelDeprecatedIndexes({ + model, + apply: true }); - const indexNames = await getIndexNames(collectionName); expect(report.items).toEqual([ expect.objectContaining({ - action: 'skip_mismatch', - applied: false, - collectionName, - indexName: 'legacy_field_1' + action: 'skip_missing', + reason: 'Deprecated index was already removed' }) ]); - expect(indexNames.has('legacy_field_1')).toBe(true); }); - it('skips same-name compound indexes when key order does not match registry', async () => { - const collectionName = getCleanupCollectionName(); - await dropCollection(collectionName); - const collection = getDb().collection(collectionName); - await collection.createIndex( - { customerField: 1, legacyField: 1 }, - { name: 'legacy_compound_1' } - ); - - const indexes: DeprecatedMongoIndexDefinition[] = [ - { - collectionName, - indexName: 'legacy_compound_1', - key: { legacyField: 1, customerField: 1 }, - deprecatedVersion: '4.15.0', - reason: 'test legacy index' - } - ]; + it('recognizes the numeric MongoDB IndexNotFound code', async () => { + const schema = new Schema({ legacyField: String }, { autoIndex: false }); + defineDeprecatedIndexes(schema, [legacyDefinition]); + const model = createModel({ schema }); + await model.collection.createIndex({ legacyField: 1 }, { name: 'legacy_field_1' }); + vi.spyOn(model.collection, 'dropIndex').mockRejectedValueOnce({ code: 27 }); - const report = await MongoIndexManager.cleanupDeprecatedIndexes({ - db: getDb(), - apply: true, - indexes + const report = await MongoIndexManager.cleanupModelDeprecatedIndexes({ + model, + apply: true }); - const indexNames = await getIndexNames(collectionName); - expect(report.items).toEqual([ - expect.objectContaining({ - action: 'skip_mismatch', - applied: false, - collectionName, - indexName: 'legacy_compound_1' - }) - ]); - expect(indexNames.has('legacy_compound_1')).toBe(true); + expect(report.items[0]).toMatchObject({ + action: 'skip_missing', + reason: 'Deprecated index was already removed' + }); }); - it('skips cleanup when replacement index is required but missing', async () => { - const collectionName = getCleanupCollectionName(); - await dropCollection(collectionName); - const collection = getDb().collection(collectionName); - await collection.createIndex({ legacyField: 1 }, { name: 'legacy_field_1' }); - - const indexes: DeprecatedMongoIndexDefinition[] = [ - { - collectionName, - indexName: 'legacy_field_1', - key: { legacyField: 1 }, - deprecatedVersion: '4.15.0', - replacementIndexNames: ['current_field_1'], - reason: 'test legacy index' - } - ]; + it('captures unexpected inspection errors in the cleanup report', async () => { + const schema = new Schema({ legacyField: String }, { autoIndex: false }); + defineDeprecatedIndexes(schema, [legacyDefinition]); + const model = createModel({ schema }); + vi.spyOn(model.collection, 'indexes').mockRejectedValueOnce(new Error('inspection failed')); - const report = await MongoIndexManager.cleanupDeprecatedIndexes({ - db: getDb(), + const report = await MongoIndexManager.cleanupModelDeprecatedIndexes({ + model, apply: true, - indexes + logger }); - const indexNames = await getIndexNames(collectionName); expect(report.items).toEqual([ expect.objectContaining({ - action: 'skip_missing_replacement', - applied: false, - collectionName, + action: 'error', + error: 'inspection failed', indexName: 'legacy_field_1' }) ]); - expect(indexNames.has('legacy_field_1')).toBe(true); + expect(logger.error).toHaveBeenCalledWith( + 'Failed to cleanup deprecated MongoDB index', + expect.objectContaining({ error: 'inspection failed' }) + ); }); - it('skips cleanup until every replacement index exists', async () => { - const collectionName = getCleanupCollectionName(); - await dropCollection(collectionName); - const collection = getDb().collection(collectionName); - await collection.createIndex({ legacyField: 1 }, { name: 'legacy_field_1' }); - await collection.createIndex({ currentField: 1 }, { name: 'current_field_1' }); - - const indexes: DeprecatedMongoIndexDefinition[] = [ - { - collectionName, - indexName: 'legacy_field_1', - key: { legacyField: 1 }, - deprecatedVersion: '4.15.0', - replacementIndexNames: ['current_field_1', 'current_field_2'], - reason: 'test legacy index' - } - ]; + it('normalizes non-Error cleanup failures into report messages', async () => { + const schema = new Schema({ legacyField: String }, { autoIndex: false }); + defineDeprecatedIndexes(schema, [legacyDefinition]); + const model = createModel({ schema }); + await model.collection.createIndex({ legacyField: 1 }, { name: 'legacy_field_1' }); + vi.spyOn(model.collection, 'dropIndex').mockRejectedValueOnce('drop failed'); - const report = await MongoIndexManager.cleanupDeprecatedIndexes({ - db: getDb(), - apply: true, - indexes + const report = await MongoIndexManager.cleanupModelDeprecatedIndexes({ + model, + apply: true }); - const indexNames = await getIndexNames(collectionName); - expect(report.items).toEqual([ - expect.objectContaining({ - action: 'skip_missing_replacement', - applied: false, - collectionName, - indexName: 'legacy_field_1', - missingReplacementIndexNames: ['current_field_2'] - }) - ]); - expect(indexNames.has('legacy_field_1')).toBe(true); + expect(report.items[0]).toMatchObject({ action: 'error', error: 'drop failed' }); }); - it('reports missing deprecated indexes and formats the report', async () => { - const collectionName = getCleanupCollectionName(); - await dropCollection(collectionName); - - const report = await MongoIndexManager.cleanupDeprecatedIndexes({ - db: getDb(), - apply: false, - indexes: [ + it('summarizes every cleanup action and formats error details', () => { + const report = { + apply: true, + items: [ { - collectionName, - indexName: 'missing_1', - key: { missing: 1 }, - deprecatedVersion: '4.15.0', - reason: 'test missing index' + collectionName: 'test_collection', + indexName: 'legacy_drop_1', + action: 'drop' as const, + applied: false, + reason: 'Can drop', + error: 'test error' + }, + { + collectionName: 'test_collection', + indexName: 'legacy_mismatch_1', + action: 'skip_mismatch' as const, + applied: false, + reason: 'Mismatch' + }, + { + collectionName: 'test_collection', + indexName: 'legacy_error_1', + action: 'error' as const, + applied: false, + reason: 'Error' } ] - }); + }; - expect(report.items).toEqual([ - expect.objectContaining({ - action: 'skip_missing', - collectionName, - indexName: 'missing_1' - }) - ]); expect(MongoIndexManager.summarizeCleanupReport(report)).toEqual({ - total: 1, + total: 3, dropped: 0, - droppable: 0, - skippedMissing: 1, - skippedMismatch: 0, - skippedMissingReplacement: 0, - errors: 0 + droppable: 1, + skippedMissing: 0, + skippedMismatch: 1, + errors: 1 }); - expect(MongoIndexManager.formatCleanupReport(report)).toContain( - `${collectionName}.missing_1 version=4.15.0` - ); + expect(MongoIndexManager.formatCleanupReport(report)).toContain('error=test error'); }); }); diff --git a/packages/service/test/common/mongo/schemaIndexes.test.ts b/packages/service/test/common/mongo/schemaIndexes.test.ts new file mode 100644 index 000000000000..ec4822a29667 --- /dev/null +++ b/packages/service/test/common/mongo/schemaIndexes.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from 'vitest'; +import { Schema } from '@fastgpt/service/common/mongo'; +import { + getDeprecatedIndexes, + defineDeprecatedIndexes +} from '@fastgpt/service/common/mongo/schemaIndexes'; + +describe('Schema deprecated MongoDB indexes', () => { + it('returns an empty list for a Schema without declarations', () => { + expect(getDeprecatedIndexes(new Schema())).toEqual([]); + }); + + it('stores declarations on the Schema and appends later declarations', () => { + const schema = new Schema(); + defineDeprecatedIndexes(schema, [ + { + indexName: 'legacy_a_1', + key: { legacyA: 1 } + } + ]); + defineDeprecatedIndexes(schema, [ + { + indexName: 'legacy_b_1', + key: { legacyB: 1 } + } + ]); + + expect(getDeprecatedIndexes(schema).map((index) => index.indexName)).toEqual([ + 'legacy_a_1', + 'legacy_b_1' + ]); + }); + + it('rejects duplicate index names within one Schema', () => { + const schema = new Schema(); + const definition = { + indexName: 'legacy_1', + key: { legacy: 1 } + } as const; + defineDeprecatedIndexes(schema, [definition]); + + expect(() => defineDeprecatedIndexes(schema, [definition])).toThrow( + 'Duplicate deprecated MongoDB index declaration: legacy_1' + ); + }); +}); diff --git a/packages/service/test/core/ai/sandbox/infrastructure/instance/schema.test.ts b/packages/service/test/core/ai/sandbox/infrastructure/instance/schema.test.ts index 09f527c3b986..7753c933f885 100644 --- a/packages/service/test/core/ai/sandbox/infrastructure/instance/schema.test.ts +++ b/packages/service/test/core/ai/sandbox/infrastructure/instance/schema.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest'; import { MongoSandboxInstance } from '@fastgpt/service/core/ai/sandbox/infrastructure/instance/schema'; +import { getSchemaDeprecatedMongoIndexes } from '@fastgpt/service/common/mongo'; describe('MongoSandboxInstance schema indexes', () => { it('declares provider sandbox uniqueness for remote resource records', () => { @@ -21,6 +22,17 @@ describe('MongoSandboxInstance schema indexes', () => { expect(legacyIndex).toBeUndefined(); }); + it('declares replaced sandbox indexes as deprecated on the same Schema', () => { + const deprecatedIndexes = getSchemaDeprecatedMongoIndexes(MongoSandboxInstance.schema); + + expect(deprecatedIndexes.map((index) => index.indexName)).toEqual([ + 'provider_1_appId_1_userId_1_chatId_1', + 'appId_1_chatId_1', + 'metadata.skillId_1', + 'type_1_chatId_1' + ]); + }); + it('declares source lookup index for migrated sandbox instances', () => { const indexes = MongoSandboxInstance.schema.indexes(); const sourceChatIndex = indexes.find( diff --git a/packages/service/test/core/chat/schema.test.ts b/packages/service/test/core/chat/schema.test.ts index 023c7af40bc3..63528b2c45ee 100644 --- a/packages/service/test/core/chat/schema.test.ts +++ b/packages/service/test/core/chat/schema.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest'; import { MongoChat } from '@fastgpt/service/core/chat/chatSchema'; import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema'; import { MongoChatItemResponse } from '@fastgpt/service/core/chat/chatItemResponseSchema'; +import { getSchemaDeprecatedMongoIndexes } from '@fastgpt/service/common/mongo'; const hasIndex = ( indexes: ReturnType, @@ -37,6 +38,16 @@ describe('chat schema indexes', () => { expect(sourceAwareIndex?.[1]?.unique).toBe(true); }); + it('declares the replaced chat identity index as deprecated on the same Schema', () => { + expect(getSchemaDeprecatedMongoIndexes(MongoChat.schema)).toEqual([ + expect.objectContaining({ + indexName: 'appId_1_chatId_1', + key: { appId: 1, chatId: 1 }, + options: { unique: true } + }) + ]); + }); + it('keeps legacy chat item read and pagination indexes', () => { const indexes = MongoChatItem.schema.indexes(); diff --git a/packages/service/test/env.test.ts b/packages/service/test/env.test.ts index c4139fca4187..e4f056fdc42a 100644 --- a/packages/service/test/env.test.ts +++ b/packages/service/test/env.test.ts @@ -14,7 +14,6 @@ const originalEnv = { PRO_TOKEN: process.env.PRO_TOKEN, VITEST: process.env.VITEST, NODE_ENV: process.env.NODE_ENV, - MONGO_INDEX_SYNC_MODE: process.env.MONGO_INDEX_SYNC_MODE, AGENT_SANDBOX_PROVIDER: process.env.AGENT_SANDBOX_PROVIDER, AGENT_SANDBOX_SEALOS_BASEURL: process.env.AGENT_SANDBOX_SEALOS_BASEURL, AGENT_SANDBOX_SEALOS_TOKEN: process.env.AGENT_SANDBOX_SEALOS_TOKEN, @@ -43,7 +42,6 @@ describe('serviceEnv', () => { vi.stubEnv('PRO_TOKEN', originalEnv.PRO_TOKEN); vi.stubEnv('VITEST', originalEnv.VITEST); vi.stubEnv('NODE_ENV', originalEnv.NODE_ENV); - vi.stubEnv('MONGO_INDEX_SYNC_MODE', originalEnv.MONGO_INDEX_SYNC_MODE); vi.stubEnv('AGENT_SANDBOX_PROVIDER', originalEnv.AGENT_SANDBOX_PROVIDER); vi.stubEnv('AGENT_SANDBOX_SEALOS_BASEURL', originalEnv.AGENT_SANDBOX_SEALOS_BASEURL); vi.stubEnv('AGENT_SANDBOX_SEALOS_TOKEN', originalEnv.AGENT_SANDBOX_SEALOS_TOKEN); @@ -75,56 +73,6 @@ describe('serviceEnv', () => { }); }); - it('parses MongoDB index sync mode during service env init', async () => { - vi.stubEnv('FILE_TOKEN_KEY', 'filetokenkey'); - vi.stubEnv('AES256_SECRET_KEY', 'fastgptsecret'); - vi.stubEnv('INVOKE_TOKEN_SECRET', validInvokeTokenSecret); - - vi.stubEnv('MONGO_INDEX_SYNC_MODE', undefined); - await expect(importServiceEnv()).resolves.toMatchObject({ - serviceEnv: { - MONGO_INDEX_SYNC_MODE: 'create' - } - }); - - vi.stubEnv('MONGO_INDEX_SYNC_MODE', ''); - await expect(importServiceEnv()).resolves.toMatchObject({ - serviceEnv: { - MONGO_INDEX_SYNC_MODE: 'create' - } - }); - - vi.stubEnv('MONGO_INDEX_SYNC_MODE', 'off'); - await expect(importServiceEnv()).resolves.toMatchObject({ - serviceEnv: { - MONGO_INDEX_SYNC_MODE: 'off' - } - }); - - vi.stubEnv('MONGO_INDEX_SYNC_MODE', 'sync'); - await expect(importServiceEnv()).resolves.toMatchObject({ - serviceEnv: { - MONGO_INDEX_SYNC_MODE: 'sync' - } - }); - - vi.stubEnv('MONGO_INDEX_SYNC_MODE', 'dryRun'); - await expect(importServiceEnv()).resolves.toMatchObject({ - serviceEnv: { - MONGO_INDEX_SYNC_MODE: 'dryRun' - } - }); - }); - - it('rejects invalid MongoDB index sync mode during service env init', async () => { - vi.stubEnv('FILE_TOKEN_KEY', 'filetokenkey'); - vi.stubEnv('AES256_SECRET_KEY', 'fastgptsecret'); - vi.stubEnv('INVOKE_TOKEN_SECRET', validInvokeTokenSecret); - vi.stubEnv('MONGO_INDEX_SYNC_MODE', 'migrate'); - - await expect(importServiceEnv()).rejects.toThrow('Invalid environment variables'); - }); - it('rejects invalid SYSTEM_MAX_STRING_LENGTH_M during service env init', async () => { vi.stubEnv('FILE_TOKEN_KEY', 'filetokenkey'); vi.stubEnv('AES256_SECRET_KEY', 'fastgptsecret'); diff --git a/projects/app/src/instrumentation-node.ts b/projects/app/src/instrumentation-node.ts index 2dca7db4906e..2a001c5e386a 100644 --- a/projects/app/src/instrumentation-node.ts +++ b/projects/app/src/instrumentation-node.ts @@ -99,7 +99,6 @@ export async function registerNodeInstrumentation() { connectMongo({ db: connectionMongo, url: MONGO_URL, - cleanupDeprecatedIndexes: true, connectedCb: () => startMongoWatch() }), logger, diff --git a/projects/marketplace/src/env.ts b/projects/marketplace/src/env.ts index 44f31abf6e54..7e5dd7de789f 100644 --- a/projects/marketplace/src/env.ts +++ b/projects/marketplace/src/env.ts @@ -15,7 +15,6 @@ export const marketplaceEnv = createEnv({ COMMUNITY_AUTH_TOKEN: z.string().optional(), MONGODB_URI: z.string().optional().default(''), DB_MAX_LINK: IntSchema.default(20), - MONGO_INDEX_SYNC_MODE: z.enum(['off', 'sync', 'create', 'dryRun']).default('create'), // 对象存储。保持与主项目 packages/service/env.ts 同名,并兼容 marketplace 旧 S3_* 变量。 STORAGE_VENDOR: StorageVendorSchema.default('minio'), diff --git a/projects/marketplace/src/service/mongo/index.ts b/projects/marketplace/src/service/mongo/index.ts index 26dbfdec0117..492670d084b7 100644 --- a/projects/marketplace/src/service/mongo/index.ts +++ b/projects/marketplace/src/service/mongo/index.ts @@ -37,16 +37,14 @@ const syncMongoIndex = async (model: Model) => { } try { - await MongoIndexManager.runModelIndexMode({ + await MongoIndexManager.syncModelIndexes({ model, - mode: marketplaceEnv.MONGO_INDEX_SYNC_MODE, logger }); - } catch (error: any) { + } catch (error) { logger.error('Failed to ensure MongoDB indexes', { modelName: model.modelName, collectionName: model.collection.collectionName, - mode: marketplaceEnv.MONGO_INDEX_SYNC_MODE, error }); } @@ -93,21 +91,6 @@ export async function connectMongo(db: Mongoose, url: string): Promise heartbeatFrequencyMS: 5000 // 5s 进行一次健康检查 }); - if (marketplaceEnv.MONGO_INDEX_SYNC_MODE === 'sync') { - const mongoDb = db.connection.db; - if (!mongoDb) { - throw new Error('MongoDB connection has no database instance after connecting'); - } - - await MongoIndexManager.waitForModelIndexTasks(db.connection); - await MongoIndexManager.runDeprecatedIndexCleanupOnce({ - db: mongoDb, - cleanupKey: MongoIndexManager.getConnectionCleanupKey(db.connection), - apply: true, - logger - }); - } - return db; } catch (error) { logger.error('Mongo connect error', { error }); diff --git a/projects/marketplace/test/env.test.ts b/projects/marketplace/test/env.test.ts index dda4b2fa3620..de24670dc235 100644 --- a/projects/marketplace/test/env.test.ts +++ b/projects/marketplace/test/env.test.ts @@ -1,7 +1,6 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; const originalCommunityAuthToken = process.env.COMMUNITY_AUTH_TOKEN; -const originalMongoIndexSyncMode = process.env.MONGO_INDEX_SYNC_MODE; const importEnv = async () => { vi.resetModules(); @@ -11,45 +10,6 @@ const importEnv = async () => { describe('marketplace env', () => { afterEach(() => { vi.stubEnv('COMMUNITY_AUTH_TOKEN', originalCommunityAuthToken); - vi.stubEnv('MONGO_INDEX_SYNC_MODE', originalMongoIndexSyncMode); - }); - - it('defaults MONGO_INDEX_SYNC_MODE to create when it is not configured', async () => { - vi.stubEnv('MONGO_INDEX_SYNC_MODE', undefined); - - const { marketplaceEnv } = await importEnv(); - - expect(marketplaceEnv.MONGO_INDEX_SYNC_MODE).toBe('create'); - }); - - it('defaults MONGO_INDEX_SYNC_MODE to create when it is empty', async () => { - vi.stubEnv('MONGO_INDEX_SYNC_MODE', ''); - - const { marketplaceEnv } = await importEnv(); - - expect(marketplaceEnv.MONGO_INDEX_SYNC_MODE).toBe('create'); - }); - - it('parses explicit MONGO_INDEX_SYNC_MODE values', async () => { - vi.stubEnv('MONGO_INDEX_SYNC_MODE', 'off'); - - const { marketplaceEnv } = await importEnv(); - - expect(marketplaceEnv.MONGO_INDEX_SYNC_MODE).toBe('off'); - }); - - it('parses destructive MongoDB index sync mode', async () => { - vi.stubEnv('MONGO_INDEX_SYNC_MODE', 'sync'); - - const { marketplaceEnv } = await importEnv(); - - expect(marketplaceEnv.MONGO_INDEX_SYNC_MODE).toBe('sync'); - }); - - it('rejects invalid MONGO_INDEX_SYNC_MODE values', async () => { - vi.stubEnv('MONGO_INDEX_SYNC_MODE', 'full'); - - await expect(importEnv()).rejects.toThrow('Invalid marketplace environment variables'); }); it('parses optional community auth token', async () => { From b5be9ec66c476917618882a13258b8f8c5499907 Mon Sep 17 00:00:00 2001 From: Xianquan Date: Mon, 20 Jul 2026 15:25:19 +0800 Subject: [PATCH 5/5] refactor: reduce mongo index sync logs --- .../common/mongo-index-sync-strategy.md | 8 +- AGENTS.md | 4 +- packages/service/common/mongo/indexManager.ts | 77 +++++-------------- .../test/common/mongo/indexManager.test.ts | 32 +++++--- 4 files changed, 44 insertions(+), 77 deletions(-) diff --git a/.agents/design/common/mongo-index-sync-strategy.md b/.agents/design/common/mongo-index-sync-strategy.md index 86151edffcab..b3c9e8f93850 100644 --- a/.agents/design/common/mongo-index-sync-strategy.md +++ b/.agents/design/common/mongo-index-sync-strategy.md @@ -170,10 +170,10 @@ registerDeprecatedMongoIndexes(ChatSchema, [ ### 4. 日志分级 -- `debug`：每个 model 输出完整 diff、schema 外索引名称和 schema-local 废弃项扫描明细。 -- `info`：每个 model 输出索引同步结果摘要，并记录实际 drop 的废弃索引。 -- `warn`：输出需要运维关注但不阻塞启动的问题，例如检测到 schema 外索引或废弃索引定义不匹配。 -- `error`：输出索引创建或废弃索引清理失败，保留 model、collection 和 error 信息。 +- 无索引变化时不输出日志，避免每个 model 启动时重复打印空结果。 +- `info`：仅在实际创建或删除索引时输出一条精简摘要。 +- `warn`：输出 schema 外索引或废弃索引定义不匹配，只保留 collection 和索引名称。 +- `error`：输出索引创建或废弃索引清理失败，只保留定位所需字段和错误信息。 ### 5. 本次交付边界 diff --git a/AGENTS.md b/AGENTS.md index 6d995f2ed109..b9627d8b1fcb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -92,7 +92,7 @@ FastGPT 是一个 AI Agent 构建平台,通过 Flow 提供开箱即用的数据 ### MongoDB Schema 与索引维护 - 每当新增、修改、删除或重命名 MongoDB/Mongoose Schema 字段、索引定义、唯一约束、TTL、partialFilterExpression、collation 等索引相关配置时，必须同步检查是否有 FastGPT 旧版本创建的索引不再被当前 Schema 使用。 -- 如果历史索引可能继续影响写入约束、查询计划或存储成本，应在所属 Schema 文件中通过 `registerDeprecatedMongoIndexes` 紧邻当前索引声明登记删除定义，并补充/调整 `packages/service/test/common/mongo/indexManager.test.ts` 的清理行为覆盖。定义只包含旧索引的 name、key，以及需要参与精确匹配的关键 options。 +- 如果历史索引可能继续影响写入约束、查询计划或存储成本，应在所属 Schema 文件中通过 `defineDeprecatedIndexes` 紧邻当前索引声明登记删除定义，并补充/调整 `packages/service/test/common/mongo/indexManager.test.ts` 的清理行为覆盖。定义只包含旧索引的 name、key，以及需要参与精确匹配的关键 options。 - 不要登记客户自建索引、无法确认来源的索引，或仅凭当前 Schema 未声明就推断为废弃的索引。主动同步只允许删除 FastGPT 明确创建过、明确废弃且与 Schema 本地声明精确匹配的历史索引。 ### API 入参校验 @@ -145,7 +145,7 @@ function agent_loop(用户需求){ 提出问题，让用户提供答案; 调整需求文档; } - + // 2. 开发文档编写 while(开发文档编写未完成){ 编写开发文档; diff --git a/packages/service/common/mongo/indexManager.ts b/packages/service/common/mongo/indexManager.ts index 0f516fe63d18..b058ddd985cc 100644 --- a/packages/service/common/mongo/indexManager.ts +++ b/packages/service/common/mongo/indexManager.ts @@ -134,19 +134,11 @@ export class MongoIndexManager { logger = defaultLogger }: SyncModelIndexesParams): Promise { const inspection = await MongoIndexManager.inspectModelIndexes(model); - const logData = MongoIndexManager.buildIndexSyncLogData(inspection); - - logger.debug('MongoDB index diff inspected', { - ...logData, - schemaExternalIndexNames: inspection.toDrop, - toCreate: inspection.toCreate - }); if (inspection.toDrop.length > 0) { logger.warn('Detected MongoDB indexes not declared by FastGPT schema', { - ...logData, - schemaExternalIndexNames: inspection.toDrop, - cleanupPolicy: 'only_schema_registered_deprecated_indexes_can_be_dropped' + collectionName: inspection.collectionName, + indexNames: inspection.toDrop }); } @@ -161,17 +153,15 @@ export class MongoIndexManager { ...inspection, cleanupReport }; + const cleanupSummary = MongoIndexManager.summarizeCleanupReport(cleanupReport); - logger.info('MongoDB indexes synchronized', { - ...logData, - cleanupSummary: MongoIndexManager.summarizeCleanupReport(cleanupReport) - }); - logger.debug('MongoDB index synchronization detail', { - ...logData, - schemaExternalIndexNames: inspection.toDrop, - toCreate: inspection.toCreate, - cleanupReport - }); + if (inspection.toCreate.length > 0 || cleanupSummary.dropped > 0) { + logger.info('MongoDB indexes synchronized', { + collectionName: inspection.collectionName, + created: inspection.toCreate.length, + dropped: cleanupSummary.dropped + }); + } return result; } @@ -199,13 +189,6 @@ export class MongoIndexManager { return { apply, items }; } - logger?.debug('MongoDB deprecated index cleanup started', { - modelName: model.modelName, - collectionName, - apply, - deprecatedIndexCount: definitions.length - }); - for (const definition of definitions) { try { const currentIndexes = (await model.collection.indexes().catch((error) => { @@ -223,7 +206,6 @@ export class MongoIndexManager { action: 'skip_missing', reason: 'Deprecated index does not exist' }); - logger?.debug('Deprecated MongoDB index does not exist', item); items.push(item); continue; } @@ -235,12 +217,9 @@ export class MongoIndexManager { action: 'skip_mismatch', reason: 'Index definition does not match Schema declaration' }); - logger?.warn('Skip deprecated MongoDB index cleanup because definition mismatched', { - ...item, - expectedKey: definition.key, - actualKey: targetIndex.key, - expectedOptions: MongoIndexManager.getExpectedOptions(definition), - actualOptions: MongoIndexManager.getActualOptions(targetIndex) + logger?.warn('Deprecated MongoDB index definition mismatched', { + collectionName, + indexName: definition.indexName }); items.push(item); continue; @@ -257,7 +236,6 @@ export class MongoIndexManager { action: 'skip_missing', reason: 'Deprecated index was already removed' }); - logger?.debug('Deprecated MongoDB index was already removed', item); items.push(item); continue; } @@ -272,11 +250,6 @@ export class MongoIndexManager { applied: apply, reason: apply ? 'Deprecated index dropped' : 'Deprecated index can be dropped' }); - if (apply) { - logger?.info('Dropped deprecated MongoDB index', item); - } else { - logger?.debug('Deprecated MongoDB index can be dropped', item); - } items.push(item); } catch (error) { const item = MongoIndexManager.buildCleanupItem({ @@ -286,18 +259,15 @@ export class MongoIndexManager { reason: 'Failed to inspect or cleanup deprecated index', error: MongoIndexManager.getErrorMessage(error) }); - logger?.error('Failed to cleanup deprecated MongoDB index', item); + logger?.error('Failed to cleanup deprecated MongoDB index', { + collectionName, + indexName: definition.indexName, + error: item.error + }); items.push(item); } } - logger?.debug('MongoDB deprecated index cleanup completed', { - modelName: model.modelName, - collectionName, - apply, - summary: MongoIndexManager.summarizeCleanupReport({ apply, items }) - }); - return { apply, items }; } @@ -354,17 +324,6 @@ export class MongoIndexManager { return lines.join('\n'); } - private static buildIndexSyncLogData( - result: Pick - ) { - return { - modelName: result.modelName, - collectionName: result.collectionName, - toCreateCount: result.toCreate.length, - schemaExternalIndexCount: result.toDrop.length - }; - } - private static normalizeForCompare( value: unknown, { sortObjectKeys }: { sortObjectKeys: boolean } diff --git a/packages/service/test/common/mongo/indexManager.test.ts b/packages/service/test/common/mongo/indexManager.test.ts index 3fb706b48ee2..e18949916d8e 100644 --- a/packages/service/test/common/mongo/indexManager.test.ts +++ b/packages/service/test/common/mongo/indexManager.test.ts @@ -65,10 +65,16 @@ describe('MongoIndexManager.syncModelIndexes', () => { ]); expect(logger.warn).toHaveBeenCalledWith( 'Detected MongoDB indexes not declared by FastGPT schema', - expect.objectContaining({ - schemaExternalIndexNames: expect.arrayContaining(['legacy_field_1', 'customer_custom_1']) - }) + { + collectionName: model.collection.collectionName, + indexNames: expect.arrayContaining(['legacy_field_1', 'customer_custom_1']) + } ); + expect(logger.info).toHaveBeenCalledWith('MongoDB indexes synchronized', { + collectionName: model.collection.collectionName, + created: 1, + dropped: 1 + }); }); it('does not delete schema-external indexes when the Schema has no deprecated declarations', async () => { @@ -78,12 +84,15 @@ describe('MongoIndexManager.syncModelIndexes', () => { ); schema.index({ currentField: 1 }, { name: 'current_field_1' }); const model = createModel({ schema }); + await model.collection.createIndex({ currentField: 1 }, { name: 'current_field_1' }); await model.collection.createIndex({ customerField: 1 }, { name: 'customer_custom_1' }); - const result = await MongoIndexManager.syncModelIndexes({ model }); + const result = await MongoIndexManager.syncModelIndexes({ model, logger }); expect(await getIndexNames(model)).toContain('customer_custom_1'); expect(result.cleanupReport.items).toEqual([]); + expect(logger.info).not.toHaveBeenCalled(); + expect(logger.debug).not.toHaveBeenCalled(); }); it('reuses an in-flight task for concurrent calls on the same Model', async () => { @@ -147,10 +156,8 @@ describe('MongoIndexManager.cleanupModelDeprecatedIndexes', () => { expect.objectContaining({ action: 'drop', applied: false, indexName: 'legacy_field_1' }) ]); expect(await getIndexNames(model)).toContain('legacy_field_1'); - expect(logger.debug).toHaveBeenCalledWith( - 'Deprecated MongoDB index can be dropped', - expect.objectContaining({ indexName: 'legacy_field_1' }) - ); + expect(logger.info).not.toHaveBeenCalled(); + expect(logger.debug).not.toHaveBeenCalled(); }); it('preserves same-name indexes when key, key order, or options do not match', async () => { @@ -275,10 +282,11 @@ describe('MongoIndexManager.cleanupModelDeprecatedIndexes', () => { indexName: 'legacy_field_1' }) ]); - expect(logger.error).toHaveBeenCalledWith( - 'Failed to cleanup deprecated MongoDB index', - expect.objectContaining({ error: 'inspection failed' }) - ); + expect(logger.error).toHaveBeenCalledWith('Failed to cleanup deprecated MongoDB index', { + collectionName: model.collection.collectionName, + indexName: 'legacy_field_1', + error: 'inspection failed' + }); }); it('normalizes non-Error cleanup failures into report messages', async () => {