From 0ff466cf1edfe7776e83ea49140e863a4d0c55de Mon Sep 17 00:00:00 2001 From: elioxue Date: Mon, 20 Jul 2026 09:57:08 +0800 Subject: [PATCH] =?UTF-8?q?feat(ragkit):=20=E4=BA=91=E7=AB=AF=E5=90=91?= =?UTF-8?q?=E9=87=8F=E5=90=8E=E7=AB=AF=E5=AE=B9=E9=94=99=E4=B8=8E=20batch?= =?UTF-8?q?=5Fsize=20=E9=80=82=E9=85=8D=EF=BC=880.2.1=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - backend.py: 默认批大小 16→10（DashScope text-embedding-v4 兼容模式硬上限），支持 cloud config 的 batch_size 覆盖 - pipeline.py + ragkit.py: 云端调用失败（密钥失效/网络异常/sidecar 崩溃）捕获为 vector_error，自动降级词汇+元数据路，不再整体崩溃，stderr 透出失败详情 - README.md + query/SKILL.md: 补各平台 API key 设置、强制走云端、batch_size 调优、vector_error 降级信号说明 - tests: 新增降级不崩与 batch 覆盖用例，批大小边界改为 10 - 定版 0.2.1（plugin.json + marketplace.json 同步） Co-Authored-By: Claude Opus 4.8 (1M context) --- .agents/plugins/marketplace.json | 2 +- .claude-plugin/marketplace.json | 2 +- .codebuddy-plugin/marketplace.json | 2 +- .kimi-plugin/marketplace.json | 2 +- plugins/ragkit/.claude-plugin/plugin.json | 2 +- plugins/ragkit/.codebuddy-plugin/plugin.json | 2 +- plugins/ragkit/.codex-plugin/plugin.json | 2 +- plugins/ragkit/.kimi-plugin/plugin.json | 2 +- plugins/ragkit/CHANGELOG.md | 14 +++++ plugins/ragkit/README.md | 62 +++++++++++++++++--- plugins/ragkit/scripts/rag/backend.py | 10 +++- plugins/ragkit/scripts/rag/pipeline.py | 12 +++- plugins/ragkit/scripts/ragkit.py | 3 + plugins/ragkit/skills/query/SKILL.md | 1 + plugins/ragkit/tests/conftest.py | 6 +- plugins/ragkit/tests/test_backend.py | 36 +++++++++++- plugins/ragkit/tests/test_query.py | 21 +++++++ 17 files changed, 159 insertions(+), 22 deletions(-) diff --git a/.agents/plugins/marketplace.json b/.agents/plugins/marketplace.json index 0d2a5c6..428ba60 100644 --- a/.agents/plugins/marketplace.json +++ b/.agents/plugins/marketplace.json @@ -6,6 +6,6 @@ { "name": "specode", "source": "./plugins/specode", "version": "6.5.0", "description": "Lightweight spec-driven workflow orchestration shell — walks a host agent through requirements → design → tasks → execute → acceptance, delegating each phase to superpowers skills with a first-class specode-native fallback, and landing 4 fixed docs per spec (requirements / design / tasks / implementation-log). Bundles dedicated intake and execute skills (the execution tail — selector, engine dispatch, acceptance — is manually triggerable anytime via /specode:execute), a zero-import task-swarm handoff for parallel execution, and optional locate-oriented experience retrieval.", "category": "development", "homepage": "https://github.com/qxbyte/pluginhub", "repository": "https://github.com/qxbyte/pluginhub", "license": "MIT", "keywords": ["spec","specification","workflow"] }, { "name": "task-swarm", "source": "./plugins/task-swarm", "version": "0.12.0", "description": "Standalone multi-agent orchestration driven by a pipeline.yml — semantic task groups with cross-group concurrency, forked coders, and per-group reviewer + validator loops (state.json is the single source of truth). specode delegates its execution phase here; also runnable directly via /task-swarm:swarm.", "category": "development", "homepage": "https://github.com/qxbyte/pluginhub", "repository": "https://github.com/qxbyte/pluginhub", "license": "MIT", "keywords": ["task-swarm","multi-agent","orchestration","pipeline"] }, { "name": "obsidian-wiki","source": "./plugins/obsidian-wiki","version": "2.2.0", "description": "Maintain an Obsidian LLM-Wiki via three skills — a deterministic structure layer (wiki-struct: Home tree / per-dir READMEs / partition pages), content curation (wiki-curate: ingest / curate / lint), and a unified orchestrator (wiki-orchestrate). Generic code + per-vault config in the home-dir registry ~/.config/obsidian-wiki/ (fallback: /.wiki/config.json), zero hardcoded structure.", "category": "productivity", "homepage": "https://github.com/qxbyte/pluginhub", "repository": "https://github.com/qxbyte/pluginhub", "license": "MIT", "keywords": ["obsidian","llm-wiki","knowledge-base","notes","curation"] }, - { "name": "ragkit", "source": "./plugins/ragkit", "version": "0.2.0", "description": "Standalone knowledge-base RAG plugin (vector/lexical/metadata multi-channel recall).", "category": "database", "homepage": "https://github.com/qxbyte/pluginhub", "repository": "https://github.com/qxbyte/pluginhub", "license": "MIT", "keywords": ["rag","knowledge-base","vector-search","embedding"] } + { "name": "ragkit", "source": "./plugins/ragkit", "version": "0.2.1", "description": "Standalone knowledge-base RAG plugin (vector/lexical/metadata multi-channel recall).", "category": "database", "homepage": "https://github.com/qxbyte/pluginhub", "repository": "https://github.com/qxbyte/pluginhub", "license": "MIT", "keywords": ["rag","knowledge-base","vector-search","embedding"] } ] } diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index d4be780..1ebe781 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -58,7 +58,7 @@ { "name": "ragkit", "source": "./plugins/ragkit", - "version": "0.2.0", + "version": "0.2.1", "description": "Standalone knowledge-base RAG plugin (vector/lexical/metadata multi-channel recall).", "category": "database", "homepage": "https://github.com/qxbyte/pluginhub", diff --git a/.codebuddy-plugin/marketplace.json b/.codebuddy-plugin/marketplace.json index 0d2a5c6..428ba60 100644 --- a/.codebuddy-plugin/marketplace.json +++ b/.codebuddy-plugin/marketplace.json @@ -6,6 +6,6 @@ { "name": "specode", "source": "./plugins/specode", "version": "6.5.0", "description": "Lightweight spec-driven workflow orchestration shell — walks a host agent through requirements → design → tasks → execute → acceptance, delegating each phase to superpowers skills with a first-class specode-native fallback, and landing 4 fixed docs per spec (requirements / design / tasks / implementation-log). Bundles dedicated intake and execute skills (the execution tail — selector, engine dispatch, acceptance — is manually triggerable anytime via /specode:execute), a zero-import task-swarm handoff for parallel execution, and optional locate-oriented experience retrieval.", "category": "development", "homepage": "https://github.com/qxbyte/pluginhub", "repository": "https://github.com/qxbyte/pluginhub", "license": "MIT", "keywords": ["spec","specification","workflow"] }, { "name": "task-swarm", "source": "./plugins/task-swarm", "version": "0.12.0", "description": "Standalone multi-agent orchestration driven by a pipeline.yml — semantic task groups with cross-group concurrency, forked coders, and per-group reviewer + validator loops (state.json is the single source of truth). specode delegates its execution phase here; also runnable directly via /task-swarm:swarm.", "category": "development", "homepage": "https://github.com/qxbyte/pluginhub", "repository": "https://github.com/qxbyte/pluginhub", "license": "MIT", "keywords": ["task-swarm","multi-agent","orchestration","pipeline"] }, { "name": "obsidian-wiki","source": "./plugins/obsidian-wiki","version": "2.2.0", "description": "Maintain an Obsidian LLM-Wiki via three skills — a deterministic structure layer (wiki-struct: Home tree / per-dir READMEs / partition pages), content curation (wiki-curate: ingest / curate / lint), and a unified orchestrator (wiki-orchestrate). Generic code + per-vault config in the home-dir registry ~/.config/obsidian-wiki/ (fallback: /.wiki/config.json), zero hardcoded structure.", "category": "productivity", "homepage": "https://github.com/qxbyte/pluginhub", "repository": "https://github.com/qxbyte/pluginhub", "license": "MIT", "keywords": ["obsidian","llm-wiki","knowledge-base","notes","curation"] }, - { "name": "ragkit", "source": "./plugins/ragkit", "version": "0.2.0", "description": "Standalone knowledge-base RAG plugin (vector/lexical/metadata multi-channel recall).", "category": "database", "homepage": "https://github.com/qxbyte/pluginhub", "repository": "https://github.com/qxbyte/pluginhub", "license": "MIT", "keywords": ["rag","knowledge-base","vector-search","embedding"] } + { "name": "ragkit", "source": "./plugins/ragkit", "version": "0.2.1", "description": "Standalone knowledge-base RAG plugin (vector/lexical/metadata multi-channel recall).", "category": "database", "homepage": "https://github.com/qxbyte/pluginhub", "repository": "https://github.com/qxbyte/pluginhub", "license": "MIT", "keywords": ["rag","knowledge-base","vector-search","embedding"] } ] } diff --git a/.kimi-plugin/marketplace.json b/.kimi-plugin/marketplace.json index 0d2a5c6..428ba60 100644 --- a/.kimi-plugin/marketplace.json +++ b/.kimi-plugin/marketplace.json @@ -6,6 +6,6 @@ { "name": "specode", "source": "./plugins/specode", "version": "6.5.0", "description": "Lightweight spec-driven workflow orchestration shell — walks a host agent through requirements → design → tasks → execute → acceptance, delegating each phase to superpowers skills with a first-class specode-native fallback, and landing 4 fixed docs per spec (requirements / design / tasks / implementation-log). Bundles dedicated intake and execute skills (the execution tail — selector, engine dispatch, acceptance — is manually triggerable anytime via /specode:execute), a zero-import task-swarm handoff for parallel execution, and optional locate-oriented experience retrieval.", "category": "development", "homepage": "https://github.com/qxbyte/pluginhub", "repository": "https://github.com/qxbyte/pluginhub", "license": "MIT", "keywords": ["spec","specification","workflow"] }, { "name": "task-swarm", "source": "./plugins/task-swarm", "version": "0.12.0", "description": "Standalone multi-agent orchestration driven by a pipeline.yml — semantic task groups with cross-group concurrency, forked coders, and per-group reviewer + validator loops (state.json is the single source of truth). specode delegates its execution phase here; also runnable directly via /task-swarm:swarm.", "category": "development", "homepage": "https://github.com/qxbyte/pluginhub", "repository": "https://github.com/qxbyte/pluginhub", "license": "MIT", "keywords": ["task-swarm","multi-agent","orchestration","pipeline"] }, { "name": "obsidian-wiki","source": "./plugins/obsidian-wiki","version": "2.2.0", "description": "Maintain an Obsidian LLM-Wiki via three skills — a deterministic structure layer (wiki-struct: Home tree / per-dir READMEs / partition pages), content curation (wiki-curate: ingest / curate / lint), and a unified orchestrator (wiki-orchestrate). Generic code + per-vault config in the home-dir registry ~/.config/obsidian-wiki/ (fallback: /.wiki/config.json), zero hardcoded structure.", "category": "productivity", "homepage": "https://github.com/qxbyte/pluginhub", "repository": "https://github.com/qxbyte/pluginhub", "license": "MIT", "keywords": ["obsidian","llm-wiki","knowledge-base","notes","curation"] }, - { "name": "ragkit", "source": "./plugins/ragkit", "version": "0.2.0", "description": "Standalone knowledge-base RAG plugin (vector/lexical/metadata multi-channel recall).", "category": "database", "homepage": "https://github.com/qxbyte/pluginhub", "repository": "https://github.com/qxbyte/pluginhub", "license": "MIT", "keywords": ["rag","knowledge-base","vector-search","embedding"] } + { "name": "ragkit", "source": "./plugins/ragkit", "version": "0.2.1", "description": "Standalone knowledge-base RAG plugin (vector/lexical/metadata multi-channel recall).", "category": "database", "homepage": "https://github.com/qxbyte/pluginhub", "repository": "https://github.com/qxbyte/pluginhub", "license": "MIT", "keywords": ["rag","knowledge-base","vector-search","embedding"] } ] } diff --git a/plugins/ragkit/.claude-plugin/plugin.json b/plugins/ragkit/.claude-plugin/plugin.json index 295d761..e1e70c1 100644 --- a/plugins/ragkit/.claude-plugin/plugin.json +++ b/plugins/ragkit/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "ragkit", - "version": "0.2.0", + "version": "0.2.1", "description": "Standalone knowledge-base RAG: chunk + embed + multi-channel recall (vector/lexical/metadata, RRF-fused) over distill-style knowledge-base/ directories.", "author": { "name": "xueqiang", diff --git a/plugins/ragkit/.codebuddy-plugin/plugin.json b/plugins/ragkit/.codebuddy-plugin/plugin.json index a55befd..b330dce 100644 --- a/plugins/ragkit/.codebuddy-plugin/plugin.json +++ b/plugins/ragkit/.codebuddy-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "ragkit", - "version": "0.2.0", + "version": "0.2.1", "description": "Standalone knowledge-base RAG: chunk + embed + multi-channel recall (vector/lexical/metadata, RRF-fused) over distill-style knowledge-base/ directories.", "author": { "name": "xueqiang", "email": "xueqiang361@gmail.com" }, "homepage": "https://github.com/qxbyte/pluginhub", diff --git a/plugins/ragkit/.codex-plugin/plugin.json b/plugins/ragkit/.codex-plugin/plugin.json index 169f78e..9be637b 100644 --- a/plugins/ragkit/.codex-plugin/plugin.json +++ b/plugins/ragkit/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "ragkit", - "version": "0.2.0", + "version": "0.2.1", "description": "Standalone knowledge-base RAG: chunk + embed + multi-channel recall (vector/lexical/metadata, RRF-fused) over distill-style knowledge-base/ directories.", "author": { "name": "xueqiang", diff --git a/plugins/ragkit/.kimi-plugin/plugin.json b/plugins/ragkit/.kimi-plugin/plugin.json index 1c39da8..353c579 100644 --- a/plugins/ragkit/.kimi-plugin/plugin.json +++ b/plugins/ragkit/.kimi-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "ragkit", - "version": "0.2.0", + "version": "0.2.1", "description": "Standalone knowledge-base RAG: chunk + embed + multi-channel recall (vector/lexical/metadata, RRF-fused) over distill-style knowledge-base/ directories.", "author": { "name": "xueqiang", diff --git a/plugins/ragkit/CHANGELOG.md b/plugins/ragkit/CHANGELOG.md index 4c40f93..2c51450 100644 --- a/plugins/ragkit/CHANGELOG.md +++ b/plugins/ragkit/CHANGELOG.md @@ -2,6 +2,20 @@ ## Unreleased +## 0.2.1 (2026-07-20) — 云端后端兼容与容错 + +### Fixed + +- **DashScope 兼容模式批大小超限**（`backend.py`）：`_BATCH` 由 16 降为 **10**。通义 `text-embedding-v4` 兼容模式单请求硬上限为 10 条，原值 16 会导致 embed/query 报 `400 InvalidParameter`（`batch size ... should not be larger than 10`）。 +- **云端向量调用失败导致 query 崩溃**（`pipeline.py` + `ragkit.py`）：密钥失效 / 网络抖动 / sidecar 异常时，`_vector_rank` 原会抛异常使整个 query 中断；现捕获为新状态 `vector_error`，**自动降级到词汇+元数据路**，stderr 打印固定提示 + 失败详情，退出码仍为 0，与文档承诺的「无后端时降级仍可用」一致。 +- **Windows 下 CLI 测试编码崩溃**（`tests/conftest.py`）：`run_cli` 显式 `encoding="utf-8"` + 子进程 `PYTHONIOENCODING`，修复 GBK 默认解码对 UTF-8 中文/emoji 输出的 `UnicodeDecodeError`。 + +### Added + +- **云端 `batch_size` 覆盖**（`backend.py`）：`cloud` 配置可加 `"batch_size": ` 覆盖默认 10，供 OpenAI 等高上限端点调大以减少请求次数。 +- **文档：API key 全局设置 + 强制走云端**（`README.md`、`skills/query/SKILL.md`）：补充各平台临时/永久设密钥命令（含 Windows `setx`）、本地已缓存时用 `"backend": "cloud"` 强制云端、`vector_error` 降级信号说明。 +- **测试**：`test_pipeline_degrades_on_vector_backend_error`（降级不崩）、`test_cloud_encode_respects_batch_size_override`（batch 覆盖切分），批大小边界测试同步改为 `_BATCH=10`。共 47 passed。 + ## 0.2.0 (2026-07-20) — 多宿主适配：bootstrap 去宿主绑定 + CodeBuddy/Codex/Kimi 独立 manifest ragkit 现在同时面向 Claude Code / CodeBuddy / Codex / Kimi 四个宿主。 diff --git a/plugins/ragkit/README.md b/plugins/ragkit/README.md index 921ff71..7f64f0a 100644 --- a/plugins/ragkit/README.md +++ b/plugins/ragkit/README.md @@ -44,18 +44,66 @@ export DASHSCOPE_API_KEY=<你的密钥> 内置 preset（`--provider` 可选值）： -| preset | model | key_env | -| --- | --- | --- | -| `openai` | text-embedding-3-small | `OPENAI_API_KEY` | -| `qwen` | text-embedding-v4 | `DASHSCOPE_API_KEY` | -| `zhipu` | embedding-3 | `ZHIPUAI_API_KEY` | -| `voyage` | voyage-3 | `VOYAGE_API_KEY` | -| `azure` | text-embedding-3-small | `AZURE_OPENAI_API_KEY` | +| preset | model | key_env | 单请求上限 | +| --- | --- | --- | --- | +| `openai` | text-embedding-3-small | `OPENAI_API_KEY` | 高（可调大 `batch_size`） | +| `qwen` | text-embedding-v4 | `DASHSCOPE_API_KEY` | **10 条/请求**（硬上限） | +| `zhipu` | embedding-3 | `ZHIPUAI_API_KEY` | 视文档 | +| `voyage` | voyage-3 | `VOYAGE_API_KEY` | 视文档 | +| `azure` | text-embedding-3-small | `AZURE_OPENAI_API_KEY` | 高 | 自定义端点：`--base-url --model --key-env `（任何 OpenAI 兼容接口均可）。 密钥只通过环境变量传入，**不落盘**。 +#### 设置 API key（按平台） + +密钥读取自 `key_env` 指定的环境变量。**临时**（仅当前终端会话有效）： + +```sh +# macOS / Linux (bash/zsh) +export DASHSCOPE_API_KEY=<你的密钥> +``` +```powershell +# Windows PowerShell +$env:DASHSCOPE_API_KEY = "<你的密钥>" +``` + +**永久 / 全局**（新开终端也生效，推荐）： + +```powershell +# Windows —— 写入用户环境变量，设完需【新开】终端才生效 +setx DASHSCOPE_API_KEY "<你的密钥>" +``` +```sh +# macOS / Linux —— 追加到 shell 配置后 source 生效 +echo 'export DASHSCOPE_API_KEY=<你的密钥>' >> ~/.zshrc && source ~/.zshrc +``` + +> ⚠️ 未设置密钥时后端解析会跳过云端；若本地模型也未缓存则降级为词汇+元数据路。 +> query 时云端调用失败（密钥失效/网络异常）不再崩溃，**自动降级**并在 stderr 提示。 + +#### 强制走云端（本地模型已缓存时） + +后端解析优先级为「本地已缓存 > 云端」。**若本地已装过模型、但想强制用云端**，在 `<知识库路径>/.ragkit/config.json` 显式指定： + +```json +{ + "backend": "cloud", + "cloud": { "provider": "qwen", "base_url": "...", "model": "text-embedding-v4", "key_env": "DASHSCOPE_API_KEY" } +} +``` + +#### 单请求批大小（batch_size） + +云端每次请求最多上传 `batch_size` 条文本，默认 **10**（DashScope text-embedding-v4 兼容模式硬上限，超出返回 `400 InvalidParameter`）。OpenAI 等端点上限更高，可在 `cloud` 配置里加 `"batch_size": 64` 减少请求次数、加快 embed： + +```json +{ "cloud": { "provider": "openai", "...": "...", "batch_size": 64 } } +``` + +> 切换后端或改 batch 后，需 `embed --rebuild` 重建索引（向量维度/模型空间不同，旧向量不可混用）。 + ### 无 uv 时的回退 仅需 numpy，可用系统/项目虚拟环境回退： diff --git a/plugins/ragkit/scripts/rag/backend.py b/plugins/ragkit/scripts/rag/backend.py index 852945d..f2c205b 100644 --- a/plugins/ragkit/scripts/rag/backend.py +++ b/plugins/ragkit/scripts/rag/backend.py @@ -12,7 +12,10 @@ DEFAULT_LOCAL_MODEL = "Qwen/Qwen3-Embedding-0.6B" EXIT_NO_BACKEND = 3 -_BATCH = 16 +# 单请求最多上传多少条文本。默认 10：DashScope（通义 text-embedding-v4）兼容模式 +# 硬上限即 10 条，超出返回 400 InvalidParameter。OpenAI 等端点上限更高，可在 +# 云端 config 里加 "batch_size": 覆盖以减少请求次数。 +_BATCH = 10 PRESETS = { "openai": {"base_url": "https://api.openai.com/v1", @@ -151,9 +154,10 @@ def _encode_dummy(texts: list[str]): def _encode_cloud(opts: dict, texts: list[str]): url = opts["base_url"].rstrip("/") + "/embeddings" key = os.environ[opts["key_env"]] + batch_size = int(opts.get("batch_size") or _BATCH) vecs: list = [] - for start in range(0, len(texts), _BATCH): - batch = texts[start:start + _BATCH] + for start in range(0, len(texts), batch_size): + batch = texts[start:start + batch_size] req = urllib.request.Request( url, method="POST", headers={"Authorization": f"Bearer {key}", diff --git a/plugins/ragkit/scripts/rag/pipeline.py b/plugins/ragkit/scripts/rag/pipeline.py index 379fd24..db2d93c 100644 --- a/plugins/ragkit/scripts/rag/pipeline.py +++ b/plugins/ragkit/scripts/rag/pipeline.py @@ -10,6 +10,9 @@ DEFAULT_TOP = 8 ALL_CHANNELS = ["lexical", "metadata", "vector"] +# _vector_rank 里捕获到的向量后端异常详情，供 query_pipeline 透出到输出（stderr 提示用）。 +_VECTOR_ERROR_DETAIL: dict = {"msg": ""} + def query_pipeline(kb_root: Path, query: str, top: int = DEFAULT_TOP, channels_filter: list[str] | None = None) -> dict: @@ -37,11 +40,14 @@ def query_pipeline(kb_root: Path, query: str, top: int = DEFAULT_TOP, rankings["metadata"] = [k for k, _ in md] channel_scores["metadata"] = dict(md) if "vector" in active: + _VECTOR_ERROR_DETAIL["msg"] = "" state, vec = _vector_rank(kb_root, chunks, focus) out["vector_channel"] = state if state == "ok": rankings["vector"] = [k for k, _ in vec] channel_scores["vector"] = dict(vec) + elif state == "vector_error": + out["vector_error_detail"] = _VECTOR_ERROR_DETAIL["msg"] fused = fuse.rrf_fuse(rankings)[:top] for row in fused: d = docs[row["knowledge_id"]] @@ -69,7 +75,11 @@ def _vector_rank(kb_root: Path, chunks: list[dict], focus_text: str): return "no_backend", [] if backend.model_id(kind, opts) != stored: return "model_mismatch", [] - q = backend.encode(kind, opts, [focus_text])[0] + try: + q = backend.encode(kind, opts, [focus_text])[0] + except Exception as exc: # 云端认证失败/网络抖动/sidecar 崩溃：不整体崩，降级到词汇+元数据路 + _VECTOR_ERROR_DETAIL["msg"] = f"{type(exc).__name__}: {exc}"[:200] + return "vector_error", [] sims = vectors @ q order = np.argsort(-sims)[:TOP_CHUNK_HITS] best: dict[str, float] = {} diff --git a/plugins/ragkit/scripts/ragkit.py b/plugins/ragkit/scripts/ragkit.py index f98a6ed..f9bff36 100644 --- a/plugins/ragkit/scripts/ragkit.py +++ b/plugins/ragkit/scripts/ragkit.py @@ -24,6 +24,7 @@ "no_vectors": "RagKit：索引缺向量（上次 embed 无可用后端），已降级词汇+元数据路；补好后端后重跑 embed。", "no_backend": "RagKit：无可用向量后端，已降级词汇+元数据路。", "model_mismatch": "RagKit：当前后端与索引向量模型不一致，向量路已跳过；请 embed --rebuild。", + "vector_error": "RagKit：向量后端调用失败（密钥无效/网络异常等），已降级词汇+元数据路；结果仍有效。请检查密钥或网络后重试。", "skipped": "", } @@ -42,6 +43,8 @@ def cmd_query(args: argparse.Namespace) -> int: print(note, file=sys.stderr) if out["vector_channel"] == "no_backend": print(backend.no_backend_block(), file=sys.stderr) + elif out["vector_channel"] == "vector_error" and out.get("vector_error_detail"): + print(f" 详情：{out['vector_error_detail']}", file=sys.stderr) if args.json: print(json.dumps(out, ensure_ascii=False, indent=1)) else: diff --git a/plugins/ragkit/skills/query/SKILL.md b/plugins/ragkit/skills/query/SKILL.md index 4e86d24..e724ffc 100644 --- a/plugins/ragkit/skills/query/SKILL.md +++ b/plugins/ragkit/skills/query/SKILL.md @@ -29,5 +29,6 @@ sh ../../scripts/run.sh ../../scripts/ragkit.py \ ## 降级与错误（stderr 信号，固定文案） - `无可用向量后端`：结果仍有效（词汇+元数据路）；stderr 会带一个 ╭─ RagKit ─╮ 提示块，**原样转述给用户**（不改写、不省略）。 +- `向量后端调用失败`（`vector_error`）：云端密钥失效/网络异常等，query **不崩溃**，已自动降级词汇+元数据路，结果仍有效；stderr 带失败详情，转述并提示用户检查密钥（环境变量是否设对、是否用了完整 key）或网络后重试。 - `索引不存在`：提示用户先跑 `/ragkit:embed`。 - `model_mismatch` / `index_stale`：提示用户重跑 `/ragkit:embed`（必要时 `--rebuild`）。 diff --git a/plugins/ragkit/tests/conftest.py b/plugins/ragkit/tests/conftest.py index a74dacb..bd453a4 100644 --- a/plugins/ragkit/tests/conftest.py +++ b/plugins/ragkit/tests/conftest.py @@ -75,9 +75,13 @@ def run_cli(): """Invoke ragkit.py as a CLI subprocess with the (monkeypatched) env.""" def _run(*args): + # Windows 默认用 GBK 解码子进程输出，遇到 UTF-8 中文/emoji 会 UnicodeDecodeError； + # 显式 encoding=utf-8 + 子进程 PYTHONIOENCODING 保证跨平台一致。 + env = os.environ.copy() + env["PYTHONIOENCODING"] = "utf-8" return subprocess.run( [sys.executable, str(SCRIPTS / "ragkit.py"), *args], - capture_output=True, text=True, env=os.environ.copy(), + capture_output=True, text=True, encoding="utf-8", env=env, ) return _run diff --git a/plugins/ragkit/tests/test_backend.py b/plugins/ragkit/tests/test_backend.py index 51b1314..e4fbeb8 100644 --- a/plugins/ragkit/tests/test_backend.py +++ b/plugins/ragkit/tests/test_backend.py @@ -76,6 +76,24 @@ def log_message(self, *a): pass +class _CountingHandler(BaseHTTPRequestHandler): + """记录每个请求的 input 条数，验证 batch_size 切分。""" + batch_sizes: list = [] + + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + _CountingHandler.batch_sizes.append(len(body["input"])) + data = [{"index": i, "embedding": [1.0, float(i)]} for i in range(len(body["input"]))] + payload = json.dumps({"data": data}).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *a): + pass + + def test_cloud_encode_openai_compat(monkeypatch): _Handler.seen_auth.clear() server = HTTPServer(("127.0.0.1", 0), _Handler) @@ -97,8 +115,22 @@ def test_cloud_encode_batching_and_index_order(monkeypatch): monkeypatch.setenv("TEST_RAGKIT_KEY", "sk-test") opts = {"base_url": f"http://127.0.0.1:{server.server_port}", "model": "m", "key_env": "TEST_RAGKIT_KEY"} - texts = [f"text-{i}" for i in range(17)] # crosses the _BATCH=16 boundary + texts = [f"text-{i}" for i in range(23)] # crosses the _BATCH=10 boundary (3 批) vecs = backend.encode("cloud", opts, texts) server.shutdown() - assert vecs.shape == (17, 2) + assert vecs.shape == (23, 2) assert np.allclose(np.linalg.norm(vecs, axis=1), 1.0, atol=1e-5) + + +def test_cloud_encode_respects_batch_size_override(monkeypatch): + """config 里的 batch_size 覆盖默认 _BATCH（如 OpenAI 端点可调大）。""" + _Handler.seen_auth.clear() + _Handler.batch_sizes = [] + server = HTTPServer(("127.0.0.1", 0), _CountingHandler) + threading.Thread(target=server.serve_forever, daemon=True).start() + monkeypatch.setenv("TEST_RAGKIT_KEY", "sk-test") + opts = {"base_url": f"http://127.0.0.1:{server.server_port}", + "model": "m", "key_env": "TEST_RAGKIT_KEY", "batch_size": 5} + backend.encode("cloud", opts, [f"t-{i}" for i in range(12)]) # 12 条 / 5 = 3 批 + server.shutdown() + assert _CountingHandler.batch_sizes == [5, 5, 2] diff --git a/plugins/ragkit/tests/test_query.py b/plugins/ragkit/tests/test_query.py index 6611bd0..1949fbf 100644 --- a/plugins/ragkit/tests/test_query.py +++ b/plugins/ragkit/tests/test_query.py @@ -37,6 +37,27 @@ def test_pipeline_no_index(kb): assert out["results"] == [] +def test_pipeline_degrades_on_vector_backend_error(kb, monkeypatch): + """云端认证失败/网络抖动时不整体崩溃，降级到词汇+元数据路。""" + _build_index(kb) # 索引带 dummy 向量，backend 解析仍为 dummy + + def _boom(*a, **k): + raise urllib_error() + + monkeypatch.setattr(backend, "encode", _boom) + out = query_pipeline(kb, "银行账号脱敏规则") + assert out["vector_channel"] == "vector_error" + assert "vector_error_detail" in out + # 关键：词汇/元数据路仍返回结果，query 不崩 + assert out["results"][0]["knowledge_id"] == "114371-mask-rule" + assert "vector" not in out["results"][0]["ranked_by"] + + +def urllib_error(): + import urllib.error + return urllib.error.HTTPError("http://x/embeddings", 401, "Unauthorized", {}, None) + + def test_cli_query_json(kb, run_cli): _build_index(kb) res = run_cli("query", "银行账号脱敏", "--kb", str(kb), "--json")