From 6706f8cc126b8220317a153f58d200bbe7fde2e6 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 21 Aug 2026 16:36:49 +0900 Subject: [PATCH 1/4] Add partial family continuation (#5101) --- DEVELOPER_GUIDE.md | 2 + TESTING_GUIDE.md | 4 + USER_GUIDE.md | 8 +- changelog.d/unreleased/5101.added.md | 25 ++ src/CodeIndex/Cli/CliFlagSchema.cs | 2 +- .../Cli/JsonEnvelopeWrapper.Bounded.cs | 169 ++++++++++- src/CodeIndex/Cli/ProjectionFieldRegistry.cs | 8 +- .../Cli/QueryCommandRunner.Discovery.cs | 91 +++++- .../Database/DbReader.SymbolSearchList.cs | 10 +- .../DbReader.SymbolSearchListQueryBuilder.cs | 2 +- .../DbReader.SymbolSearchMultiQuery.cs | 4 +- .../DbReader.SymbolSearchQueryPlan.cs | 2 + .../Database/DbReader.SymbolSearchSql.cs | 36 ++- src/CodeIndex/Models/QueryResults.cs | 14 + ...PartialFamilyContinuationIssue5101Tests.cs | 277 ++++++++++++++++++ 15 files changed, 626 insertions(+), 28 deletions(-) create mode 100644 changelog.d/unreleased/5101.added.md create mode 100644 tests/CodeIndex.Tests/QueryCommandRunnerPartialFamilyContinuationIssue5101Tests.cs diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index afcdd3b67..0bc16053a 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -2218,6 +2218,7 @@ catalog and unknown-field errors are generated from that same schema. | Compact location envelope | CLI `--format compact` location output uses a versioned envelope with `api_version`, returned `count`, conservative limit-based `truncated` / `truncation` metadata, applied `query_context`, and lightweight `results` rows. | | Grouped search totals | `search --format grouped` derives `total_matches` / `matched_count`, `total_groups`, and `total_files` from the complete bounded query rather than the displayed page. `grouped_match_count` counts rows supplied to returned groups, `emitted_match_count` counts rows left after per-file grouping limits, and `omitted_match_count`, `truncated`, `has_more`, and `continuation_action` describe incomplete output. | | Bounded high-volume responses | `search`, `definition`, `find`, `status`, `hotspots`, `references`, `callers`, `callees`, `symbols`, `files`, `languages`, `impact`, and `map` accept shared bounded-response controls where their schema exposes them. Newly emitted opaque `--cursor ` values bind the offset to the command/query/filter selection and index generation; legacy `response:v1::` cursors remain accepted for transition. Reuse with changed selection or generation fails with restart-required guidance. `search --format compact`, `symbols --format compact`, and `files --format compact` auto-select the bounded contract, while `search --json=array --json-envelope` provides the opt-in array envelope and `languages --json` selects it when paging or `--max-json-bytes` is requested. Existing compact roots and location rows remain compatible while adding shared metadata. Metadata reports `returned_count`, authoritative `total_count` where available, `omitted_count`, `remaining_count`, `cursor_offset`, `page_limit`, `has_more`, `next_cursor`, `result_stable_at`, `pagination_window_limit`, and `pagination_window_exhausted`. The safety window is 10,000 rows; exhaustion suppresses `next_cursor` rather than returning a cursor that the next request would reject. Pageable commands pass the cursor offset into their database/scan layer instead of serializing an `offset + limit` prefix. `find --all` partial scans encode the next path/line in the opaque cursor so replay continues after the last scanned line. `hotspots` and `impact` page their active primary nested collection as `results`, identify it with `metadata.primary_collection`, and retain scalar/container evidence in `metadata.response_context`; dotted fields such as `callers.path,callers.depth` select that collection and project its rows. The final newline is included in `--max-json-bytes`, and trailing whole rows are removed until the complete envelope fits. `definition` remains metadata-only by default; explicit `--body` content is retained for `body`, `body_content`, or `all`, and suppressed when the projection excludes it. `map --sections` remains its section-level projection, while dotted bounded fields page a selected array section with section-specific totals and scalar projections skip unused ranked arrays. | +| Grouped partial-family continuation | A grouped `symbols` row reports authoritative family-member total, returned, omitted, and remaining counts while materializing at most 50 members. Its opaque recovery/next cursors bind the normalized symbol selection, family key/ID, member offset, and index generation. SQL applies the member offset before JSON aggregation, so continuation never loads the complete family into managed memory. Compact and projected output preserve the recovery metadata even when they omit `family_members`; array, NDJSON, envelope, and byte-bounded paths serialize the same row contract. | | Bounded outline responses | `outline` opts into the shared bounded-response contract only when `--max-json-bytes` is present. The wrapper extracts complete projected symbol rows, preserves hierarchy and deterministic order, reports authoritative returned / total / omitted counts, includes the final newline in its UTF-8 measurement, and emits a bound `response:v2` continuation cursor. An undersized minimum envelope produces one typed `E028_RESPONSE_BUDGET_TOO_SMALL` object on stdout with empty stderr and actionable byte fields. Uncapped outline JSON retains its existing root shape and outline cursor contract. | | Bounded unused responses | `unused` opts into the shared bounded-response contract only when `--max-json-bytes` is present. The wrapper extracts the canonical `symbols` rows, applies the cursor offset in the unused query layer, recomputes returned bucket / confidence / contract-domain counts after byte trimming, and includes an optional `by_bucket` view in the same whole-response UTF-8 budget. Compact mode projects smaller audit rows. Continuation cursors bind the effective audit filters, bucket mode, ordering, and index generation; an undersized one-row envelope returns one typed `E028_RESPONSE_BUDGET_TOO_SMALL` object on stdout with empty stderr and actionable byte fields. Uncapped JSON, compact summaries, and the legacy unused cursor remain unchanged. | | MCP outline pages | MCP `outline` routes `fields`, `sort`, `limit`, and `cursor` through `QueryCommandRunner.BuildOutlinePage`, so its projection aliases, derived sort fields, stable tie-breakers, `page:v1` query fingerprint, and generation validation remain the CLI outline contract rather than a second MCP-specific implementation. The default page is 100 rows and the MCP-wide maximum is 200. `maxBytes` measures the fully enriched serialized `structuredContent`; a binary search rebuilds the page with fewer complete rows and therefore regenerates `next_cursor` from the actual returned count. A budget that cannot hold metadata plus one row fails instead of returning a zero-progress cursor. Default MCP symbol serialization remains backward-compatible, while explicit projection fields use the CLI snake_case names. | @@ -6152,6 +6153,7 @@ total / cursor / truncation、partial-family metadata、definition body の pagi | compact location envelope | CLI の `--format compact` location output は、`api_version`、返却 `count`、limit 到達を基準にした保守的な `truncated` / `truncation` metadata、適用済み `query_context`、軽量な `results` row を持つ versioned envelope です。 | | grouped search の総数 | `search --format grouped` の `total_matches` / `matched_count`、`total_groups`、`total_files` は、表示 page ではなく上限適用前の query 全体から算出します。`grouped_match_count` は返却 group に渡した row 数、`emitted_match_count` は file ごとの上限適用後に残った row 数を表し、`omitted_match_count`、`truncated`、`has_more`、`continuation_action` が未完了出力を示します。 | | 高ボリューム応答の bounded 契約 | `search`、`definition`、`find`、`status`、`hotspots`、`references`、`callers`、`callees`、`symbols`、`files`、`languages`、`impact`、`map` は、それぞれの schema が公開する共通 bounded-response control に対応します。新しく発行する opaque な `--cursor ` は offset を command / query / filter と index generation に束縛し、移行用に legacy の `response:v1::` も受理します。選択条件または generation を変えて再利用すると restart-required の案内付きで失敗します。`search --format compact`、`symbols --format compact`、`files --format compact` は bounded 契約を自動選択し、`search --json=array --json-envelope` は opt-in の array envelope、`languages --json` は paging または `--max-json-bytes` 指定時に同じ契約を使います。既存 compact の root と location row は維持したまま共通 metadata を追加します。metadata は `returned_count`、取得可能な場合は authoritative な `total_count`、`omitted_count`、`remaining_count`、`cursor_offset`、`page_limit`、`has_more`、`next_cursor`、`result_stable_at`、`pagination_window_limit`、`pagination_window_exhausted` を返します。safety window は 10,000 row で、上限到達時は次の request が拒否する cursor を返さず `next_cursor` を抑止します。pageable command は `offset + limit` 件を serialize せず、cursor offset を database / scan layer へ渡します。`find --all` の partial scan cursor は次の path / line を保持し、再利用時は最後に scan した line の次から継続します。`hotspots` と `impact` は active な主要 nested collection を `results` としてページングし、`metadata.primary_collection` でその名前を示し、scalar / container evidence は `metadata.response_context` に保持します。`callers.path,callers.depth` のような dotted field で collection と row field を同時に選べます。`--max-json-bytes` は最後の改行を含み、完全な envelope が収まるまで末尾の完全な row を省略します。`definition` は既定で metadata-only のままで、明示的な `--body` は `body`、`body_content`、`all` で保持し、それ以外の projection では materialize 前に抑止します。`map --sections` は section-level projection として残り、dotted な bounded field は選択した array section を section 固有の総件数付きでページングし、scalar projection は不要な ranking array を構築しません。 | +| grouped partial-family の継続取得 | grouped `symbols` row は authoritative な family-member の総数、返却数、省略数、残数を報告し、materialize する member を最大 50 件に保ちます。opaque な recovery / next cursor は正規化済み symbol 選択、family key / ID、member offset、index generation に束縛されます。SQL が JSON aggregation 前に member offset を適用するため、継続取得時も family 全体を managed memory へ読み込みません。compact / projected 出力は `family_members` を省略しても recovery metadata を維持し、array、NDJSON、envelope、byte-bounded の各経路は同じ row 契約を serialize します。 | | bounded outline 応答 | `outline` は `--max-json-bytes` がある場合だけ共通 bounded-response 契約を選択します。wrapper は projection 済みの完全な symbol row を抽出し、階層と決定的な順序を維持し、authoritative な返却 / 総 / 省略件数を報告します。UTF-8 計測には最後の改行を含め、束縛済みの `response:v2` continuation cursor を発行します。最小 envelope が収まらない場合は、実行可能な byte field を持つ型付きの `E028_RESPONSE_BUDGET_TOO_SMALL` object 1 件を stdout に出し、stderr を空に保ちます。上限なしの outline JSON は既存の root shape と outline cursor 契約を維持します。 | | bounded unused 応答 | `unused` は `--max-json-bytes` がある場合だけ共通 bounded-response 契約を選択します。wrapper は canonical な `symbols` row を抽出し、unused query layer で cursor offset を適用し、byte trimming 後の返却 bucket / confidence / contract-domain 件数を再計算します。任意の `by_bucket` view も同じ応答全体の UTF-8 budget に含め、compact mode はより小さな audit row へ projection します。continuation cursor は有効な audit filter、bucket mode、ordering、index generation に束縛されます。1 row を含む最小 envelope が収まらない場合は、実行可能な byte field を持つ型付きの `E028_RESPONSE_BUDGET_TOO_SMALL` object 1 件を stdout に出し、stderr を空に保ちます。上限なしの JSON、compact summary、legacy unused cursor は既存契約を維持します。 | | MCP outline page | MCP `outline` は `fields`、`sort`、`limit`、`cursor` を `QueryCommandRunner.BuildOutlinePage` へ渡すため、projection alias、派生 sort field、安定した tie-breaker、`page:v1` query fingerprint、generation validation は第 2 の MCP 固有実装ではなく CLI outline 契約のままです。既定 page は 100 row、MCP 共通の上限は 200 row です。`maxBytes` は enrichment 済みの `structuredContent` 全体を serialize した byte 数で計測し、binary search で完全な row 数を減らして page を再構築するため、`next_cursor` も実際の返却件数から再生成されます。metadata と 1 row が収まらない budget は、進捗しない cursor を返さず失敗します。既定の MCP symbol serialization は後方互換を維持し、明示的な projection field は CLI の snake_case 名を使います。 | diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 4e609aef5..4faacdea8 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -638,6 +638,8 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result keeps explicit constructors split across partial declarations in one fixture. Preserve the separate `definition` / `goto --kind function` assertions and the grouped `--kind class --group-partials` assertion so callable constructor results cannot merge back into logical type-family results. - `QueryCommandRunnerTests.PartialCanonicalRepresentative_UsesSemanticRulesAndExposesFamilyNavigation_Issue4914`, `PartialCanonicalRepresentative_GroupsSplitModifierAndRanksLeadingEvidence_Issue4914`, `PartialCanonicalRepresentative_PersistsSplitModifierAndLeadingSemanticEvidence_Issue4914`, `PartialCanonicalRepresentative_RespectsFileLocalAndLexedEvidence_Issue4914`, `PartialCanonicalRepresentative_HonorsCaseSensitiveProjectMarkerCasing_Issue4914Review`, `GroupPartials_DegradesSafelyWhenCSharpFamilyContractIsStale_Issue4914`, and `PartialCanonicalRepresentative_FallsBackToDesignerPathForOldDatabaseMetadata_Issue4914` keep canonical partial-family ranking and navigation in one contract. Preserve partial classes, records (stored under the existing C# class kind) whose base lists repeat the declared name, file-local partial hosts that group only within one source file, partial types and containing generic types with distinct arity, case-sensitive project-marker casing when resolving nullable custom types, and declaration/implementation partial methods whose parameter attributes, names, quoted or commented defaults, declaration/type-position comments, comments between the identifier/generic parameters/parameter list, predefined and `dynamic` / `object` aliases, nullable value-type equivalents, predefined reference-type nullable annotations, explicitly global-rooted framework aliases with verbatim segments, shadowable unrooted `System` types, meaningful global-root qualification, qualified type leaves that collide with method generic-parameter names, verbatim method or type escapes, or user-type casing differ; lexer-confirmed identifier columns and semantic evidence when same-name attribute calls or documentation-like block-comment decoys precede the declaration; preceding-line `partial` modifiers across blank or C# comment trivia, modifiers after a balanced same-line attribute list, attributes bound only to their same-line declaration occurrence, repeated same-line declarations with distinct navigation columns, and blank-detached XML documentation that must not rank across extraction-worker transport, post-extraction hook cloning, and DB persistence; conservative physical-row fallback for stale family metadata whose arity-less keys would otherwise collide, while the Issue4850 LSP constructor fixture still distinguishes stale-index partial type targets; the non-C# function-column fallback; generated metadata and designer-name fallback; reversed insertion order and rebuild-stable family IDs; a semantic primary that remains primary after a path rename; normalized identifier-aligned member columns and normalized verbatim-identifier navigation ranges; a bounded family list that retains a late-sorting representative; grouped `definition` metadata; canonical `goto`; and physical `goto --all`. Keep the legacy-column fixture separate so schema initialization cannot silently restore the metadata being tested as absent. +- `QueryCommandRunnerTests.GroupPartials_FamilyMembersContinueWithoutGapsAndRejectInvalidCursors_Issue5101` + keeps two same-named families separate while walking a 105-member family in stable 50-member pages without duplicates or gaps. Preserve recovery replay, compact metadata without the nested list, metadata-only byte-bounded recovery, raw NDJSON, field discovery, and malformed, selection-mismatch, and stale-generation cursor failures in the same fixture; perform the generation mutation last. - `LspServerTests.cs` LSP JSON-RPC behavior and indexed editor semantics. Reference parity coverage must compare LSP locations with the matching CLI candidate bundle, including overload identity and both `includeDeclaration` states. Keep document/workspace identifier ranges and explicit-versus-inferred inlay hints in one source-semantics fixture so persisted-column anchoring, source confirmation, and hint suppression cannot drift independently. Lifecycle transport coverage must stage initialization before sending shutdown/exit, use a caller-owned stream whose read after the exit frame cannot be interrupted by cancellation, and assert the shutdown response, success versus exit-before-shutdown status, and absence of any post-exit read. Keep finite EOF, malformed-frame, and caller-cancellation coverage alongside it so early transport completion cannot weaken those paths. @@ -1729,6 +1731,8 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" は partial declaration の別ファイルに分かれた明示 constructor を 1 つの fixture にまとめます。callable constructor result が logical type-family result に再び混在しないよう、`definition` / `goto --kind function` の検証と、`--kind class --group-partials` による grouped type の検証を分離したまま維持してください。 - `QueryCommandRunnerTests.PartialCanonicalRepresentative_UsesSemanticRulesAndExposesFamilyNavigation_Issue4914`、`PartialCanonicalRepresentative_GroupsSplitModifierAndRanksLeadingEvidence_Issue4914`、`PartialCanonicalRepresentative_PersistsSplitModifierAndLeadingSemanticEvidence_Issue4914`、`PartialCanonicalRepresentative_RespectsFileLocalAndLexedEvidence_Issue4914`、`GroupPartials_DegradesSafelyWhenCSharpFamilyContractIsStale_Issue4914`、`PartialCanonicalRepresentative_FallsBackToDesignerPathForOldDatabaseMetadata_Issue4914` は canonical な partial-family ranking と navigation を1つの契約として維持します。partial class、base list で宣言名が再登場する record（既存の C# class kind として保存）、同一 source file 内だけで集約する file-local partial host、arity が異なる partial type 自身と外側 generic type、parameter attribute・名前・quote または comment を含む default・declaration / type 内の comment・identifier / generic parameter / parameter list 間の comment・predefined alias と `dynamic` / `object` alias・nullable value type の同値表記・predefined reference type の nullable annotation・verbatim segment を含む明示的に global-rooted な framework alias・shadow 可能な root のない `System` type・意味のある global-root 修飾・method generic parameter 名と衝突する qualified type leaf・method または type の verbatim escape・user type の大文字小文字が異なる宣言 / 実装の partial method、同名 attribute call や documentation 風 block-comment decoy が宣言より前にある場合も lexer で確認した identifier column と semantic evidence を使うこと、C# comment を含む前の行へ分割された `partial` 修飾子、balanced な同一行 attribute list 後の modifier、正しい同一行 declaration occurrence だけに属する attribute、別々の navigation column を持つ同一行の反復宣言、先行 XML documentation が extraction-worker 搬送、post-extraction hook の clone、DB 保存を越えて残ること、blank-detached XML documentation は rank しないこと、arity のない key が衝突する stale family metadata では物理 row へ保守的に fallback しつつ Issue4850 の LSP constructor fixture では stale-index の partial type target を区別すること、non-C# function の column fallback、generated metadata と designer filename fallback、挿入順を逆転した再構築でも安定する family ID、path rename 後も primary であり続ける semantic declaration、正規化後の identifier に揃った member column と正規化後の verbatim identifier に揃った navigation range、遅い path の representative を保持する上限付き family list、grouped `definition` metadata、canonical `goto`、物理位置を返す `goto --all` を維持してください。欠落 metadata が schema initialization によって暗黙に復元されないよう、legacy-column fixture は独立したままにしてください。 +- `QueryCommandRunnerTests.GroupPartials_FamilyMembersContinueWithoutGapsAndRejectInvalidCursors_Issue5101` + は同名の2 family を分離したまま、105 member の family を50件ずつ安定順で走査し、重複も欠落もないことを検証します。同じ fixture で recovery replay、nested list を省略する compact metadata、metadata-only の byte-bounded recovery、raw NDJSON、field discovery、malformed・選択条件 mismatch・index generation stale の cursor failure を維持し、generation の変更は最後に行ってください。 - `LspServerTests.cs` LSP の JSON-RPC 挙動と indexed editor semantics のテスト。reference parity coverage では、overload identity と `includeDeclaration` の両状態を含め、LSP location と対応する CLI candidate bundle を比較してください。document/workspace の identifier range と explicit/inferred inlay hint は 1 つの source-semantics fixture にまとめ、保存済み column の anchoring、source 上の確認、hint 抑制が別々に drift しないようにします。 lifecycle transport coverage では、shutdown/exit の送信前に initialization を stage し、exit frame 後の read が cancellation では中断されない caller-owned stream を使い、shutdown response、success と shutdown 前 exit の status の違い、post-exit read が一度も始まらないことを検証してください。早期 transport completion によって他経路が弱まらないよう、finite EOF、malformed frame、caller cancellation の coverage も並行して維持します。 diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 4e0870b5d..3b37f4492 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -1964,8 +1964,8 @@ lack generated-file metadata use generated/designer filename conventions. | Grouped output | Meaning | |---|---| -| Family metadata | `definition_sites` is the physical declaration count. Rows also expose `partial_family_id`, `representative_reason`, and up to 50 stable `family_members`. | -| Member cap | The bounded member list always retains the representative and uses identifier-aligned columns after a verbatim `@`; `family_members_truncated` marks additional sites. | +| Family metadata | `definition_sites` is the physical declaration count. Rows also expose `partial_family_id`, `representative_reason`, authoritative `family_member_total_count`, page counts (`returned`, `omitted`, and `remaining`), and up to 50 stable `family_members`. | +| Member cap and continuation | The bounded member list always retains the representative and uses identifier-aligned columns after a verbatim `@`; `family_members_truncated` marks additional sites. Reuse `family_members_next_cursor` unchanged with the same symbol query, filters, and ordering to fetch the next page. `family_members_recovery_cursor` restarts at the first family page and remains available when compact output or `--fields` omits the nested list, including under a byte budget. Family cursors are bound to the family identity and index generation; changed selection is rejected as a mismatch and a refreshed index is rejected as stale. | | `goto` | Uses the canonical representative and returns family metadata in LSP-shaped JSON by default. Use `goto --all` for every physical location. | | Counts | JSON returns `logical_count`, `physical_count`, and `physical_file_count`. Human summaries distinguish rows shown after `--limit` from query-wide logical and physical totals. | | Sorted symbols | Uses the family's maximum rank metric while retaining the canonical representative, keeping `--sort` monotonic before `--limit`. | @@ -5492,8 +5492,8 @@ metadata がない旧 database では generated/designer filename 規約へ fall | 集約後の出力 | 意味 | |---|---| -| family metadata | `definition_sites` は物理宣言数です。row は `partial_family_id`、`representative_reason`、安定順で最大 50 件の `family_members` も公開します。 | -| member 上限 | 上限付き list は representative を必ず残し、verbatim `@` より後ろの identifier に column を揃えます。追加 site がある場合は `family_members_truncated` が true です。 | +| family metadata | `definition_sites` は物理宣言数です。row は `partial_family_id`、`representative_reason`、authoritative な `family_member_total_count`、page 件数（`returned`、`omitted`、`remaining`）、安定順で最大 50 件の `family_members` も公開します。 | +| member 上限と継続取得 | 上限付き list は representative を必ず残し、verbatim `@` より後ろの identifier に column を揃えます。追加 site がある場合は `family_members_truncated` が true です。同じ symbol query、filter、order で `family_members_next_cursor` を変更せず再利用すると次 page を取得できます。byte budget の指定時を含め、compact 出力または `--fields` により nested list を省略しても `family_members_recovery_cursor` から family の先頭 page を再取得できます。family cursor は family identity と index generation に束縛され、選択条件を変えると mismatch、index を更新すると stale として拒否されます。 | | `goto` | 既定では canonical representative と family metadata を LSP 形式の JSON で返します。全物理 location には `goto --all` を使います。 | | count | JSON は `logical_count`、`physical_count`、`physical_file_count` を返します。human summary は `--limit` 後の表示行数と query 全体の論理・物理総数を区別します。 | | sorted `symbols` | family 内の最大 rank metric と canonical representative を使い、`--limit` 前の `--sort` 順序を単調に保ちます。 | diff --git a/changelog.d/unreleased/5101.added.md b/changelog.d/unreleased/5101.added.md new file mode 100644 index 000000000..7ff01be09 --- /dev/null +++ b/changelog.d/unreleased/5101.added.md @@ -0,0 +1,25 @@ +--- +category: added +issues: + - 5101 +affected: + - src/CodeIndex/Cli/CliFlagSchema.cs + - src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs + - src/CodeIndex/Cli/ProjectionFieldRegistry.cs + - src/CodeIndex/Cli/QueryCommandRunner.Discovery.cs + - src/CodeIndex/Database/DbReader.SymbolSearchList.cs + - src/CodeIndex/Database/DbReader.SymbolSearchSql.cs + - src/CodeIndex/Models/QueryResults.cs + - tests/CodeIndex.Tests/QueryCommandRunnerPartialFamilyContinuationIssue5101Tests.cs + - USER_GUIDE.md + - DEVELOPER_GUIDE.md + - TESTING_GUIDE.md +--- + +## English + +- **Grouped partial-symbol families support bounded continuation (#5101)** — `symbols --group-partials` now reports authoritative family-member totals and recovery/next cursors while retaining the 50-member materialization cap. + +## 日本語 + +- **grouped partial-symbol family の上限付き継続取得に対応しました (#5101)** — `symbols --group-partials` は50 member の materialize 上限を維持しながら、authoritative な family-member 総数と recovery / next cursor を返すようになりました。 diff --git a/src/CodeIndex/Cli/CliFlagSchema.cs b/src/CodeIndex/Cli/CliFlagSchema.cs index 35aee24b5..21d8bf705 100644 --- a/src/CodeIndex/Cli/CliFlagSchema.cs +++ b/src/CodeIndex/Cli/CliFlagSchema.cs @@ -531,7 +531,7 @@ private static IReadOnlyList BuildAll() new() { Name = "--duplicate-threshold", ValuePlaceholder = "", Description = "Issue-drafts: explicit duplicate-preflight minimum score from 0 to 1", PrimaryCommands = Set("search", "suggestions") }, new() { Name = "--issue-title", ValuePlaceholder = "", Description = "Search issue-drafts: override the title for an ad hoc search draft", PrimaryCommands = Set("search") }, new() { Name = "--issue-label", ValuePlaceholder = "<label>", Description = "Search issue-drafts: add a label hint; repeat or comma-separate values", PrimaryCommands = Set("search") }, - new() { Name = "--cursor", ValuePlaceholder = "<cursor>", Description = "Opaque continuation cursor returned as next_cursor and bound to its query, options, and index generation; find cursors resume at match boundaries", PrimaryCommands = Set(CursorCommands) }, + new() { Name = "--cursor", ValuePlaceholder = "<cursor>", Description = "Opaque continuation cursor returned as next_cursor (or grouped symbols family_members_next_cursor) and bound to its query, options, family when applicable, and index generation; find cursors resume at match boundaries", PrimaryCommands = Set(CursorCommands) }, new() { Name = "--status", ValueDomain = Values(["all", "draft", "submitted_pending_triage", "open_in_upstream", "resolved_in_upstream", "wont_fix", "duplicate", "superseded", "submitted", "unsubmitted"]), Description = "Suggestions: filter by suggestion status", PrimaryCommands = Set("suggestions") }, new() { Name = "--category", ValueDomain = Values(["symbol_extraction", "reference_extraction", "search_ranking", "language_support", "output_format", "crash_report", "unexpected_error", "other"]), Description = "Suggestions: filter by category", PrimaryCommands = Set("suggestions") }, new() { Name = "--agent", ValuePlaceholder = "<agent>", Description = "Suggestions: filter by agent", PrimaryCommands = Set("suggestions") }, diff --git a/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs b/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs index f89dee75b..1e3c18891 100644 --- a/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs +++ b/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs @@ -315,7 +315,10 @@ private static int RunBoundedResponse( controls.ResumeLine, controls.ResumeFileOrdinal, controls.ResumeMatchOrdinal, - controls.ResumeByteOffset); + controls.ResumeByteOffset, + controls.PartialFamilyKey, + controls.PartialFamilyId, + controls.FamilyMemberOffset); using var executionScope = EnterBoundedExecution(executionContext); exitCode = runInner(innerArgs); } @@ -1824,6 +1827,9 @@ private static bool TryParseBoundedResponseControls( int? resumeFileOrdinal = null; int? resumeMatchOrdinal = null; int? resumeByteOffset = null; + string? partialFamilyKey = null; + string? partialFamilyId = null; + int? familyMemberOffset = null; if (cursor is not null && !TryParseResponseCursor( cursor, @@ -1834,7 +1840,10 @@ private static bool TryParseBoundedResponseControls( out resumeLine, out resumeFileOrdinal, out resumeMatchOrdinal, - out resumeByteOffset)) + out resumeByteOffset, + out partialFamilyKey, + out partialFamilyId, + out familyMemberOffset)) { controls = default!; error = "cursor_malformed: --cursor must be an opaque response:v2 cursor returned as next_cursor."; @@ -1852,7 +1861,10 @@ private static bool TryParseBoundedResponseControls( resumeLine, resumeFileOrdinal, resumeMatchOrdinal, - resumeByteOffset); + resumeByteOffset, + partialFamilyKey, + partialFamilyId, + familyMemberOffset); return true; } @@ -1980,7 +1992,10 @@ internal static string FormatResponseCursor( int? resumeLine = null, int? resumeFileOrdinal = null, int? resumeMatchOrdinal = null, - int? resumeByteOffset = null) + int? resumeByteOffset = null, + string? partialFamilyKey = null, + string? partialFamilyId = null, + int? familyMemberOffset = null) { var payload = new JsonObject { @@ -1998,6 +2013,20 @@ internal static string FormatResponseCursor( payload["resume_match_ordinal"] = resumeMatchOrdinal.Value; if (resumeByteOffset.HasValue) payload["resume_byte_offset"] = resumeByteOffset.Value; + if (partialFamilyKey is not null + && partialFamilyId is not null + && familyMemberOffset.HasValue) + { + payload["partial_family_key"] = partialFamilyKey; + payload["partial_family_id"] = partialFamilyId; + payload["family_member_offset"] = familyMemberOffset.Value; + payload["family_member_integrity"] = BuildPartialFamilyCursorIntegrity( + queryFingerprint, + generationFingerprint, + partialFamilyKey, + partialFamilyId, + familyMemberOffset.Value); + } var encoded = Convert.ToBase64String(Encoding.UTF8.GetBytes(payload.ToJsonString())) .TrimEnd('=') .Replace('+', '-') @@ -2015,6 +2044,33 @@ internal static bool TryParseResponseCursor( out int? resumeFileOrdinal, out int? resumeMatchOrdinal, out int? resumeByteOffset) + => TryParseResponseCursor( + cursor, + out offset, + out queryFingerprint, + out generationFingerprint, + out resumePath, + out resumeLine, + out resumeFileOrdinal, + out resumeMatchOrdinal, + out resumeByteOffset, + out _, + out _, + out _); + + internal static bool TryParseResponseCursor( + string cursor, + out int offset, + out string? queryFingerprint, + out string? generationFingerprint, + out string? resumePath, + out int? resumeLine, + out int? resumeFileOrdinal, + out int? resumeMatchOrdinal, + out int? resumeByteOffset, + out string? partialFamilyKey, + out string? partialFamilyId, + out int? familyMemberOffset) { offset = 0; queryFingerprint = null; @@ -2024,6 +2080,9 @@ internal static bool TryParseResponseCursor( resumeFileOrdinal = null; resumeMatchOrdinal = null; resumeByteOffset = null; + partialFamilyKey = null; + partialFamilyId = null; + familyMemberOffset = null; if (cursor.StartsWith(LegacyResponseCursorPrefix, StringComparison.Ordinal)) { var remainder = cursor[LegacyResponseCursorPrefix.Length..]; @@ -2101,9 +2160,43 @@ internal static bool TryParseResponseCursor( } resumeByteOffset = parsedByteOffset; } + partialFamilyKey = ReadString(payload, "partial_family_key"); + partialFamilyId = ReadString(payload, "partial_family_id"); + var partialFamilyIntegrity = ReadString(payload, "family_member_integrity"); + if (payload.ContainsKey("family_member_offset")) + { + if (payload["family_member_offset"] is not JsonValue familyOffsetValue + || !familyOffsetValue.TryGetValue<int>(out var parsedFamilyOffset)) + { + return false; + } + familyMemberOffset = parsedFamilyOffset; + } var extendedResumeFieldsPresent = resumeFileOrdinal.HasValue || resumeMatchOrdinal.HasValue || resumeByteOffset.HasValue; + var partialFamilyFieldsPresent = partialFamilyKey is not null + || partialFamilyId is not null + || familyMemberOffset.HasValue + || partialFamilyIntegrity is not null; + var partialFamilyFieldsValid = !partialFamilyFieldsPresent + || offset == 0 + && partialFamilyKey is { Length: > 0 and <= 4096 } + && partialFamilyId is { Length: > 0 and <= 128 } + && familyMemberOffset is >= 0 + && string.Equals( + partialFamilyId, + LogicalPartialSymbolGrouper.BuildPartialFamilyId(partialFamilyKey), + StringComparison.Ordinal) + && string.Equals( + partialFamilyIntegrity, + BuildPartialFamilyCursorIntegrity( + queryFingerprint!, + generationFingerprint!, + partialFamilyKey, + partialFamilyId, + familyMemberOffset.Value), + StringComparison.Ordinal); return IsCursorFingerprint(queryFingerprint) && IsCursorFingerprint(generationFingerprint) && (resumePath is null || resumePath.Length <= 4096) @@ -2114,9 +2207,25 @@ internal static bool TryParseResponseCursor( && resumeFileOrdinal is >= 0 && resumeByteOffset is >= 0 && resumeMatchOrdinal is null or >= 0) - && (!resumeMatchOrdinal.HasValue || resumeByteOffset.HasValue); + && (!resumeMatchOrdinal.HasValue || resumeByteOffset.HasValue) + && partialFamilyFieldsValid; } + private static string BuildPartialFamilyCursorIntegrity( + string queryFingerprint, + string generationFingerprint, + string partialFamilyKey, + string partialFamilyId, + int familyMemberOffset) + => BuildResponseValueFingerprint(string.Join( + '\0', + "partial-family-members:v1", + queryFingerprint, + generationFingerprint, + partialFamilyKey, + partialFamilyId, + familyMemberOffset.ToString(CultureInfo.InvariantCulture))); + private static bool IsCursorFingerprint(string? fingerprint) => fingerprint is { Length: 16 } && fingerprint.All(Uri.IsHexDigit); @@ -2231,7 +2340,10 @@ private sealed record BoundedResponseControls( int? ResumeLine, int? ResumeFileOrdinal, int? ResumeMatchOrdinal, - int? ResumeByteOffset) + int? ResumeByteOffset, + string? PartialFamilyKey, + string? PartialFamilyId, + int? FamilyMemberOffset) { public IReadOnlyList<string>? EffectiveFields( string command, @@ -2346,6 +2458,23 @@ internal static (string Cursor, string? ResultStableAt) BuildFindResumeCursor( snapshot.ResultStableAt); } + internal static string BuildPartialFamilyMembersCursor( + string[] args, + DbReader reader, + string partialFamilyKey, + string partialFamilyId, + int familyMemberOffset) + { + var snapshot = BuildResponseSnapshot(reader); + return FormatResponseCursor( + offset: 0, + BuildResponseFingerprint("symbols", args), + snapshot.GenerationFingerprint, + partialFamilyKey: partialFamilyKey, + partialFamilyId: partialFamilyId, + familyMemberOffset: familyMemberOffset); + } + internal static int GetBoundedResponseOffset(string command) { var execution = BoundedExecution.Value; @@ -2355,6 +2484,29 @@ internal static int GetBoundedResponseOffset(string command) : 0; } + internal static PartialFamilyContinuation? GetPartialFamilyContinuation(string command) + { + var execution = BoundedExecution.Value; + if (execution is null + || !string.Equals(execution.Command, CanonicalizeCommandName(command), StringComparison.Ordinal) + || execution.PartialFamilyKey is null + || execution.PartialFamilyId is null + || !execution.FamilyMemberOffset.HasValue) + { + return null; + } + + return new PartialFamilyContinuation( + execution.PartialFamilyKey, + execution.PartialFamilyId, + execution.FamilyMemberOffset.Value); + } + + internal readonly record struct PartialFamilyContinuation( + string PartialFamilyKey, + string PartialFamilyId, + int FamilyMemberOffset); + internal static bool ShouldMaterializeBody(string command) { var execution = BoundedExecution.Value; @@ -2533,7 +2685,10 @@ private sealed record BoundedExecutionContext( int? ResumeLine, int? ResumeFileOrdinal, int? ResumeMatchOrdinal, - int? ResumeByteOffset) + int? ResumeByteOffset, + string? PartialFamilyKey, + string? PartialFamilyId, + int? FamilyMemberOffset) { public int? ReportedTotalCount { get; set; } public bool ReportedTotalCountAuthoritative { get; set; } diff --git a/src/CodeIndex/Cli/ProjectionFieldRegistry.cs b/src/CodeIndex/Cli/ProjectionFieldRegistry.cs index 3b46c8a38..816ed1ebb 100644 --- a/src/CodeIndex/Cli/ProjectionFieldRegistry.cs +++ b/src/CodeIndex/Cli/ProjectionFieldRegistry.cs @@ -444,7 +444,13 @@ private static ProjectionCommandFieldSchema CreateCallGraphSchema(string command private static ProjectionCommandFieldSchema CreateSymbolsSchema() => Create( "symbols", - ["path", "line", "kind", "name"], + [ + "path", "line", "kind", "name", "definition_sites", "partial_family_id", + "representative_reason", "family_members_truncated", "family_member_total_count", + "family_member_total_count_authoritative", "family_member_returned_count", + "family_member_omitted_count", "family_member_remaining_count", + "family_members_recovery_cursor", "family_members_next_cursor", + ], builder => builder .Fields(GetJsonFieldNames<SymbolResult>()) .Alias("file", "path")); diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Discovery.cs b/src/CodeIndex/Cli/QueryCommandRunner.Discovery.cs index 0a6c46009..26906d118 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Discovery.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Discovery.cs @@ -180,9 +180,36 @@ public static int RunSymbols(string[] cmdArgs, JsonSerializerOptions jsonOptions return CommandExitCodes.Success; } + var partialFamilyContinuation = JsonEnvelopeWrapper.GetPartialFamilyContinuation("symbols"); var results = options.GroupPartials - ? reader.SearchSymbols(symbolQueries, options.Limit, options.Kind, options.Lang, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, options.Since, exact, visibilityFilters: options.VisibilityFilters, excludeVisibilityFilters: options.ExcludeVisibilityFilters, sortMode: options.SymbolSortMode, groupPartials: true, offset: JsonEnvelopeWrapper.GetBoundedResponseOffset("symbols")) + ? reader.SearchSymbols( + symbolQueries, + options.Limit, + options.Kind, + options.Lang, + options.PathPatterns, + options.ExcludePaths, + options.ExcludeTests, + options.Since, + exact, + visibilityFilters: options.VisibilityFilters, + excludeVisibilityFilters: options.ExcludeVisibilityFilters, + sortMode: options.SymbolSortMode, + groupPartials: true, + offset: JsonEnvelopeWrapper.GetBoundedResponseOffset("symbols"), + partialFamilyKey: partialFamilyContinuation?.PartialFamilyKey, + familyMemberOffset: partialFamilyContinuation?.FamilyMemberOffset ?? 0) : reader.SearchSymbols(symbolQueries, options.Limit, options.Kind, options.Lang, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, options.Since, exact, visibilityFilters: options.VisibilityFilters, excludeVisibilityFilters: options.ExcludeVisibilityFilters, sortMode: options.SymbolSortMode, offset: JsonEnvelopeWrapper.GetBoundedResponseOffset("symbols")); + if (options.GroupPartials) + { + AddPartialFamilyContinuationMetadata( + effectiveCmdArgs, + reader, + results, + partialFamilyContinuation?.FamilyMemberOffset ?? 0); + if (partialFamilyContinuation.HasValue) + JsonEnvelopeWrapper.ReportBoundedResponseTotal("symbols", results.Count, authoritative: true); + } var hasExactPredicate = exact && symbolQueries is { Count: > 0 }; var exactSignal = reader.GetSymbolsExactQuerySignal(options.Lang, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, options.Since); Func<SymbolResult, JsonNode?> rowFactory = @@ -337,6 +364,48 @@ public static int RunSymbols(string[] cmdArgs, JsonSerializerOptions jsonOptions }); } + private static void AddPartialFamilyContinuationMetadata( + string[] commandArgs, + DbReader reader, + IReadOnlyList<SymbolResult> results, + int familyMemberOffset) + { + foreach (var result in results) + { + if (result.DefinitionSites is not { } totalCount + || result.FamilyMembers is not { } members + || result.LogicalPartialKey is null + || result.PartialFamilyId is null) + { + continue; + } + + var returnedCount = members.Count; + result.FamilyMemberTotalCount = totalCount; + result.FamilyMemberTotalCountAuthoritative = true; + result.FamilyMemberReturnedCount = returnedCount; + result.FamilyMemberOmittedCount = Math.Max(0, totalCount - returnedCount); + result.FamilyMemberRemainingCount = Math.Max( + 0, + totalCount - familyMemberOffset - returnedCount); + result.FamilyMembersRecoveryCursor = JsonEnvelopeWrapper.BuildPartialFamilyMembersCursor( + commandArgs, + reader, + result.LogicalPartialKey, + result.PartialFamilyId, + familyMemberOffset: 0); + if (result.FamilyMemberRemainingCount > 0) + { + result.FamilyMembersNextCursor = JsonEnvelopeWrapper.BuildPartialFamilyMembersCursor( + commandArgs, + reader, + result.LogicalPartialKey, + result.PartialFamilyId, + familyMemberOffset + returnedCount); + } + } + } + private static string[] ExpandCompactAlias(string[] args) { if (!args.Any(arg => string.Equals(arg, "--compact", StringComparison.Ordinal))) @@ -1101,6 +1170,26 @@ private static bool JsonFitsByteLimit(string json, int maxJsonBytes) row["structural_rank_penalty"] = result.StructuralRankPenalty.Value; if (result.DefinitionSites.HasValue) row["definition_sites"] = result.DefinitionSites.Value; + if (result.PartialFamilyId is not null) + row["partial_family_id"] = result.PartialFamilyId; + if (result.RepresentativeReason is not null) + row["representative_reason"] = result.RepresentativeReason; + if (result.FamilyMembersTruncated) + row["family_members_truncated"] = true; + if (result.FamilyMemberTotalCount.HasValue) + row["family_member_total_count"] = result.FamilyMemberTotalCount.Value; + if (result.FamilyMemberTotalCountAuthoritative.HasValue) + row["family_member_total_count_authoritative"] = result.FamilyMemberTotalCountAuthoritative.Value; + if (result.FamilyMemberReturnedCount.HasValue) + row["family_member_returned_count"] = result.FamilyMemberReturnedCount.Value; + if (result.FamilyMemberOmittedCount.HasValue) + row["family_member_omitted_count"] = result.FamilyMemberOmittedCount.Value; + if (result.FamilyMemberRemainingCount.HasValue) + row["family_member_remaining_count"] = result.FamilyMemberRemainingCount.Value; + if (result.FamilyMembersRecoveryCursor is not null) + row["family_members_recovery_cursor"] = result.FamilyMembersRecoveryCursor; + if (result.FamilyMembersNextCursor is not null) + row["family_members_next_cursor"] = result.FamilyMembersNextCursor; if (result.SizeLines.HasValue) row["size_lines"] = result.SizeLines.Value; if (result.ComplexityScore.HasValue) diff --git a/src/CodeIndex/Database/DbReader.SymbolSearchList.cs b/src/CodeIndex/Database/DbReader.SymbolSearchList.cs index af743d787..7ac05b2b5 100644 --- a/src/CodeIndex/Database/DbReader.SymbolSearchList.cs +++ b/src/CodeIndex/Database/DbReader.SymbolSearchList.cs @@ -8,7 +8,7 @@ public partial class DbReader /// 複数名前パターン（OR結合）でシンボルを検索。空/null なら他フィルタに一致する全シンボルを返す。 /// <paramref name="exact"/> が true の場合、部分一致ではなく大文字小文字を無視した完全一致になる。 /// </summary> - public List<SymbolResult> SearchSymbols(IReadOnlyList<string>? queries, int limit = 20, string? kind = null, string? lang = null, IReadOnlyList<string>? pathPatterns = null, IReadOnlyList<string>? excludePathPatterns = null, bool excludeTests = false, DateTime? since = null, bool exact = false, IReadOnlyList<string>? visibilityFilters = null, IReadOnlyList<string>? excludeVisibilityFilters = null, SymbolSortMode sortMode = SymbolSortMode.Name, int? startLine = null, int? endLine = null, bool groupPartials = false, int offset = 0) + public List<SymbolResult> SearchSymbols(IReadOnlyList<string>? queries, int limit = 20, string? kind = null, string? lang = null, IReadOnlyList<string>? pathPatterns = null, IReadOnlyList<string>? excludePathPatterns = null, bool excludeTests = false, DateTime? since = null, bool exact = false, IReadOnlyList<string>? visibilityFilters = null, IReadOnlyList<string>? excludeVisibilityFilters = null, SymbolSortMode sortMode = SymbolSortMode.Name, int? startLine = null, int? endLine = null, bool groupPartials = false, int offset = 0, string? partialFamilyKey = null, int familyMemberOffset = 0) { var plan = SymbolSearchQueryPlanBuilder.Build(new SymbolSearchQueryPlan { @@ -28,6 +28,8 @@ public List<SymbolResult> SearchSymbols(IReadOnlyList<string>? queries, int limi EndLine = endLine, GroupPartials = groupPartials, Offset = offset, + PartialFamilyKey = partialFamilyKey, + FamilyMemberOffset = Math.Max(0, familyMemberOffset), }); return ExecuteSymbolSearchList(plan); } @@ -54,6 +56,12 @@ private List<SymbolResult> ExecuteSymbolSearchList(SymbolSearchQueryPlan plan) SymbolSearchQueryBinder.BindFilters(this, cmd, plan, includeLineRange: true); SqliteCommandPolicy.Add(cmd, "@limit", plan.Limit); SqliteCommandPolicy.Add(cmd, "@offset", Math.Max(0, plan.Offset)); + if (plan.GroupPartials) + { + SqliteCommandPolicy.Add(cmd, "@familyMemberOffset", plan.FamilyMemberOffset); + if (plan.PartialFamilyKey is not null) + SqliteCommandPolicy.Add(cmd, "@partialFamilyKey", plan.PartialFamilyKey); + } using var reader = cmd.ExecuteTrackedReader(); return SymbolSearchRowProjector.ReadAll(reader, plan); diff --git a/src/CodeIndex/Database/DbReader.SymbolSearchListQueryBuilder.cs b/src/CodeIndex/Database/DbReader.SymbolSearchListQueryBuilder.cs index 8cb3d1681..6ba75d2e2 100644 --- a/src/CodeIndex/Database/DbReader.SymbolSearchListQueryBuilder.cs +++ b/src/CodeIndex/Database/DbReader.SymbolSearchListQueryBuilder.cs @@ -53,7 +53,7 @@ private string BuildSymbolSearchListSql(SymbolSearchQueryPlan plan) includeLineRange: true); if (plan.GroupPartials) { - sql = BuildLogicalPartialSymbolQuery(sql, plan.SortMode); + sql = BuildLogicalPartialSymbolQuery(sql, plan); } else { diff --git a/src/CodeIndex/Database/DbReader.SymbolSearchMultiQuery.cs b/src/CodeIndex/Database/DbReader.SymbolSearchMultiQuery.cs index ff3f69063..725653747 100644 --- a/src/CodeIndex/Database/DbReader.SymbolSearchMultiQuery.cs +++ b/src/CodeIndex/Database/DbReader.SymbolSearchMultiQuery.cs @@ -26,7 +26,9 @@ private List<SymbolResult> ExecuteMultiQuerySymbolSearch(SymbolSearchQueryPlan p plan.SortMode, plan.StartLine, plan.EndLine, - plan.GroupPartials)); + plan.GroupPartials, + partialFamilyKey: plan.PartialFamilyKey, + familyMemberOffset: plan.FamilyMemberOffset)); } var seen = new HashSet<long?>(); diff --git a/src/CodeIndex/Database/DbReader.SymbolSearchQueryPlan.cs b/src/CodeIndex/Database/DbReader.SymbolSearchQueryPlan.cs index 374e9c7ab..518ededc6 100644 --- a/src/CodeIndex/Database/DbReader.SymbolSearchQueryPlan.cs +++ b/src/CodeIndex/Database/DbReader.SymbolSearchQueryPlan.cs @@ -20,6 +20,8 @@ private sealed record SymbolSearchQueryPlan public int? EndLine { get; init; } public bool GroupPartials { get; init; } public int Offset { get; init; } + public string? PartialFamilyKey { get; init; } + public int FamilyMemberOffset { get; init; } } private static class SymbolSearchQueryPlanBuilder diff --git a/src/CodeIndex/Database/DbReader.SymbolSearchSql.cs b/src/CodeIndex/Database/DbReader.SymbolSearchSql.cs index fd441c41f..aa5632d07 100644 --- a/src/CodeIndex/Database/DbReader.SymbolSearchSql.cs +++ b/src/CodeIndex/Database/DbReader.SymbolSearchSql.cs @@ -5,9 +5,12 @@ public partial class DbReader private static string GetGenericSymbolRankNamePenaltySql(string nameSql) => $"CASE WHEN lower({nameSql}) IN {GenericSymbolRankNamesSql} THEN {GenericSymbolRankNamePenaltySqlLiteral} ELSE 1.0 END"; - private static string BuildLogicalPartialSymbolQuery(string matchingSymbolsSql, SymbolSortMode sortMode) + private static string BuildLogicalPartialSymbolQuery(string matchingSymbolsSql, SymbolSearchQueryPlan plan) { - var orderBy = BuildLogicalPartialSortOrderBy(sortMode); + var orderBy = BuildLogicalPartialSortOrderBy(plan.SortMode); + var partialFamilyFilter = plan.PartialFamilyKey is null + ? string.Empty + : "WHERE logical_partial_key = @partialFamilyKey"; return $@" WITH matching_symbols AS ( {matchingSymbolsSql} @@ -32,6 +35,7 @@ PARTITION BY logical_partial_key ) AS family_member_row_number, COUNT(*) OVER (PARTITION BY logical_partial_key) AS logical_definition_sites FROM matching_symbols + {partialFamilyFilter} ), family_ranked_symbols AS ( SELECT ranked_symbols.*, @@ -40,8 +44,22 @@ PARTITION BY logical_partial_key ) AS representative_member_row_number FROM ranked_symbols ), - logical_symbols AS ( + family_paged_symbols AS ( SELECT family_ranked_symbols.*, + CASE + WHEN representative_member_row_number <= {LogicalPartialSymbolGrouper.FamilyMemberLimit} + THEN family_member_row_number + WHEN family_member_row_number = representative_member_row_number + THEN {LogicalPartialSymbolGrouper.FamilyMemberLimit} + WHEN family_member_row_number >= {LogicalPartialSymbolGrouper.FamilyMemberLimit} + AND family_member_row_number < representative_member_row_number + THEN family_member_row_number + 1 + ELSE family_member_row_number + END AS family_member_page_number + FROM family_ranked_symbols + ), + logical_symbols AS ( + SELECT family_paged_symbols.*, MAX(reference_count) OVER (PARTITION BY logical_partial_key) AS logical_reference_count, MAX(hotspot_score) OVER (PARTITION BY logical_partial_key) AS logical_hotspot_score, MAX(ranking_reference_score) OVER (PARTITION BY logical_partial_key) AS logical_ranking_reference_score, @@ -73,18 +91,14 @@ logical_symbols AS ( 'identifier_start_column', identifier_start_column, 'generated', canonical_generated_rank )) FILTER (WHERE - family_member_row_number <= CASE - WHEN representative_member_row_number <= {LogicalPartialSymbolGrouper.FamilyMemberLimit} - THEN {LogicalPartialSymbolGrouper.FamilyMemberLimit} - ELSE {LogicalPartialSymbolGrouper.FamilyMemberLimit - 1} - END - OR logical_row_number = 1 + family_member_page_number > @familyMemberOffset + AND family_member_page_number <= @familyMemberOffset + {LogicalPartialSymbolGrouper.FamilyMemberLimit} ) OVER ( PARTITION BY logical_partial_key - ORDER BY path COLLATE BINARY, start_line, stable_start_column, symbol_id + ORDER BY family_member_page_number ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING ) AS logical_family_members_json - FROM family_ranked_symbols + FROM family_paged_symbols ) SELECT path, lang, kind, sub_kind, name, line, start_line, start_column, end_line, diff --git a/src/CodeIndex/Models/QueryResults.cs b/src/CodeIndex/Models/QueryResults.cs index 2488a68dc..3d5d0098a 100644 --- a/src/CodeIndex/Models/QueryResults.cs +++ b/src/CodeIndex/Models/QueryResults.cs @@ -396,6 +396,20 @@ public class SymbolResult public List<PartialFamilyMember>? FamilyMembers { get; set; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public bool FamilyMembersTruncated { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? FamilyMemberTotalCount { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? FamilyMemberTotalCountAuthoritative { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? FamilyMemberReturnedCount { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? FamilyMemberOmittedCount { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? FamilyMemberRemainingCount { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FamilyMembersRecoveryCursor { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FamilyMembersNextCursor { get; set; } [JsonIgnore] internal bool? IsGeneratedCode { get; set; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerPartialFamilyContinuationIssue5101Tests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerPartialFamilyContinuationIssue5101Tests.cs new file mode 100644 index 000000000..53d422f2d --- /dev/null +++ b/tests/CodeIndex.Tests/QueryCommandRunnerPartialFamilyContinuationIssue5101Tests.cs @@ -0,0 +1,277 @@ +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using CodeIndex.Cli; +using CodeIndex.Database; + +namespace CodeIndex.Tests; + +public partial class QueryCommandRunnerTests +{ + [Fact] + public void GroupPartials_FamilyMembersContinueWithoutGapsAndRejectInvalidCursors_Issue5101() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_partial_family_cursor_issue5101"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + for (var index = 0; index < 105; index++) + { + TestProjectHelper.InsertIndexedFile( + dbPath, + $"src/One/Wide.{index:D3}.cs", + "csharp", + index == 104 + ? "namespace Demo.One;\npublic partial class Wide : BaseWide { }" + : "namespace Demo.One;\npublic partial class Wide { }"); + } + for (var index = 0; index < 2; index++) + { + TestProjectHelper.InsertIndexedFile( + dbPath, + $"src/Two/Wide.{index:D3}.cs", + "csharp", + "namespace Demo.Two;\npublic partial class Wide { }"); + } + MarkGraphAndFoldReady(dbPath); + + var commonArgs = new[] + { + "symbols", "Wide", "--db", dbPath, "--json=array", "--exact-name", + "--lang", "csharp", "--kind", "class", "--group-partials", "--limit", "5", + }; + var pageArgs = commonArgs.Concat([ + "--fields", + "path,name,partial_family_id,representative_reason,family_members,family_members_truncated," + + "family_member_total_count,family_member_total_count_authoritative," + + "family_member_returned_count,family_member_omitted_count," + + "family_member_remaining_count,family_members_recovery_cursor,family_members_next_cursor", + ]).ToArray(); + + var firstPage = RunIssue5101FamilyPage(pageArgs, cursor: null, expectedTotal: 105); + Assert.Equal(2, firstPage.TopLevelTotalCount); + Assert.Equal([2, 105], firstPage.AvailableFamilyTotals.Order().ToArray()); + Assert.Equal(50, firstPage.MemberIds.Count); + Assert.Equal(55, firstPage.OmittedCount); + Assert.Equal(55, firstPage.RemainingCount); + Assert.True(firstPage.TotalCountAuthoritative); + Assert.Contains("src/One/Wide.104.cs", firstPage.RepresentativePaths); + Assert.NotNull(firstPage.RecoveryCursor); + Assert.NotNull(firstPage.NextCursor); + + var memberIds = new List<long>(firstPage.MemberIds); + var cursor = firstPage.NextCursor; + var pageCount = 1; + while (cursor is not null) + { + var page = RunIssue5101FamilyPage(pageArgs, cursor, expectedTotal: 105); + pageCount++; + Assert.True(pageCount <= 3, "partial-family cursor did not make forward progress"); + Assert.Equal(1, page.TopLevelTotalCount); + Assert.Equal(firstPage.PartialFamilyId, page.PartialFamilyId); + memberIds.AddRange(page.MemberIds); + cursor = page.NextCursor; + } + + Assert.Equal(3, pageCount); + Assert.Equal(105, memberIds.Count); + Assert.Equal(105, memberIds.Distinct().Count()); + + var replayedFirstPage = RunIssue5101FamilyPage( + pageArgs, + firstPage.RecoveryCursor, + expectedTotal: 105); + Assert.Equal(firstPage.MemberIds, replayedFirstPage.MemberIds); + + var (arrayExitCode, arrayStdout, arrayStderr) = CaptureConsole(() => + ProgramRunner.Run(commonArgs, _jsonOptions, "1.0.0-test")); + Assert.Equal(CommandExitCodes.Success, arrayExitCode); + Assert.Equal(string.Empty, arrayStderr); + using (var arrayDocument = JsonDocument.Parse(arrayStdout)) + { + var arrayFamily = Assert.Single( + arrayDocument.RootElement.EnumerateArray(), + row => row.GetProperty("family_member_total_count").GetInt32() == 105); + Assert.Equal(50, arrayFamily.GetProperty("family_members").GetArrayLength()); + Assert.NotNull(arrayFamily.GetProperty("family_members_next_cursor").GetString()); + } + + var (compactExitCode, compactStdout, compactStderr) = CaptureConsole(() => + ProgramRunner.Run( + commonArgs.Concat(["--compact", "--max-json-bytes", "20000"]).ToArray(), + _jsonOptions, + "1.0.0-test")); + Assert.Equal(CommandExitCodes.Success, compactExitCode); + Assert.Equal(string.Empty, compactStderr); + using (var compactDocument = JsonDocument.Parse(compactStdout)) + { + var compactFamily = Assert.Single( + compactDocument.RootElement.GetProperty("symbols").EnumerateArray(), + row => row.TryGetProperty("family_member_total_count", out var total) + && total.GetInt32() == 105); + Assert.False(compactFamily.TryGetProperty("family_members", out _)); + Assert.NotNull(compactFamily.GetProperty("family_members_recovery_cursor").GetString()); + Assert.NotNull(compactFamily.GetProperty("family_members_next_cursor").GetString()); + } + + var metadataOnlyArgs = commonArgs.Concat([ + "--fields", + "path,partial_family_id,family_member_total_count,family_member_returned_count," + + "family_member_omitted_count,family_members_recovery_cursor,family_members_next_cursor", + "--max-json-bytes", "12000", + ]).ToArray(); + var metadataOnlyPage = RunIssue5101FamilyPage( + metadataOnlyArgs, + cursor: null, + expectedTotal: 105, + expectMembers: false); + var recoveredPage = RunIssue5101FamilyPage( + pageArgs, + metadataOnlyPage.RecoveryCursor, + expectedTotal: 105); + Assert.Equal(firstPage.MemberIds, recoveredPage.MemberIds); + + var (ndjsonExitCode, ndjsonStdout, ndjsonStderr) = CaptureConsole(() => + QueryCommandRunner.RunSymbols( + commonArgs.Skip(1).Where(arg => arg != "--json=array").Concat(["--json=ndjson"]).ToArray(), + _jsonOptions)); + Assert.Equal(CommandExitCodes.Success, ndjsonExitCode); + Assert.Equal(string.Empty, ndjsonStderr); + var ndjsonRows = ndjsonStdout + .Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries) + .Select(line => JsonNode.Parse(line)!.AsObject()) + .Where(row => !row.ContainsKey("terminal_record")) + .ToList(); + var ndjsonFamily = Assert.Single( + ndjsonRows, + row => row["family_member_total_count"]!.GetValue<int>() == 105); + Assert.Equal(50, ndjsonFamily["family_member_returned_count"]!.GetValue<int>()); + Assert.NotNull(ndjsonFamily["family_members_next_cursor"]?.GetValue<string>()); + + var (fieldsExitCode, fieldsStdout, fieldsStderr) = CaptureConsole(() => + ProgramRunner.Run(["symbols", "--fields", "list"], _jsonOptions, "1.0.0-test")); + Assert.Equal(CommandExitCodes.Success, fieldsExitCode); + Assert.Equal(string.Empty, fieldsStderr); + using (var fieldsDocument = JsonDocument.Parse(fieldsStdout)) + { + var validFields = fieldsDocument.RootElement.GetProperty("valid_fields") + .EnumerateArray() + .Select(field => field.GetString()) + .ToHashSet(StringComparer.Ordinal); + Assert.Contains("family_member_total_count", validFields); + Assert.Contains("family_members_recovery_cursor", validFields); + Assert.Contains("family_members_next_cursor", validFields); + } + + AssertIssue5101CursorFailure( + pageArgs.Concat([ + "--cursor", + MutateIssue5101Cursor( + firstPage.NextCursor!, + payload => payload["family_member_offset"] = 51), + ]).ToArray(), + "cursor_malformed"); + AssertIssue5101CursorFailure( + pageArgs.Concat(["--path", "src/One/*", "--cursor", firstPage.NextCursor!]).ToArray(), + "cursor_mismatch"); + + TestProjectHelper.InsertIndexedFile( + dbPath, + "src/GenerationChange.cs", + "csharp", + "namespace Demo; internal sealed class GenerationChange { }"); + MarkGraphAndFoldReady(dbPath); + AssertIssue5101CursorFailure( + pageArgs.Concat(["--cursor", firstPage.NextCursor!]).ToArray(), + "cursor_stale"); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + + private FamilyPageIssue5101 RunIssue5101FamilyPage( + string[] args, + string? cursor, + int expectedTotal, + bool expectMembers = true) + { + var effectiveArgs = cursor is null ? args : args.Concat(["--cursor", cursor]).ToArray(); + var (exitCode, stdout, stderr) = CaptureConsole(() => + ProgramRunner.Run(effectiveArgs, _jsonOptions, "1.0.0-test")); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(string.Empty, stderr); + using var document = JsonDocument.Parse(stdout); + var results = document.RootElement.GetProperty("results").EnumerateArray().ToList(); + var family = Assert.Single( + results, + row => row.GetProperty("family_member_total_count").GetInt32() == expectedTotal); + var members = expectMembers + ? family.GetProperty("family_members").EnumerateArray().ToList() + : []; + if (!expectMembers) + Assert.False(family.TryGetProperty("family_members", out _)); + + return new FamilyPageIssue5101( + TopLevelTotalCount: document.RootElement.GetProperty("metadata").GetProperty("total_count").GetInt32(), + AvailableFamilyTotals: results + .Select(row => row.GetProperty("family_member_total_count").GetInt32()) + .ToList(), + PartialFamilyId: family.GetProperty("partial_family_id").GetString()!, + MemberIds: members.Select(member => member.GetProperty("symbol_id").GetInt64()).ToList(), + RepresentativePaths: members + .Where(member => member.TryGetProperty("representative", out var representative) + && representative.GetBoolean()) + .Select(member => member.GetProperty("path").GetString()!) + .ToList(), + TotalCountAuthoritative: family.TryGetProperty("family_member_total_count_authoritative", out var authoritative) + && authoritative.GetBoolean(), + OmittedCount: family.GetProperty("family_member_omitted_count").GetInt32(), + RemainingCount: family.TryGetProperty("family_member_remaining_count", out var remaining) + ? remaining.GetInt32() + : 0, + RecoveryCursor: family.GetProperty("family_members_recovery_cursor").GetString(), + NextCursor: family.TryGetProperty("family_members_next_cursor", out var nextCursor) + ? nextCursor.GetString() + : null); + } + + private void AssertIssue5101CursorFailure(string[] args, string expectedError) + { + var (exitCode, stdout, stderr) = CaptureConsole(() => + ProgramRunner.Run(args, _jsonOptions, "1.0.0-test")); + + Assert.Equal(CommandExitCodes.UsageError, exitCode); + Assert.Equal(string.Empty, stdout); + Assert.Contains(expectedError, stderr, StringComparison.Ordinal); + } + + private static string MutateIssue5101Cursor(string cursor, Action<JsonObject> mutate) + { + const string prefix = "response:v2:"; + Assert.StartsWith(prefix, cursor, StringComparison.Ordinal); + var encoded = cursor[prefix.Length..].Replace('-', '+').Replace('_', '/'); + encoded += new string('=', (4 - encoded.Length % 4) % 4); + var payload = JsonNode.Parse(Encoding.UTF8.GetString(Convert.FromBase64String(encoded)))!.AsObject(); + mutate(payload); + var mutated = Convert.ToBase64String(Encoding.UTF8.GetBytes(payload.ToJsonString())) + .TrimEnd('=') + .Replace('+', '-') + .Replace('/', '_'); + return prefix + mutated; + } + + private sealed record FamilyPageIssue5101( + int TopLevelTotalCount, + IReadOnlyList<int> AvailableFamilyTotals, + string PartialFamilyId, + IReadOnlyList<long> MemberIds, + IReadOnlyList<string> RepresentativePaths, + bool TotalCountAuthoritative, + int OmittedCount, + int RemainingCount, + string? RecoveryCursor, + string? NextCursor); +} From 36afc97a90ee3c3349d8168f333f25fc7e36ddbc Mon Sep 17 00:00:00 2001 From: Widthdom <widthdom@gmail.com> Date: Fri, 21 Aug 2026 17:06:41 +0900 Subject: [PATCH 2/4] Harden partial family continuation (#5101) --- USER_GUIDE.md | 4 +- .../Cli/JsonEnvelopeWrapper.Bounded.cs | 39 ++++++++++- src/CodeIndex/Cli/ProjectionFieldRegistry.cs | 16 ++++- .../Cli/QueryCommandRunner.Discovery.cs | 54 ++++++++++----- ...PartialFamilyContinuationIssue5101Tests.cs | 69 ++++++++++++++++++- 5 files changed, 155 insertions(+), 27 deletions(-) diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 3b37f4492..50bf740ca 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -1965,7 +1965,7 @@ lack generated-file metadata use generated/designer filename conventions. | Grouped output | Meaning | |---|---| | Family metadata | `definition_sites` is the physical declaration count. Rows also expose `partial_family_id`, `representative_reason`, authoritative `family_member_total_count`, page counts (`returned`, `omitted`, and `remaining`), and up to 50 stable `family_members`. | -| Member cap and continuation | The bounded member list always retains the representative and uses identifier-aligned columns after a verbatim `@`; `family_members_truncated` marks additional sites. Reuse `family_members_next_cursor` unchanged with the same symbol query, filters, and ordering to fetch the next page. `family_members_recovery_cursor` restarts at the first family page and remains available when compact output or `--fields` omits the nested list, including under a byte budget. Family cursors are bound to the family identity and index generation; changed selection is rejected as a mismatch and a refreshed index is rejected as stale. | +| Member cap and continuation | The first bounded member page always retains the representative and uses identifier-aligned columns after a verbatim `@`; `family_members_truncated` marks additional sites. Reuse `family_members_next_cursor` unchanged with the same symbol query, filters, and ordering to fetch the next page. `family_members_recovery_cursor` restarts at the first family page and remains available when compact output or `--fields` omits the nested list, including under a byte budget. Family cursors are bound to the family identity and index generation; changed selection is rejected as a mismatch and a refreshed index is rejected as stale. | | `goto` | Uses the canonical representative and returns family metadata in LSP-shaped JSON by default. Use `goto --all` for every physical location. | | Counts | JSON returns `logical_count`, `physical_count`, and `physical_file_count`. Human summaries distinguish rows shown after `--limit` from query-wide logical and physical totals. | | Sorted symbols | Uses the family's maximum rank metric while retaining the canonical representative, keeping `--sort` monotonic before `--limit`. | @@ -5493,7 +5493,7 @@ metadata がない旧 database では generated/designer filename 規約へ fall | 集約後の出力 | 意味 | |---|---| | family metadata | `definition_sites` は物理宣言数です。row は `partial_family_id`、`representative_reason`、authoritative な `family_member_total_count`、page 件数（`returned`、`omitted`、`remaining`）、安定順で最大 50 件の `family_members` も公開します。 | -| member 上限と継続取得 | 上限付き list は representative を必ず残し、verbatim `@` より後ろの identifier に column を揃えます。追加 site がある場合は `family_members_truncated` が true です。同じ symbol query、filter、order で `family_members_next_cursor` を変更せず再利用すると次 page を取得できます。byte budget の指定時を含め、compact 出力または `--fields` により nested list を省略しても `family_members_recovery_cursor` から family の先頭 page を再取得できます。family cursor は family identity と index generation に束縛され、選択条件を変えると mismatch、index を更新すると stale として拒否されます。 | +| member 上限と継続取得 | 上限付き list の先頭 page は representative を必ず残し、verbatim `@` より後ろの identifier に column を揃えます。追加 site がある場合は `family_members_truncated` が true です。同じ symbol query、filter、order で `family_members_next_cursor` を変更せず再利用すると次 page を取得できます。byte budget の指定時を含め、compact 出力または `--fields` により nested list を省略しても `family_members_recovery_cursor` から family の先頭 page を再取得できます。family cursor は family identity と index generation に束縛され、選択条件を変えると mismatch、index を更新すると stale として拒否されます。 | | `goto` | 既定では canonical representative と family metadata を LSP 形式の JSON で返します。全物理 location には `goto --all` を使います。 | | count | JSON は `logical_count`、`physical_count`、`physical_file_count` を返します。human summary は `--limit` 後の表示行数と query 全体の論理・物理総数を区別します。 | | sorted `symbols` | family 内の最大 rank metric と canonical representative を使い、`--limit` 前の `--sort` 順序を単調に保ちます。 | diff --git a/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs b/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs index 1e3c18891..0b76dd44b 100644 --- a/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs +++ b/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs @@ -24,6 +24,7 @@ internal static partial class JsonEnvelopeWrapper private const string ResponseCursorPrefix = "response:v2:"; private static readonly AsyncLocal<BoundedExecutionContext?> BoundedExecution = new(); internal static Action? ResponseSnapshotValidatedForTesting { get; set; } + internal static Action? PartialFamilyPageReadForTesting { get; set; } private static readonly HashSet<string> BoundedResponseCommands = ProjectionFieldRegistry.SupportedCommands.ToHashSet(StringComparer.Ordinal); @@ -2350,7 +2351,10 @@ private sealed record BoundedResponseControls( string? primaryCollection, bool statusExplainRequest) { - var preserveFullDiscoveryRows = command is "search" or "languages"; + var preserveFullDiscoveryRows = command is "search" or "languages" + || command == "symbols" + && PartialFamilyKey is not null + && !Compact; var selected = Fields ?? (statusExplainRequest ? ProjectionFieldRegistry.GetStatusExplainCompactFields() @@ -2460,12 +2464,11 @@ internal static (string Cursor, string? ResultStableAt) BuildFindResumeCursor( internal static string BuildPartialFamilyMembersCursor( string[] args, - DbReader reader, + PartialFamilyCursorSnapshot snapshot, string partialFamilyKey, string partialFamilyId, int familyMemberOffset) { - var snapshot = BuildResponseSnapshot(reader); return FormatResponseCursor( offset: 0, BuildResponseFingerprint("symbols", args), @@ -2475,6 +2478,33 @@ internal static string BuildPartialFamilyMembersCursor( familyMemberOffset: familyMemberOffset); } + internal static PartialFamilyCursorSnapshot CapturePartialFamilyCursorSnapshot( + DbReader reader) + => new(BuildResponseSnapshot(reader).GenerationFingerprint); + + internal static int? ValidatePartialFamilyCursorSnapshot( + string dbPath, + bool dbPathExplicit, + PartialFamilyCursorSnapshot snapshot) + { + PartialFamilyPageReadForTesting?.Invoke(); + var completedSnapshot = SafeReadResponseSnapshot( + dbPath, + dbPathExplicit, + appVersion: "partial-family"); + if (string.Equals( + snapshot.GenerationFingerprint, + completedSnapshot.GenerationFingerprint, + StringComparison.Ordinal)) + { + return null; + } + + return WriteBoundedResponseUsageError( + "The index generation changed while this partial-family page was being read.", + "Restart pagination without --cursor after the active index refresh completes."); + } + internal static int GetBoundedResponseOffset(string command) { var execution = BoundedExecution.Value; @@ -2507,6 +2537,9 @@ internal readonly record struct PartialFamilyContinuation( string PartialFamilyId, int FamilyMemberOffset); + internal readonly record struct PartialFamilyCursorSnapshot( + string GenerationFingerprint); + internal static bool ShouldMaterializeBody(string command) { var execution = BoundedExecution.Value; diff --git a/src/CodeIndex/Cli/ProjectionFieldRegistry.cs b/src/CodeIndex/Cli/ProjectionFieldRegistry.cs index 816ed1ebb..bb177ac27 100644 --- a/src/CodeIndex/Cli/ProjectionFieldRegistry.cs +++ b/src/CodeIndex/Cli/ProjectionFieldRegistry.cs @@ -256,11 +256,14 @@ private static JsonObject CreateDiscoveryDocument( private static ProjectionCommandFieldSchema CreateInspectSchema() { var definitionFields = GetJsonFieldNames<DefinitionResult>() - .Where(field => !string.Equals(field, "content", StringComparison.Ordinal)) + .Where(field => !string.Equals(field, "content", StringComparison.Ordinal) + && !IsSymbolsOnlyPartialFamilyContinuationField(field)) .Concat(["content_omitted", "content_omitted_reason"]) .Distinct(StringComparer.Ordinal) .ToArray(); - var nearbyFields = GetJsonFieldNames<SymbolResult>().ToArray(); + var nearbyFields = GetJsonFieldNames<SymbolResult>() + .Where(field => !IsSymbolsOnlyPartialFamilyContinuationField(field)) + .ToArray(); var referenceFields = GetJsonFieldNames<ReferenceResult>() .Where(field => !IsInspectGraphBodyField(field)) .ToArray(); @@ -310,6 +313,15 @@ private static bool IsInspectGraphBodyField(string field) => string.Equals(field, "body_content", StringComparison.Ordinal) || field.StartsWith("body_", StringComparison.Ordinal); + private static bool IsSymbolsOnlyPartialFamilyContinuationField(string field) + => field is "family_member_total_count" + or "family_member_total_count_authoritative" + or "family_member_returned_count" + or "family_member_omitted_count" + or "family_member_remaining_count" + or "family_members_recovery_cursor" + or "family_members_next_cursor"; + private static ProjectionCommandFieldSchema CreateSearchSchema() => Create( "search", diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Discovery.cs b/src/CodeIndex/Cli/QueryCommandRunner.Discovery.cs index 26906d118..32d509679 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Discovery.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Discovery.cs @@ -92,9 +92,10 @@ public static int RunSymbols(string[] cmdArgs, JsonSerializerOptions jsonOptions string? ndjsonTerminalLine = null; return WithDb(options, jsonOptions, reader => { + var partialFamilyContinuation = JsonEnvelopeWrapper.GetPartialFamilyContinuation("symbols"); QueryCountResult? partialPhysicalCounts = null; QueryCountResult? partialLogicalCounts = null; - if (options.GroupPartials) + if (options.GroupPartials && !partialFamilyContinuation.HasValue) { partialPhysicalCounts = reader.CountSearchSymbolsTotal(symbolQueries, options.Kind, options.Lang, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, options.Since, exact, visibilityFilters: options.VisibilityFilters, excludeVisibilityFilters: options.ExcludeVisibilityFilters); partialLogicalCounts = reader.CountSearchSymbolsTotal(symbolQueries, options.Kind, options.Lang, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, options.Since, exact, visibilityFilters: options.VisibilityFilters, excludeVisibilityFilters: options.ExcludeVisibilityFilters, groupPartials: true); @@ -180,7 +181,9 @@ public static int RunSymbols(string[] cmdArgs, JsonSerializerOptions jsonOptions return CommandExitCodes.Success; } - var partialFamilyContinuation = JsonEnvelopeWrapper.GetPartialFamilyContinuation("symbols"); + var partialFamilyCursorSnapshot = options.GroupPartials + ? JsonEnvelopeWrapper.CapturePartialFamilyCursorSnapshot(reader) + : (JsonEnvelopeWrapper.PartialFamilyCursorSnapshot?)null; var results = options.GroupPartials ? reader.SearchSymbols( symbolQueries, @@ -204,11 +207,16 @@ public static int RunSymbols(string[] cmdArgs, JsonSerializerOptions jsonOptions { AddPartialFamilyContinuationMetadata( effectiveCmdArgs, - reader, + partialFamilyCursorSnapshot!.Value, results, partialFamilyContinuation?.FamilyMemberOffset ?? 0); if (partialFamilyContinuation.HasValue) + { JsonEnvelopeWrapper.ReportBoundedResponseTotal("symbols", results.Count, authoritative: true); + partialLogicalCounts = new QueryCountResult( + results.Count, + results.Select(result => result.Path).Distinct(StringComparer.Ordinal).Count()); + } } var hasExactPredicate = exact && symbolQueries is { Count: > 0 }; var exactSignal = reader.GetSymbolsExactQuerySignal(options.Lang, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, options.Since); @@ -216,18 +224,28 @@ public static int RunSymbols(string[] cmdArgs, JsonSerializerOptions jsonOptions result => ToSymbolDiscoveryJsonNode(result, jsonOptions, options.OutputFormat == OutputFormatCompact); var rowExactSignal = hasExactPredicate ? exactSignal : (ExactQuerySignal?)null; var multiNameExactHint = symbolQueries != null && symbolQueries.Count > 1; - var exactZeroHint = multiNameExactHint - ? BuildExactZeroHint( - exact, - () => reader.AnySearchSymbols(symbolQueries, options.Kind, options.Lang, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, options.Since, exact: false, visibilityFilters: options.VisibilityFilters, excludeVisibilityFilters: options.ExcludeVisibilityFilters), - () => reader.SearchSymbols(symbolQueries, Math.Min(options.Limit, ExactZeroHintSampleLimit), options.Kind, options.Lang, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, options.Since, exact: false, visibilityFilters: options.VisibilityFilters, excludeVisibilityFilters: options.ExcludeVisibilityFilters), - r => r.Name) - : BuildExactZeroHint( - exact && symbolQueries != null && symbolQueries.Count > 0, - () => reader.CountSearchSymbols(symbolQueries, ExactZeroHintProbeLimit, options.Kind, options.Lang, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, options.Since, exact: false, visibilityFilters: options.VisibilityFilters, excludeVisibilityFilters: options.ExcludeVisibilityFilters) > 0, - () => reader.CountSearchSymbols(symbolQueries, options.Limit, options.Kind, options.Lang, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, options.Since, exact: false, visibilityFilters: options.VisibilityFilters, excludeVisibilityFilters: options.ExcludeVisibilityFilters), - () => reader.SearchSymbols(symbolQueries, Math.Min(options.Limit, ExactZeroHintSampleLimit), options.Kind, options.Lang, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, options.Since, exact: false, visibilityFilters: options.VisibilityFilters, excludeVisibilityFilters: options.ExcludeVisibilityFilters), - r => r.Name); + var exactZeroHint = partialFamilyContinuation.HasValue + ? null + : multiNameExactHint + ? BuildExactZeroHint( + exact, + () => reader.AnySearchSymbols(symbolQueries, options.Kind, options.Lang, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, options.Since, exact: false, visibilityFilters: options.VisibilityFilters, excludeVisibilityFilters: options.ExcludeVisibilityFilters), + () => reader.SearchSymbols(symbolQueries, Math.Min(options.Limit, ExactZeroHintSampleLimit), options.Kind, options.Lang, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, options.Since, exact: false, visibilityFilters: options.VisibilityFilters, excludeVisibilityFilters: options.ExcludeVisibilityFilters), + r => r.Name) + : BuildExactZeroHint( + exact && symbolQueries != null && symbolQueries.Count > 0, + () => reader.CountSearchSymbols(symbolQueries, ExactZeroHintProbeLimit, options.Kind, options.Lang, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, options.Since, exact: false, visibilityFilters: options.VisibilityFilters, excludeVisibilityFilters: options.ExcludeVisibilityFilters) > 0, + () => reader.CountSearchSymbols(symbolQueries, options.Limit, options.Kind, options.Lang, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, options.Since, exact: false, visibilityFilters: options.VisibilityFilters, excludeVisibilityFilters: options.ExcludeVisibilityFilters), + () => reader.SearchSymbols(symbolQueries, Math.Min(options.Limit, ExactZeroHintSampleLimit), options.Kind, options.Lang, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, options.Since, exact: false, visibilityFilters: options.VisibilityFilters, excludeVisibilityFilters: options.ExcludeVisibilityFilters), + r => r.Name); + if (partialFamilyCursorSnapshot.HasValue + && JsonEnvelopeWrapper.ValidatePartialFamilyCursorSnapshot( + options.DbPath, + options.DbPathExplicit, + partialFamilyCursorSnapshot.Value) is { } snapshotError) + { + return snapshotError; + } WriteExactSymbolWarningIfNeeded(hasExactPredicate, options.Json, exactSignal, reader, options); if (results.Count == 0) { @@ -366,7 +384,7 @@ public static int RunSymbols(string[] cmdArgs, JsonSerializerOptions jsonOptions private static void AddPartialFamilyContinuationMetadata( string[] commandArgs, - DbReader reader, + JsonEnvelopeWrapper.PartialFamilyCursorSnapshot snapshot, IReadOnlyList<SymbolResult> results, int familyMemberOffset) { @@ -390,7 +408,7 @@ private static void AddPartialFamilyContinuationMetadata( totalCount - familyMemberOffset - returnedCount); result.FamilyMembersRecoveryCursor = JsonEnvelopeWrapper.BuildPartialFamilyMembersCursor( commandArgs, - reader, + snapshot, result.LogicalPartialKey, result.PartialFamilyId, familyMemberOffset: 0); @@ -398,7 +416,7 @@ private static void AddPartialFamilyContinuationMetadata( { result.FamilyMembersNextCursor = JsonEnvelopeWrapper.BuildPartialFamilyMembersCursor( commandArgs, - reader, + snapshot, result.LogicalPartialKey, result.PartialFamilyId, familyMemberOffset + returnedCount); diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerPartialFamilyContinuationIssue5101Tests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerPartialFamilyContinuationIssue5101Tests.cs index 53d422f2d..e9f266d78 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerPartialFamilyContinuationIssue5101Tests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerPartialFamilyContinuationIssue5101Tests.cs @@ -87,14 +87,21 @@ public void GroupPartials_FamilyMembersContinueWithoutGapsAndRejectInvalidCursor ProgramRunner.Run(commonArgs, _jsonOptions, "1.0.0-test")); Assert.Equal(CommandExitCodes.Success, arrayExitCode); Assert.Equal(string.Empty, arrayStderr); + string arrayNextCursor; using (var arrayDocument = JsonDocument.Parse(arrayStdout)) { var arrayFamily = Assert.Single( arrayDocument.RootElement.EnumerateArray(), row => row.GetProperty("family_member_total_count").GetInt32() == 105); Assert.Equal(50, arrayFamily.GetProperty("family_members").GetArrayLength()); - Assert.NotNull(arrayFamily.GetProperty("family_members_next_cursor").GetString()); + arrayNextCursor = arrayFamily.GetProperty("family_members_next_cursor").GetString()!; } + var arrayContinuation = RunIssue5101FamilyPage( + commonArgs, + arrayNextCursor, + expectedTotal: 105); + Assert.Equal(50, arrayContinuation.MemberIds.Count); + Assert.Equal(5, arrayContinuation.RemainingCount); var (compactExitCode, compactStdout, compactStderr) = CaptureConsole(() => ProgramRunner.Run( @@ -146,7 +153,13 @@ public void GroupPartials_FamilyMembersContinueWithoutGapsAndRejectInvalidCursor ndjsonRows, row => row["family_member_total_count"]!.GetValue<int>() == 105); Assert.Equal(50, ndjsonFamily["family_member_returned_count"]!.GetValue<int>()); - Assert.NotNull(ndjsonFamily["family_members_next_cursor"]?.GetValue<string>()); + var ndjsonNextCursor = ndjsonFamily["family_members_next_cursor"]!.GetValue<string>(); + var ndjsonContinuation = RunIssue5101FamilyPage( + commonArgs.Where(arg => arg != "--json=array").Concat(["--json=ndjson"]).ToArray(), + ndjsonNextCursor, + expectedTotal: 105); + Assert.Equal(50, ndjsonContinuation.MemberIds.Count); + Assert.Equal(5, ndjsonContinuation.RemainingCount); var (fieldsExitCode, fieldsStdout, fieldsStderr) = CaptureConsole(() => ProgramRunner.Run(["symbols", "--fields", "list"], _jsonOptions, "1.0.0-test")); @@ -163,6 +176,33 @@ public void GroupPartials_FamilyMembersContinueWithoutGapsAndRejectInvalidCursor Assert.Contains("family_members_next_cursor", validFields); } + var (inspectFieldsExitCode, inspectFieldsStdout, inspectFieldsStderr) = CaptureConsole(() => + ProgramRunner.Run(["inspect", "--fields", "list"], _jsonOptions, "1.0.0-test")); + Assert.Equal(CommandExitCodes.Success, inspectFieldsExitCode); + Assert.Equal(string.Empty, inspectFieldsStderr); + using (var inspectFieldsDocument = JsonDocument.Parse(inspectFieldsStdout)) + { + var validFields = inspectFieldsDocument.RootElement.GetProperty("valid_fields") + .EnumerateArray() + .Select(field => field.GetString()) + .ToHashSet(StringComparer.Ordinal); + Assert.DoesNotContain("definitions.family_member_total_count", validFields); + Assert.DoesNotContain("definitions.family_members_next_cursor", validFields); + Assert.DoesNotContain("nearby_symbols.family_member_total_count", validFields); + Assert.DoesNotContain("nearby_symbols.family_members_next_cursor", validFields); + } + var (inspectUnsupportedExitCode, inspectUnsupportedStdout, inspectUnsupportedStderr) = CaptureConsole(() => + ProgramRunner.Run( + [ + "inspect", "Wide", "--db", dbPath, "--group-partials", + "--fields", "definitions.family_member_total_count", + ], + _jsonOptions, + "1.0.0-test")); + Assert.Equal(CommandExitCodes.UsageError, inspectUnsupportedExitCode); + Assert.Equal(string.Empty, inspectUnsupportedStderr); + Assert.Contains("Unknown --fields value", inspectUnsupportedStdout, StringComparison.Ordinal); + AssertIssue5101CursorFailure( pageArgs.Concat([ "--cursor", @@ -175,6 +215,31 @@ public void GroupPartials_FamilyMembersContinueWithoutGapsAndRejectInvalidCursor pageArgs.Concat(["--path", "src/One/*", "--cursor", firstPage.NextCursor!]).ToArray(), "cursor_mismatch"); + var snapshotRaceHookInvoked = false; + JsonEnvelopeWrapper.PartialFamilyPageReadForTesting = () => + { + snapshotRaceHookInvoked = true; + TestProjectHelper.InsertIndexedFile( + dbPath, + "src/SnapshotRace.cs", + "csharp", + "namespace Demo; internal sealed class SnapshotRace { }"); + MarkGraphAndFoldReady(dbPath); + }; + try + { + var (raceExitCode, raceStdout, raceStderr) = CaptureConsole(() => + ProgramRunner.Run(commonArgs, _jsonOptions, "1.0.0-test")); + Assert.True(snapshotRaceHookInvoked); + Assert.Equal(CommandExitCodes.UsageError, raceExitCode); + Assert.Equal(string.Empty, raceStdout); + Assert.Contains("index generation changed", raceStderr, StringComparison.Ordinal); + } + finally + { + JsonEnvelopeWrapper.PartialFamilyPageReadForTesting = null; + } + TestProjectHelper.InsertIndexedFile( dbPath, "src/GenerationChange.cs", From 6fa9550e43735bb97eb865d91266f61730b67627 Mon Sep 17 00:00:00 2001 From: Widthdom <widthdom@gmail.com> Date: Fri, 21 Aug 2026 17:36:18 +0900 Subject: [PATCH 3/4] Use fixed-size family cursor identity (#5101) --- DEVELOPER_GUIDE.md | 4 +- TESTING_GUIDE.md | 4 +- .../Cli/JsonEnvelopeWrapper.Bounded.cs | 47 +++++-------------- .../Cli/QueryCommandRunner.Discovery.cs | 5 +- ...bContext.ConnectionFunctionRegistration.cs | 4 ++ .../Database/DbReader.SymbolSearchList.cs | 8 ++-- .../DbReader.SymbolSearchMultiQuery.cs | 2 +- .../DbReader.SymbolSearchQueryPlan.cs | 2 +- .../Database/DbReader.SymbolSearchSql.cs | 4 +- ...PartialFamilyContinuationIssue5101Tests.cs | 26 ++++++---- 10 files changed, 48 insertions(+), 58 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 0bc16053a..074aca8b1 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -2218,7 +2218,7 @@ catalog and unknown-field errors are generated from that same schema. | Compact location envelope | CLI `--format compact` location output uses a versioned envelope with `api_version`, returned `count`, conservative limit-based `truncated` / `truncation` metadata, applied `query_context`, and lightweight `results` rows. | | Grouped search totals | `search --format grouped` derives `total_matches` / `matched_count`, `total_groups`, and `total_files` from the complete bounded query rather than the displayed page. `grouped_match_count` counts rows supplied to returned groups, `emitted_match_count` counts rows left after per-file grouping limits, and `omitted_match_count`, `truncated`, `has_more`, and `continuation_action` describe incomplete output. | | Bounded high-volume responses | `search`, `definition`, `find`, `status`, `hotspots`, `references`, `callers`, `callees`, `symbols`, `files`, `languages`, `impact`, and `map` accept shared bounded-response controls where their schema exposes them. Newly emitted opaque `--cursor <response:v2:...>` values bind the offset to the command/query/filter selection and index generation; legacy `response:v1:<offset>:<fingerprint>` cursors remain accepted for transition. Reuse with changed selection or generation fails with restart-required guidance. `search --format compact`, `symbols --format compact`, and `files --format compact` auto-select the bounded contract, while `search --json=array --json-envelope` provides the opt-in array envelope and `languages --json` selects it when paging or `--max-json-bytes` is requested. Existing compact roots and location rows remain compatible while adding shared metadata. Metadata reports `returned_count`, authoritative `total_count` where available, `omitted_count`, `remaining_count`, `cursor_offset`, `page_limit`, `has_more`, `next_cursor`, `result_stable_at`, `pagination_window_limit`, and `pagination_window_exhausted`. The safety window is 10,000 rows; exhaustion suppresses `next_cursor` rather than returning a cursor that the next request would reject. Pageable commands pass the cursor offset into their database/scan layer instead of serializing an `offset + limit` prefix. `find --all` partial scans encode the next path/line in the opaque cursor so replay continues after the last scanned line. `hotspots` and `impact` page their active primary nested collection as `results`, identify it with `metadata.primary_collection`, and retain scalar/container evidence in `metadata.response_context`; dotted fields such as `callers.path,callers.depth` select that collection and project its rows. The final newline is included in `--max-json-bytes`, and trailing whole rows are removed until the complete envelope fits. `definition` remains metadata-only by default; explicit `--body` content is retained for `body`, `body_content`, or `all`, and suppressed when the projection excludes it. `map --sections` remains its section-level projection, while dotted bounded fields page a selected array section with section-specific totals and scalar projections skip unused ranked arrays. | -| Grouped partial-family continuation | A grouped `symbols` row reports authoritative family-member total, returned, omitted, and remaining counts while materializing at most 50 members. Its opaque recovery/next cursors bind the normalized symbol selection, family key/ID, member offset, and index generation. SQL applies the member offset before JSON aggregation, so continuation never loads the complete family into managed memory. Compact and projected output preserve the recovery metadata even when they omit `family_members`; array, NDJSON, envelope, and byte-bounded paths serialize the same row contract. | +| Grouped partial-family continuation | A grouped `symbols` row reports authoritative family-member total, returned, omitted, and remaining counts while materializing at most 50 members. Its opaque recovery/next cursors bind the normalized symbol selection, fixed-size family ID, member offset, and index generation; they never embed the potentially unbounded family key. SQL applies the family-ID filter and member offset before JSON aggregation, so continuation never loads the complete family into managed memory. Compact and projected output preserve the recovery metadata even when they omit `family_members`; array, NDJSON, envelope, and byte-bounded paths serialize the same row contract. | | Bounded outline responses | `outline` opts into the shared bounded-response contract only when `--max-json-bytes` is present. The wrapper extracts complete projected symbol rows, preserves hierarchy and deterministic order, reports authoritative returned / total / omitted counts, includes the final newline in its UTF-8 measurement, and emits a bound `response:v2` continuation cursor. An undersized minimum envelope produces one typed `E028_RESPONSE_BUDGET_TOO_SMALL` object on stdout with empty stderr and actionable byte fields. Uncapped outline JSON retains its existing root shape and outline cursor contract. | | Bounded unused responses | `unused` opts into the shared bounded-response contract only when `--max-json-bytes` is present. The wrapper extracts the canonical `symbols` rows, applies the cursor offset in the unused query layer, recomputes returned bucket / confidence / contract-domain counts after byte trimming, and includes an optional `by_bucket` view in the same whole-response UTF-8 budget. Compact mode projects smaller audit rows. Continuation cursors bind the effective audit filters, bucket mode, ordering, and index generation; an undersized one-row envelope returns one typed `E028_RESPONSE_BUDGET_TOO_SMALL` object on stdout with empty stderr and actionable byte fields. Uncapped JSON, compact summaries, and the legacy unused cursor remain unchanged. | | MCP outline pages | MCP `outline` routes `fields`, `sort`, `limit`, and `cursor` through `QueryCommandRunner.BuildOutlinePage`, so its projection aliases, derived sort fields, stable tie-breakers, `page:v1` query fingerprint, and generation validation remain the CLI outline contract rather than a second MCP-specific implementation. The default page is 100 rows and the MCP-wide maximum is 200. `maxBytes` measures the fully enriched serialized `structuredContent`; a binary search rebuilds the page with fewer complete rows and therefore regenerates `next_cursor` from the actual returned count. A budget that cannot hold metadata plus one row fails instead of returning a zero-progress cursor. Default MCP symbol serialization remains backward-compatible, while explicit projection fields use the CLI snake_case names. | @@ -6153,7 +6153,7 @@ total / cursor / truncation、partial-family metadata、definition body の pagi | compact location envelope | CLI の `--format compact` location output は、`api_version`、返却 `count`、limit 到達を基準にした保守的な `truncated` / `truncation` metadata、適用済み `query_context`、軽量な `results` row を持つ versioned envelope です。 | | grouped search の総数 | `search --format grouped` の `total_matches` / `matched_count`、`total_groups`、`total_files` は、表示 page ではなく上限適用前の query 全体から算出します。`grouped_match_count` は返却 group に渡した row 数、`emitted_match_count` は file ごとの上限適用後に残った row 数を表し、`omitted_match_count`、`truncated`、`has_more`、`continuation_action` が未完了出力を示します。 | | 高ボリューム応答の bounded 契約 | `search`、`definition`、`find`、`status`、`hotspots`、`references`、`callers`、`callees`、`symbols`、`files`、`languages`、`impact`、`map` は、それぞれの schema が公開する共通 bounded-response control に対応します。新しく発行する opaque な `--cursor <response:v2:...>` は offset を command / query / filter と index generation に束縛し、移行用に legacy の `response:v1:<offset>:<fingerprint>` も受理します。選択条件または generation を変えて再利用すると restart-required の案内付きで失敗します。`search --format compact`、`symbols --format compact`、`files --format compact` は bounded 契約を自動選択し、`search --json=array --json-envelope` は opt-in の array envelope、`languages --json` は paging または `--max-json-bytes` 指定時に同じ契約を使います。既存 compact の root と location row は維持したまま共通 metadata を追加します。metadata は `returned_count`、取得可能な場合は authoritative な `total_count`、`omitted_count`、`remaining_count`、`cursor_offset`、`page_limit`、`has_more`、`next_cursor`、`result_stable_at`、`pagination_window_limit`、`pagination_window_exhausted` を返します。safety window は 10,000 row で、上限到達時は次の request が拒否する cursor を返さず `next_cursor` を抑止します。pageable command は `offset + limit` 件を serialize せず、cursor offset を database / scan layer へ渡します。`find --all` の partial scan cursor は次の path / line を保持し、再利用時は最後に scan した line の次から継続します。`hotspots` と `impact` は active な主要 nested collection を `results` としてページングし、`metadata.primary_collection` でその名前を示し、scalar / container evidence は `metadata.response_context` に保持します。`callers.path,callers.depth` のような dotted field で collection と row field を同時に選べます。`--max-json-bytes` は最後の改行を含み、完全な envelope が収まるまで末尾の完全な row を省略します。`definition` は既定で metadata-only のままで、明示的な `--body` は `body`、`body_content`、`all` で保持し、それ以外の projection では materialize 前に抑止します。`map --sections` は section-level projection として残り、dotted な bounded field は選択した array section を section 固有の総件数付きでページングし、scalar projection は不要な ranking array を構築しません。 | -| grouped partial-family の継続取得 | grouped `symbols` row は authoritative な family-member の総数、返却数、省略数、残数を報告し、materialize する member を最大 50 件に保ちます。opaque な recovery / next cursor は正規化済み symbol 選択、family key / ID、member offset、index generation に束縛されます。SQL が JSON aggregation 前に member offset を適用するため、継続取得時も family 全体を managed memory へ読み込みません。compact / projected 出力は `family_members` を省略しても recovery metadata を維持し、array、NDJSON、envelope、byte-bounded の各経路は同じ row 契約を serialize します。 | +| grouped partial-family の継続取得 | grouped `symbols` row は authoritative な family-member の総数、返却数、省略数、残数を報告し、materialize する member を最大 50 件に保ちます。opaque な recovery / next cursor は正規化済み symbol 選択、固定長の family ID、member offset、index generation に束縛され、長さが無制限になり得る family key 自体は埋め込みません。SQL が JSON aggregation 前に family-ID filter と member offset を適用するため、継続取得時も family 全体を managed memory へ読み込みません。compact / projected 出力は `family_members` を省略しても recovery metadata を維持し、array、NDJSON、envelope、byte-bounded の各経路は同じ row 契約を serialize します。 | | bounded outline 応答 | `outline` は `--max-json-bytes` がある場合だけ共通 bounded-response 契約を選択します。wrapper は projection 済みの完全な symbol row を抽出し、階層と決定的な順序を維持し、authoritative な返却 / 総 / 省略件数を報告します。UTF-8 計測には最後の改行を含め、束縛済みの `response:v2` continuation cursor を発行します。最小 envelope が収まらない場合は、実行可能な byte field を持つ型付きの `E028_RESPONSE_BUDGET_TOO_SMALL` object 1 件を stdout に出し、stderr を空に保ちます。上限なしの outline JSON は既存の root shape と outline cursor 契約を維持します。 | | bounded unused 応答 | `unused` は `--max-json-bytes` がある場合だけ共通 bounded-response 契約を選択します。wrapper は canonical な `symbols` row を抽出し、unused query layer で cursor offset を適用し、byte trimming 後の返却 bucket / confidence / contract-domain 件数を再計算します。任意の `by_bucket` view も同じ応答全体の UTF-8 budget に含め、compact mode はより小さな audit row へ projection します。continuation cursor は有効な audit filter、bucket mode、ordering、index generation に束縛されます。1 row を含む最小 envelope が収まらない場合は、実行可能な byte field を持つ型付きの `E028_RESPONSE_BUDGET_TOO_SMALL` object 1 件を stdout に出し、stderr を空に保ちます。上限なしの JSON、compact summary、legacy unused cursor は既存契約を維持します。 | | MCP outline page | MCP `outline` は `fields`、`sort`、`limit`、`cursor` を `QueryCommandRunner.BuildOutlinePage` へ渡すため、projection alias、派生 sort field、安定した tie-breaker、`page:v1` query fingerprint、generation validation は第 2 の MCP 固有実装ではなく CLI outline 契約のままです。既定 page は 100 row、MCP 共通の上限は 200 row です。`maxBytes` は enrichment 済みの `structuredContent` 全体を serialize した byte 数で計測し、binary search で完全な row 数を減らして page を再構築するため、`next_cursor` も実際の返却件数から再生成されます。metadata と 1 row が収まらない budget は、進捗しない cursor を返さず失敗します。既定の MCP symbol serialization は後方互換を維持し、明示的な projection field は CLI の snake_case 名を使います。 | diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 4faacdea8..b7ea4a4be 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -639,7 +639,7 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result - `QueryCommandRunnerTests.PartialCanonicalRepresentative_UsesSemanticRulesAndExposesFamilyNavigation_Issue4914`, `PartialCanonicalRepresentative_GroupsSplitModifierAndRanksLeadingEvidence_Issue4914`, `PartialCanonicalRepresentative_PersistsSplitModifierAndLeadingSemanticEvidence_Issue4914`, `PartialCanonicalRepresentative_RespectsFileLocalAndLexedEvidence_Issue4914`, `PartialCanonicalRepresentative_HonorsCaseSensitiveProjectMarkerCasing_Issue4914Review`, `GroupPartials_DegradesSafelyWhenCSharpFamilyContractIsStale_Issue4914`, and `PartialCanonicalRepresentative_FallsBackToDesignerPathForOldDatabaseMetadata_Issue4914` keep canonical partial-family ranking and navigation in one contract. Preserve partial classes, records (stored under the existing C# class kind) whose base lists repeat the declared name, file-local partial hosts that group only within one source file, partial types and containing generic types with distinct arity, case-sensitive project-marker casing when resolving nullable custom types, and declaration/implementation partial methods whose parameter attributes, names, quoted or commented defaults, declaration/type-position comments, comments between the identifier/generic parameters/parameter list, predefined and `dynamic` / `object` aliases, nullable value-type equivalents, predefined reference-type nullable annotations, explicitly global-rooted framework aliases with verbatim segments, shadowable unrooted `System` types, meaningful global-root qualification, qualified type leaves that collide with method generic-parameter names, verbatim method or type escapes, or user-type casing differ; lexer-confirmed identifier columns and semantic evidence when same-name attribute calls or documentation-like block-comment decoys precede the declaration; preceding-line `partial` modifiers across blank or C# comment trivia, modifiers after a balanced same-line attribute list, attributes bound only to their same-line declaration occurrence, repeated same-line declarations with distinct navigation columns, and blank-detached XML documentation that must not rank across extraction-worker transport, post-extraction hook cloning, and DB persistence; conservative physical-row fallback for stale family metadata whose arity-less keys would otherwise collide, while the Issue4850 LSP constructor fixture still distinguishes stale-index partial type targets; the non-C# function-column fallback; generated metadata and designer-name fallback; reversed insertion order and rebuild-stable family IDs; a semantic primary that remains primary after a path rename; normalized identifier-aligned member columns and normalized verbatim-identifier navigation ranges; a bounded family list that retains a late-sorting representative; grouped `definition` metadata; canonical `goto`; and physical `goto --all`. Keep the legacy-column fixture separate so schema initialization cannot silently restore the metadata being tested as absent. - `QueryCommandRunnerTests.GroupPartials_FamilyMembersContinueWithoutGapsAndRejectInvalidCursors_Issue5101` - keeps two same-named families separate while walking a 105-member family in stable 50-member pages without duplicates or gaps. Preserve recovery replay, compact metadata without the nested list, metadata-only byte-bounded recovery, raw NDJSON, field discovery, and malformed, selection-mismatch, and stale-generation cursor failures in the same fixture; perform the generation mutation last. + keeps two same-named families separate while walking a 105-member family with an over-4,096-character identity in stable 50-member pages without duplicates or gaps. Preserve the fixed-size cursor identity, recovery replay, compact metadata without the nested list, metadata-only byte-bounded recovery, raw NDJSON, field discovery, and malformed, selection-mismatch, and stale-generation cursor failures in the same fixture; perform the generation mutation last. - `LspServerTests.cs` LSP JSON-RPC behavior and indexed editor semantics. Reference parity coverage must compare LSP locations with the matching CLI candidate bundle, including overload identity and both `includeDeclaration` states. Keep document/workspace identifier ranges and explicit-versus-inferred inlay hints in one source-semantics fixture so persisted-column anchoring, source confirmation, and hint suppression cannot drift independently. Lifecycle transport coverage must stage initialization before sending shutdown/exit, use a caller-owned stream whose read after the exit frame cannot be interrupted by cancellation, and assert the shutdown response, success versus exit-before-shutdown status, and absence of any post-exit read. Keep finite EOF, malformed-frame, and caller-cancellation coverage alongside it so early transport completion cannot weaken those paths. @@ -1732,7 +1732,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" - `QueryCommandRunnerTests.PartialCanonicalRepresentative_UsesSemanticRulesAndExposesFamilyNavigation_Issue4914`、`PartialCanonicalRepresentative_GroupsSplitModifierAndRanksLeadingEvidence_Issue4914`、`PartialCanonicalRepresentative_PersistsSplitModifierAndLeadingSemanticEvidence_Issue4914`、`PartialCanonicalRepresentative_RespectsFileLocalAndLexedEvidence_Issue4914`、`GroupPartials_DegradesSafelyWhenCSharpFamilyContractIsStale_Issue4914`、`PartialCanonicalRepresentative_FallsBackToDesignerPathForOldDatabaseMetadata_Issue4914` は canonical な partial-family ranking と navigation を1つの契約として維持します。partial class、base list で宣言名が再登場する record（既存の C# class kind として保存）、同一 source file 内だけで集約する file-local partial host、arity が異なる partial type 自身と外側 generic type、parameter attribute・名前・quote または comment を含む default・declaration / type 内の comment・identifier / generic parameter / parameter list 間の comment・predefined alias と `dynamic` / `object` alias・nullable value type の同値表記・predefined reference type の nullable annotation・verbatim segment を含む明示的に global-rooted な framework alias・shadow 可能な root のない `System` type・意味のある global-root 修飾・method generic parameter 名と衝突する qualified type leaf・method または type の verbatim escape・user type の大文字小文字が異なる宣言 / 実装の partial method、同名 attribute call や documentation 風 block-comment decoy が宣言より前にある場合も lexer で確認した identifier column と semantic evidence を使うこと、C# comment を含む前の行へ分割された `partial` 修飾子、balanced な同一行 attribute list 後の modifier、正しい同一行 declaration occurrence だけに属する attribute、別々の navigation column を持つ同一行の反復宣言、先行 XML documentation が extraction-worker 搬送、post-extraction hook の clone、DB 保存を越えて残ること、blank-detached XML documentation は rank しないこと、arity のない key が衝突する stale family metadata では物理 row へ保守的に fallback しつつ Issue4850 の LSP constructor fixture では stale-index の partial type target を区別すること、non-C# function の column fallback、generated metadata と designer filename fallback、挿入順を逆転した再構築でも安定する family ID、path rename 後も primary であり続ける semantic declaration、正規化後の identifier に揃った member column と正規化後の verbatim identifier に揃った navigation range、遅い path の representative を保持する上限付き family list、grouped `definition` metadata、canonical `goto`、物理位置を返す `goto --all` を維持してください。欠落 metadata が schema initialization によって暗黙に復元されないよう、legacy-column fixture は独立したままにしてください。 - `QueryCommandRunnerTests.GroupPartials_FamilyMembersContinueWithoutGapsAndRejectInvalidCursors_Issue5101` - は同名の2 family を分離したまま、105 member の family を50件ずつ安定順で走査し、重複も欠落もないことを検証します。同じ fixture で recovery replay、nested list を省略する compact metadata、metadata-only の byte-bounded recovery、raw NDJSON、field discovery、malformed・選択条件 mismatch・index generation stale の cursor failure を維持し、generation の変更は最後に行ってください。 + は同名の2 family を分離したまま、4,096文字を超える identity を持つ105 member の family を50件ずつ安定順で走査し、重複も欠落もないことを検証します。同じ fixture で固定長 cursor identity、recovery replay、nested list を省略する compact metadata、metadata-only の byte-bounded recovery、raw NDJSON、field discovery、malformed・選択条件 mismatch・index generation stale の cursor failure を維持し、generation の変更は最後に行ってください。 - `LspServerTests.cs` LSP の JSON-RPC 挙動と indexed editor semantics のテスト。reference parity coverage では、overload identity と `includeDeclaration` の両状態を含め、LSP location と対応する CLI candidate bundle を比較してください。document/workspace の identifier range と explicit/inferred inlay hint は 1 つの source-semantics fixture にまとめ、保存済み column の anchoring、source 上の確認、hint 抑制が別々に drift しないようにします。 lifecycle transport coverage では、shutdown/exit の送信前に initialization を stage し、exit frame 後の read が cancellation では中断されない caller-owned stream を使い、shutdown response、success と shutdown 前 exit の status の違い、post-exit read が一度も始まらないことを検証してください。早期 transport completion によって他経路が弱まらないよう、finite EOF、malformed frame、caller cancellation の coverage も並行して維持します。 diff --git a/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs b/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs index 0b76dd44b..e8debbdc1 100644 --- a/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs +++ b/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs @@ -317,7 +317,6 @@ private static int RunBoundedResponse( controls.ResumeFileOrdinal, controls.ResumeMatchOrdinal, controls.ResumeByteOffset, - controls.PartialFamilyKey, controls.PartialFamilyId, controls.FamilyMemberOffset); using var executionScope = EnterBoundedExecution(executionContext); @@ -1828,7 +1827,6 @@ private static bool TryParseBoundedResponseControls( int? resumeFileOrdinal = null; int? resumeMatchOrdinal = null; int? resumeByteOffset = null; - string? partialFamilyKey = null; string? partialFamilyId = null; int? familyMemberOffset = null; if (cursor is not null @@ -1842,7 +1840,6 @@ private static bool TryParseBoundedResponseControls( out resumeFileOrdinal, out resumeMatchOrdinal, out resumeByteOffset, - out partialFamilyKey, out partialFamilyId, out familyMemberOffset)) { @@ -1863,7 +1860,6 @@ private static bool TryParseBoundedResponseControls( resumeFileOrdinal, resumeMatchOrdinal, resumeByteOffset, - partialFamilyKey, partialFamilyId, familyMemberOffset); return true; @@ -1994,7 +1990,6 @@ internal static string FormatResponseCursor( int? resumeFileOrdinal = null, int? resumeMatchOrdinal = null, int? resumeByteOffset = null, - string? partialFamilyKey = null, string? partialFamilyId = null, int? familyMemberOffset = null) { @@ -2014,17 +2009,14 @@ internal static string FormatResponseCursor( payload["resume_match_ordinal"] = resumeMatchOrdinal.Value; if (resumeByteOffset.HasValue) payload["resume_byte_offset"] = resumeByteOffset.Value; - if (partialFamilyKey is not null - && partialFamilyId is not null + if (partialFamilyId is not null && familyMemberOffset.HasValue) { - payload["partial_family_key"] = partialFamilyKey; payload["partial_family_id"] = partialFamilyId; payload["family_member_offset"] = familyMemberOffset.Value; payload["family_member_integrity"] = BuildPartialFamilyCursorIntegrity( queryFingerprint, generationFingerprint, - partialFamilyKey, partialFamilyId, familyMemberOffset.Value); } @@ -2056,7 +2048,6 @@ internal static bool TryParseResponseCursor( out resumeMatchOrdinal, out resumeByteOffset, out _, - out _, out _); internal static bool TryParseResponseCursor( @@ -2069,7 +2060,6 @@ internal static bool TryParseResponseCursor( out int? resumeFileOrdinal, out int? resumeMatchOrdinal, out int? resumeByteOffset, - out string? partialFamilyKey, out string? partialFamilyId, out int? familyMemberOffset) { @@ -2081,7 +2071,6 @@ internal static bool TryParseResponseCursor( resumeFileOrdinal = null; resumeMatchOrdinal = null; resumeByteOffset = null; - partialFamilyKey = null; partialFamilyId = null; familyMemberOffset = null; if (cursor.StartsWith(LegacyResponseCursorPrefix, StringComparison.Ordinal)) @@ -2161,7 +2150,6 @@ internal static bool TryParseResponseCursor( } resumeByteOffset = parsedByteOffset; } - partialFamilyKey = ReadString(payload, "partial_family_key"); partialFamilyId = ReadString(payload, "partial_family_id"); var partialFamilyIntegrity = ReadString(payload, "family_member_integrity"); if (payload.ContainsKey("family_member_offset")) @@ -2176,26 +2164,19 @@ internal static bool TryParseResponseCursor( var extendedResumeFieldsPresent = resumeFileOrdinal.HasValue || resumeMatchOrdinal.HasValue || resumeByteOffset.HasValue; - var partialFamilyFieldsPresent = partialFamilyKey is not null - || partialFamilyId is not null + var partialFamilyFieldsPresent = partialFamilyId is not null || familyMemberOffset.HasValue || partialFamilyIntegrity is not null; var partialFamilyFieldsValid = !partialFamilyFieldsPresent || offset == 0 - && partialFamilyKey is { Length: > 0 and <= 4096 } - && partialFamilyId is { Length: > 0 and <= 128 } + && IsPartialFamilyId(partialFamilyId) && familyMemberOffset is >= 0 - && string.Equals( - partialFamilyId, - LogicalPartialSymbolGrouper.BuildPartialFamilyId(partialFamilyKey), - StringComparison.Ordinal) && string.Equals( partialFamilyIntegrity, BuildPartialFamilyCursorIntegrity( queryFingerprint!, generationFingerprint!, - partialFamilyKey, - partialFamilyId, + partialFamilyId!, familyMemberOffset.Value), StringComparison.Ordinal); return IsCursorFingerprint(queryFingerprint) @@ -2209,24 +2190,29 @@ internal static bool TryParseResponseCursor( && resumeByteOffset is >= 0 && resumeMatchOrdinal is null or >= 0) && (!resumeMatchOrdinal.HasValue || resumeByteOffset.HasValue) + && !payload.ContainsKey("partial_family_key") && partialFamilyFieldsValid; } private static string BuildPartialFamilyCursorIntegrity( string queryFingerprint, string generationFingerprint, - string partialFamilyKey, string partialFamilyId, int familyMemberOffset) => BuildResponseValueFingerprint(string.Join( '\0', - "partial-family-members:v1", + "partial-family-members:v2", queryFingerprint, generationFingerprint, - partialFamilyKey, partialFamilyId, familyMemberOffset.ToString(CultureInfo.InvariantCulture))); + private static bool IsPartialFamilyId(string? value) + => value is { Length: 32 } + && value.StartsWith("partial:", StringComparison.Ordinal) + && value["partial:".Length..].All(static character => + character is >= '0' and <= '9' or >= 'a' and <= 'f'); + private static bool IsCursorFingerprint(string? fingerprint) => fingerprint is { Length: 16 } && fingerprint.All(Uri.IsHexDigit); @@ -2342,7 +2328,6 @@ private sealed record BoundedResponseControls( int? ResumeFileOrdinal, int? ResumeMatchOrdinal, int? ResumeByteOffset, - string? PartialFamilyKey, string? PartialFamilyId, int? FamilyMemberOffset) { @@ -2353,7 +2338,7 @@ private sealed record BoundedResponseControls( { var preserveFullDiscoveryRows = command is "search" or "languages" || command == "symbols" - && PartialFamilyKey is not null + && PartialFamilyId is not null && !Compact; var selected = Fields ?? (statusExplainRequest @@ -2465,7 +2450,6 @@ internal static (string Cursor, string? ResultStableAt) BuildFindResumeCursor( internal static string BuildPartialFamilyMembersCursor( string[] args, PartialFamilyCursorSnapshot snapshot, - string partialFamilyKey, string partialFamilyId, int familyMemberOffset) { @@ -2473,7 +2457,6 @@ internal static string BuildPartialFamilyMembersCursor( offset: 0, BuildResponseFingerprint("symbols", args), snapshot.GenerationFingerprint, - partialFamilyKey: partialFamilyKey, partialFamilyId: partialFamilyId, familyMemberOffset: familyMemberOffset); } @@ -2519,7 +2502,6 @@ internal static int GetBoundedResponseOffset(string command) var execution = BoundedExecution.Value; if (execution is null || !string.Equals(execution.Command, CanonicalizeCommandName(command), StringComparison.Ordinal) - || execution.PartialFamilyKey is null || execution.PartialFamilyId is null || !execution.FamilyMemberOffset.HasValue) { @@ -2527,13 +2509,11 @@ internal static int GetBoundedResponseOffset(string command) } return new PartialFamilyContinuation( - execution.PartialFamilyKey, execution.PartialFamilyId, execution.FamilyMemberOffset.Value); } internal readonly record struct PartialFamilyContinuation( - string PartialFamilyKey, string PartialFamilyId, int FamilyMemberOffset); @@ -2719,7 +2699,6 @@ private sealed record BoundedExecutionContext( int? ResumeFileOrdinal, int? ResumeMatchOrdinal, int? ResumeByteOffset, - string? PartialFamilyKey, string? PartialFamilyId, int? FamilyMemberOffset) { diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Discovery.cs b/src/CodeIndex/Cli/QueryCommandRunner.Discovery.cs index 32d509679..cfa7e9e2a 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Discovery.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Discovery.cs @@ -200,7 +200,7 @@ public static int RunSymbols(string[] cmdArgs, JsonSerializerOptions jsonOptions sortMode: options.SymbolSortMode, groupPartials: true, offset: JsonEnvelopeWrapper.GetBoundedResponseOffset("symbols"), - partialFamilyKey: partialFamilyContinuation?.PartialFamilyKey, + partialFamilyId: partialFamilyContinuation?.PartialFamilyId, familyMemberOffset: partialFamilyContinuation?.FamilyMemberOffset ?? 0) : reader.SearchSymbols(symbolQueries, options.Limit, options.Kind, options.Lang, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, options.Since, exact, visibilityFilters: options.VisibilityFilters, excludeVisibilityFilters: options.ExcludeVisibilityFilters, sortMode: options.SymbolSortMode, offset: JsonEnvelopeWrapper.GetBoundedResponseOffset("symbols")); if (options.GroupPartials) @@ -392,7 +392,6 @@ private static void AddPartialFamilyContinuationMetadata( { if (result.DefinitionSites is not { } totalCount || result.FamilyMembers is not { } members - || result.LogicalPartialKey is null || result.PartialFamilyId is null) { continue; @@ -409,7 +408,6 @@ private static void AddPartialFamilyContinuationMetadata( result.FamilyMembersRecoveryCursor = JsonEnvelopeWrapper.BuildPartialFamilyMembersCursor( commandArgs, snapshot, - result.LogicalPartialKey, result.PartialFamilyId, familyMemberOffset: 0); if (result.FamilyMemberRemainingCount > 0) @@ -417,7 +415,6 @@ private static void AddPartialFamilyContinuationMetadata( result.FamilyMembersNextCursor = JsonEnvelopeWrapper.BuildPartialFamilyMembersCursor( commandArgs, snapshot, - result.LogicalPartialKey, result.PartialFamilyId, familyMemberOffset + returnedCount); } diff --git a/src/CodeIndex/Database/DbContext.ConnectionFunctionRegistration.cs b/src/CodeIndex/Database/DbContext.ConnectionFunctionRegistration.cs index 29a20a5fe..d26222494 100644 --- a/src/CodeIndex/Database/DbContext.ConnectionFunctionRegistration.cs +++ b/src/CodeIndex/Database/DbContext.ConnectionFunctionRegistration.cs @@ -149,6 +149,10 @@ private static void RegisterCSharpPartialIdentityFunctions(SqliteConnection conn "csharp_partial_declaration_identity", (string? signature) => LogicalPartialSymbolGrouper.BuildCanonicalDeclarationIdentity(signature), isDeterministic: true); + connection.CreateFunction( + "codeindex_partial_family_id", + (string? key) => key is null ? null : LogicalPartialSymbolGrouper.BuildPartialFamilyId(key), + isDeterministic: true); RegisterCSharpPartialDeclaration(connection); } diff --git a/src/CodeIndex/Database/DbReader.SymbolSearchList.cs b/src/CodeIndex/Database/DbReader.SymbolSearchList.cs index 7ac05b2b5..ef5f1fb9b 100644 --- a/src/CodeIndex/Database/DbReader.SymbolSearchList.cs +++ b/src/CodeIndex/Database/DbReader.SymbolSearchList.cs @@ -8,7 +8,7 @@ public partial class DbReader /// 複数名前パターン（OR結合）でシンボルを検索。空/null なら他フィルタに一致する全シンボルを返す。 /// <paramref name="exact"/> が true の場合、部分一致ではなく大文字小文字を無視した完全一致になる。 /// </summary> - public List<SymbolResult> SearchSymbols(IReadOnlyList<string>? queries, int limit = 20, string? kind = null, string? lang = null, IReadOnlyList<string>? pathPatterns = null, IReadOnlyList<string>? excludePathPatterns = null, bool excludeTests = false, DateTime? since = null, bool exact = false, IReadOnlyList<string>? visibilityFilters = null, IReadOnlyList<string>? excludeVisibilityFilters = null, SymbolSortMode sortMode = SymbolSortMode.Name, int? startLine = null, int? endLine = null, bool groupPartials = false, int offset = 0, string? partialFamilyKey = null, int familyMemberOffset = 0) + public List<SymbolResult> SearchSymbols(IReadOnlyList<string>? queries, int limit = 20, string? kind = null, string? lang = null, IReadOnlyList<string>? pathPatterns = null, IReadOnlyList<string>? excludePathPatterns = null, bool excludeTests = false, DateTime? since = null, bool exact = false, IReadOnlyList<string>? visibilityFilters = null, IReadOnlyList<string>? excludeVisibilityFilters = null, SymbolSortMode sortMode = SymbolSortMode.Name, int? startLine = null, int? endLine = null, bool groupPartials = false, int offset = 0, string? partialFamilyId = null, int familyMemberOffset = 0) { var plan = SymbolSearchQueryPlanBuilder.Build(new SymbolSearchQueryPlan { @@ -28,7 +28,7 @@ public List<SymbolResult> SearchSymbols(IReadOnlyList<string>? queries, int limi EndLine = endLine, GroupPartials = groupPartials, Offset = offset, - PartialFamilyKey = partialFamilyKey, + PartialFamilyId = partialFamilyId, FamilyMemberOffset = Math.Max(0, familyMemberOffset), }); return ExecuteSymbolSearchList(plan); @@ -59,8 +59,8 @@ private List<SymbolResult> ExecuteSymbolSearchList(SymbolSearchQueryPlan plan) if (plan.GroupPartials) { SqliteCommandPolicy.Add(cmd, "@familyMemberOffset", plan.FamilyMemberOffset); - if (plan.PartialFamilyKey is not null) - SqliteCommandPolicy.Add(cmd, "@partialFamilyKey", plan.PartialFamilyKey); + if (plan.PartialFamilyId is not null) + SqliteCommandPolicy.Add(cmd, "@partialFamilyId", plan.PartialFamilyId); } using var reader = cmd.ExecuteTrackedReader(); diff --git a/src/CodeIndex/Database/DbReader.SymbolSearchMultiQuery.cs b/src/CodeIndex/Database/DbReader.SymbolSearchMultiQuery.cs index 725653747..63c7f8ff5 100644 --- a/src/CodeIndex/Database/DbReader.SymbolSearchMultiQuery.cs +++ b/src/CodeIndex/Database/DbReader.SymbolSearchMultiQuery.cs @@ -27,7 +27,7 @@ private List<SymbolResult> ExecuteMultiQuerySymbolSearch(SymbolSearchQueryPlan p plan.StartLine, plan.EndLine, plan.GroupPartials, - partialFamilyKey: plan.PartialFamilyKey, + partialFamilyId: plan.PartialFamilyId, familyMemberOffset: plan.FamilyMemberOffset)); } diff --git a/src/CodeIndex/Database/DbReader.SymbolSearchQueryPlan.cs b/src/CodeIndex/Database/DbReader.SymbolSearchQueryPlan.cs index 518ededc6..64fc91da3 100644 --- a/src/CodeIndex/Database/DbReader.SymbolSearchQueryPlan.cs +++ b/src/CodeIndex/Database/DbReader.SymbolSearchQueryPlan.cs @@ -20,7 +20,7 @@ private sealed record SymbolSearchQueryPlan public int? EndLine { get; init; } public bool GroupPartials { get; init; } public int Offset { get; init; } - public string? PartialFamilyKey { get; init; } + public string? PartialFamilyId { get; init; } public int FamilyMemberOffset { get; init; } } diff --git a/src/CodeIndex/Database/DbReader.SymbolSearchSql.cs b/src/CodeIndex/Database/DbReader.SymbolSearchSql.cs index aa5632d07..54d492987 100644 --- a/src/CodeIndex/Database/DbReader.SymbolSearchSql.cs +++ b/src/CodeIndex/Database/DbReader.SymbolSearchSql.cs @@ -8,9 +8,9 @@ private static string GetGenericSymbolRankNamePenaltySql(string nameSql) private static string BuildLogicalPartialSymbolQuery(string matchingSymbolsSql, SymbolSearchQueryPlan plan) { var orderBy = BuildLogicalPartialSortOrderBy(plan.SortMode); - var partialFamilyFilter = plan.PartialFamilyKey is null + var partialFamilyFilter = plan.PartialFamilyId is null ? string.Empty - : "WHERE logical_partial_key = @partialFamilyKey"; + : "WHERE codeindex_partial_family_id(logical_partial_key) = @partialFamilyId"; return $@" WITH matching_symbols AS ( {matchingSymbolsSql} diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerPartialFamilyContinuationIssue5101Tests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerPartialFamilyContinuationIssue5101Tests.cs index e9f266d78..3a950043d 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerPartialFamilyContinuationIssue5101Tests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerPartialFamilyContinuationIssue5101Tests.cs @@ -15,6 +15,7 @@ public void GroupPartials_FamilyMembersContinueWithoutGapsAndRejectInvalidCursor try { var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + var longNamespace = "N" + new string('x', 4_999); for (var index = 0; index < 105; index++) { TestProjectHelper.InsertIndexedFile( @@ -22,8 +23,8 @@ public void GroupPartials_FamilyMembersContinueWithoutGapsAndRejectInvalidCursor $"src/One/Wide.{index:D3}.cs", "csharp", index == 104 - ? "namespace Demo.One;\npublic partial class Wide : BaseWide { }" - : "namespace Demo.One;\npublic partial class Wide { }"); + ? $"namespace {longNamespace};\npublic partial class Wide : BaseWide {{ }}" + : $"namespace {longNamespace};\npublic partial class Wide {{ }}"); } for (var index = 0; index < 2; index++) { @@ -58,6 +59,10 @@ public void GroupPartials_FamilyMembersContinueWithoutGapsAndRejectInvalidCursor Assert.Contains("src/One/Wide.104.cs", firstPage.RepresentativePaths); Assert.NotNull(firstPage.RecoveryCursor); Assert.NotNull(firstPage.NextCursor); + Assert.InRange(firstPage.NextCursor!.Length, 1, 1_024); + var nextCursorPayload = DecodeIssue5101Cursor(firstPage.NextCursor); + Assert.False(nextCursorPayload.ContainsKey("partial_family_key")); + Assert.Equal(firstPage.PartialFamilyId, nextCursorPayload["partial_family_id"]!.GetValue<string>()); var memberIds = new List<long>(firstPage.MemberIds); var cursor = firstPage.NextCursor; @@ -315,17 +320,22 @@ private void AssertIssue5101CursorFailure(string[] args, string expectedError) private static string MutateIssue5101Cursor(string cursor, Action<JsonObject> mutate) { - const string prefix = "response:v2:"; - Assert.StartsWith(prefix, cursor, StringComparison.Ordinal); - var encoded = cursor[prefix.Length..].Replace('-', '+').Replace('_', '/'); - encoded += new string('=', (4 - encoded.Length % 4) % 4); - var payload = JsonNode.Parse(Encoding.UTF8.GetString(Convert.FromBase64String(encoded)))!.AsObject(); + var payload = DecodeIssue5101Cursor(cursor); mutate(payload); var mutated = Convert.ToBase64String(Encoding.UTF8.GetBytes(payload.ToJsonString())) .TrimEnd('=') .Replace('+', '-') .Replace('/', '_'); - return prefix + mutated; + return "response:v2:" + mutated; + } + + private static JsonObject DecodeIssue5101Cursor(string cursor) + { + const string prefix = "response:v2:"; + Assert.StartsWith(prefix, cursor, StringComparison.Ordinal); + var encoded = cursor[prefix.Length..].Replace('-', '+').Replace('_', '/'); + encoded += new string('=', (4 - encoded.Length % 4) % 4); + return JsonNode.Parse(Encoding.UTF8.GetString(Convert.FromBase64String(encoded)))!.AsObject(); } private sealed record FamilyPageIssue5101( From 1cdb9b7b85589197e4fbca9e43ef3a961e7bdf2a Mon Sep 17 00:00:00 2001 From: Widthdom <widthdom@gmail.com> Date: Fri, 21 Aug 2026 17:58:54 +0900 Subject: [PATCH 4/4] Preserve partial family recovery projections (#5101) --- .../Cli/JsonEnvelopeWrapper.Bounded.cs | 40 +++++++++++++++++-- ...PartialFamilyContinuationIssue5101Tests.cs | 3 +- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs b/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs index e8debbdc1..dee87aa3a 100644 --- a/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs +++ b/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs @@ -71,6 +71,19 @@ internal static partial class JsonEnvelopeWrapper "unused_contract_domain", ]; + private static readonly string[] PartialFamilyContinuationProjectionFields = + [ + "partial_family_id", + "family_members_truncated", + "family_member_total_count", + "family_member_total_count_authoritative", + "family_member_returned_count", + "family_member_omitted_count", + "family_member_remaining_count", + "family_members_recovery_cursor", + "family_members_next_cursor", + ]; + internal static bool ShouldAutoWrapBoundedResponse(string command, string[] args) { if (!BoundedResponseCommands.Contains(command) @@ -380,7 +393,8 @@ private static int RunBoundedResponse( var effectiveFields = controls.EffectiveFields( command, extraction.PrimaryCollection, - suppressRuntimeMetadata); + suppressRuntimeMetadata, + groupedSymbolsRequest: command == "symbols" && HasArgument(args, "--group-partials")); var pageItems = availableItems .Take(controls.PageLimit) .Select(item => ProjectResponseItem( @@ -2334,7 +2348,8 @@ private sealed record BoundedResponseControls( public IReadOnlyList<string>? EffectiveFields( string command, string? primaryCollection, - bool statusExplainRequest) + bool statusExplainRequest, + bool groupedSymbolsRequest) { var preserveFullDiscoveryRows = command is "search" or "languages" || command == "symbols" @@ -2358,11 +2373,28 @@ private sealed record BoundedResponseControls( .Select(field => field[(primaryCollection.Length + 1)..]) .ToList(); if (dotted.Count > 0) - return dotted; + return PreservePartialFamilyContinuationFields( + dotted, + command, + groupedSymbolsRequest); if (selected.Contains(primaryCollection, StringComparer.Ordinal)) return null; - return selected.Where(field => !field.Contains('.')).ToList(); + return PreservePartialFamilyContinuationFields( + selected.Where(field => !field.Contains('.')).ToList(), + command, + groupedSymbolsRequest); } + + private static IReadOnlyList<string> PreservePartialFamilyContinuationFields( + IReadOnlyList<string> selected, + string command, + bool groupedSymbolsRequest) + => command == "symbols" && groupedSymbolsRequest + ? selected + .Concat(PartialFamilyContinuationProjectionFields) + .Distinct(StringComparer.Ordinal) + .ToArray() + : selected; } private sealed record ResponseExtraction( diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerPartialFamilyContinuationIssue5101Tests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerPartialFamilyContinuationIssue5101Tests.cs index 3a950043d..c81c0024b 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerPartialFamilyContinuationIssue5101Tests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerPartialFamilyContinuationIssue5101Tests.cs @@ -128,8 +128,7 @@ public void GroupPartials_FamilyMembersContinueWithoutGapsAndRejectInvalidCursor var metadataOnlyArgs = commonArgs.Concat([ "--fields", - "path,partial_family_id,family_member_total_count,family_member_returned_count," - + "family_member_omitted_count,family_members_recovery_cursor,family_members_next_cursor", + "path", "--max-json-bytes", "12000", ]).ToArray(); var metadataOnlyPage = RunIssue5101FamilyPage(