From 4d62adb78ed88005a0ec08f04997f8428b5f9614 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 21 Aug 2026 22:36:39 +0900 Subject: [PATCH 1/3] Surface unsupported index diagnostics (#5100) --- AGENT_GUIDE.md | 2 +- DEVELOPER_GUIDE.md | 4 +- README.md | 14 ++- USER_GUIDE.md | 58 +++++++++ changelog.d/unreleased/5100.changed.md | 21 ++++ src/CodeIndex/Cli/ConsoleUi.cs | 1 + .../ExportImportCommandRunner.ArchiveScope.cs | 6 +- .../Cli/IndexCommandRunner.DryRun.cs | 43 +++++++ .../Cli/IndexCommandRunner.FullScan.Output.cs | 54 ++++++++- src/CodeIndex/Cli/JsonOutputContracts.cs | 18 +++ src/CodeIndex/Cli/ProjectionFieldRegistry.cs | 5 + .../Database/DbContext.SchemaMetadata.cs | 8 +- src/CodeIndex/Database/DbReader.Status.cs | 11 ++ .../Database/DbWriter.StatusMetadata.cs | 12 +- .../Database/UnknownExtensionClassifier.cs | 21 +++- .../FileIndexer.DirectoryFiltering.cs | 8 +- .../Scanning/FileIndexer.FileAcceptance.cs | 2 +- src/CodeIndex/Models/QueryResults.cs | 19 ++- tests/CodeIndex.Tests/ConsoleUiTests.cs | 12 ++ .../DocumentationStatusContractTests.cs | 5 + .../ExportImportCommandRunnerTests.cs | 6 + tests/CodeIndex.Tests/FileIndexerTests.cs | 6 +- .../IndexCommandRunnerDryRunTests.cs | 23 +++- .../IndexCommandRunnerTests.cs | 110 +++++++++++++++++- .../JsonStringListCodecTests.cs | 35 ++++++ 25 files changed, 467 insertions(+), 37 deletions(-) create mode 100644 changelog.d/unreleased/5100.changed.md diff --git a/AGENT_GUIDE.md b/AGENT_GUIDE.md index 075975b729..b73a22068a 100644 --- a/AGENT_GUIDE.md +++ b/AGENT_GUIDE.md @@ -143,7 +143,7 @@ CI watching must be bounded. Do not loop indefinitely. ## Status Contract -- `status --json` and related JSON/MCP payloads currently expose the trust fields documented in `README.md` and `DEVELOPER_GUIDE.md`, including `fold_ready`, `fold_ready_reason`, `graph_table_available`, `graph_data_current`, `index_complete`, `index_incomplete_reasons`, `issues_table_available`, `file_issues_data_current`, `migration_in_progress`, `sql_graph_contract_ready`, `sql_graph_contract_degraded_reason`, `hotspot_family_ready`, `hotspot_family_degraded_reason`, `language_readiness`, `csharp_symbol_name_ready`, `csharp_metadata_target_ready`, `csharp_metadata_target_degraded_reason`, `indexed_head_commit`, `workspace_verified_head_sha`, `worktree_head_changed`, `indexed_head_sha`, `indexed_head_branch`, `indexed_head_timestamp`, `commits_ahead_of_indexed_head`, `head_freshness`, `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`, `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `extractors`, `git_executable`, `path_case_sensitive`, `data_dir`, `data_dir_source`, `data_dir_mode`, `db_file_mode`, `database_permission_policy`, `database_permission_diagnostics`, `mac_profile`, `mac_profile_diagnostics`, `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `busy_timeout_ms`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`), `prepared_command_cache` (`count`, `capacity`, `hit_count`, `miss_count`, `eviction_count`), `maintenance_guidance`, WAL checkpoint diagnostics (`read_only_fallback`, `wal_checkpoint_attempted`, `wal_checkpoint_succeeded`, `wal_checkpoint_skipped_reason`, `wal_checkpoint_failure_reason`, `wal_checkpoint_busy`, `wal_checkpoint_log_page_count`, `wal_checkpoint_checkpointed_page_count`, `wal_checkpoint_remaining_page_count`, `read_only_immutable_fallback`, `wal_stale_snapshot_risk`, `wal_stale_snapshot_reason`), `symbol_kinds`, `symbols_by_language`, status kind cap metadata (`symbol_kind_limit`, `symbol_kind_name_limit`, `symbol_kind_total_count`, `symbol_kind_omitted_count`, `symbol_kind_names_truncated`, `symbols_by_language_kind_total_counts`, `symbols_by_language_kind_omitted_counts`, `symbols_by_language_kind_names_truncated`), `process`, `last_index_run`, `last_failed_or_partial_index_run`, `last_failed_or_partial_index_run.progress_persisted`, `last_failed_or_partial_index_run.recovery_hint`, `last_failed_or_partial_index_run.file_errors`, `last_workspace_freshened_at`, `hooks`, `hook_diagnostics`, `trust_overrides`, MCP-only `mcp_session`, `mcp.rate_limit.bucket_limit`, `mcp.rate_limit.bucket_limit_rejection_count`, and the `status --check`-only `stale_after_seconds` / `index_age_seconds` threshold audit fields and `repair_commands`. +- `status --json` and related JSON/MCP payloads currently expose the trust fields documented in `README.md` and `DEVELOPER_GUIDE.md`, including `fold_ready`, `fold_ready_reason`, `graph_table_available`, `graph_data_current`, `index_complete`, `index_incomplete_reasons`, `issues_table_available`, `file_issues_data_current`, `migration_in_progress`, `sql_graph_contract_ready`, `sql_graph_contract_degraded_reason`, `hotspot_family_ready`, `hotspot_family_degraded_reason`, `language_readiness`, `csharp_symbol_name_ready`, `csharp_metadata_target_ready`, `csharp_metadata_target_degraded_reason`, `indexed_head_commit`, `workspace_verified_head_sha`, `worktree_head_changed`, `indexed_head_sha`, `indexed_head_branch`, `indexed_head_timestamp`, `commits_ahead_of_indexed_head`, `head_freshness`, `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`, `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `unknown_extension_group_count`, `unknown_extension_groups_truncated`, `unknown_extension_group_limit`, `unknown_extension_group_omitted_count`, `unknown_extension_guidance`, `extractors`, `git_executable`, `path_case_sensitive`, `data_dir`, `data_dir_source`, `data_dir_mode`, `db_file_mode`, `database_permission_policy`, `database_permission_diagnostics`, `mac_profile`, `mac_profile_diagnostics`, `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `busy_timeout_ms`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`), `prepared_command_cache` (`count`, `capacity`, `hit_count`, `miss_count`, `eviction_count`), `maintenance_guidance`, WAL checkpoint diagnostics (`read_only_fallback`, `wal_checkpoint_attempted`, `wal_checkpoint_succeeded`, `wal_checkpoint_skipped_reason`, `wal_checkpoint_failure_reason`, `wal_checkpoint_busy`, `wal_checkpoint_log_page_count`, `wal_checkpoint_checkpointed_page_count`, `wal_checkpoint_remaining_page_count`, `read_only_immutable_fallback`, `wal_stale_snapshot_risk`, `wal_stale_snapshot_reason`), `symbol_kinds`, `symbols_by_language`, status kind cap metadata (`symbol_kind_limit`, `symbol_kind_name_limit`, `symbol_kind_total_count`, `symbol_kind_omitted_count`, `symbol_kind_names_truncated`, `symbols_by_language_kind_total_counts`, `symbols_by_language_kind_omitted_counts`, `symbols_by_language_kind_names_truncated`), `process`, `last_index_run`, `last_failed_or_partial_index_run`, `last_failed_or_partial_index_run.progress_persisted`, `last_failed_or_partial_index_run.recovery_hint`, `last_failed_or_partial_index_run.file_errors`, `last_workspace_freshened_at`, `hooks`, `hook_diagnostics`, `trust_overrides`, MCP-only `mcp_session`, `mcp.rate_limit.bucket_limit`, `mcp.rate_limit.bucket_limit_rejection_count`, and the `status --check`-only `stale_after_seconds` / `index_age_seconds` threshold audit fields and `repair_commands`. - `status --check` repair actions are structured by `name`, `action`, `args`, `mutation_class`, `safety_class`, and `safety_notes`. Preserve compatibility `reason` as the first trigger and ordered `reasons` as the complete trigger set. Deduplicate only exact structured identities; different targets, options, actions, mutation classes, or safety semantics must remain distinct. JSON and human output must use the same deduplicated order. Human output must preserve platform-aware shell quoting, visibly escape control characters to keep each repair action on one diagnostic line, and leave structured JSON `args` unchanged. - `maintenance_guidance.fts_optimization` is the shared, read-only recommendation contract for status, explain, optimize preview, and optimize execution. Keep `recommended`, `action`, `reason`, `threshold_writes`, `observed_writes`, and `state` synchronized; stale or unavailable snapshots must not recommend mutation. - A successful CLI or MCP `index --rebuild` evaluates the shared freelist warning threshold after the index transaction commits. Incremental-auto-vacuum databases may run bounded `PRAGMA incremental_vacuum`; legacy databases must skip automatic reclaim instead of running a full `VACUUM`. Preserve the immediate index-result and persisted `last_index_run.rebuild_reclaim` telemetry, stable states/reasons, before/after ratios and byte/page counts, and the rule that reclaim failure never reclassifies an already committed index run as failed. diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 2df80a2527..f1b2c61f98 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -2255,7 +2255,7 @@ catalog and unknown-field errors are generated from that same schema. | Readiness and graph trust | `fold_ready`, `fold_ready_reason`, `graph_table_available`, `graph_data_current`, `reference_extraction_limits`, `reference_graph_complete`, `reference_graph_incomplete_reasons`, `reference_extraction_cap_hits`, `index_complete`, `index_incomplete_reasons`, `issues_table_available`, `file_issues_data_current`, `migration_in_progress`, `sql_graph_contract_ready`, `sql_graph_contract_degraded_reason`, `hotspot_family_ready`, `hotspot_family_degraded_reason`, `language_readiness`, `csharp_symbol_name_ready`, `csharp_metadata_target_ready`, `csharp_metadata_target_degraded_reason`. | | Workspace and HEAD freshness | `indexed_head_commit`, `worktree_head_changed`, `indexed_head_sha`, `indexed_head_branch`, `indexed_head_timestamp`, `commits_ahead_of_indexed_head`, `head_freshness`. | | Version and forward compatibility | `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`. | -| Unknown-extension and runtime diagnostics | `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `extractors`, `hooks`, `hook_diagnostics`, `trust_overrides`, `path_case_sensitive`, `data_dir_mode`, `mac_profile`, `mac_profile_diagnostics`, `stale_after_seconds`, `index_age_seconds`, `query_context.check_mode`, `query_context.stale_after_seconds`, `last_index_run.reference_extraction_cap_hits`, `last_failed_or_partial_index_run`, `last_failed_or_partial_index_run.progress_persisted`, `last_failed_or_partial_index_run.recovery_hint`, `last_failed_or_partial_index_run.file_errors`. | +| Unknown-extension and runtime diagnostics | `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `unknown_extension_group_count`, `unknown_extension_groups_truncated`, `unknown_extension_group_limit`, `unknown_extension_group_omitted_count`, `unknown_extension_guidance`, `extractors`, `hooks`, `hook_diagnostics`, `trust_overrides`, `path_case_sensitive`, `data_dir_mode`, `mac_profile`, `mac_profile_diagnostics`, `stale_after_seconds`, `index_age_seconds`, `query_context.check_mode`, `query_context.stale_after_seconds`, `last_index_run.reference_extraction_cap_hits`, `last_failed_or_partial_index_run`, `last_failed_or_partial_index_run.progress_persisted`, `last_failed_or_partial_index_run.recovery_hint`, `last_failed_or_partial_index_run.file_errors`. | | Database maintenance | `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `busy_timeout_ms`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`), `prepared_command_cache` (`count`, `capacity`, `hit_count`, `miss_count`, `eviction_count`), `maintenance_guidance`. | | Database size attribution | `database_size_attribution` (`available`, `measurement`, `unavailable_reason`, physical main/WAL/SHM sizes, logical/object/freelist/residual reconciliation, page-type and payload/overhead subtotals, and bounded `top_objects`). | | Remediation fields | `degraded_root_cause`, `degraded_reason`, `recommended_action`, `alternative_action`, `readiness_degradations`, `repair_commands`. | @@ -6191,7 +6191,7 @@ total / cursor / truncation、partial-family metadata、definition body の pagi | readiness / graph trust | `fold_ready`, `fold_ready_reason`, `graph_table_available`, `graph_data_current`, `reference_extraction_limits`, `reference_graph_complete`, `reference_graph_incomplete_reasons`, `reference_extraction_cap_hits`, `index_complete`, `index_incomplete_reasons`, `issues_table_available`, `file_issues_data_current`, `migration_in_progress`, `sql_graph_contract_ready`, `sql_graph_contract_degraded_reason`, `hotspot_family_ready`, `hotspot_family_degraded_reason`, `language_readiness`, `csharp_symbol_name_ready`, `csharp_metadata_target_ready`, `csharp_metadata_target_degraded_reason`。 | | workspace / HEAD freshness | `indexed_head_commit`, `worktree_head_changed`, `indexed_head_sha`, `indexed_head_branch`, `indexed_head_timestamp`, `commits_ahead_of_indexed_head`, `head_freshness`。 | | version / forward compatibility | `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`。 | -| unknown-extension / runtime diagnostics | `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `extractors`, `hooks`, `hook_diagnostics`, `trust_overrides`, `path_case_sensitive`, `data_dir_mode`, `mac_profile`, `mac_profile_diagnostics`, `stale_after_seconds`, `index_age_seconds`, `query_context.check_mode`, `query_context.stale_after_seconds`, `last_index_run.reference_extraction_cap_hits`, `last_failed_or_partial_index_run`, `last_failed_or_partial_index_run.progress_persisted`, `last_failed_or_partial_index_run.recovery_hint`, `last_failed_or_partial_index_run.file_errors`。 | +| unknown-extension / runtime diagnostics | `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `unknown_extension_group_count`, `unknown_extension_groups_truncated`, `unknown_extension_group_limit`, `unknown_extension_group_omitted_count`, `unknown_extension_guidance`, `extractors`, `hooks`, `hook_diagnostics`, `trust_overrides`, `path_case_sensitive`, `data_dir_mode`, `mac_profile`, `mac_profile_diagnostics`, `stale_after_seconds`, `index_age_seconds`, `query_context.check_mode`, `query_context.stale_after_seconds`, `last_index_run.reference_extraction_cap_hits`, `last_failed_or_partial_index_run`, `last_failed_or_partial_index_run.progress_persisted`, `last_failed_or_partial_index_run.recovery_hint`, `last_failed_or_partial_index_run.file_errors`。 | | database maintenance | `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `busy_timeout_ms`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`), `prepared_command_cache` (`count`, `capacity`, `hit_count`, `miss_count`, `eviction_count`), `maintenance_guidance`。 | | database size attribution | `database_size_attribution`（`available`、`measurement`、`unavailable_reason`、物理 main/WAL/SHM size、論理/object/freelist/residual の再照合、page type と payload/overhead の小計、上限付き `top_objects`）。 | | remediation fields | `degraded_root_cause`, `degraded_reason`, `recommended_action`, `alternative_action`, `readiness_degradations`, `repair_commands`。 | diff --git a/README.md b/README.md index c19a38131f..2be213903c 100644 --- a/README.md +++ b/README.md @@ -168,7 +168,7 @@ visible here as a compact compatibility index. | Workspace and HEAD freshness | `indexed_head_commit`, `worktree_head_changed`, `indexed_head_sha`, `indexed_head_branch`, `indexed_head_timestamp`, `commits_ahead_of_indexed_head`, `head_freshness`. | | Workspace-check path samples | `workspace_check.changed_files`, `workspace_check.missing_files`, `workspace_check.outside_sparse_cone_files`, `workspace_check.unindexed_files`, `workspace_check.unverifiable_files`, and `workspace_check.scan_errors`, each paired with authoritative `*_count`, `*_truncated`, `*_path_limit`, and `*_omitted_count` fields. | | Version compatibility | `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`. | -| Extension and extractor diagnostics | `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `extractors`, `hooks`, `hook_diagnostics`. | +| Extension and extractor diagnostics | `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `unknown_extension_group_count`, `unknown_extension_groups_truncated`, `unknown_extension_group_limit`, `unknown_extension_group_omitted_count`, `unknown_extension_guidance`, `extractors`, `hooks`, `hook_diagnostics`. | | Runtime trust and permissions | `trust_overrides`, `git_executable`, `path_case_sensitive`, `data_dir_mode`, `db_file_mode`, `database_permission_policy`, `database_permission_diagnostics`, `mac_profile`, `mac_profile_diagnostics`. | | Check context and run diagnostics | `stale_after_seconds`, `index_age_seconds`, `query_context.check_mode`, `query_context.stale_after_seconds`, `process`, `last_index_run`, `last_workspace_freshened_at`, `last_failed_or_partial_index_run`. | | Last-run detail | `last_index_run.bytes_read_skipped_file_count`, `last_index_run.bytes_read_incomplete`, `last_index_run.diagnostics`, `last_index_run.diagnostic_count`, `last_index_run.diagnostics_truncated`, `last_index_run.reference_extraction_cap_hits`, `last_index_run.rebuild_reclaim`, `last_failed_or_partial_index_run.progress_persisted`, `last_failed_or_partial_index_run.recovery_hint`, `last_failed_or_partial_index_run.file_errors`. | @@ -178,6 +178,11 @@ visible here as a compact compatibility index. | Remediation | `degraded_root_cause`, `degraded_reason`, `recommended_action`, `alternative_action`, `readiness_degradations`, `repair_commands`. | | MCP-only session diagnostics | `mcp_session`, `mcp_session.metrics`, `queue_capacity`, `queue_depth`, `queued_event_count`, `written_event_count`, `dropped_event_count`, `queue_full_drop_count`, `serialization_failure_count`, `write_failure_count`, `rotation_failure_count`, `batch_flush_count`, `consecutive_failure_count`, `recovery_count`, `next_retry_at`, `last_recovery_at`, `last_failure`, `mcp_session.audit_log`, `queued_record_count`, `written_record_count`, `mcp.rate_limit.bucket_limit`, `mcp.rate_limit.bucket_limit_rejection_count`. | +Full index completion, the watch initial scan, and full-scan dry runs also report +the unknown-language file count, the top 10 extension groups, explicit omission +metadata, and remediation guidance. `status --compact` includes the persisted +equivalent from the most recent successful full scan. + Use `cdidx status --explain ` for bounded field guidance. Detailed semantics, repair-action structure, readiness degradation, SQLite/WAL handling, and MCP diagnostics live in the [Developer Guide](DEVELOPER_GUIDE.md#ai-integration); @@ -379,7 +384,7 @@ field group を表に残します。 | workspace / HEAD freshness | `indexed_head_commit`、`worktree_head_changed`、`indexed_head_sha`、`indexed_head_branch`、`indexed_head_timestamp`、`commits_ahead_of_indexed_head`、`head_freshness`。 | | workspace-check の path sample | `workspace_check.changed_files`、`workspace_check.missing_files`、`workspace_check.outside_sparse_cone_files`、`workspace_check.unindexed_files`、`workspace_check.unverifiable_files`、`workspace_check.scan_errors`。各一覧には authoritative な `*_count`、`*_truncated`、`*_path_limit`、`*_omitted_count` が対応します。 | | version compatibility | `index_writer_version`、`index_newer_than_reader`、`index_newer_than_reader_reason`。 | -| extension / extractor diagnostics | `unknown_extension_file_count`、`unknown_extension_files`、`unknown_extension_files_truncated`、`unknown_extension_file_path_limit`、`unknown_extension_extension_counts`、`unknown_extension_category_counts`、`unknown_extension_groups`、`extractors`、`hooks`、`hook_diagnostics`。 | +| extension / extractor diagnostics | `unknown_extension_file_count`、`unknown_extension_files`、`unknown_extension_files_truncated`、`unknown_extension_file_path_limit`、`unknown_extension_extension_counts`、`unknown_extension_category_counts`、`unknown_extension_groups`、`unknown_extension_group_count`、`unknown_extension_groups_truncated`、`unknown_extension_group_limit`、`unknown_extension_group_omitted_count`、`unknown_extension_guidance`、`extractors`、`hooks`、`hook_diagnostics`。 | | runtime trust / permissions | `trust_overrides`、`git_executable`、`path_case_sensitive`、`data_dir_mode`、`db_file_mode`、`database_permission_policy`、`database_permission_diagnostics`、`mac_profile`、`mac_profile_diagnostics`。 | | check context / run diagnostics | `stale_after_seconds`、`index_age_seconds`、`query_context.check_mode`、`query_context.stale_after_seconds`、`process`、`last_index_run`、`last_workspace_freshened_at`、`last_failed_or_partial_index_run`。 | | last-run detail | `last_index_run.bytes_read_skipped_file_count`、`last_index_run.bytes_read_incomplete`、`last_index_run.diagnostics`、`last_index_run.diagnostic_count`、`last_index_run.diagnostics_truncated`、`last_index_run.reference_extraction_cap_hits`、`last_index_run.rebuild_reclaim`、`last_failed_or_partial_index_run.progress_persisted`、`last_failed_or_partial_index_run.recovery_hint`、`last_failed_or_partial_index_run.file_errors`。 | @@ -389,6 +394,11 @@ field group を表に残します。 | remediation | `degraded_root_cause`、`degraded_reason`、`recommended_action`、`alternative_action`、`readiness_degradations`、`repair_commands`。 | | MCP-only session diagnostics | `mcp_session`、`mcp_session.metrics`、`queue_capacity`、`queue_depth`、`queued_event_count`、`written_event_count`、`dropped_event_count`、`queue_full_drop_count`、`serialization_failure_count`、`write_failure_count`、`rotation_failure_count`、`batch_flush_count`、`consecutive_failure_count`、`recovery_count`、`next_retry_at`、`last_recovery_at`、`last_failure`、`mcp_session.audit_log`、`queued_record_count`、`written_record_count`、`mcp.rate_limit.bucket_limit`、`mcp.rate_limit.bucket_limit_rejection_count`。 | +全体 index の完了時、watch の初回 scan、全体 scan の dry-run でも、言語未対応 +ファイル数、上位 10 個の拡張子 group、明示的な省略 metadata、対処 guidance を +返します。`status --compact` には、直近に成功した全体 scan で永続化された同等の +診断が含まれます。 + 上限付きの field guidance は `cdidx status --explain ` で確認できます。 repair action、readiness degradation、SQLite/WAL、MCP diagnostic の詳細は [開発者ガイド](DEVELOPER_GUIDE.md#ai連携)、日常的な使い方は diff --git a/USER_GUIDE.md b/USER_GUIDE.md index acb6da786e..d6933638d4 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -2765,6 +2765,37 @@ The database reflects the working tree at the time of the last index. After swit | File deleted after checkout | Purged from DB | | File added after checkout | Indexed as new | +## Unsupported-file diagnostics + +A successful full index is still successful when files lack a language mapping, +but completion output no longer makes those omissions look like an empty project. +Human output reports the total and the top 10 normalized extension groups; JSON +adds `unknown_extension_file_count`, `unknown_extension_groups`, group count, +limit, truncation/omission metadata, diagnostic scope, lower-bound state, and +`unknown_extension_guidance`. Group samples use the same normalized relative +paths as full status and remain capped at five paths per group. Uppercase suffixes +are normalized, multi-dot aliases are resolved before an unsupported file is +grouped, and an unsupported extensionless file uses ``. + +The warning count increases once when at least one group needs language or +structural extraction support. Files classified as intentional metadata/assets +with `ignore_configuration` remain visible but do not create that warning. +Ignored files are excluded before classification, and a valid workspace mapping +or extractor registration removes the diagnostic on the next full scan. Inspect +an extension with `cdidx languages --extension --json`, then add a +trusted mapping to `.cdidx-langmap.yaml`, register an extractor, or add an ignore +rule for intentional non-code files. + +Full-scan dry runs report the same workspace-authoritative diagnostics without +writing. Scoped dry runs use `unknown_extension_diagnostics_scope: "candidate_scope"`; +a reached candidate-path limit also sets +`unknown_extension_file_count_lower_bound: true`. The initial scan for +`index --watch` uses the normal full-scan completion contract. Non-dry scoped +updates (`--files`, `--commits`, or `--changed-between`) do not claim a new +workspace inventory; persisted `status` diagnostics continue to describe the +most recent successful full scan. `status --compact` includes those bounded +persisted diagnostics by default. + ## Supported languages CLI JSON and MCP `languages` responses share one catalog snapshot and expose `language_capability_counts`. Every count carries an explicit `scope`, `capability`, `count`, and `available` field. The `catalog` scope describes the complete runtime catalog, `matched_catalog` describes rows remaining after language/capability/indexed filters but before pagination, and `indexed_workspace` describes catalog languages currently present in the configured database. Capabilities are reported separately for detection, symbol extraction, reference extraction, outline, and graph queries, so catalog size and indexed-language totals are never interchangeable. @@ -6278,6 +6309,33 @@ indexing はファイル単位の SQLite transaction を commit します。長 | checkout後にファイル削除 | DBからパージ | | checkout後にファイル追加 | 新規インデックス | +## 未対応ファイルの診断 + +言語 mapping がないファイルを含む全体 index も成功扱いですが、完了出力ではその +省略を空の project と区別して表示します。human 出力は総数と正規化済み拡張子 group +の上位 10 件を表示します。JSON は `unknown_extension_file_count`、 +`unknown_extension_groups`、group 総数、上限、切り詰め/省略 metadata、診断 scope、 +lower-bound 状態、`unknown_extension_guidance` を返します。group の sample path は +full status と同じ正規化済み相対 path を使い、1 group あたり 5 件に制限されます。 +大文字 suffix は正規化され、複合拡張子 alias は未対応 group 化の前に解決されます。 +未対応の拡張子なしファイルは `` で表します。 + +言語または構造抽出の対応が必要な group が 1 件以上ある場合、warning 件数を 1 増やします。 +意図的な metadata/asset として `ignore_configuration` に分類されたファイルは可視化 +しますが、その warning は発生させません。ignore 済みファイルは分類前に除外され、 +有効な workspace mapping または extractor 登録後の次回全体 scan では診断が消えます。 +`cdidx languages --extension --json` で拡張子を確認し、信頼済み mapping を +`.cdidx-langmap.yaml` に追加するか、extractor を登録するか、意図的な非 code ファイルを +ignore rule に追加してください。 + +全体 scan の dry-run は書き込みを行わず、同じ workspace-authoritative な診断を返します。 +scoped dry-run は `unknown_extension_diagnostics_scope: "candidate_scope"` を使い、candidate +path 上限に達した場合は `unknown_extension_file_count_lower_bound: true` も設定します。 +`index --watch` の初回 scan は通常の全体 scan 完了契約を使います。非 dry-run の scoped +update（`--files`、`--commits`、`--changed-between`）は新しい workspace inventory を +主張せず、永続化済み `status` 診断は直近に成功した全体 scan を表し続けます。 +`status --compact` はその上限付き永続化診断を既定で含めます。 + ## 対応言語 CLI JSON と MCP の `languages` response は同じ catalog snapshot を共有し、`language_capability_counts` を返します。各件数には `scope`、`capability`、`count`、`available` が明示されます。`catalog` scope は runtime catalog 全体、`matched_catalog` は言語・capability・indexed filter 適用後かつ pagination 前の行、`indexed_workspace` は設定済み database に現在存在する catalog 言語を表します。detection、symbol extraction、reference extraction、outline、graph query の capability は個別に集計されるため、catalog size と indexed-language total を同じ件数として扱うことはできません。 diff --git a/changelog.d/unreleased/5100.changed.md b/changelog.d/unreleased/5100.changed.md new file mode 100644 index 0000000000..9eb524a875 --- /dev/null +++ b/changelog.d/unreleased/5100.changed.md @@ -0,0 +1,21 @@ +--- +category: changed +issues: + - 5100 +affected: + - src/CodeIndex/Cli/ConsoleUi.cs + - src/CodeIndex/Cli/IndexCommandRunner.FullScan.Output.cs + - src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs + - src/CodeIndex/Database/UnknownExtensionClassifier.cs + - src/CodeIndex/Models/QueryResults.cs + - README.md + - USER_GUIDE.md +--- + +## English + +- **Index completion now surfaces unsupported-language coverage loss (#5100)** — successful full scans, watch initialization, dry runs, and compact status report bounded extension groups, omission metadata, and actionable mapping or extractor guidance; ignored files and valid custom mappings keep their expected behavior. + +## 日本語 + +- **index 完了時に未対応言語による coverage loss を表示するようになりました (#5100)** — 成功した全体 scan、watch 初期化、dry-run、compact status は、上限付き拡張子 group、省略 metadata、mapping または extractor の対処 guidance を返し、ignore 済みファイルと有効な custom mapping の既存動作は維持します。 diff --git a/src/CodeIndex/Cli/ConsoleUi.cs b/src/CodeIndex/Cli/ConsoleUi.cs index c78d46f373..ff1b0c443a 100644 --- a/src/CodeIndex/Cli/ConsoleUi.cs +++ b/src/CodeIndex/Cli/ConsoleUi.cs @@ -162,6 +162,7 @@ private static readonly (string Command, string Usage)[] CommandUsageLines = private static readonly (string Command, string Note)[] CommandUsageNotes = [ ("index", "--rebuild deletes the existing index before a full rescan. Interactive terminals prompt unless --yes or --force bypasses confirmation; when stdin is redirected, either flag is required."), + ("index", "Full scans, full-scan dry runs, and watch initialization report bounded unknown-language extension groups and remediation guidance; scoped updates preserve the last successful full-scan status inventory."), ("mcp", "--json is not supported; MCP requests and responses are JSON-RPC over the selected transport."), ("mcp", "stdio transport uses one UTF-8 JSON-RPC object per LF-delimited line, not LSP Content-Length framing; lifecycle diagnostics go to stderr."), ("mcp", "HTTP requires bearer authentication by default; --allow-unauthenticated-http is an explicit unsafe loopback-only opt-in."), diff --git a/src/CodeIndex/Cli/ExportImportCommandRunner.ArchiveScope.cs b/src/CodeIndex/Cli/ExportImportCommandRunner.ArchiveScope.cs index 24be49163c..ebf62ab72c 100644 --- a/src/CodeIndex/Cli/ExportImportCommandRunner.ArchiveScope.cs +++ b/src/CodeIndex/Cli/ExportImportCommandRunner.ArchiveScope.cs @@ -230,7 +230,11 @@ OR key IN ( 'unknown_extension_file_path_limit', 'unknown_extension_extension_counts_json', 'unknown_extension_category_counts_json', - 'unknown_extension_groups_json') + 'unknown_extension_groups_json', + 'unknown_extension_group_count', + 'unknown_extension_groups_truncated', + 'unknown_extension_group_limit', + 'unknown_extension_group_omitted_count') """; return Convert.ToInt64(command.ExecuteScalar(), CultureInfo.InvariantCulture) != 0; } diff --git a/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs b/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs index 61979bc0af..fc24d8b477 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs @@ -121,6 +121,7 @@ private static int RunDryRun( var csharpWorkspaceEstimateUnavailable = false; var unsupportedTotal = 0; var unknownExtensionTotal = 0; + var unknownExtensionPaths = new List(); using var symbolExtractionWorker = new LazyDisposable( () => new SymbolExtractionWorkerClient(options.MaxFileSizeBytes)); var normalizedUpdatePaths = options.UpdateFiles.Count > 0 @@ -213,6 +214,7 @@ void RecordDryRunScanErrors(IEnumerable scanErrors) if (authoritativeFullScan) { unknownExtensionTotal = dryScanMetadata.UnknownExtensionFiles.Count; + unknownExtensionPaths.AddRange(dryScanMetadata.UnknownExtensionFiles); unsupportedTotal = CountUnsupportedNonIndexablePaths(dryScanMetadata); } @@ -301,7 +303,11 @@ void RecordDryRunScanErrors(IEnumerable scanErrors) if (!probe.Supported) { if (probe.UnknownExtension) + { unknownExtensionTotal++; + if (!authoritativeFullScan) + unknownExtensionPaths.Add(displayRelativePath); + } else if (probe.Unsupported) unsupportedTotal++; @@ -499,6 +505,22 @@ void RecordDryRunScanErrors(IEnumerable scanErrors) var estimatedTableMutations = mutationEstimates.BuildValues(); var estimatedTableMutationDetails = mutationEstimates.BuildDetails(); + var unknownExtensionClassification = UnknownExtensionClassifier.Classify(unknownExtensionPaths); + var unknownExtensionGroups = unknownExtensionClassification.Groups + .Take(UnknownExtensionClassifier.MaxCompletionGroups) + .ToList(); + var unknownExtensionGroupOmittedCount = Math.Max( + 0, + unknownExtensionClassification.GroupCount - unknownExtensionGroups.Count); + var unknownExtensionWarning = unknownExtensionClassification.ActionableFileCount > 0 + ? $"{unknownExtensionClassification.ActionableFileCount} file(s) were excluded because no language mapping or extractor was available. {UnknownExtensionClassifier.Guidance}" + : null; + if (unknownExtensionWarning != null) + { + warningCount++; + if (warningSamples.Count < DryRunWarningSampleLimit) + warningSamples.Add(new CliJsonMessage("", unknownExtensionWarning)); + } if (options.MemoryTrace) { @@ -524,6 +546,17 @@ void RecordDryRunScanErrors(IEnumerable scanErrors) ProjectedReferenceCapHits = projectedReferenceCapHits, UnsupportedTotal = unsupportedTotal, UnknownExtensionTotal = unknownExtensionTotal, + UnknownExtensionFileCount = unknownExtensionTotal, + UnknownExtensionGroups = unknownExtensionGroups.Count > 0 ? unknownExtensionGroups : null, + UnknownExtensionGroupCount = unknownExtensionClassification.GroupCount, + UnknownExtensionGroupsTruncated = unknownExtensionGroupOmittedCount > 0, + UnknownExtensionGroupLimit = UnknownExtensionClassifier.MaxCompletionGroups, + UnknownExtensionGroupOmittedCount = unknownExtensionGroupOmittedCount, + UnknownExtensionDiagnosticsScope = authoritativeFullScan ? "workspace" : "candidate_scope", + UnknownExtensionFileCountLowerBound = candidatePathsTruncated, + UnknownExtensionGuidance = unknownExtensionTotal > 0 + ? UnknownExtensionClassifier.Guidance + : null, CandidatePathLimit = dryRunPathLimit, CandidatePathsProcessed = candidatePathsProcessed, CandidatePathsTruncated = candidatePathsTruncated, @@ -558,6 +591,14 @@ void RecordDryRunScanErrors(IEnumerable scanErrors) { var lowerBound = candidatePathsTruncated ? " (truncated; totals are lower bounds)" : string.Empty; CommandOutputWriter.WriteLine($"Dry run: {dryFileCount} indexable files inspected{lowerBound}"); + if (unknownExtensionTotal > 0) + { + CommandOutputWriter.WriteLine($" unknown extensions {unknownExtensionTotal,6}{(candidatePathsTruncated ? " (lower bound)" : string.Empty)}"); + foreach (var group in unknownExtensionGroups) + CommandOutputWriter.WriteLine($" {group.Extension}: {ConsoleUi.FormatNumber(group.Count)} ({group.RecommendedAction})"); + if (unknownExtensionGroupOmittedCount > 0) + CommandOutputWriter.WriteLine($" ... {ConsoleUi.FormatNumber(unknownExtensionGroupOmittedCount)} more extension groups"); + } if (candidatePathsTruncated) CommandOutputWriter.WriteLine($" candidate paths processed {candidatePathsProcessed.ToString("N0", System.Globalization.CultureInfo.InvariantCulture)} of limit {dryRunPathLimit.ToString("N0", System.Globalization.CultureInfo.InvariantCulture)}"); CommandOutputWriter.WriteLine($" projected updates {projectedFileUpdates,6}"); @@ -586,6 +627,8 @@ void RecordDryRunScanErrors(IEnumerable scanErrors) CommandOutputWriter.WriteLine( $" language detection {detection.Path}: {detection.Language} ({detection.Source}, confidence {detection.Confidence})"); } + if (unknownExtensionWarning != null) + ConsoleUi.PrintWarning(unknownExtensionWarning); } return CommandExitCodes.Success; } diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Output.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Output.cs index 2f835000be..a149784ebc 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Output.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Output.cs @@ -75,6 +75,22 @@ private static int WriteFullScanFinalOutput(FullScanFinalOutputContext output) output.Warnings++; } output.Warnings += AddPostExtractionHookWarnings(output.PostExtractionHooks, output.WarningList); + var unknownExtensionClassification = UnknownExtensionClassifier.Classify( + output.ScanResult.UnknownExtensionFiles); + var unknownExtensionGroups = unknownExtensionClassification.Groups + .Take(UnknownExtensionClassifier.MaxCompletionGroups) + .ToList(); + var unknownExtensionGroupOmittedCount = Math.Max( + 0, + unknownExtensionClassification.GroupCount - unknownExtensionGroups.Count); + var unknownExtensionWarning = unknownExtensionClassification.ActionableFileCount > 0 + ? $"{unknownExtensionClassification.ActionableFileCount} file(s) were excluded because no language mapping or extractor was available. {UnknownExtensionClassifier.Guidance}" + : null; + if (unknownExtensionWarning != null) + { + output.WarningList.Add(new CliJsonMessage("", unknownExtensionWarning)); + output.Warnings++; + } var (totalFiles, totalChunks, totalSymbols, totalReferences) = output.StartedWithNoIndexedFiles && !output.ScanHadErrors && output.Errors == 0 ? (output.FreshCountFiles, output.FreshCountChunks, output.FreshCountSymbols, output.FreshCountReferences) @@ -132,6 +148,17 @@ private static int WriteFullScanFinalOutput(FullScanFinalOutputContext output) { Status = output.Errors > 0 ? "partial" : "success", Mode = output.Options.Rebuild ? "rebuild" : "incremental", + UnknownExtensionFileCount = output.ScanResult.UnknownExtensionFiles.Count, + UnknownExtensionGroups = unknownExtensionGroups.Count > 0 ? unknownExtensionGroups : null, + UnknownExtensionGroupCount = unknownExtensionClassification.GroupCount, + UnknownExtensionGroupsTruncated = unknownExtensionGroupOmittedCount > 0, + UnknownExtensionGroupLimit = UnknownExtensionClassifier.MaxCompletionGroups, + UnknownExtensionGroupOmittedCount = unknownExtensionGroupOmittedCount, + UnknownExtensionDiagnosticsScope = "workspace", + UnknownExtensionFileCountLowerBound = false, + UnknownExtensionGuidance = output.ScanResult.UnknownExtensionFiles.Count > 0 + ? UnknownExtensionClassifier.Guidance + : null, Summary = new IndexFullScanSummaryJsonResult { FilesTotal = totalFiles, @@ -211,13 +238,26 @@ private static int WriteFullScanFinalOutput(FullScanFinalOutputContext output) CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine("Refs", ConsoleUi.FormatNumber(totalReferences), indent: " ")); if (output.Skipped > 0) CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine("Skipped", $"{ConsoleUi.FormatNumber(output.Skipped)} (unchanged)", indent: " ")); if (output.ScanResult.DanglingSymlinks.Count > 0) CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine("Dangling symlinks", $"{ConsoleUi.FormatNumber(output.ScanResult.DanglingSymlinks.Count)} output.Skipped", indent: " ")); - if (output.Options.Verbose && output.ScanResult.UnknownExtensionFiles.Count > 0) + if (output.ScanResult.UnknownExtensionFiles.Count > 0) { - CommandOutputWriter.WriteLine($" Unknown extension files: {ConsoleUi.FormatNumber(output.ScanResult.UnknownExtensionFiles.Count)}"); - foreach (var relPath in output.ScanResult.UnknownExtensionFiles.Take(5)) - CommandOutputWriter.WriteLine($" {relPath}"); - if (output.ScanResult.UnknownExtensionFiles.Count > 5) - CommandOutputWriter.WriteLine($" ... {ConsoleUi.FormatNumber(output.ScanResult.UnknownExtensionFiles.Count - 5)} more"); + CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine( + "Unknown extensions", + ConsoleUi.FormatNumber(output.ScanResult.UnknownExtensionFiles.Count), + indent: " ")); + foreach (var group in unknownExtensionGroups) + { + CommandOutputWriter.WriteLine( + $" {group.Extension}: {ConsoleUi.FormatNumber(group.Count)} ({group.RecommendedAction})"); + } + if (unknownExtensionGroupOmittedCount > 0) + CommandOutputWriter.WriteLine($" ... {ConsoleUi.FormatNumber(unknownExtensionGroupOmittedCount)} more extension groups"); + if (output.Options.Verbose) + { + foreach (var relPath in output.ScanResult.UnknownExtensionFiles.Take(5)) + CommandOutputWriter.WriteLine($" sample: {relPath}"); + if (output.ScanResult.UnknownExtensionFiles.Count > 5) + CommandOutputWriter.WriteLine($" ... {ConsoleUi.FormatNumber(output.ScanResult.UnknownExtensionFiles.Count - 5)} more paths"); + } } if (output.Warnings > 0) CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine("Warnings", ConsoleUi.FormatNumber(output.Warnings), indent: " ")); if (output.Errors > 0) CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine("Errors", ConsoleUi.FormatNumber(output.Errors), indent: " ")); @@ -243,6 +283,8 @@ private static int WriteFullScanFinalOutput(FullScanFinalOutputContext output) ConsoleUi.FormatBytes(reclaimedBytes), indent: " ")); } + if (unknownExtensionWarning != null) + ConsoleUi.PrintWarning(unknownExtensionWarning); CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine("Elapsed", ConsoleUi.FormatDuration(output.Stopwatch.Elapsed, output.Options.DurationFormat), indent: " ")); CommandOutputWriter.WriteLine(); if (output.Errors > 0) diff --git a/src/CodeIndex/Cli/JsonOutputContracts.cs b/src/CodeIndex/Cli/JsonOutputContracts.cs index 54b01822c0..1d680676c4 100644 --- a/src/CodeIndex/Cli/JsonOutputContracts.cs +++ b/src/CodeIndex/Cli/JsonOutputContracts.cs @@ -862,6 +862,15 @@ internal sealed class IndexDryRunJsonResult : IVersionedJsonResult public int ProjectedReferenceCapHits { get; init; } public int UnsupportedTotal { get; init; } public int UnknownExtensionTotal { get; init; } + public int UnknownExtensionFileCount { get; init; } + public IReadOnlyList? UnknownExtensionGroups { get; init; } + public int UnknownExtensionGroupCount { get; init; } + public bool UnknownExtensionGroupsTruncated { get; init; } + public int UnknownExtensionGroupLimit { get; init; } + public int UnknownExtensionGroupOmittedCount { get; init; } + public string UnknownExtensionDiagnosticsScope { get; init; } = string.Empty; + public bool UnknownExtensionFileCountLowerBound { get; init; } + public string? UnknownExtensionGuidance { get; init; } public int CandidatePathLimit { get; init; } public int CandidatePathsProcessed { get; init; } public bool CandidatePathsTruncated { get; init; } @@ -1043,6 +1052,15 @@ internal sealed class IndexFullScanJsonResult : IVersionedJsonResult public string Status { get; init; } = string.Empty; public string Mode { get; init; } = string.Empty; public IndexFullScanSummaryJsonResult Summary { get; init; } = new(); + public int UnknownExtensionFileCount { get; init; } + public IReadOnlyList? UnknownExtensionGroups { get; init; } + public int UnknownExtensionGroupCount { get; init; } + public bool UnknownExtensionGroupsTruncated { get; init; } + public int UnknownExtensionGroupLimit { get; init; } + public int UnknownExtensionGroupOmittedCount { get; init; } + public string UnknownExtensionDiagnosticsScope { get; init; } = "workspace"; + public bool UnknownExtensionFileCountLowerBound { get; init; } + public string? UnknownExtensionGuidance { get; init; } public IndexSymbolKindFilterJsonResult SymbolKindFilter { get; init; } = new(); public bool GraphTableAvailable { get; init; } public bool GraphDataCurrent { get; init; } diff --git a/src/CodeIndex/Cli/ProjectionFieldRegistry.cs b/src/CodeIndex/Cli/ProjectionFieldRegistry.cs index d19e693a8c..58a039d1f9 100644 --- a/src/CodeIndex/Cli/ProjectionFieldRegistry.cs +++ b/src/CodeIndex/Cli/ProjectionFieldRegistry.cs @@ -369,6 +369,11 @@ private static ProjectionCommandFieldSchema CreateStatusSchema() "api_version", "files", "chunks", "symbols", "references", "indexed_at", "git_head", "git_is_dirty", "head_freshness", "version", "graph_table_available", "hotspot_family_ready", "summary", + "unknown_extension_file_count", "unknown_extension_extension_counts", + "unknown_extension_category_counts", "unknown_extension_groups", + "unknown_extension_group_count", "unknown_extension_groups_truncated", + "unknown_extension_group_limit", "unknown_extension_group_omitted_count", + "unknown_extension_guidance", "workspace_check.checked", "workspace_check.matches_workspace", "workspace_check.reason", "workspace_check.changed_file_count", "workspace_check.changed_files_truncated", "workspace_check.changed_files_path_limit", "workspace_check.changed_files_omitted_count", diff --git a/src/CodeIndex/Database/DbContext.SchemaMetadata.cs b/src/CodeIndex/Database/DbContext.SchemaMetadata.cs index 03600e29f7..c016f7bfa7 100644 --- a/src/CodeIndex/Database/DbContext.SchemaMetadata.cs +++ b/src/CodeIndex/Database/DbContext.SchemaMetadata.cs @@ -172,10 +172,10 @@ public static string GetDynamicReferenceGraphContractVersionMetaKey(string lang) public const string IndexCompletenessMetaKey = "index_completeness"; public const string IndexIncompleteReasonsMetaKey = "index_incomplete_reasons_json"; // Issue #1585: count of files seen by the most recent successful full-repository scan - // whose non-empty extension did not map to a known language. This is a scan coverage + // whose filename or extension did not map to a known language. This is a scan coverage // signal, not an indexed-file count, and is omitted by readers until a current index pass // has stamped it. - // Issue #1585: 直近成功した全体 scan で、非空の拡張子が既知言語に対応しなかった + // Issue #1585: 直近成功した全体 scan で、ファイル名または拡張子が既知言語に対応しなかった // ファイル数。index 済み件数ではなく scan coverage の信号であり、現行 index が stamp // するまでは reader 側で省略する。 public const string UnknownExtensionFileCountMetaKey = "unknown_extension_file_count"; @@ -185,6 +185,10 @@ public static string GetDynamicReferenceGraphContractVersionMetaKey(string lang) public const string UnknownExtensionExtensionCountsMetaKey = "unknown_extension_extension_counts_json"; public const string UnknownExtensionCategoryCountsMetaKey = "unknown_extension_category_counts_json"; public const string UnknownExtensionGroupsMetaKey = "unknown_extension_groups_json"; + public const string UnknownExtensionGroupCountMetaKey = "unknown_extension_group_count"; + public const string UnknownExtensionGroupsTruncatedMetaKey = "unknown_extension_groups_truncated"; + public const string UnknownExtensionGroupLimitMetaKey = "unknown_extension_group_limit"; + public const string UnknownExtensionGroupOmittedCountMetaKey = "unknown_extension_group_omitted_count"; public const int UnknownExtensionFilePathSampleLimit = 50; public const string BatchInProgressMetaKey = "batch_in_progress"; // Issue #1546: case-sensitivity of the workspace filesystem the most recent successful diff --git a/src/CodeIndex/Database/DbReader.Status.cs b/src/CodeIndex/Database/DbReader.Status.cs index a527466789..f8f5fc349a 100644 --- a/src/CodeIndex/Database/DbReader.Status.cs +++ b/src/CodeIndex/Database/DbReader.Status.cs @@ -134,6 +134,10 @@ GROUP BY COALESCE(f.lang, 'unknown'), s.kind var unknownExtensionExtensionCounts = UnknownExtensionClassifier.DeserializeCounts(TryGetMetaStringInternal(DbContext.UnknownExtensionExtensionCountsMetaKey)); var unknownExtensionCategoryCounts = UnknownExtensionClassifier.DeserializeCounts(TryGetMetaStringInternal(DbContext.UnknownExtensionCategoryCountsMetaKey)); var unknownExtensionGroups = UnknownExtensionClassifier.DeserializeGroups(TryGetMetaStringInternal(DbContext.UnknownExtensionGroupsMetaKey)); + var unknownExtensionGroupCount = ParseMetaLong(TryGetMetaStringInternal(DbContext.UnknownExtensionGroupCountMetaKey)); + var unknownExtensionGroupsTruncated = ParseMetaBool(TryGetMetaStringInternal(DbContext.UnknownExtensionGroupsTruncatedMetaKey)); + var unknownExtensionGroupLimit = ParseMetaLong(TryGetMetaStringInternal(DbContext.UnknownExtensionGroupLimitMetaKey)); + var unknownExtensionGroupOmittedCount = ParseMetaLong(TryGetMetaStringInternal(DbContext.UnknownExtensionGroupOmittedCountMetaKey)); if (unknownExtensionFiles != null) { unknownExtensionFilesTruncated ??= unknownExtensionFileCount.HasValue @@ -211,6 +215,13 @@ GROUP BY COALESCE(f.lang, 'unknown'), s.kind UnknownExtensionExtensionCounts = unknownExtensionExtensionCounts, UnknownExtensionCategoryCounts = unknownExtensionCategoryCounts, UnknownExtensionGroups = unknownExtensionGroups, + UnknownExtensionGroupCount = unknownExtensionGroupCount, + UnknownExtensionGroupsTruncated = unknownExtensionGroupsTruncated, + UnknownExtensionGroupLimit = unknownExtensionGroupLimit, + UnknownExtensionGroupOmittedCount = unknownExtensionGroupOmittedCount, + UnknownExtensionGuidance = unknownExtensionFileCount > 0 + ? UnknownExtensionClassifier.Guidance + : null, IndexedAt = freshness.IndexedAt, LastWorkspaceFreshenedAt = lastIndexRun?.StartedAt ?? indexedHeadTimestamp?.UtcDateTime, LatestModified = freshness.LatestModified, diff --git a/src/CodeIndex/Database/DbWriter.StatusMetadata.cs b/src/CodeIndex/Database/DbWriter.StatusMetadata.cs index 3424b9e609..7a6a13bfb1 100644 --- a/src/CodeIndex/Database/DbWriter.StatusMetadata.cs +++ b/src/CodeIndex/Database/DbWriter.StatusMetadata.cs @@ -76,7 +76,11 @@ public void WriteUnknownExtensionFileMetadata(IReadOnlyList paths) (DbContext.UnknownExtensionFilePathLimitMetaKey, DbContext.UnknownExtensionFilePathSampleLimit.ToString(System.Globalization.CultureInfo.InvariantCulture)), (DbContext.UnknownExtensionExtensionCountsMetaKey, "{}"), (DbContext.UnknownExtensionCategoryCountsMetaKey, "{}"), - (DbContext.UnknownExtensionGroupsMetaKey, "[]")); + (DbContext.UnknownExtensionGroupsMetaKey, "[]"), + (DbContext.UnknownExtensionGroupCountMetaKey, "0"), + (DbContext.UnknownExtensionGroupsTruncatedMetaKey, false.ToString(System.Globalization.CultureInfo.InvariantCulture)), + (DbContext.UnknownExtensionGroupLimitMetaKey, UnknownExtensionClassifier.MaxPersistedGroups.ToString(System.Globalization.CultureInfo.InvariantCulture)), + (DbContext.UnknownExtensionGroupOmittedCountMetaKey, "0")); return; } @@ -91,6 +95,10 @@ public void WriteUnknownExtensionFileMetadata(IReadOnlyList paths) (DbContext.UnknownExtensionFilePathLimitMetaKey, DbContext.UnknownExtensionFilePathSampleLimit.ToString(System.Globalization.CultureInfo.InvariantCulture)), (DbContext.UnknownExtensionExtensionCountsMetaKey, UnknownExtensionClassifier.SerializeCounts(classification.ExtensionCounts)), (DbContext.UnknownExtensionCategoryCountsMetaKey, UnknownExtensionClassifier.SerializeCounts(classification.CategoryCounts)), - (DbContext.UnknownExtensionGroupsMetaKey, UnknownExtensionClassifier.SerializeGroups(classification.Groups))); + (DbContext.UnknownExtensionGroupsMetaKey, UnknownExtensionClassifier.SerializeGroups(classification.Groups)), + (DbContext.UnknownExtensionGroupCountMetaKey, classification.GroupCount.ToString(System.Globalization.CultureInfo.InvariantCulture)), + (DbContext.UnknownExtensionGroupsTruncatedMetaKey, classification.GroupsTruncated.ToString(System.Globalization.CultureInfo.InvariantCulture)), + (DbContext.UnknownExtensionGroupLimitMetaKey, classification.GroupLimit.ToString(System.Globalization.CultureInfo.InvariantCulture)), + (DbContext.UnknownExtensionGroupOmittedCountMetaKey, classification.GroupOmittedCount.ToString(System.Globalization.CultureInfo.InvariantCulture))); } } diff --git a/src/CodeIndex/Database/UnknownExtensionClassifier.cs b/src/CodeIndex/Database/UnknownExtensionClassifier.cs index 55434da51f..c23feaa32d 100644 --- a/src/CodeIndex/Database/UnknownExtensionClassifier.cs +++ b/src/CodeIndex/Database/UnknownExtensionClassifier.cs @@ -6,12 +6,24 @@ namespace CodeIndex.Database; internal sealed record UnknownExtensionClassification( Dictionary ExtensionCounts, Dictionary CategoryCounts, - List Groups); + List Groups, + int GroupCount, + long ActionableFileCount) +{ + public bool GroupsTruncated => GroupCount > Groups.Count; + public int GroupLimit => UnknownExtensionClassifier.MaxPersistedGroups; + public int GroupOmittedCount => Math.Max(0, GroupCount - Groups.Count); +} internal static class UnknownExtensionClassifier { internal const int MaxPersistedGroups = 128; + internal const int MaxCompletionGroups = 10; internal const int MaxSamplePathsPerGroup = 5; + internal const string Guidance = + "Inspect an extension with `cdidx languages --extension --json`; " + + "then add a trusted mapping in `.cdidx-langmap.yaml`, register an extractor, " + + "or ignore intentional non-code files."; private const int MaxDictionaryEntries = 256; private const int MaxRawJsonCharacters = 512 * 1024; private const int MaxDecodedStringCharacters = 64 * 1024; @@ -29,6 +41,7 @@ public static UnknownExtensionClassification Classify(IReadOnlyList path var extensionCounts = new Dictionary(StringComparer.Ordinal); var categoryCounts = new Dictionary(StringComparer.Ordinal); var groups = new Dictionary(StringComparer.Ordinal); + var actionableFileCount = 0L; foreach (var path in paths.OrderBy(p => p, StringComparer.Ordinal)) { @@ -37,6 +50,8 @@ public static UnknownExtensionClassification Classify(IReadOnlyList path var extension = GetExtensionKey(path); var (category, action) = ClassifyPath(path, extension); + if (!string.Equals(action, "ignore_configuration", StringComparison.Ordinal)) + actionableFileCount++; Increment(extensionCounts, extension); Increment(categoryCounts, category); @@ -70,7 +85,9 @@ public static UnknownExtensionClassification Classify(IReadOnlyList path return new UnknownExtensionClassification( OrderCounts(extensionCounts), OrderCounts(categoryCounts), - orderedGroups); + orderedGroups, + groups.Count, + actionableFileCount); } public static string SerializeCounts(IReadOnlyDictionary counts) diff --git a/src/CodeIndex/Indexer/Scanning/FileIndexer.DirectoryFiltering.cs b/src/CodeIndex/Indexer/Scanning/FileIndexer.DirectoryFiltering.cs index bb10443168..a0e221cb70 100644 --- a/src/CodeIndex/Indexer/Scanning/FileIndexer.DirectoryFiltering.cs +++ b/src/CodeIndex/Indexer/Scanning/FileIndexer.DirectoryFiltering.cs @@ -4,12 +4,12 @@ namespace CodeIndex.Indexer; public partial class FileIndexer { - private static bool HasUnknownExtension(string filePath) + private static bool HasUnknownLanguageMapping(string filePath) { var extension = Path.GetExtension(filePath); - return !string.IsNullOrEmpty(extension) - && !LangMap.ContainsKey(extension) - && !ExtractorPluginRegistry.TryGetLanguageForExtension(extension, out _); + return string.IsNullOrEmpty(extension) + || (!LangMap.ContainsKey(extension) + && !ExtractorPluginRegistry.TryGetLanguageForExtension(extension, out _)); } private bool IsInternalIndexArtifactPath(string relativePath) diff --git a/src/CodeIndex/Indexer/Scanning/FileIndexer.FileAcceptance.cs b/src/CodeIndex/Indexer/Scanning/FileIndexer.FileAcceptance.cs index 0fdfa608dc..d831449486 100644 --- a/src/CodeIndex/Indexer/Scanning/FileIndexer.FileAcceptance.cs +++ b/src/CodeIndex/Indexer/Scanning/FileIndexer.FileAcceptance.cs @@ -122,7 +122,7 @@ private bool TryAcceptSupportedScannedFile( { var relativeFile = ToRelativePath(file); scanState.RecordNonIndexablePath(relativeFile); - if (HasUnknownExtension(file) && !IsInternalIndexArtifactPath(relativeFile)) + if (HasUnknownLanguageMapping(file) && !IsInternalIndexArtifactPath(relativeFile)) scanState.RecordUnknownExtensionFile(relativeFile); return false; } diff --git a/src/CodeIndex/Models/QueryResults.cs b/src/CodeIndex/Models/QueryResults.cs index 99fdd2812f..2a7097392f 100644 --- a/src/CodeIndex/Models/QueryResults.cs +++ b/src/CodeIndex/Models/QueryResults.cs @@ -1483,9 +1483,9 @@ public class StatusResult public long References { get; set; } /// /// Number of non-indexed files from the most recent successful full-repository scan whose - /// non-empty extension did not map to a known language. Null on legacy DBs or before the + /// filename or extension did not map to a known language. Null on legacy DBs or before the /// current scanner has stamped this coverage signal (Issue #1585). - /// 直近成功した全体 scan で、非空の拡張子が既知言語に対応しなかった未 index ファイル数。 + /// 直近成功した全体 scan で、ファイル名または拡張子が既知言語に対応しなかった未 index ファイル数。 /// 旧 DB や現行 scanner による stamp 前は null。 /// [JsonPropertyName("unknown_extension_file_count")] @@ -1509,6 +1509,21 @@ public class StatusResult [JsonPropertyName("unknown_extension_groups")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public List? UnknownExtensionGroups { get; set; } + [JsonPropertyName("unknown_extension_group_count")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? UnknownExtensionGroupCount { get; set; } + [JsonPropertyName("unknown_extension_groups_truncated")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? UnknownExtensionGroupsTruncated { get; set; } + [JsonPropertyName("unknown_extension_group_limit")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? UnknownExtensionGroupLimit { get; set; } + [JsonPropertyName("unknown_extension_group_omitted_count")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? UnknownExtensionGroupOmittedCount { get; set; } + [JsonPropertyName("unknown_extension_guidance")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? UnknownExtensionGuidance { get; set; } public DateTime? IndexedAt { get; set; } /// /// Timestamp of the most recent successful index/update run that freshened workspace diff --git a/tests/CodeIndex.Tests/ConsoleUiTests.cs b/tests/CodeIndex.Tests/ConsoleUiTests.cs index 1c69603fe8..c9385595da 100644 --- a/tests/CodeIndex.Tests/ConsoleUiTests.cs +++ b/tests/CodeIndex.Tests/ConsoleUiTests.cs @@ -14,6 +14,18 @@ namespace CodeIndex.Tests; [Collection("SQLite pool sensitive")] public class ConsoleUiTests { + [Fact] + public void PrintCommandUsage_IndexExplainsUnknownLanguageDiagnostics_Issue5100() + { + var (printed, stdout, stderr) = ConsoleCapture.Capture(() => + ConsoleUi.PrintCommandUsage("index") ? 1 : 0); + + Assert.Equal(1, printed); + Assert.Equal(string.Empty, stderr); + Assert.Contains("unknown-language extension groups", stdout, StringComparison.Ordinal); + Assert.Contains("scoped updates preserve", stdout, StringComparison.Ordinal); + } + [Fact] public void PrintCommandUsage_DedicatedParsersOnlyEmitAuthoritativeParentOptionLists_Issues4571_4861() { diff --git a/tests/CodeIndex.Tests/DocumentationStatusContractTests.cs b/tests/CodeIndex.Tests/DocumentationStatusContractTests.cs index e53bbc0e79..ad16cf3a72 100644 --- a/tests/CodeIndex.Tests/DocumentationStatusContractTests.cs +++ b/tests/CodeIndex.Tests/DocumentationStatusContractTests.cs @@ -41,6 +41,11 @@ public class DocumentationStatusContractTests "unknown_extension_extension_counts", "unknown_extension_category_counts", "unknown_extension_groups", + "unknown_extension_group_count", + "unknown_extension_groups_truncated", + "unknown_extension_group_limit", + "unknown_extension_group_omitted_count", + "unknown_extension_guidance", "extractors", "hooks", "hook_diagnostics", diff --git a/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs b/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs index 020bc6bb29..5d9ebf37cd 100644 --- a/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs @@ -2435,6 +2435,12 @@ private static void SetArchiveTrustMetadata(string dbPath, int filesScanned) writer.SetMeta(DbContext.UnknownExtensionExtensionCountsMetaKey, "{\".bar\":1,\".foo\":1}"); writer.SetMeta(DbContext.UnknownExtensionCategoryCountsMetaKey, "{\"language_support\":2}"); writer.SetMeta(DbContext.UnknownExtensionGroupsMetaKey, "[]"); + writer.SetMeta(DbContext.UnknownExtensionGroupCountMetaKey, "2"); + writer.SetMeta(DbContext.UnknownExtensionGroupsTruncatedMetaKey, bool.FalseString); + writer.SetMeta( + DbContext.UnknownExtensionGroupLimitMetaKey, + UnknownExtensionClassifier.MaxPersistedGroups.ToString(CultureInfo.InvariantCulture)); + writer.SetMeta(DbContext.UnknownExtensionGroupOmittedCountMetaKey, "0"); } private static void AssertPartialArchiveStatus(string dbPath) diff --git a/tests/CodeIndex.Tests/FileIndexerTests.cs b/tests/CodeIndex.Tests/FileIndexerTests.cs index 904df8f966..c3d5c9b4c5 100644 --- a/tests/CodeIndex.Tests/FileIndexerTests.cs +++ b/tests/CodeIndex.Tests/FileIndexerTests.cs @@ -6044,7 +6044,7 @@ public void ScanFilesDetailed_SkipsMalformedIgnoreRulesWithoutAborting() } [Fact] - public void ScanFilesDetailed_SeparatesUnknownExtensionsFromOtherNonIndexableFiles() + public void ScanFilesDetailed_SeparatesUnmappedLanguageFilesFromOtherNonIndexableFiles_Issue5100() { using var project = TestProjectHelper.CreateTempProjectScope("codeindex_test"); var tempDir = project.Root; @@ -6069,7 +6069,7 @@ public void ScanFilesDetailed_SeparatesUnknownExtensionsFromOtherNonIndexableFil var indexer = new FileIndexer(tempDir); var scanResult = indexer.ScanFilesDetailed(); - Assert.Equal(["data.mystery"], scanResult.UnknownExtensionFiles); + Assert.Equal(["data.mystery", "tool"], scanResult.UnknownExtensionFiles); Assert.Equal("csharp", scanResult.FileLanguages[appPath]); Assert.Equal("python", scanResult.FileLanguages[scriptPath]); Assert.DoesNotContain(toolPath, scanResult.FileLanguages.Keys); @@ -6077,7 +6077,7 @@ public void ScanFilesDetailed_SeparatesUnknownExtensionsFromOtherNonIndexableFil Assert.DoesNotContain(ignoredPath, scanResult.FileLanguages.Keys); Assert.Contains("data.mystery", scanResult.NonIndexablePaths); Assert.Contains("tool", scanResult.NonIndexablePaths); - Assert.DoesNotContain("tool", scanResult.UnknownExtensionFiles); + Assert.Contains("tool", scanResult.UnknownExtensionFiles); Assert.DoesNotContain("ignored.mystery", scanResult.UnknownExtensionFiles); } diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerDryRunTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerDryRunTests.cs index d96e3f48d0..26c1c21341 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerDryRunTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerDryRunTests.cs @@ -310,7 +310,7 @@ public void Run_DryRun_JsonCapsFileSamples() } [Fact] - public void Run_DryRun_PathLimitTruncatesCandidateProcessing() + public void Run_DryRun_PathLimitTruncatesCandidateProcessing_Issue5100() { var projectRoot = CreateTempProject(); try @@ -329,6 +329,7 @@ public void Run_DryRun_PathLimitTruncatesCandidateProcessing() Assert.Equal(2, json.GetProperty("candidate_paths_processed").GetInt32()); Assert.True(json.GetProperty("candidate_paths_truncated").GetBoolean()); Assert.True(json.GetProperty("totals_lower_bound").GetBoolean()); + Assert.True(json.GetProperty("unknown_extension_file_count_lower_bound").GetBoolean()); Assert.True(json.GetProperty("file_samples_truncated").GetBoolean()); Assert.Equal(0, json.GetProperty("errors_total").GetInt32()); Assert.Equal( @@ -980,7 +981,7 @@ checksum TEXT } [Fact] - public void Run_DryRun_WithFiles_ReportsProjectedUpdatesAndDeletes_Issue5091() + public void Run_DryRun_WithFiles_ReportsProjectedUpdatesAndDeletes_Issue5091_Issue5100() { var projectRoot = CreateTempProject(); try @@ -1013,6 +1014,9 @@ public void Run_DryRun_WithFiles_ReportsProjectedUpdatesAndDeletes_Issue5091() Assert.Equal(1, json.GetProperty("projected_file_deletes").GetInt32()); Assert.Equal(0, json.GetProperty("projected_file_purges").GetInt32()); Assert.Equal(0, json.GetProperty("unknown_extension_total").GetInt32()); + Assert.Equal("candidate_scope", json.GetProperty("unknown_extension_diagnostics_scope").GetString()); + Assert.False(json.GetProperty("unknown_extension_file_count_lower_bound").GetBoolean()); + Assert.Equal(0, json.GetProperty("unknown_extension_group_count").GetInt32()); Assert.Equal(0, json.GetProperty("unsupported_total").GetInt32()); var mutations = json.GetProperty("estimated_table_mutations"); Assert.True(mutations.GetProperty("files").GetInt64() >= 2); @@ -1408,7 +1412,7 @@ public void Run_DryRun_WithFiles_RejectsUnsupportedExtensionRenameAtomically_Iss } [Fact] - public void Run_DryRun_FullScan_ReportsProjectedPurgesWithoutWriting() + public void Run_DryRun_FullScan_ReportsProjectedPurgesAndUnknownExtensionsWithoutWriting_Issue5100() { var projectRoot = CreateTempProject(); try @@ -1431,9 +1435,22 @@ public void Run_DryRun_FullScan_ReportsProjectedPurgesWithoutWriting() Assert.Equal(0, json.GetProperty("projected_file_deletes").GetInt32()); Assert.Equal(1, json.GetProperty("projected_file_purges").GetInt32()); Assert.Equal(1, json.GetProperty("unknown_extension_total").GetInt32()); + Assert.Equal(1, json.GetProperty("unknown_extension_file_count").GetInt32()); + Assert.Equal("workspace", json.GetProperty("unknown_extension_diagnostics_scope").GetString()); + Assert.False(json.GetProperty("unknown_extension_file_count_lower_bound").GetBoolean()); + Assert.Equal(1, json.GetProperty("unknown_extension_group_count").GetInt32()); + Assert.Equal(".unknownext", json.GetProperty("unknown_extension_groups")[0].GetProperty("extension").GetString()); + Assert.Contains("languages --extension", json.GetProperty("unknown_extension_guidance").GetString()); + Assert.Equal(1, json.GetProperty("warnings_total").GetInt32()); Assert.True(json.TryGetProperty("unsupported_total", out _)); Assert.Equal(1, json.GetProperty("estimated_table_mutations").GetProperty("files").GetInt64()); Assert.Equal(2, CountRows(dbPath, "files")); + + var (humanExitCode, stdout, stderr) = RunAndCaptureStreams([projectRoot, "--dry-run"]); + Assert.Equal(CommandExitCodes.Success, humanExitCode); + Assert.Contains("unknown extensions", stdout); + Assert.Contains(".unknownext: 1 (language_support)", stdout); + Assert.Contains("languages --extension", stderr); } finally { diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs index 2f7ae926d8..3553f995f0 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs @@ -5825,7 +5825,7 @@ public void Run_ExplicitDb_PersistsIndexedProjectRootMetadata() } [Fact] - public void Run_VerboseReportsUnknownExtensionCountAndStatusJsonStampsCount() + public void Run_CompletionAndStatusReportBoundedUnknownExtensionDiagnostics_Issue5100() { var projectRoot = CreateTempProject(); try @@ -5838,13 +5838,17 @@ public void Run_VerboseReportsUnknownExtensionCountAndStatusJsonStampsCount() var (exitCode, stdout, stderr) = RunAndCaptureStreams([projectRoot, "--verbose"]); var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); var (statusExitCode, statusJson) = RunStatusAndCaptureJson(["--db", dbPath, "--json"]); + var (jsonExitCode, completionJson) = RunAndCaptureJson([projectRoot, "--json"]); + var (compactExitCode, compactJson) = RunProgramAndCaptureJson( + ["status", "--db", dbPath, "--compact", "--max-json-bytes", "50000"], + projectRoot); Assert.Equal(CommandExitCodes.Success, exitCode); - Assert.Equal(string.Empty, stderr); - Assert.Contains("Unknown extension files: 3", stdout); - Assert.Contains(".cdidxignore", stdout); - Assert.Contains("data.unmapped", stdout); - Assert.Contains("notes.mystery", stdout); + Assert.Contains("Unknown extensions", stdout); + Assert.Contains(".mystery: 1 (language_support)", stdout); + Assert.Contains("sample: data.unmapped", stdout); + Assert.Contains("2 file(s) were excluded", stderr); + Assert.Contains("languages --extension", stderr); Assert.Equal(CommandExitCodes.Success, statusExitCode); Assert.Equal(3, statusJson.GetProperty("unknown_extension_file_count").GetInt64()); Assert.False(statusJson.GetProperty("unknown_extension_files_truncated").GetBoolean()); @@ -5873,6 +5877,59 @@ public void Run_VerboseReportsUnknownExtensionCountAndStatusJsonStampsCount() Assert.Equal("language_support", group.GetProperty("recommended_action").GetString()); Assert.Equal(1, group.GetProperty("count").GetInt64()); }); + Assert.Equal(3, statusJson.GetProperty("unknown_extension_group_count").GetInt64()); + Assert.False(statusJson.GetProperty("unknown_extension_groups_truncated").GetBoolean()); + Assert.Equal(UnknownExtensionClassifier.MaxPersistedGroups, statusJson.GetProperty("unknown_extension_group_limit").GetInt64()); + Assert.Equal(0, statusJson.GetProperty("unknown_extension_group_omitted_count").GetInt64()); + Assert.Contains(".cdidx-langmap.yaml", statusJson.GetProperty("unknown_extension_guidance").GetString()); + + Assert.Equal(CommandExitCodes.Success, jsonExitCode); + Assert.Equal(3, completionJson.GetProperty("unknown_extension_file_count").GetInt32()); + Assert.Equal("workspace", completionJson.GetProperty("unknown_extension_diagnostics_scope").GetString()); + Assert.False(completionJson.GetProperty("unknown_extension_file_count_lower_bound").GetBoolean()); + Assert.Equal(3, completionJson.GetProperty("unknown_extension_group_count").GetInt32()); + Assert.Equal(UnknownExtensionClassifier.MaxCompletionGroups, completionJson.GetProperty("unknown_extension_group_limit").GetInt32()); + Assert.Equal(3, completionJson.GetProperty("unknown_extension_groups").GetArrayLength()); + Assert.Equal(1, completionJson.GetProperty("summary").GetProperty("warnings").GetInt32()); + + Assert.Equal(CommandExitCodes.Success, compactExitCode); + var compactResult = Assert.Single(compactJson.GetProperty("results").EnumerateArray()); + Assert.Equal(3, compactResult.GetProperty("unknown_extension_file_count").GetInt64()); + Assert.Equal(3, compactResult.GetProperty("unknown_extension_groups").GetArrayLength()); + Assert.Equal(3, compactResult.GetProperty("unknown_extension_group_count").GetInt64()); + Assert.Contains("languages --extension", compactResult.GetProperty("unknown_extension_guidance").GetString()); + } + finally + { + DeleteDirectory(projectRoot); + } + } + + [Fact] + public void Run_UnsupportedOnlyPreservesCaseAndExtensionlessGroups_Issue5100() + { + var projectRoot = CreateTempProject(); + try + { + File.WriteAllText(Path.Combine(projectRoot, "source.MYSTERY"), "unknown extension\n"); + File.WriteAllText(Path.Combine(projectRoot, "archive.part.UNMAPPED"), "unknown extension\n"); + File.WriteAllText(Path.Combine(projectRoot, "extensionless_source"), "unknown extension\n"); + for (var index = 0; index < 9; index++) + File.WriteAllText(Path.Combine(projectRoot, $"source.unknown{index:D3}"), "unknown extension\n"); + + var (exitCode, json) = RunAndCaptureJson([projectRoot, "--json"]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal("success", json.GetProperty("status").GetString()); + Assert.Equal(0, json.GetProperty("summary").GetProperty("files_total").GetInt64()); + Assert.Equal(12, json.GetProperty("unknown_extension_file_count").GetInt32()); + var groups = json.GetProperty("unknown_extension_groups").EnumerateArray().ToArray(); + Assert.Equal(10, groups.Length); + Assert.Contains(groups, group => group.GetProperty("extension").GetString() == ".mystery"); + Assert.Equal(12, json.GetProperty("unknown_extension_group_count").GetInt32()); + Assert.True(json.GetProperty("unknown_extension_groups_truncated").GetBoolean()); + Assert.Equal(2, json.GetProperty("unknown_extension_group_omitted_count").GetInt32()); + Assert.Equal(1, json.GetProperty("summary").GetProperty("warnings").GetInt32()); } finally { @@ -5880,6 +5937,47 @@ public void Run_VerboseReportsUnknownExtensionCountAndStatusJsonStampsCount() } } + [Fact] + public void Run_IgnoredOnlyDoesNotWarnAndCustomMappingsClearDiagnostics_Issue5100() + { + var ignoredRoot = CreateTempProject(); + var mappedRoot = CreateTempProject(); + LanguageMapOverrides.ClearEffectiveMapCacheForTesting(); + try + { + File.WriteAllText(Path.Combine(ignoredRoot, ".cdidxignore"), "ignored.mystery\n"); + File.WriteAllText(Path.Combine(ignoredRoot, "ignored.mystery"), "ignored\n"); + + var (ignoredExitCode, ignoredJson) = RunAndCaptureJson([ignoredRoot, "--json"]); + + Assert.Equal(CommandExitCodes.Success, ignoredExitCode); + Assert.Equal(1, ignoredJson.GetProperty("unknown_extension_file_count").GetInt32()); + Assert.Equal("ignore_configuration", ignoredJson.GetProperty("unknown_extension_groups")[0].GetProperty("recommended_action").GetString()); + Assert.Equal(0, ignoredJson.GetProperty("summary").GetProperty("warnings").GetInt32()); + + File.WriteAllText( + Path.Combine(mappedRoot, LanguageMapOverrides.WorkspaceFileName), + "entries:\n - extension: \".custom\"\n language: \"csharp\"\n - extension: \".kts.in\"\n language: \"kotlin\"\n"); + File.WriteAllText(Path.Combine(mappedRoot, "App.CUSTOM"), "public class App { }\n"); + File.WriteAllText(Path.Combine(mappedRoot, "build.KTS.IN"), "fun main() = Unit\n"); + + var (mappedExitCode, mappedJson) = RunAndCaptureJson([mappedRoot, "--json"]); + + Assert.Equal(CommandExitCodes.Success, mappedExitCode); + Assert.Equal(0, mappedJson.GetProperty("unknown_extension_file_count").GetInt32()); + Assert.Equal(0, mappedJson.GetProperty("summary").GetProperty("warnings").GetInt32()); + Assert.True( + !mappedJson.TryGetProperty("unknown_extension_guidance", out var mappedGuidance) + || mappedGuidance.ValueKind == JsonValueKind.Null); + } + finally + { + LanguageMapOverrides.ClearEffectiveMapCacheForTesting(); + DeleteDirectory(ignoredRoot); + DeleteDirectory(mappedRoot); + } + } + [Fact] public void Run_StatusJsonCapsUnknownExtensionPathSample() { diff --git a/tests/CodeIndex.Tests/JsonStringListCodecTests.cs b/tests/CodeIndex.Tests/JsonStringListCodecTests.cs index 4ee9e6a102..66b0c9cb01 100644 --- a/tests/CodeIndex.Tests/JsonStringListCodecTests.cs +++ b/tests/CodeIndex.Tests/JsonStringListCodecTests.cs @@ -124,6 +124,41 @@ public void WriteUnknownExtensionFileMetadata_EmptyPathsStampsEmptyMetadata() Assert.Empty(Assert.IsType>(status.UnknownExtensionExtensionCounts)); Assert.Empty(Assert.IsType>(status.UnknownExtensionCategoryCounts)); Assert.Empty(Assert.IsType>(status.UnknownExtensionGroups)); + Assert.Equal(0, status.UnknownExtensionGroupCount); + Assert.False(status.UnknownExtensionGroupsTruncated); + Assert.Equal(UnknownExtensionClassifier.MaxPersistedGroups, status.UnknownExtensionGroupLimit); + Assert.Equal(0, status.UnknownExtensionGroupOmittedCount); + Assert.Null(status.UnknownExtensionGuidance); + } + finally + { + TestProjectHelper.DeleteFile(dbPath); + } + } + + [Fact] + public void WriteUnknownExtensionFileMetadata_ReportsOmittedGroups_Issue5100() + { + var dbPath = Path.Combine(Path.GetTempPath(), $"codeindex_unknown_groups_{Guid.NewGuid():N}.db"); + try + { + using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); + db.InitializeSchema(); + var writer = new DbWriter(db.Connection); + var paths = Enumerable.Range(0, UnknownExtensionClassifier.MaxPersistedGroups + 2) + .Select(index => $"source-{index:D3}.unknown{index:D3}") + .ToArray(); + + writer.WriteUnknownExtensionFileMetadata(paths); + + var status = new DbReader(db.Connection).GetStatus(); + Assert.Equal(paths.Length, status.UnknownExtensionGroupCount); + Assert.True(status.UnknownExtensionGroupsTruncated); + Assert.Equal(UnknownExtensionClassifier.MaxPersistedGroups, status.UnknownExtensionGroupLimit); + Assert.Equal(2, status.UnknownExtensionGroupOmittedCount); + Assert.Equal( + UnknownExtensionClassifier.MaxPersistedGroups, + Assert.IsType>(status.UnknownExtensionGroups).Count); } finally { From e5281cfcc1fba1d1e5f7f0e58800d8d9f7f71575 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 21 Aug 2026 23:03:28 +0900 Subject: [PATCH 2/3] Address adversarial review findings (#5100) --- changelog.d/unreleased/5100.changed.md | 1 + .../ExportImportCommandRunner.ArchiveScope.cs | 6 +++- .../Cli/IndexCommandRunner.DryRun.cs | 6 ++-- .../Cli/IndexCommandRunner.FullScan.Output.cs | 5 ++-- ...xCommandRunner.FullScan.SnapshotFailure.cs | 29 ++++++++++++++++++- .../Cli/IndexCommandRunner.FullScan.cs | 1 + .../ExportImportCommandRunnerTests.cs | 4 +++ .../IndexCommandRunnerDryRunTests.cs | 9 ++++-- .../IndexCommandRunnerFullScanTests.cs | 6 +++- 9 files changed, 58 insertions(+), 9 deletions(-) diff --git a/changelog.d/unreleased/5100.changed.md b/changelog.d/unreleased/5100.changed.md index 9eb524a875..78b44094a0 100644 --- a/changelog.d/unreleased/5100.changed.md +++ b/changelog.d/unreleased/5100.changed.md @@ -5,6 +5,7 @@ issues: affected: - src/CodeIndex/Cli/ConsoleUi.cs - src/CodeIndex/Cli/IndexCommandRunner.FullScan.Output.cs + - src/CodeIndex/Cli/IndexCommandRunner.FullScan.SnapshotFailure.cs - src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs - src/CodeIndex/Database/UnknownExtensionClassifier.cs - src/CodeIndex/Models/QueryResults.cs diff --git a/src/CodeIndex/Cli/ExportImportCommandRunner.ArchiveScope.cs b/src/CodeIndex/Cli/ExportImportCommandRunner.ArchiveScope.cs index ebf62ab72c..dc416d40c3 100644 --- a/src/CodeIndex/Cli/ExportImportCommandRunner.ArchiveScope.cs +++ b/src/CodeIndex/Cli/ExportImportCommandRunner.ArchiveScope.cs @@ -329,7 +329,11 @@ OR key IN ( 'unknown_extension_file_path_limit', 'unknown_extension_extension_counts_json', 'unknown_extension_category_counts_json', - 'unknown_extension_groups_json'); + 'unknown_extension_groups_json', + 'unknown_extension_group_count', + 'unknown_extension_groups_truncated', + 'unknown_extension_group_limit', + 'unknown_extension_group_omitted_count'); INSERT INTO codeindex_meta(key, value) VALUES (@indexCompletenessKey, 'incomplete') diff --git a/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs b/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs index fc24d8b477..7de408cc97 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs @@ -512,6 +512,8 @@ void RecordDryRunScanErrors(IEnumerable scanErrors) var unknownExtensionGroupOmittedCount = Math.Max( 0, unknownExtensionClassification.GroupCount - unknownExtensionGroups.Count); + var unknownExtensionFileCountLowerBound = candidatePathsTruncated + || (authoritativeFullScan && dryScanMetadata.HadErrors); var unknownExtensionWarning = unknownExtensionClassification.ActionableFileCount > 0 ? $"{unknownExtensionClassification.ActionableFileCount} file(s) were excluded because no language mapping or extractor was available. {UnknownExtensionClassifier.Guidance}" : null; @@ -553,7 +555,7 @@ void RecordDryRunScanErrors(IEnumerable scanErrors) UnknownExtensionGroupLimit = UnknownExtensionClassifier.MaxCompletionGroups, UnknownExtensionGroupOmittedCount = unknownExtensionGroupOmittedCount, UnknownExtensionDiagnosticsScope = authoritativeFullScan ? "workspace" : "candidate_scope", - UnknownExtensionFileCountLowerBound = candidatePathsTruncated, + UnknownExtensionFileCountLowerBound = unknownExtensionFileCountLowerBound, UnknownExtensionGuidance = unknownExtensionTotal > 0 ? UnknownExtensionClassifier.Guidance : null, @@ -593,7 +595,7 @@ void RecordDryRunScanErrors(IEnumerable scanErrors) CommandOutputWriter.WriteLine($"Dry run: {dryFileCount} indexable files inspected{lowerBound}"); if (unknownExtensionTotal > 0) { - CommandOutputWriter.WriteLine($" unknown extensions {unknownExtensionTotal,6}{(candidatePathsTruncated ? " (lower bound)" : string.Empty)}"); + CommandOutputWriter.WriteLine($" unknown extensions {unknownExtensionTotal,6}{(unknownExtensionFileCountLowerBound ? " (lower bound)" : string.Empty)}"); foreach (var group in unknownExtensionGroups) CommandOutputWriter.WriteLine($" {group.Extension}: {ConsoleUi.FormatNumber(group.Count)} ({group.RecommendedAction})"); if (unknownExtensionGroupOmittedCount > 0) diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Output.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Output.cs index a149784ebc..f45c362865 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Output.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Output.cs @@ -155,7 +155,7 @@ private static int WriteFullScanFinalOutput(FullScanFinalOutputContext output) UnknownExtensionGroupLimit = UnknownExtensionClassifier.MaxCompletionGroups, UnknownExtensionGroupOmittedCount = unknownExtensionGroupOmittedCount, UnknownExtensionDiagnosticsScope = "workspace", - UnknownExtensionFileCountLowerBound = false, + UnknownExtensionFileCountLowerBound = output.ScanHadErrors, UnknownExtensionGuidance = output.ScanResult.UnknownExtensionFiles.Count > 0 ? UnknownExtensionClassifier.Guidance : null, @@ -242,7 +242,8 @@ private static int WriteFullScanFinalOutput(FullScanFinalOutputContext output) { CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine( "Unknown extensions", - ConsoleUi.FormatNumber(output.ScanResult.UnknownExtensionFiles.Count), + ConsoleUi.FormatNumber(output.ScanResult.UnknownExtensionFiles.Count) + + (output.ScanHadErrors ? " (lower bound)" : string.Empty), indent: " ")); foreach (var group in unknownExtensionGroups) { diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.SnapshotFailure.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.SnapshotFailure.cs index 4ffdebb194..2e32f80b68 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.SnapshotFailure.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.SnapshotFailure.cs @@ -21,6 +21,7 @@ private sealed class FullScanSnapshotFailureContext internal string? PriorFoldFingerprint { get; init; } internal required List MemorySamples { get; init; } internal required IReadOnlyDictionary LanguageCounts { get; init; } + internal required IReadOnlyList UnknownExtensionFiles { get; init; } internal int FilesCount { get; init; } internal int Skipped { get; init; } internal int DanglingSymlinkCount { get; init; } @@ -83,6 +84,21 @@ private static int WriteFullScanSnapshotFailure( || (persistedCSharpFiles && failure.PriorMetadataTargetCsharpMatchesCurrent); var foldReady = (failure.PriorReadiness & DbContext.FoldReadyFlag) != 0; var memoryTimeline = BuildMemoryTimeline(failure.MemorySamples); + var unknownExtensionClassification = UnknownExtensionClassifier.Classify( + failure.UnknownExtensionFiles); + var unknownExtensionGroups = unknownExtensionClassification.Groups + .Take(UnknownExtensionClassifier.MaxCompletionGroups) + .ToList(); + var unknownExtensionGroupOmittedCount = Math.Max( + 0, + unknownExtensionClassification.GroupCount - unknownExtensionGroups.Count); + var warningCount = failure.Warnings; + if (unknownExtensionClassification.ActionableFileCount > 0) + { + var warning = $"{unknownExtensionClassification.ActionableFileCount} file(s) were excluded because no language mapping or extractor was available. {UnknownExtensionClassifier.Guidance}"; + failure.WarningList.Add(new CliJsonMessage("", warning)); + warningCount++; + } if (failure.Options.Json) { @@ -90,6 +106,17 @@ private static int WriteFullScanSnapshotFailure( { Status = "partial", Mode = failure.Options.Rebuild ? "rebuild" : "incremental", + UnknownExtensionFileCount = failure.UnknownExtensionFiles.Count, + UnknownExtensionGroups = unknownExtensionGroups.Count > 0 ? unknownExtensionGroups : null, + UnknownExtensionGroupCount = unknownExtensionClassification.GroupCount, + UnknownExtensionGroupsTruncated = unknownExtensionGroupOmittedCount > 0, + UnknownExtensionGroupLimit = UnknownExtensionClassifier.MaxCompletionGroups, + UnknownExtensionGroupOmittedCount = unknownExtensionGroupOmittedCount, + UnknownExtensionDiagnosticsScope = "workspace", + UnknownExtensionFileCountLowerBound = true, + UnknownExtensionGuidance = failure.UnknownExtensionFiles.Count > 0 + ? UnknownExtensionClassifier.Guidance + : null, Summary = new IndexFullScanSummaryJsonResult { FilesTotal = totalFiles, @@ -100,7 +127,7 @@ private static int WriteFullScanSnapshotFailure( FilesSkipped = failure.Skipped, FilesPurged = 0, DanglingSymlinksSkipped = failure.DanglingSymlinkCount, - Warnings = failure.Warnings, + Warnings = warningCount, Errors = failure.Errors, SymbolsDroppedByKindFilter = failure.SymbolsDroppedByKindFilter, }, diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs index 0dca0e7b36..a88041c135 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs @@ -456,6 +456,7 @@ bool FullScanJavaScriptTypeScriptConfigChanged() PriorFoldFingerprint = priorFoldFingerprint, MemorySamples = memorySamples, LanguageCounts = languageCounts, + UnknownExtensionFiles = scanResult.UnknownExtensionFiles, FilesCount = files.Count, Skipped = preWriteSelection.Skipped, DanglingSymlinkCount = scanResult.DanglingSymlinks.Count, diff --git a/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs b/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs index 5d9ebf37cd..59455d0cf6 100644 --- a/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs @@ -2461,6 +2461,10 @@ private static void AssertPartialArchiveStatus(string dbPath) !status.RootElement.TryGetProperty("head_freshness", out var headFreshness) || headFreshness.GetProperty("state").GetString() != "head_current"); Assert.False(status.RootElement.TryGetProperty("unknown_extension_file_count", out _)); + Assert.False(status.RootElement.TryGetProperty("unknown_extension_group_count", out _)); + Assert.False(status.RootElement.TryGetProperty("unknown_extension_groups_truncated", out _)); + Assert.False(status.RootElement.TryGetProperty("unknown_extension_group_limit", out _)); + Assert.False(status.RootElement.TryGetProperty("unknown_extension_group_omitted_count", out _)); Assert.True( !status.RootElement.TryGetProperty("last_index_run", out var lastIndexRun) || lastIndexRun.ValueKind == JsonValueKind.Null); diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerDryRunTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerDryRunTests.cs index 26c1c21341..392f687638 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerDryRunTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerDryRunTests.cs @@ -1459,7 +1459,7 @@ public void Run_DryRun_FullScan_ReportsProjectedPurgesAndUnknownExtensionsWithou } [Fact] - public void Run_DryRun_FullScan_ReportsUnreadableDirectory() + public void Run_DryRun_FullScan_ReportsUnreadableDirectory_Issue5100() { if (OperatingSystem.IsWindows()) return; @@ -1470,6 +1470,7 @@ public void Run_DryRun_FullScan_ReportsUnreadableDirectory() { Directory.CreateDirectory(secretDir); File.WriteAllText(Path.Combine(secretDir, "a.cs"), "public class A { }\n"); + File.WriteAllText(Path.Combine(projectRoot, "notes.unknownext"), "unknown language\n"); SetUnixPermissions(secretDir, UnixFileMode.None); var (exitCode, json) = RunAndCaptureJson([projectRoot, "--dry-run", "--json"]); @@ -1477,11 +1478,15 @@ public void Run_DryRun_FullScan_ReportsUnreadableDirectory() Assert.Equal(CommandExitCodes.Success, exitCode); Assert.Equal("dry_run", json.GetProperty("status").GetString()); Assert.Equal(0, json.GetProperty("files_total").GetInt32()); + Assert.Equal(1, json.GetProperty("unknown_extension_file_count").GetInt32()); + Assert.True(json.GetProperty("unknown_extension_file_count_lower_bound").GetBoolean()); Assert.Equal("secret", json.GetProperty("errors")[0].GetProperty("file").GetString()); Assert.Equal("Could not scan directory due to permissions.", json.GetProperty("errors")[0].GetProperty("message").GetString()); - var (humanExitCode, _, stderr) = RunAndCaptureStreams([projectRoot, "--dry-run"]); + var (humanExitCode, stdout, stderr) = RunAndCaptureStreams([projectRoot, "--dry-run"]); Assert.Equal(CommandExitCodes.Success, humanExitCode); + Assert.Contains("unknown extensions", stdout); + Assert.Contains("(lower bound)", stdout); Assert.Contains("secret", stderr); Assert.Contains("Could not scan directory due to permissions.", stderr); } diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs index 331df92099..f98045b61b 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs @@ -3120,7 +3120,7 @@ public void Run_FullScan_RebuildFailureRetainsPersistedReadinessForReclassifiedL } [Fact] - public void Run_FullScan_PartialDiscoveryRetainsKnownLanguageReadinessFailures() + public void Run_FullScan_PartialDiscoveryRetainsKnownLanguageReadinessFailures_Issue5100() { if (OperatingSystem.IsWindows()) return; @@ -3137,6 +3137,7 @@ public void Run_FullScan_PartialDiscoveryRetainsKnownLanguageReadinessFailures() File.WriteAllText( Path.Combine(projectRoot, "Contract.cs"), "public interface IContract { }\n"); + File.WriteAllText(Path.Combine(projectRoot, "notes.unknownext"), "unknown language\n"); Directory.CreateDirectory(unreadableDirectory); File.WriteAllText(Path.Combine(unreadableDirectory, "blocked.py"), "print('blocked')\n"); originalMode = File.GetUnixFileMode(unreadableDirectory); @@ -3152,6 +3153,9 @@ public void Run_FullScan_PartialDiscoveryRetainsKnownLanguageReadinessFailures() Assert.Equal(CommandExitCodes.PartialResult, exitCode); Assert.True(json.GetProperty("summary").GetProperty("errors").GetInt32() >= 2); + Assert.True( + json.GetProperty("unknown_extension_file_count_lower_bound").GetBoolean(), + json.GetRawText()); Assert.False( json.GetProperty("sql_graph_contract_ready").GetBoolean(), json.GetRawText()); From 139dc987bf5ee8376ab0e1d20bcaec15fee41025 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 21 Aug 2026 23:58:25 +0900 Subject: [PATCH 3/3] Address final adversarial review findings (#5100) --- AGENT_GUIDE.md | 2 +- DEVELOPER_GUIDE.md | 4 +- USER_GUIDE.md | 21 +++++- changelog.d/unreleased/5100.changed.md | 16 ++++- .../ExportImportCommandRunner.ArchiveScope.cs | 4 ++ .../Cli/ExportImportCommandRunner.Manifest.cs | 19 ++++-- .../Cli/IndexCommandRunner.DryRun.cs | 63 ++++++++++------- .../Cli/IndexCommandRunner.FullScan.Output.cs | 4 +- ...xCommandRunner.FullScan.SnapshotFailure.cs | 4 +- .../Database/DbContext.SchemaMetadata.cs | 3 + src/CodeIndex/Database/DbReader.Status.cs | 53 +++++++++++---- src/CodeIndex/Database/DbReader.cs | 1 + .../Database/DbWriter.StatusMetadata.cs | 4 ++ .../Database/UnknownExtensionClassifier.cs | 24 ++++++- .../Indexer/Scanning/FileContentLoader.cs | 24 +++++++ .../FileIndexer.ContentLoadingContracts.cs | 10 +++ .../FileIndexer.DirectoryEnumeration.cs | 2 + .../Scanning/FileIndexer.FileAcceptance.cs | 57 +++++++++++++++- .../ExportImportCommandRunnerTests.cs | 10 +++ tests/CodeIndex.Tests/FileIndexerTests.cs | 5 ++ .../IndexCommandRunnerDryRunTests.cs | 1 + .../IndexCommandRunnerTests.cs | 3 + .../JsonStringListCodecTests.cs | 67 +++++++++++++++++++ tests/CodeIndex.Tests/ProgramCliTests.cs | 4 ++ 24 files changed, 347 insertions(+), 58 deletions(-) diff --git a/AGENT_GUIDE.md b/AGENT_GUIDE.md index b73a22068a..ce2d28a719 100644 --- a/AGENT_GUIDE.md +++ b/AGENT_GUIDE.md @@ -159,7 +159,7 @@ CI watching must be bounded. Do not loop indefinitely. - Successful CLI full/update indexing, immediate status/workspace status, and MCP indexing/status must derive `index_complete`, `index_incomplete_reasons`, `reference_graph_complete`, and `reference_graph_incomplete_reasons` from the same persisted-readiness snapshot. Symbols-only runs and persisted file-size, symbol-count, reference-count, extractor-failure, or reference-cap evidence make the generation incomplete. Legacy databases keep the complete compatibility default only when persisted rows do not prove an omission. - `index_writer_version` records the `cdidx` version that last wrote to the DB (stamped into `codeindex_meta` as `cdidx_writer_version` on every full scan, update, and MCP index). `index_newer_than_reader` flips to `true` whenever any persisted numeric contract stamp in `codeindex_meta` (or unknown `PRAGMA user_version` readiness bits) exceeds the current binary's compiled maximum, so an older CLI re-opening a DB written by a newer CLI degrades loudly with an audit trail instead of silently dropping back to text-search fallbacks. `index_newer_than_reader_reason` enumerates the specific newer-than-reader stamps. - `status` separates latest-write provenance (`indexed_head_sha`, `indexed_head_branch`, `indexed_head_timestamp`, `commits_ahead_of_indexed_head`) from whole-workspace verification (`workspace_verified_head_sha`) and the full-scan compatibility baseline (`indexed_head_commit`). Every successful full or partial run updates the latest-write triple; only a full scan or a Git-scoped refresh reconciled from the prior verified baseline advances whole-workspace verification. Scoped mutations persist their affected paths until that reconciled refresh revisits them, including net-zero Git ranges; incomplete pending-path coverage fails closed until a normal full scan succeeds. `worktree_head_changed`, status, map, and MCP use that verified value, falling back conservatively to `indexed_head_commit` for legacy DBs. `head_freshness.state=fresh` requires `status --check` to match the workspace, `fresh_but_incomplete` keeps matching-workspace freshness distinct from incomplete extraction coverage, and `state=head_current` only means the runtime HEAD matches the `indexed_head` selected by `indexed_head_source`. -- `status` also surfaces unknown-extension scan coverage via `unknown_extension_file_count`, stamped by successful full-repository index runs (`cdidx index ` and MCP `index_project`) as the number of non-indexed files with non-empty extensions that do not map to a known language. Current scans also stamp `unknown_extension_files` as a path sample bounded by `unknown_extension_file_path_limit` items and the string-list decoded-character budget, `unknown_extension_files_truncated` when more paths existed than were emitted for either bound, and `unknown_extension_file_path_limit` as the item cap rather than a guarantee that that many paths are returned. Newer scans also expose `unknown_extension_extension_counts`, `unknown_extension_category_counts`, and `unknown_extension_groups`; groups classify common non-code buckets such as repository metadata, licenses, binary assets, configuration, structural metadata, and language-support candidates, and include `recommended_action` values of `ignore_configuration`, `first_class_structural_extraction`, or `language_support`. These fields are omitted on legacy DBs or before a current full scan has stamped them. +- `status` also surfaces unsupported-language scan coverage via `unknown_extension_file_count`, stamped by successful full-repository index runs (`cdidx index ` and MCP `index_project`) as the number of readable, text-like, non-indexed files whose filename or extension does not map to a known language. Extensionless files without a recognized shebang are included as ``; binary, oversized, unreadable, explicitly ignored, and internal index-artifact files are excluded. Current scans also stamp a diagnostics contract version and `unknown_extension_files` as a path sample bounded by `unknown_extension_file_path_limit` items and the string-list decoded-character budget, `unknown_extension_files_truncated` when more paths existed than were emitted for either bound, and `unknown_extension_file_path_limit` as the item cap rather than a guarantee that that many paths are returned. Newer scans also expose `unknown_extension_extension_counts`, `unknown_extension_category_counts`, and `unknown_extension_groups`; groups classify common non-code buckets such as repository metadata, licenses, binary assets, configuration, structural metadata, and language-support candidates, and include `recommended_action` values of `ignore_configuration`, `first_class_structural_extraction`, or `language_support`. These fields are omitted on legacy DBs, when the persisted diagnostics version is absent or unsupported, or before a current full scan has stamped them. Guidance for actionable extensionless files starts with adding a recognized shebang or renaming the file with a supported extension. - `status` also surfaces extractor plugin and pattern-config runtime diagnostics via `extractors`, including loaded counts, the zero parent `retained_load_context_count`, the isolated-worker `load_context_lifecycle`, skipped file counts, and a bounded diagnostics list for incompatible or malformed plugin/pattern files. `extractors.pattern_configs[]` reports accepted sidecars with sanitized `path`, `source` (`workspace` or `user`), normalized `language`, and `rule_count`; workspace discovery stops at its explicit root and path identity follows the live filesystem case policy. Pattern rules and their 128-rule budget live in immutable workspace snapshots; reindex replaces the owning snapshot, and timeout cooldowns/diagnostics remain workspace-scoped. `extractors.snapshot_scope` and highest-first `registration_precedence` expose snapshot selection and the fixed `built_in > user_plugin > user_pattern > workspace_plugin > workspace_pattern` resolution order; replacing one workspace snapshot never mutates another. Diagnostic paths, categories, and messages are sanitized before output; `diagnostics[].category` is the stable machine-readable failure code. - `status` surfaces worker-discovered post-extraction hook manifests and callback budgets through `hooks[]`, including stable assembly-qualified `hooks[].id`, `hooks[].callback_budget_ms`, and the worker-only `hooks[].load_context_lifecycle`. Assembly loading, module initialization, type inspection, and constructor validation occur only in deadline-, memory-, and output-bounded discovery workers, which are terminated after returning a manifest. `hook_diagnostics[]` reports sanitized discovery and callback diagnostics such as candidate-limit truncation, assembly load failure, constructor failure, callback failure, and timeout; `hook_diagnostics[].category` is the stable machine-readable failure code and `hook_diagnostics[].hook_id` links a concrete-hook failure to the corresponding manifest. Index runs enforce `CDIDX_HOOK_CALLBACK_BUDGET_MS` (default: 5000 ms) on scratch copies, discard timed-out mutations, and disable only the timed-out assembly-qualified hook ID for the remainder of the current run. - `status` also surfaces accepted trust-boundary environment overrides through `trust_overrides[]`. Entries include `kind`, `environment_variable`, sanitized `value`, optional sanitized `path`, and `message`; current entries cover `CDIDX_TRUST_WORKSPACE_PLUGINS` workspace plugin discovery, `CDIDX_HOOKS_DIR` hook directory overrides, and the absolute `CDIDX_GIT_EXECUTABLE` executable override. `git_executable` reports the selected source, acceptance, stable reason, sanitized path, owner-only-write result, Unix mode, owner category, owner/ancestor trust (including POSIX sticky-ancestor and Windows ACL policy), and bounded `git --version` execution-probe result even when an explicit Git override is rejected. Build `git_executable` and its matching `trust_overrides[]` entry from the same resolution snapshot. Keep this visible runtime field registered in `status --explain` and in both README status-field tables. diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 5e2642e14c..cf9af649cd 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -2535,7 +2535,7 @@ Process exit codes are coarse (`0` success including valid zero-row queries, `1` - **Path-aware narrowing and ranking** — `search`, `definition`, `references`, `callers`, `callees`, `symbols`, and `files` share path include/exclude filters plus `--exclude-tests`. Read queries prefer source files over tests/docs, and full-text search boosts exact symbol-name and path matches to surface likely implementation files first. - **Compact search snippets for AI** — `search --json` and MCP `search` return match-centered snippets with explicit snippet ranges, match lines, highlights, context counts, `truncated_line_count`, and `truncation_context` instead of whole chunks. `truncation_context.char_counts` and `truncation_context.total_chars` expose the omitted character counts behind each clamped snippet line, while truncated highlights also carry `truncated_char_counts`. `--snippet-lines` lets clients trade recall for smaller payloads, and `--max-line-width` (CLI) / `maxLineWidth` (MCP) routes each snippet line through the same `LineWidthFormatter.ClampLine` contract used by `find` / `references` / `excerpt` / `inspect` so hits inside minified / transpiled / generated single-line files no longer return hundreds of KB per result unless the caller explicitly sets `0`; clamped lines carry `...(+N)...` markers and `highlights[].truncated` / `highlights[].original_line_length`. - **Repo map for first-pass orientation** — `map` aggregates languages, modules, top files, file hot spots, and likely entrypoints from indexed data so AI clients can decide where to look before issuing precise queries. Entrypoint inference now falls back to known top-level entry files when symbol extraction does not produce an explicit `Main`-style symbol. -- **Freshness metadata for trust decisions** — `status` exposes whole-workspace freshness and git state, plus trust metadata such as `sql_graph_contract_ready` / `sql_graph_contract_degraded_reason`, `hotspot_family_ready` / `hotspot_family_degraded_reason`, forward-compatibility audit fields (`index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason` — see "Forward-compatibility readiness audit"), and fold remediation fields (`fold_ready_reason`, `degraded_reason`, `recommended_action`, `alternative_action`) so AI clients can tell up front whether SQL graph/dependency/impact answers, duplicate-name hotspot families, and Unicode `--exact` are authoritative. CLI `status --json` and MCP `status` both populate those fold remediation fields when `fold_ready=false`. It also carries `unknown_extension_file_count`, a capped `unknown_extension_files` path sample, `unknown_extension_files_truncated`, and `unknown_extension_file_path_limit` after a current full-repository scan so extension-table coverage gaps are visible and actionable even when those files were excluded from indexing. When those fold remediation fields are derived from an explicit read-only `file:` DB URI, they are normalized back to a writable filesystem path for both absolute (`file:///...?...`) and relative (`file:codeindex.db?...`) forms instead of echoing the read-only URI into commands that would fail. `cdidx index` JSON/human readiness output also surfaces the same trust bits, keeping the post-index readiness summary aligned with `status`. `impact` / MCP `impact_analysis` also mirror the SQL graph-contract signal in JSON so stale SQL rows do not masquerade as authoritative zero-impact answers. `inspect` / MCP `analyze_symbol` and `references` / MCP `references` now mirror that same SQL graph-contract signal whenever SQL-backed graph reads contribute to their payloads, so stale SQL rows do not look like authoritative hits or zero-result answers there either. `map` keeps `indexed_at` / `latest_modified` scoped to the filtered result set and also exposes `workspace_indexed_at` / `workspace_latest_modified` for whole-workspace freshness. `inspect` mirrors those whole-workspace timestamps and git fields so symbol-oriented AI flows can make trust decisions without a separate `status` call. `files` exposes per-file checksum plus modified/indexed timestamps. File-column migrations are applied opportunistically for older DBs, and read paths are designed to avoid crashing if in-place migration is unavailable. CLI and MCP zero-result JSON responses for `search`, `files`, `symbols`, `definition`, `references`, `callers`, `callees`, `deps`, `unused`, `hotspots`, and `impact` include `indexed_file_count`, `indexed_at`, and `freshness_available`. `indexed_at:null` with `freshness_available=true` means the index is empty, while `freshness_available=false` means a legacy/read-only DB could not expose freshness timestamps and `freshness_degraded_reason` explains why. **HEAD-aware staleness signal**: every successful `cdidx index` full scan stamps the captured `git HEAD` into `codeindex_meta` so subsequent runs can compare it against the workspace HEAD. A successful full scan reconciles and purges the whole workspace, advances `workspace_verified_head_sha`, and returns `head_changed: false`; a partial scan keeps verification at the prior baseline and exposes `head_changed` / `prior_indexed_head_commit` / `current_head_commit` / `head_change_notice` with normal full-scan recovery guidance. `status --check` compares against the same verified baseline through `workspace_check.head_changed`. Git-scoped refreshes can advance verification only after unioning the prior baseline-to-current paths, while explicit `--files` updates preserve it. Non-git workspaces and legacy DBs with no captured baseline skip the comparison instead of false-positive flagging. +- **Freshness metadata for trust decisions** — `status` exposes whole-workspace freshness and git state, plus trust metadata such as `sql_graph_contract_ready` / `sql_graph_contract_degraded_reason`, `hotspot_family_ready` / `hotspot_family_degraded_reason`, forward-compatibility audit fields (`index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason` — see "Forward-compatibility readiness audit"), and fold remediation fields (`fold_ready_reason`, `degraded_reason`, `recommended_action`, `alternative_action`) so AI clients can tell up front whether SQL graph/dependency/impact answers, duplicate-name hotspot families, and Unicode `--exact` are authoritative. CLI `status --json` and MCP `status` both populate those fold remediation fields when `fold_ready=false`. It also carries `unknown_extension_file_count`, a capped `unknown_extension_files` path sample, `unknown_extension_files_truncated`, and `unknown_extension_file_path_limit` after a current full-repository scan so readable text files excluded for missing filename/extension language support are visible and actionable; binary, oversized, unreadable, ignored, and internal-artifact files are not counted. A diagnostics-version stamp gates all persisted fields, so legacy or unsupported semantics are omitted until a current full scan replaces them; extensionless actionable guidance recommends a recognized shebang or a supported-extension rename. When those fold remediation fields are derived from an explicit read-only `file:` DB URI, they are normalized back to a writable filesystem path for both absolute (`file:///...?...`) and relative (`file:codeindex.db?...`) forms instead of echoing the read-only URI into commands that would fail. `cdidx index` JSON/human readiness output also surfaces the same trust bits, keeping the post-index readiness summary aligned with `status`. `impact` / MCP `impact_analysis` also mirror the SQL graph-contract signal in JSON so stale SQL rows do not masquerade as authoritative zero-impact answers. `inspect` / MCP `analyze_symbol` and `references` / MCP `references` now mirror that same SQL graph-contract signal whenever SQL-backed graph reads contribute to their payloads, so stale SQL rows do not look like authoritative hits or zero-result answers there either. `map` keeps `indexed_at` / `latest_modified` scoped to the filtered result set and also exposes `workspace_indexed_at` / `workspace_latest_modified` for whole-workspace freshness. `inspect` mirrors those whole-workspace timestamps and git fields so symbol-oriented AI flows can make trust decisions without a separate `status` call. `files` exposes per-file checksum plus modified/indexed timestamps. File-column migrations are applied opportunistically for older DBs, and read paths are designed to avoid crashing if in-place migration is unavailable. CLI and MCP zero-result JSON responses for `search`, `files`, `symbols`, `definition`, `references`, `callers`, `callees`, `deps`, `unused`, `hotspots`, and `impact` include `indexed_file_count`, `indexed_at`, and `freshness_available`. `indexed_at:null` with `freshness_available=true` means the index is empty, while `freshness_available=false` means a legacy/read-only DB could not expose freshness timestamps and `freshness_degraded_reason` explains why. **HEAD-aware staleness signal**: every successful `cdidx index` full scan stamps the captured `git HEAD` into `codeindex_meta` so subsequent runs can compare it against the workspace HEAD. A successful full scan reconciles and purges the whole workspace, advances `workspace_verified_head_sha`, and returns `head_changed: false`; a partial scan keeps verification at the prior baseline and exposes `head_changed` / `prior_indexed_head_commit` / `current_head_commit` / `head_change_notice` with normal full-scan recovery guidance. `status --check` compares against the same verified baseline through `workspace_check.head_changed`. Git-scoped refreshes can advance verification only after unioning the prior baseline-to-current paths, while explicit `--files` updates preserve it. Non-git workspaces and legacy DBs with no captured baseline skip the comparison instead of false-positive flagging. - **Observable bounded workspace-check samples** — `IndexFreshnessChecker` retains at most 20 ordered entries for each changed, missing, outside-sparse-cone, unindexed, unverifiable, and scan-error list while incrementing authoritative counts independently. The serialized `workspace_check` object derives each `*_omitted_count` from that authoritative count and the returned list, exposes the shared `*_path_limit`, and marks `*_truncated` whenever paths are omitted. A list-only `--fields` projection automatically includes these signals and its count; compact status keeps the count and signals without the arrays; and byte-budget trimming removes only ordered suffix paths, recomputes the signals, and records `metadata.byte_limit_omitted_path_count`. Human check diagnostics use `coverage=sample` or `coverage=complete` so a bounded list cannot be mistaken for a complete inventory. - **Folded-key upgrade without reparse** — `backfill-fold` and MCP `backfill_fold` recompute `name_folded`, explicit-interface `display_name_folded`, and reference `*_folded` values directly from existing DB rows, then stamp `FoldReadyFlag` once verification confirms no required folded values remain NULL. This gives AI clients and users a low-cost upgrade path from pre-#86 DBs without re-reading every source file. It rewrites all folded rows when fold metadata is stale or the C# symbol-name contract changed, and stamps the current C# contract only after verification succeeds. Upgrading the explicit-interface v3 contract also requires persisted signatures for every C# method/property/event row; older databases without that reconstruction evidence must refresh C# files or rebuild rather than receiving a false-ready stamp. - **Bundled symbol analysis** — `inspect` and MCP `analyze_symbol` return definition, nearby symbols, references, callers, callees, file metadata, workspace trust metadata, and graph-support metadata in one request so AI clients can answer common symbol questions with fewer round-trips. @@ -6467,7 +6467,7 @@ USER_GUIDEの[終了コード](USER_GUIDE.md#終了コード)セクションを - **信用判断のための鮮度メタデータ** — `status` はワークスペース全体の鮮度と git 状態を返す。`map` は `indexed_at` / `latest_modified` を絞り込み結果の鮮度として維持しつつ、`workspace_indexed_at` / `workspace_latest_modified` でワークスペース全体の鮮度も返す。`inspect` も同じワークスペース鮮度と git フィールドを返すため、シンボル中心の AI フローで `status` を別途呼ばずに済む。さらに `status` は `sql_graph_contract_ready` / `sql_graph_contract_degraded_reason`、`hotspot_family_ready` / `hotspot_family_degraded_reason` に加えて、forward-compatibility 監査 (`index_writer_version`、`index_newer_than_reader`、`index_newer_than_reader_reason`、詳細は「リーダー側の forward-compatibility 監査」を参照)、および fold-only remediation 用の `fold_ready_reason`、`degraded_reason`、`recommended_action`、`alternative_action` も返すため、AI クライアントは SQL graph/dependency/impact、duplicate-name hotspot family、Unicode `--exact` のどれが authoritative か、また DB が現在の binary より新しい `cdidx` で書かれていないかを最初に判断できる。現行の全体 scan 後は `unknown_extension_file_count` も返すため、未知拡張子で index 対象外になった件数を `status` から確認できる。これらの fold-only remediation field は、明示的な read-only `file:///...?...` DB URI から導出された場合でも、失敗する read-only URI をそのままコマンドへ埋め込まず、writable な filesystem path に正規化して返す。さらに `impact` / MCP `impact_analysis` に加えて、`inspect` / MCP `analyze_symbol`、`references` / `callers` / `callees`、`deps` / `unused` / `hotspots` 系も、SQL ベースの graph/dependency read が実際に結果へ関与したときだけ `sql_graph_contract_ready` / `sql_graph_contract_degraded_reason` を反映するため、stale な SQL 行が authoritative なヒットや 0 件応答に見えてしまうのを防ぎつつ、mixed-language index 内の純粋な非SQL結果を誤って degraded 扱いしない。`files` はファイルごとの checksum・modified・indexed timestamp を返す。古いDBに対する file 列の移行は可能なら自動で行い、その場移行できない場合でも読み取り経路がクラッシュしないようにする。CLI と MCP の 0 件 JSON レスポンスは `indexed_file_count`、`indexed_at`、`freshness_available` を含む。`freshness_available=true` で `indexed_at:null` なら空インデックス、`freshness_available=false` なら legacy/read-only DB で鮮度 timestamp を取得できず、理由は `freshness_degraded_reason` に入る。**HEAD 起点の stale 検知**: `cdidx index` の full scan が成功するたびに、現時点の `git HEAD` を `codeindex_meta` に stamp し、後続実行で workspace HEAD と比較できるようにする。full scan 成功時は workspace 全体を照合・purge して `workspace_verified_head_sha` を進め、`head_changed: false` を返す。partial scan は検証値を従来基準に残し、通常の full scan を再実行する recovery guidance とともに `head_changed` / `prior_indexed_head_commit` / `current_head_commit` / `head_change_notice` を公開する。`status --check` も同じ検証基準を `workspace_check.head_changed` で比較する。Git scoped refresh は従来基準から現在 HEAD までの path を union した場合だけ検証値を進め、明示的な `--files` update は検証値を維持する。非 Git workspace と基準を記録していない legacy DB は比較自体をスキップし、false-positive な警告を出さない。 **観測可能な上限付き workspace-check sample**: `IndexFreshnessChecker` は authoritative な件数を独立して加算しながら、changed、missing、outside-sparse-cone、unindexed、unverifiable、scan-error の各一覧を決定的順序の最大20件に制限します。serialized `workspace_check` は authoritative count と返却一覧から各 `*_omitted_count` を導出し、共有の `*_path_limit` を公開し、path が省略された場合に `*_truncated` を設定します。一覧だけを選ぶ `--fields` projection は count とこれらの signal を自動的に含め、compact status は配列を省いて count と signal を維持します。byte budget による短縮は順序付き末尾 path だけを除去して signal を再計算し、`metadata.byte_limit_omitted_path_count` を記録します。human check 診断は `coverage=sample` または `coverage=complete` を表示します。 -`unknown_extension_files` は `unknown_extension_file_path_limit` 件と decoded-character budget の両方で上限付けされた未知拡張子 path sample で、`unknown_extension_files_truncated` は件数上限または decoded-character budget により未出力の path が残ったことを示します。`unknown_extension_file_path_limit` は item 上限であり、常にその件数まで返す保証ではありません。 +`unknown_extension_files` は `unknown_extension_file_path_limit` 件と decoded-character budget の両方で上限付けされた未対応言語 path sample で、`unknown_extension_files_truncated` は件数上限または decoded-character budget により未出力の path が残ったことを示します。対象は読み取り可能な text 系ファイルであり、binary、上限超過、読み取り不能、ignore 済み、内部 artifact は件数に含めません。拡張子なしの actionable file は `` として分類し、guidance は認識可能な shebang の追加または対応済み拡張子への rename を案内します。diagnostics-version stamp が無い、または未対応の legacy DB では永続化 field を省略し、現行全体 scan の成功後にだけ公開します。`unknown_extension_file_path_limit` は item 上限であり、常にその件数まで返す保証ではありません。 `extractors` は extractor plugin と pattern config の runtime health で、読み込み済み plugin assembly / pattern 件数、symbol/reference extractor 件数、parent load context が 0 であることと isolated-worker lifecycle、skip されたファイル数、上限付き diagnostics list を返します。 diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 3741cd3f89..24d4a6760b 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -2775,7 +2775,10 @@ limit, truncation/omission metadata, diagnostic scope, lower-bound state, and `unknown_extension_guidance`. Group samples use the same normalized relative paths as full status and remain capped at five paths per group. Uppercase suffixes are normalized, multi-dot aliases are resolved before an unsupported file is -grouped, and an unsupported extensionless file uses ``. +grouped, and an unsupported extensionless file uses ``. The inventory is +limited to readable, text-like files: binary and oversized files stay policy +exclusions rather than language-support warnings, and unreadable coverage makes +the reported count a lower bound. The warning count increases once when at least one group needs language or structural extraction support. Files classified as intentional metadata/assets @@ -2784,7 +2787,8 @@ Ignored files are excluded before classification, and a valid workspace mapping or extractor registration removes the diagnostic on the next full scan. Inspect an extension with `cdidx languages --extension --json`, then add a trusted mapping to `.cdidx-langmap.yaml`, register an extractor, or add an ignore -rule for intentional non-code files. +rule for intentional non-code files. For an extensionless source file, add a +recognized shebang or rename it with a supported extension. Full-scan dry runs report the same workspace-authoritative diagnostics without writing. Scoped dry runs use `unknown_extension_diagnostics_scope: "candidate_scope"`; @@ -2796,6 +2800,10 @@ workspace inventory; persisted `status` diagnostics continue to describe the most recent successful full scan. `status --compact` includes those bounded persisted diagnostics by default. +Legacy databases omit these persisted fields until a successful full scan writes +the current diagnostics-version stamp, preventing older count semantics from +being presented as current coverage. + ## Supported languages CLI JSON and MCP `languages` responses share one catalog snapshot and expose `language_capability_counts`. Every count carries an explicit `scope`, `capability`, `count`, and `available` field. The `catalog` scope describes the complete runtime catalog, `matched_catalog` describes rows remaining after language/capability/indexed filters but before pagination, and `indexed_workspace` describes catalog languages currently present in the configured database. Capabilities are reported separately for detection, symbol extraction, reference extraction, outline, and graph queries, so catalog size and indexed-language totals are never interchangeable. @@ -6319,6 +6327,9 @@ lower-bound 状態、`unknown_extension_guidance` を返します。group の sa full status と同じ正規化済み相対 path を使い、1 group あたり 5 件に制限されます。 大文字 suffix は正規化され、複合拡張子 alias は未対応 group 化の前に解決されます。 未対応の拡張子なしファイルは `` で表します。 +inventory は読み取り可能な text 系ファイルだけを対象にします。binary と上限超過 +ファイルは言語対応 warning ではなく policy 除外のまま扱い、読み取り不能な範囲が +ある場合は件数を lower bound として示します。 言語または構造抽出の対応が必要な group が 1 件以上ある場合、warning 件数を 1 増やします。 意図的な metadata/asset として `ignore_configuration` に分類されたファイルは可視化 @@ -6326,7 +6337,8 @@ full status と同じ正規化済み相対 path を使い、1 group あたり 5 有効な workspace mapping または extractor 登録後の次回全体 scan では診断が消えます。 `cdidx languages --extension --json` で拡張子を確認し、信頼済み mapping を `.cdidx-langmap.yaml` に追加するか、extractor を登録するか、意図的な非 code ファイルを -ignore rule に追加してください。 +ignore rule に追加してください。拡張子なしの source ファイルには、認識可能な shebang +を追加するか、対応済み拡張子を持つ名前へ変更してください。 全体 scan の dry-run は書き込みを行わず、同じ workspace-authoritative な診断を返します。 scoped dry-run は `unknown_extension_diagnostics_scope: "candidate_scope"` を使い、candidate @@ -6336,6 +6348,9 @@ update（`--files`、`--commits`、`--changed-between`）は新しい workspace 主張せず、永続化済み `status` 診断は直近に成功した全体 scan を表し続けます。 `status --compact` はその上限付き永続化診断を既定で含めます。 +legacy DB では、成功した全体 scan が現行 diagnostics-version stamp を書き込むまで +これらの永続化 field を省略し、旧来の件数 semantics を現行 coverage として表示しません。 + ## 対応言語 CLI JSON と MCP の `languages` response は同じ catalog snapshot を共有し、`language_capability_counts` を返します。各件数には `scope`、`capability`、`count`、`available` が明示されます。`catalog` scope は runtime catalog 全体、`matched_catalog` は言語・capability・indexed filter 適用後かつ pagination 前の行、`indexed_workspace` は設定済み database に現在存在する catalog 言語を表します。detection、symbol extraction、reference extraction、outline、graph query の capability は個別に集計されるため、catalog size と indexed-language total を同じ件数として扱うことはできません。 diff --git a/changelog.d/unreleased/5100.changed.md b/changelog.d/unreleased/5100.changed.md index 78b44094a0..43199e8bea 100644 --- a/changelog.d/unreleased/5100.changed.md +++ b/changelog.d/unreleased/5100.changed.md @@ -7,16 +7,28 @@ affected: - src/CodeIndex/Cli/IndexCommandRunner.FullScan.Output.cs - src/CodeIndex/Cli/IndexCommandRunner.FullScan.SnapshotFailure.cs - src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs + - src/CodeIndex/Cli/ExportImportCommandRunner.ArchiveScope.cs + - src/CodeIndex/Cli/ExportImportCommandRunner.Manifest.cs + - src/CodeIndex/Database/DbContext.SchemaMetadata.cs + - src/CodeIndex/Database/DbReader.Status.cs + - src/CodeIndex/Database/DbReader.cs + - src/CodeIndex/Database/DbWriter.StatusMetadata.cs - src/CodeIndex/Database/UnknownExtensionClassifier.cs + - src/CodeIndex/Indexer/Scanning/FileContentLoader.cs + - src/CodeIndex/Indexer/Scanning/FileIndexer.ContentLoadingContracts.cs + - src/CodeIndex/Indexer/Scanning/FileIndexer.DirectoryEnumeration.cs + - src/CodeIndex/Indexer/Scanning/FileIndexer.FileAcceptance.cs - src/CodeIndex/Models/QueryResults.cs + - AGENT_GUIDE.md + - DEVELOPER_GUIDE.md - README.md - USER_GUIDE.md --- ## English -- **Index completion now surfaces unsupported-language coverage loss (#5100)** — successful full scans, watch initialization, dry runs, and compact status report bounded extension groups, omission metadata, and actionable mapping or extractor guidance; ignored files and valid custom mappings keep their expected behavior. +- **Index completion now surfaces unsupported-language coverage loss (#5100)** — successful full scans, watch initialization, dry runs, and compact status report bounded extension groups, omission metadata, and actionable mapping or extractor guidance; binary/oversized policy exclusions stay out of the inventory, legacy persisted values require the current diagnostics stamp, and extensionless sources receive shebang/rename guidance. ## 日本語 -- **index 完了時に未対応言語による coverage loss を表示するようになりました (#5100)** — 成功した全体 scan、watch 初期化、dry-run、compact status は、上限付き拡張子 group、省略 metadata、mapping または extractor の対処 guidance を返し、ignore 済みファイルと有効な custom mapping の既存動作は維持します。 +- **index 完了時に未対応言語による coverage loss を表示するようになりました (#5100)** — 成功した全体 scan、watch 初期化、dry-run、compact status は、上限付き拡張子 group、省略 metadata、mapping または extractor の対処 guidance を返します。binary/上限超過の policy 除外は inventory に含めず、legacy の永続化値には現行 diagnostics stamp を要求し、拡張子なし source には shebang 追加または rename を案内します。 diff --git a/src/CodeIndex/Cli/ExportImportCommandRunner.ArchiveScope.cs b/src/CodeIndex/Cli/ExportImportCommandRunner.ArchiveScope.cs index dc416d40c3..f7e79a31c9 100644 --- a/src/CodeIndex/Cli/ExportImportCommandRunner.ArchiveScope.cs +++ b/src/CodeIndex/Cli/ExportImportCommandRunner.ArchiveScope.cs @@ -224,6 +224,8 @@ OR key IN ( 'commit_scoped_fresh_head_sha', 'last_full_scan_elapsed_ms', 'last_workspace_freshened_at', + 'unknown_extension_diagnostics_version', + 'unknown_extension_has_actionable_extensionless_files', 'unknown_extension_file_count', 'unknown_extension_file_paths_json', 'unknown_extension_files_truncated', @@ -323,6 +325,8 @@ OR key IN ( 'commit_scoped_fresh_head_sha', 'last_full_scan_elapsed_ms', 'last_workspace_freshened_at', + 'unknown_extension_diagnostics_version', + 'unknown_extension_has_actionable_extensionless_files', 'unknown_extension_file_count', 'unknown_extension_file_paths_json', 'unknown_extension_files_truncated', diff --git a/src/CodeIndex/Cli/ExportImportCommandRunner.Manifest.cs b/src/CodeIndex/Cli/ExportImportCommandRunner.Manifest.cs index 76a95e72ea..c2d8b9d758 100644 --- a/src/CodeIndex/Cli/ExportImportCommandRunner.Manifest.cs +++ b/src/CodeIndex/Cli/ExportImportCommandRunner.Manifest.cs @@ -24,7 +24,12 @@ private static ExportManifest BuildManifest( var userVersion = ReadSqliteUserVersion(connection); var projectRoot = ReadMetaString(connection, DbContext.IndexedProjectRootMetaKey); var indexedHead = ReadMetaString(connection, DbContext.IndexedHeadShaMetaKey); - var unknownExtensionFiles = ReadUnknownExtensionFileSample(connection); + var unknownExtensionDiagnosticsCurrent = + ReadMetaInt(connection, DbContext.UnknownExtensionDiagnosticsVersionMetaKey) + == DbContext.UnknownExtensionDiagnosticsVersion; + var unknownExtensionFiles = unknownExtensionDiagnosticsCurrent + ? ReadUnknownExtensionFileSample(connection) + : default; var indexCompleteness = ReadMetaString(connection, DbContext.IndexCompletenessMetaKey); var indexIncompleteReasons = ReadArchiveIncompleteReasons(connection); cancellationToken.ThrowIfCancellationRequested(); @@ -49,10 +54,16 @@ private static ExportManifest BuildManifest( CSharpSymbolNameContractVersion: ReadMetaInt(connection, DbContext.CSharpSymbolNameContractVersionMetaKey), SqlGraphContractVersion: ReadMetaInt(connection, DbContext.SqlGraphContractVersionMetaKey), HotspotFamilyVersion: ReadMetaInt(connection, DbContext.HotspotFamilyVersionMetaKey), - UnknownExtensionFileCount: ReadMetaLong(connection, DbContext.UnknownExtensionFileCountMetaKey), + UnknownExtensionFileCount: unknownExtensionDiagnosticsCurrent + ? ReadMetaLong(connection, DbContext.UnknownExtensionFileCountMetaKey) + : null, UnknownExtensionFiles: unknownExtensionFiles.Files, - UnknownExtensionFilesTruncated: ReadMetaBool(connection, DbContext.UnknownExtensionFilesTruncatedMetaKey), - UnknownExtensionFilePathLimit: ReadMetaInt(connection, DbContext.UnknownExtensionFilePathLimitMetaKey), + UnknownExtensionFilesTruncated: unknownExtensionDiagnosticsCurrent + ? ReadMetaBool(connection, DbContext.UnknownExtensionFilesTruncatedMetaKey) + : null, + UnknownExtensionFilePathLimit: unknownExtensionDiagnosticsCurrent + ? ReadMetaInt(connection, DbContext.UnknownExtensionFilePathLimitMetaKey) + : null, UnknownExtensionFileSampleCount: unknownExtensionFiles.Count, UnknownExtensionFileSampleLimit: unknownExtensionFiles.Limit, UnknownExtensionFileSampleTruncated: unknownExtensionFiles.Truncated, diff --git a/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs b/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs index 7de408cc97..6241d11f46 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs @@ -242,7 +242,12 @@ void RecordDryRunScanErrors(IEnumerable scanErrors) var knownLanguage = authoritativeFullScan ? FileIndexer.GetReusableDetectedLanguage(f, dryScanMetadata.FileLanguages) : null; - var probe = ProbeDryRunFile(dryIndexer, f, displayRelativePath, knownLanguage); + var probe = ProbeDryRunFile( + dryIndexer, + f, + displayRelativePath, + knownLanguage, + cancellationToken); if (probe.PolicySkipped) { dryFileCount++; @@ -515,7 +520,7 @@ void RecordDryRunScanErrors(IEnumerable scanErrors) var unknownExtensionFileCountLowerBound = candidatePathsTruncated || (authoritativeFullScan && dryScanMetadata.HadErrors); var unknownExtensionWarning = unknownExtensionClassification.ActionableFileCount > 0 - ? $"{unknownExtensionClassification.ActionableFileCount} file(s) were excluded because no language mapping or extractor was available. {UnknownExtensionClassifier.Guidance}" + ? $"{unknownExtensionClassification.ActionableFileCount} file(s) were excluded because no language mapping or extractor was available. {UnknownExtensionClassifier.GetGuidance(unknownExtensionClassification)}" : null; if (unknownExtensionWarning != null) { @@ -557,7 +562,7 @@ void RecordDryRunScanErrors(IEnumerable scanErrors) UnknownExtensionDiagnosticsScope = authoritativeFullScan ? "workspace" : "candidate_scope", UnknownExtensionFileCountLowerBound = unknownExtensionFileCountLowerBound, UnknownExtensionGuidance = unknownExtensionTotal > 0 - ? UnknownExtensionClassifier.Guidance + ? UnknownExtensionClassifier.GetGuidance(unknownExtensionClassification) : null, CandidatePathLimit = dryRunPathLimit, CandidatePathsProcessed = candidatePathsProcessed, @@ -995,7 +1000,8 @@ private static DryRunFileProbe ProbeDryRunFile( FileIndexer indexer, string absolutePath, string relativePath, - string? knownLanguage) + string? knownLanguage, + CancellationToken cancellationToken) { var indexability = indexer.GetFileIndexabilityForIndexing(absolutePath); if (indexability == FileIndexer.FileProbeStatus.ProbeFailed) @@ -1005,32 +1011,37 @@ private static DryRunFileProbe ProbeDryRunFile( string? reusableLanguage = knownLanguage; FileIndexer.LanguageDetectionResult? preLoadDetection = null; - var isAmbiguousExtension = FileIndexer.TryGetAmbiguousLanguageDescriptor( - Path.GetExtension(absolutePath), - out _); - if (reusableLanguage == null || isAmbiguousExtension) + try { - var detection = indexer.TryDetectLanguageForIndexing(absolutePath); - if (reusableLanguage == null) + var isAmbiguousExtension = FileIndexer.TryGetAmbiguousLanguageDescriptor( + Path.GetExtension(absolutePath), + out _); + if (reusableLanguage == null || isAmbiguousExtension) { - if (detection.Status == FileIndexer.FileProbeStatus.ProbeFailed) - return DryRunFileProbe.FromError("Could not probe file for indexability/language."); - if (detection.Status != FileIndexer.FileProbeStatus.Supported) - return string.IsNullOrEmpty(Path.GetExtension(absolutePath)) - ? DryRunFileProbe.FromUnsupported() - : DryRunFileProbe.FromUnknownExtension(); - - reusableLanguage = FileIndexer.CanReuseDetectedLanguageWithoutContent(absolutePath, detection.Language) - ? detection.Language - : null; - } + var detection = indexer.TryDetectLanguageForIndexing(absolutePath); + if (reusableLanguage == null) + { + if (detection.Status == FileIndexer.FileProbeStatus.ProbeFailed) + return DryRunFileProbe.FromError("Could not probe file for indexability/language."); + if (detection.Status != FileIndexer.FileProbeStatus.Supported) + { + return indexer.IsUnknownLanguageCoverageCandidate( + absolutePath, + relativePath, + cancellationToken) + ? DryRunFileProbe.FromUnknownExtension() + : DryRunFileProbe.FromUnsupported(); + } - if (detection.Status == FileIndexer.FileProbeStatus.Supported) - preLoadDetection = detection; - } + reusableLanguage = FileIndexer.CanReuseDetectedLanguageWithoutContent(absolutePath, detection.Language) + ? detection.Language + : null; + } + + if (detection.Status == FileIndexer.FileProbeStatus.Supported) + preLoadDetection = detection; + } - try - { var loaded = indexer.BuildLoadedRecordWithRawBytes( absolutePath, relativePath, diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Output.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Output.cs index f45c362865..87235398be 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Output.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Output.cs @@ -84,7 +84,7 @@ private static int WriteFullScanFinalOutput(FullScanFinalOutputContext output) 0, unknownExtensionClassification.GroupCount - unknownExtensionGroups.Count); var unknownExtensionWarning = unknownExtensionClassification.ActionableFileCount > 0 - ? $"{unknownExtensionClassification.ActionableFileCount} file(s) were excluded because no language mapping or extractor was available. {UnknownExtensionClassifier.Guidance}" + ? $"{unknownExtensionClassification.ActionableFileCount} file(s) were excluded because no language mapping or extractor was available. {UnknownExtensionClassifier.GetGuidance(unknownExtensionClassification)}" : null; if (unknownExtensionWarning != null) { @@ -157,7 +157,7 @@ private static int WriteFullScanFinalOutput(FullScanFinalOutputContext output) UnknownExtensionDiagnosticsScope = "workspace", UnknownExtensionFileCountLowerBound = output.ScanHadErrors, UnknownExtensionGuidance = output.ScanResult.UnknownExtensionFiles.Count > 0 - ? UnknownExtensionClassifier.Guidance + ? UnknownExtensionClassifier.GetGuidance(unknownExtensionClassification) : null, Summary = new IndexFullScanSummaryJsonResult { diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.SnapshotFailure.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.SnapshotFailure.cs index 2e32f80b68..b59811600d 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.SnapshotFailure.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.SnapshotFailure.cs @@ -95,7 +95,7 @@ private static int WriteFullScanSnapshotFailure( var warningCount = failure.Warnings; if (unknownExtensionClassification.ActionableFileCount > 0) { - var warning = $"{unknownExtensionClassification.ActionableFileCount} file(s) were excluded because no language mapping or extractor was available. {UnknownExtensionClassifier.Guidance}"; + var warning = $"{unknownExtensionClassification.ActionableFileCount} file(s) were excluded because no language mapping or extractor was available. {UnknownExtensionClassifier.GetGuidance(unknownExtensionClassification)}"; failure.WarningList.Add(new CliJsonMessage("", warning)); warningCount++; } @@ -115,7 +115,7 @@ private static int WriteFullScanSnapshotFailure( UnknownExtensionDiagnosticsScope = "workspace", UnknownExtensionFileCountLowerBound = true, UnknownExtensionGuidance = failure.UnknownExtensionFiles.Count > 0 - ? UnknownExtensionClassifier.Guidance + ? UnknownExtensionClassifier.GetGuidance(unknownExtensionClassification) : null, Summary = new IndexFullScanSummaryJsonResult { diff --git a/src/CodeIndex/Database/DbContext.SchemaMetadata.cs b/src/CodeIndex/Database/DbContext.SchemaMetadata.cs index c016f7bfa7..344b19bc2b 100644 --- a/src/CodeIndex/Database/DbContext.SchemaMetadata.cs +++ b/src/CodeIndex/Database/DbContext.SchemaMetadata.cs @@ -189,6 +189,9 @@ public static string GetDynamicReferenceGraphContractVersionMetaKey(string lang) public const string UnknownExtensionGroupsTruncatedMetaKey = "unknown_extension_groups_truncated"; public const string UnknownExtensionGroupLimitMetaKey = "unknown_extension_group_limit"; public const string UnknownExtensionGroupOmittedCountMetaKey = "unknown_extension_group_omitted_count"; + public const string UnknownExtensionDiagnosticsVersionMetaKey = "unknown_extension_diagnostics_version"; + public const string UnknownExtensionHasActionableExtensionlessFilesMetaKey = "unknown_extension_has_actionable_extensionless_files"; + public const int UnknownExtensionDiagnosticsVersion = 1; public const int UnknownExtensionFilePathSampleLimit = 50; public const string BatchInProgressMetaKey = "batch_in_progress"; // Issue #1546: case-sensitivity of the workspace filesystem the most recent successful diff --git a/src/CodeIndex/Database/DbReader.Status.cs b/src/CodeIndex/Database/DbReader.Status.cs index f8f5fc349a..7e7d4c2af9 100644 --- a/src/CodeIndex/Database/DbReader.Status.cs +++ b/src/CodeIndex/Database/DbReader.Status.cs @@ -127,17 +127,45 @@ GROUP BY COALESCE(f.lang, 'unknown'), s.kind var indexedHeadBranch = TryGetMetaStringInternal(DbContext.IndexedHeadBranchMetaKey); var indexedHeadBranchStampPresent = HasMetaKeyInternal(DbContext.IndexedHeadBranchMetaKey); var indexedHeadTimestamp = ParseMetaDateTimeOffset(TryGetMetaStringInternal(DbContext.IndexedHeadTimestampMetaKey)); - var unknownExtensionFileCount = ParseMetaLong(TryGetMetaStringInternal(DbContext.UnknownExtensionFileCountMetaKey)); - var unknownExtensionFiles = ParseMetaStringList(TryGetMetaStringInternal(DbContext.UnknownExtensionFilePathsMetaKey)); - var unknownExtensionFilesTruncated = ParseMetaBool(TryGetMetaStringInternal(DbContext.UnknownExtensionFilesTruncatedMetaKey)); - var unknownExtensionFilePathLimit = ParseMetaLong(TryGetMetaStringInternal(DbContext.UnknownExtensionFilePathLimitMetaKey)); - var unknownExtensionExtensionCounts = UnknownExtensionClassifier.DeserializeCounts(TryGetMetaStringInternal(DbContext.UnknownExtensionExtensionCountsMetaKey)); - var unknownExtensionCategoryCounts = UnknownExtensionClassifier.DeserializeCounts(TryGetMetaStringInternal(DbContext.UnknownExtensionCategoryCountsMetaKey)); - var unknownExtensionGroups = UnknownExtensionClassifier.DeserializeGroups(TryGetMetaStringInternal(DbContext.UnknownExtensionGroupsMetaKey)); - var unknownExtensionGroupCount = ParseMetaLong(TryGetMetaStringInternal(DbContext.UnknownExtensionGroupCountMetaKey)); - var unknownExtensionGroupsTruncated = ParseMetaBool(TryGetMetaStringInternal(DbContext.UnknownExtensionGroupsTruncatedMetaKey)); - var unknownExtensionGroupLimit = ParseMetaLong(TryGetMetaStringInternal(DbContext.UnknownExtensionGroupLimitMetaKey)); - var unknownExtensionGroupOmittedCount = ParseMetaLong(TryGetMetaStringInternal(DbContext.UnknownExtensionGroupOmittedCountMetaKey)); + var unknownExtensionDiagnosticsCurrent = + ParseMetaLong(TryGetMetaStringInternal(DbContext.UnknownExtensionDiagnosticsVersionMetaKey)) + == DbContext.UnknownExtensionDiagnosticsVersion; + var unknownExtensionHasActionableExtensionlessFiles = unknownExtensionDiagnosticsCurrent + ? ParseMetaBool(TryGetMetaStringInternal(DbContext.UnknownExtensionHasActionableExtensionlessFilesMetaKey)) + : null; + var unknownExtensionFileCount = unknownExtensionDiagnosticsCurrent + ? ParseMetaLong(TryGetMetaStringInternal(DbContext.UnknownExtensionFileCountMetaKey)) + : null; + var unknownExtensionFiles = unknownExtensionDiagnosticsCurrent + ? ParseMetaStringList(TryGetMetaStringInternal(DbContext.UnknownExtensionFilePathsMetaKey)) + : null; + var unknownExtensionFilesTruncated = unknownExtensionDiagnosticsCurrent + ? ParseMetaBool(TryGetMetaStringInternal(DbContext.UnknownExtensionFilesTruncatedMetaKey)) + : null; + var unknownExtensionFilePathLimit = unknownExtensionDiagnosticsCurrent + ? ParseMetaLong(TryGetMetaStringInternal(DbContext.UnknownExtensionFilePathLimitMetaKey)) + : null; + var unknownExtensionExtensionCounts = unknownExtensionDiagnosticsCurrent + ? UnknownExtensionClassifier.DeserializeCounts(TryGetMetaStringInternal(DbContext.UnknownExtensionExtensionCountsMetaKey)) + : null; + var unknownExtensionCategoryCounts = unknownExtensionDiagnosticsCurrent + ? UnknownExtensionClassifier.DeserializeCounts(TryGetMetaStringInternal(DbContext.UnknownExtensionCategoryCountsMetaKey)) + : null; + var unknownExtensionGroups = unknownExtensionDiagnosticsCurrent + ? UnknownExtensionClassifier.DeserializeGroups(TryGetMetaStringInternal(DbContext.UnknownExtensionGroupsMetaKey)) + : null; + var unknownExtensionGroupCount = unknownExtensionDiagnosticsCurrent + ? ParseMetaLong(TryGetMetaStringInternal(DbContext.UnknownExtensionGroupCountMetaKey)) + : null; + var unknownExtensionGroupsTruncated = unknownExtensionDiagnosticsCurrent + ? ParseMetaBool(TryGetMetaStringInternal(DbContext.UnknownExtensionGroupsTruncatedMetaKey)) + : null; + var unknownExtensionGroupLimit = unknownExtensionDiagnosticsCurrent + ? ParseMetaLong(TryGetMetaStringInternal(DbContext.UnknownExtensionGroupLimitMetaKey)) + : null; + var unknownExtensionGroupOmittedCount = unknownExtensionDiagnosticsCurrent + ? ParseMetaLong(TryGetMetaStringInternal(DbContext.UnknownExtensionGroupOmittedCountMetaKey)) + : null; if (unknownExtensionFiles != null) { unknownExtensionFilesTruncated ??= unknownExtensionFileCount.HasValue @@ -220,7 +248,8 @@ GROUP BY COALESCE(f.lang, 'unknown'), s.kind UnknownExtensionGroupLimit = unknownExtensionGroupLimit, UnknownExtensionGroupOmittedCount = unknownExtensionGroupOmittedCount, UnknownExtensionGuidance = unknownExtensionFileCount > 0 - ? UnknownExtensionClassifier.Guidance + ? UnknownExtensionClassifier.GetGuidance( + unknownExtensionHasActionableExtensionlessFiles == true) : null, IndexedAt = freshness.IndexedAt, LastWorkspaceFreshenedAt = lastIndexRun?.StartedAt ?? indexedHeadTimestamp?.UtcDateTime, diff --git a/src/CodeIndex/Database/DbReader.cs b/src/CodeIndex/Database/DbReader.cs index 46edc96e77..05d3ca1013 100644 --- a/src/CodeIndex/Database/DbReader.cs +++ b/src/CodeIndex/Database/DbReader.cs @@ -811,6 +811,7 @@ internal static (bool Newer, string? Reason) DetectNewerThanReaderContracts(Sqli AppendIfStoredGreater(conn, DbContext.SqlGraphContractVersionMetaKey, DbContext.SqlGraphContractVersion, "sql_graph_contract_version", newerContracts); AppendIfStoredGreater(conn, DbContext.HdlGraphContractVersionMetaKey, DbContext.HdlGraphContractVersion, "hdl_graph_contract_version", newerContracts); AppendIfStoredGreater(conn, DbContext.ReferenceIdentityContractVersionMetaKey, DbContext.ReferenceIdentityContractVersion, "reference_identity_contract_version", newerContracts); + AppendIfStoredGreater(conn, DbContext.UnknownExtensionDiagnosticsVersionMetaKey, DbContext.UnknownExtensionDiagnosticsVersion, "unknown_extension_diagnostics_version", newerContracts); AppendIfStoredGreater(conn, "fold_key_version", NameFold.Version, "fold_key_version", newerContracts); foreach (var lang in SymbolExtractor.GetExplicitReferenceGraphContractLanguages()) { diff --git a/src/CodeIndex/Database/DbWriter.StatusMetadata.cs b/src/CodeIndex/Database/DbWriter.StatusMetadata.cs index 7a6a13bfb1..37c3dacd5d 100644 --- a/src/CodeIndex/Database/DbWriter.StatusMetadata.cs +++ b/src/CodeIndex/Database/DbWriter.StatusMetadata.cs @@ -70,6 +70,8 @@ public void WriteUnknownExtensionFileMetadata(IReadOnlyList paths) if (paths.Count == 0) { SetMetaValues( + (DbContext.UnknownExtensionDiagnosticsVersionMetaKey, DbContext.UnknownExtensionDiagnosticsVersion.ToString(System.Globalization.CultureInfo.InvariantCulture)), + (DbContext.UnknownExtensionHasActionableExtensionlessFilesMetaKey, false.ToString(System.Globalization.CultureInfo.InvariantCulture)), (DbContext.UnknownExtensionFileCountMetaKey, "0"), (DbContext.UnknownExtensionFilePathsMetaKey, "[]"), (DbContext.UnknownExtensionFilesTruncatedMetaKey, false.ToString(System.Globalization.CultureInfo.InvariantCulture)), @@ -89,6 +91,8 @@ public void WriteUnknownExtensionFileMetadata(IReadOnlyList paths) DbContext.UnknownExtensionFilePathSampleLimit); var classification = UnknownExtensionClassifier.Classify(paths); SetMetaValues( + (DbContext.UnknownExtensionDiagnosticsVersionMetaKey, DbContext.UnknownExtensionDiagnosticsVersion.ToString(System.Globalization.CultureInfo.InvariantCulture)), + (DbContext.UnknownExtensionHasActionableExtensionlessFilesMetaKey, classification.HasActionableExtensionlessFiles.ToString(System.Globalization.CultureInfo.InvariantCulture)), (DbContext.UnknownExtensionFileCountMetaKey, paths.Count.ToString(System.Globalization.CultureInfo.InvariantCulture)), (DbContext.UnknownExtensionFilePathsMetaKey, JsonStringListCodec.Serialize(sample)), (DbContext.UnknownExtensionFilesTruncatedMetaKey, (paths.Count > sample.Count).ToString(System.Globalization.CultureInfo.InvariantCulture)), diff --git a/src/CodeIndex/Database/UnknownExtensionClassifier.cs b/src/CodeIndex/Database/UnknownExtensionClassifier.cs index c23feaa32d..af934747a4 100644 --- a/src/CodeIndex/Database/UnknownExtensionClassifier.cs +++ b/src/CodeIndex/Database/UnknownExtensionClassifier.cs @@ -8,7 +8,8 @@ internal sealed record UnknownExtensionClassification( Dictionary CategoryCounts, List Groups, int GroupCount, - long ActionableFileCount) + long ActionableFileCount, + bool HasActionableExtensionlessFiles) { public bool GroupsTruncated => GroupCount > Groups.Count; public int GroupLimit => UnknownExtensionClassifier.MaxPersistedGroups; @@ -24,6 +25,11 @@ internal static class UnknownExtensionClassifier "Inspect an extension with `cdidx languages --extension --json`; " + "then add a trusted mapping in `.cdidx-langmap.yaml`, register an extractor, " + "or ignore intentional non-code files."; + internal const string ExtensionlessGuidance = + "For extensionless source files, add a recognized shebang or rename them with a supported extension. " + + "For suffixed files, inspect an extension with `cdidx languages --extension --json`; " + + "then add a trusted mapping in `.cdidx-langmap.yaml`, register an extractor, " + + "or ignore intentional non-code files."; private const int MaxDictionaryEntries = 256; private const int MaxRawJsonCharacters = 512 * 1024; private const int MaxDecodedStringCharacters = 64 * 1024; @@ -42,6 +48,7 @@ public static UnknownExtensionClassification Classify(IReadOnlyList path var categoryCounts = new Dictionary(StringComparer.Ordinal); var groups = new Dictionary(StringComparer.Ordinal); var actionableFileCount = 0L; + var hasActionableExtensionlessFiles = false; foreach (var path in paths.OrderBy(p => p, StringComparer.Ordinal)) { @@ -51,7 +58,13 @@ public static UnknownExtensionClassification Classify(IReadOnlyList path var extension = GetExtensionKey(path); var (category, action) = ClassifyPath(path, extension); if (!string.Equals(action, "ignore_configuration", StringComparison.Ordinal)) + { actionableFileCount++; + hasActionableExtensionlessFiles |= string.Equals( + extension, + "", + StringComparison.Ordinal); + } Increment(extensionCounts, extension); Increment(categoryCounts, category); @@ -87,9 +100,16 @@ public static UnknownExtensionClassification Classify(IReadOnlyList path OrderCounts(categoryCounts), orderedGroups, groups.Count, - actionableFileCount); + actionableFileCount, + hasActionableExtensionlessFiles); } + public static string GetGuidance(UnknownExtensionClassification classification) + => GetGuidance(classification.HasActionableExtensionlessFiles); + + public static string GetGuidance(bool hasActionableExtensionlessFiles) + => hasActionableExtensionlessFiles ? ExtensionlessGuidance : Guidance; + public static string SerializeCounts(IReadOnlyDictionary counts) { using var buffer = new MemoryStream(); diff --git a/src/CodeIndex/Indexer/Scanning/FileContentLoader.cs b/src/CodeIndex/Indexer/Scanning/FileContentLoader.cs index 4adbbb9fe8..bd4dcb7fd4 100644 --- a/src/CodeIndex/Indexer/Scanning/FileContentLoader.cs +++ b/src/CodeIndex/Indexer/Scanning/FileContentLoader.cs @@ -40,6 +40,30 @@ internal LoadedFileContent Load( return BuildLoadedFileContent(rawFile, decoded, normalized); } + internal bool IsUnknownLanguageCoverageCandidate( + string absolutePath, + string normalizedRelativePath, + string relativePath, + CancellationToken cancellationToken) + { + var rawFile = ReadRawBytesWithSizeLimit( + absolutePath, + normalizedRelativePath, + cancellationToken); + if (IsGitLfsPointer(rawFile.Bytes)) + return false; + + if (TryFindIndexBlockingNullByte(rawFile.Bytes, out var nullByteOffset)) + { + throw new FileIndexer.BinaryFileSkippedException( + relativePath, + nullByteOffset, + $"{relativePath}: binary file skipped because it contains NULL byte at byte offset {nullByteOffset}"); + } + + return true; + } + internal static bool CanReuseRawBytesForNormalizedChecksum( string decodedContent, string? decodeWarning, diff --git a/src/CodeIndex/Indexer/Scanning/FileIndexer.ContentLoadingContracts.cs b/src/CodeIndex/Indexer/Scanning/FileIndexer.ContentLoadingContracts.cs index 44b2279bea..6d53b841ac 100644 --- a/src/CodeIndex/Indexer/Scanning/FileIndexer.ContentLoadingContracts.cs +++ b/src/CodeIndex/Indexer/Scanning/FileIndexer.ContentLoadingContracts.cs @@ -12,6 +12,16 @@ internal static bool TryDetectUtf16Encoding( out bool hasBom) => FileContentLoader.TryDetectUtf16Encoding(rawBytes, allowHeuristic, out bigEndian, out hasBom); + internal bool IsUnknownLanguageCoverageCandidate( + string absolutePath, + string relativePath, + CancellationToken cancellationToken) + => _contentLoader.IsUnknownLanguageCoverageCandidate( + absolutePath, + NormalizeIndexPath(relativePath), + relativePath, + cancellationToken); + internal sealed class BinaryFileSkippedException( string relativePath, long nullByteOffset, diff --git a/src/CodeIndex/Indexer/Scanning/FileIndexer.DirectoryEnumeration.cs b/src/CodeIndex/Indexer/Scanning/FileIndexer.DirectoryEnumeration.cs index 19bcb31dcd..aaacef8d5f 100644 --- a/src/CodeIndex/Indexer/Scanning/FileIndexer.DirectoryEnumeration.cs +++ b/src/CodeIndex/Indexer/Scanning/FileIndexer.DirectoryEnumeration.cs @@ -226,6 +226,7 @@ private bool EnumerateDirectoryEntries( scanState, activeIgnoreRules, seenFilePaths, + cancellationToken, knownAttributes: attributes, filePathCameFromDirectoryEnumeration: true)) scanState.Results.Add(entry); @@ -308,6 +309,7 @@ private void EnumerateIndexableFilesInDirectory( scanState, activeIgnoreRules, seenFilePaths, + cancellationToken, knownAttributes)) continue; diff --git a/src/CodeIndex/Indexer/Scanning/FileIndexer.FileAcceptance.cs b/src/CodeIndex/Indexer/Scanning/FileIndexer.FileAcceptance.cs index d831449486..2a20d36f51 100644 --- a/src/CodeIndex/Indexer/Scanning/FileIndexer.FileAcceptance.cs +++ b/src/CodeIndex/Indexer/Scanning/FileIndexer.FileAcceptance.cs @@ -10,6 +10,7 @@ private bool TryAcceptScannedFile( DirectoryScanState scanState, IgnoreRuleSet activeIgnoreRules, HashSet? seenFilePaths, + CancellationToken cancellationToken, FileAttributes? knownAttributes = null, bool filePathCameFromDirectoryEnumeration = false) { @@ -54,7 +55,11 @@ private bool TryAcceptScannedFile( var knownIndexability = knownAttributes.HasValue ? GetFileIndexabilityForFoundAttributes(file, knownAttributes.Value, _symlinkPolicy, _projectRoot) : (FileProbeStatus?)null; - return TryAcceptSupportedScannedFile(file, scanState, knownIndexability); + return TryAcceptSupportedScannedFile( + file, + scanState, + cancellationToken, + knownIndexability); } private static string GetSeenFilePathKey(string file, bool filePathCameFromDirectoryEnumeration) @@ -63,6 +68,7 @@ private static string GetSeenFilePathKey(string file, bool filePathCameFromDirec private bool TryAcceptSupportedScannedFile( string file, DirectoryScanState scanState, + CancellationToken cancellationToken, FileProbeStatus? knownIndexability = null) { // Use the instance symlink policy here so full scans and update paths apply the same @@ -123,7 +129,54 @@ private bool TryAcceptSupportedScannedFile( var relativeFile = ToRelativePath(file); scanState.RecordNonIndexablePath(relativeFile); if (HasUnknownLanguageMapping(file) && !IsInternalIndexArtifactPath(relativeFile)) - scanState.RecordUnknownExtensionFile(relativeFile); + { + try + { + if (IsUnknownLanguageCoverageCandidate( + file, + relativeFile, + cancellationToken)) + { + scanState.RecordUnknownExtensionFile(relativeFile); + } + } + catch (FileTooLargeSkippedException) + { + // Policy-excluded files are not language-support coverage gaps. + } + catch (BinaryFileSkippedException) + { + // Policy-excluded files are not language-support coverage gaps. + } + catch (FileNotFoundException) + { + scanState.Errors.Add(new ScanError( + relativeFile, + "Skipped file because it was deleted during scanning.", + ScanIssueSeverity.Warning)); + } + catch (DirectoryNotFoundException) + { + scanState.Errors.Add(new ScanError( + relativeFile, + "Skipped file because it was deleted during scanning.", + ScanIssueSeverity.Warning)); + } + catch (IOException) + { + scanState.Errors.Add(new ScanError( + relativeFile, + "Could not probe file for indexability/language.")); + scanState.RecordProbeFailedFilePath(relativeFile); + } + catch (UnauthorizedAccessException) + { + scanState.Errors.Add(new ScanError( + relativeFile, + "Could not probe file for indexability/language.")); + scanState.RecordProbeFailedFilePath(relativeFile); + } + } return false; } diff --git a/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs b/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs index 59455d0cf6..fbedd69fc3 100644 --- a/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs @@ -2424,6 +2424,10 @@ private static void SetArchiveTrustMetadata(string dbPath, int filesScanned) writer.SetMeta(DbContext.LastIndexRunModeMetaKey, "rebuild"); writer.SetMeta(DbContext.LastIndexRunFilesScannedMetaKey, filesScanned.ToString(CultureInfo.InvariantCulture)); writer.SetMeta(DbContext.LastIndexRunPeakMemoryMbMetaKey, "1298"); + writer.SetMeta( + DbContext.UnknownExtensionDiagnosticsVersionMetaKey, + DbContext.UnknownExtensionDiagnosticsVersion.ToString(CultureInfo.InvariantCulture)); + writer.SetMeta(DbContext.UnknownExtensionHasActionableExtensionlessFilesMetaKey, bool.FalseString); writer.SetMeta(DbContext.UnknownExtensionFileCountMetaKey, "2"); writer.SetMeta( DbContext.UnknownExtensionFilePathsMetaKey, @@ -2465,6 +2469,8 @@ private static void AssertPartialArchiveStatus(string dbPath) Assert.False(status.RootElement.TryGetProperty("unknown_extension_groups_truncated", out _)); Assert.False(status.RootElement.TryGetProperty("unknown_extension_group_limit", out _)); Assert.False(status.RootElement.TryGetProperty("unknown_extension_group_omitted_count", out _)); + Assert.Null(ReadMetaValue(dbPath, DbContext.UnknownExtensionDiagnosticsVersionMetaKey)); + Assert.Null(ReadMetaValue(dbPath, DbContext.UnknownExtensionHasActionableExtensionlessFilesMetaKey)); Assert.True( !status.RootElement.TryGetProperty("last_index_run", out var lastIndexRun) || lastIndexRun.ValueKind == JsonValueKind.Null); @@ -2489,6 +2495,10 @@ private static void SetUnknownExtensionPathSamples(string dbPath, string[] paths { using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); var writer = new DbWriter(db.Connection); + writer.SetMeta( + DbContext.UnknownExtensionDiagnosticsVersionMetaKey, + DbContext.UnknownExtensionDiagnosticsVersion.ToString(CultureInfo.InvariantCulture)); + writer.SetMeta(DbContext.UnknownExtensionHasActionableExtensionlessFilesMetaKey, bool.FalseString); writer.SetMeta(DbContext.UnknownExtensionFileCountMetaKey, paths.Length.ToString(CultureInfo.InvariantCulture)); writer.SetMeta(DbContext.UnknownExtensionFilePathsMetaKey, JsonSerializer.Serialize(paths)); writer.SetMeta(DbContext.UnknownExtensionFilesTruncatedMetaKey, bool.FalseString); diff --git a/tests/CodeIndex.Tests/FileIndexerTests.cs b/tests/CodeIndex.Tests/FileIndexerTests.cs index c3d5c9b4c5..1e0d14ad0b 100644 --- a/tests/CodeIndex.Tests/FileIndexerTests.cs +++ b/tests/CodeIndex.Tests/FileIndexerTests.cs @@ -6060,6 +6060,9 @@ public void ScanFilesDetailed_SeparatesUnmappedLanguageFilesFromOtherNonIndexabl ["data.mystery"] = "unknown extension\n", ["ignored.mystery"] = "ignored unknown extension\n", }); + File.WriteAllBytes( + TestProjectHelper.ProjectPath(tempDir, "blob.bf"), + [0x42, 0x00, 0x46]); var appPath = TestProjectHelper.ProjectPath(tempDir, "app.cs"); var scriptPath = TestProjectHelper.ProjectPath(tempDir, "script"); var toolPath = TestProjectHelper.ProjectPath(tempDir, "tool"); @@ -6078,6 +6081,8 @@ public void ScanFilesDetailed_SeparatesUnmappedLanguageFilesFromOtherNonIndexabl Assert.Contains("data.mystery", scanResult.NonIndexablePaths); Assert.Contains("tool", scanResult.NonIndexablePaths); Assert.Contains("tool", scanResult.UnknownExtensionFiles); + Assert.DoesNotContain("blob.bf", scanResult.UnknownExtensionFiles); + Assert.Contains("blob.bf", scanResult.NonIndexablePaths); Assert.DoesNotContain("ignored.mystery", scanResult.UnknownExtensionFiles); } diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerDryRunTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerDryRunTests.cs index 392f687638..b18a23d0cb 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerDryRunTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerDryRunTests.cs @@ -1425,6 +1425,7 @@ public void Run_DryRun_FullScan_ReportsProjectedPurgesAndUnknownExtensionsWithou var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); File.Delete(Path.Combine(projectRoot, "removed.cs")); File.WriteAllText(Path.Combine(projectRoot, "notes.unknownext"), "plain text\n"); + File.WriteAllBytes(Path.Combine(projectRoot, "blob.bf"), [0x42, 0x00, 0x46]); var (exitCode, json) = RunAndCaptureJson([projectRoot, "--dry-run", "--json"]); diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs index 3553f995f0..f7059a5093 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs @@ -5930,6 +5930,9 @@ public void Run_UnsupportedOnlyPreservesCaseAndExtensionlessGroups_Issue5100() Assert.True(json.GetProperty("unknown_extension_groups_truncated").GetBoolean()); Assert.Equal(2, json.GetProperty("unknown_extension_group_omitted_count").GetInt32()); Assert.Equal(1, json.GetProperty("summary").GetProperty("warnings").GetInt32()); + Assert.Contains( + "recognized shebang or rename", + json.GetProperty("unknown_extension_guidance").GetString()); } finally { diff --git a/tests/CodeIndex.Tests/JsonStringListCodecTests.cs b/tests/CodeIndex.Tests/JsonStringListCodecTests.cs index 66b0c9cb01..fd0c4cfc11 100644 --- a/tests/CodeIndex.Tests/JsonStringListCodecTests.cs +++ b/tests/CodeIndex.Tests/JsonStringListCodecTests.cs @@ -165,4 +165,71 @@ public void WriteUnknownExtensionFileMetadata_ReportsOmittedGroups_Issue5100() TestProjectHelper.DeleteFile(dbPath); } } + + [Fact] + public void GetStatus_OmitsLegacyUnknownExtensionMetadataWithoutCurrentVersion_Issue5100() + { + var dbPath = Path.Combine(Path.GetTempPath(), $"codeindex_legacy_unknown_{Guid.NewGuid():N}.db"); + try + { + using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); + db.InitializeSchema(); + var writer = new DbWriter(db.Connection); + writer.SetMetaValues( + (DbContext.UnknownExtensionFileCountMetaKey, "2"), + (DbContext.UnknownExtensionFilePathsMetaKey, "[\"legacy.foo\",\"extensionless\"]"), + (DbContext.UnknownExtensionFilesTruncatedMetaKey, bool.FalseString), + (DbContext.UnknownExtensionFilePathLimitMetaKey, "50")); + + var legacyStatus = new DbReader(db.Connection).GetStatus(); + + Assert.Null(legacyStatus.UnknownExtensionFileCount); + Assert.Null(legacyStatus.UnknownExtensionFiles); + Assert.Null(legacyStatus.UnknownExtensionGuidance); + + writer.SetMetaValues( + (DbContext.UnknownExtensionDiagnosticsVersionMetaKey, + (DbContext.UnknownExtensionDiagnosticsVersion + 1).ToString(System.Globalization.CultureInfo.InvariantCulture))); + + var futureStatus = new DbReader(db.Connection).GetStatus(); + + Assert.Null(futureStatus.UnknownExtensionFileCount); + Assert.Null(futureStatus.UnknownExtensionFiles); + Assert.Null(futureStatus.UnknownExtensionGuidance); + Assert.True(futureStatus.IndexNewerThanReader); + Assert.Contains( + "unknown_extension_diagnostics_version", + Assert.IsType(futureStatus.IndexNewerThanReaderReason)); + } + finally + { + TestProjectHelper.DeleteFile(dbPath); + } + } + + [Fact] + public void WriteUnknownExtensionFileMetadata_PersistsExtensionlessGuidance_Issue5100() + { + var dbPath = Path.Combine(Path.GetTempPath(), $"codeindex_extensionless_unknown_{Guid.NewGuid():N}.db"); + try + { + using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); + db.InitializeSchema(); + var writer = new DbWriter(db.Connection); + + writer.WriteUnknownExtensionFileMetadata(["extensionless_source", "source.foo"]); + + var status = new DbReader(db.Connection).GetStatus(); + Assert.Contains( + "recognized shebang or rename", + Assert.IsType(status.UnknownExtensionGuidance)); + Assert.Contains( + Assert.IsType>(status.UnknownExtensionGroups), + group => group.Extension == ""); + } + finally + { + TestProjectHelper.DeleteFile(dbPath); + } + } } diff --git a/tests/CodeIndex.Tests/ProgramCliTests.cs b/tests/CodeIndex.Tests/ProgramCliTests.cs index 0bbf14a13a..ddc7f53897 100644 --- a/tests/CodeIndex.Tests/ProgramCliTests.cs +++ b/tests/CodeIndex.Tests/ProgramCliTests.cs @@ -625,6 +625,10 @@ public void ExportImportArchive_SharesMetadataRichPristineAcrossSuccessPaths_Iss writer.SetMeta(DbContext.CSharpSymbolNameContractVersionMetaKey, "2"); writer.SetMeta(DbContext.SqlGraphContractVersionMetaKey, "1"); writer.SetMeta(DbContext.HotspotFamilyVersionMetaKey, "2"); + writer.SetMeta( + DbContext.UnknownExtensionDiagnosticsVersionMetaKey, + DbContext.UnknownExtensionDiagnosticsVersion.ToString(System.Globalization.CultureInfo.InvariantCulture)); + writer.SetMeta(DbContext.UnknownExtensionHasActionableExtensionlessFilesMetaKey, bool.FalseString); writer.SetMeta(DbContext.UnknownExtensionFileCountMetaKey, "2"); writer.SetMeta(DbContext.UnknownExtensionFilePathsMetaKey, "[\"tools/custom.foo\",\"docs/archive.bar\"]"); writer.SetMeta(DbContext.UnknownExtensionFilesTruncatedMetaKey, "false");