From 6df84209820718099728d96a140b7a8c1891e355 Mon Sep 17 00:00:00 2001 From: Den Rozhnovskiy Date: Thu, 5 Feb 2026 17:41:20 +0500 Subject: [PATCH 01/28] chore(UI): update html report --- codeclone/templates.py | 1304 ++++++++++++++++++++++++++-------------- pyproject.toml | 2 +- uv.lock | 2 +- 3 files changed, 870 insertions(+), 438 deletions(-) diff --git a/codeclone/templates.py b/codeclone/templates.py index d8870e0..c0c1306 100644 --- a/codeclone/templates.py +++ b/codeclone/templates.py @@ -6,161 +6,127 @@ Licensed under the MIT License. """ +from __future__ import annotations + from string import Template FONT_CSS_URL = ( - "https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700" - "&family=JetBrains+Mono:wght@400;500;600&display=swap" + "https://fonts.googleapis.com/css2?" + "family=Inter:wght@400;500;600;700&" + "family=JetBrains+Mono:wght@400;500&" + "display=swap" ) -REPORT_TEMPLATE = Template(r""" - +REPORT_TEMPLATE = Template( + r""" ${title} - - @@ -962,6 +1029,25 @@
+ +
+
+
+
+ +
+
+ +
+
+
+
@@ -970,8 +1056,21 @@
v${version}
+ '; - const container = $('.toast-container'); + const container = $$('.toast-container'); container.appendChild(toast); setTimeout(() => toast.classList.add('toast-show'), 10); @@ -1192,9 +1192,9 @@ // ========== Command Palette ========== function initCommandPalette() { - const palette = $('#command-palette'); - const input = $('#command-input'); - const results = $('#command-results'); + const palette = $$('#command-palette'); + const input = $$('#command-input'); + const results = $$('#command-results'); const commands = [ { @@ -1239,7 +1239,7 @@ icon: '🔍', label: 'Focus Search', shortcut: '/', - action: () => $('.search')?.focus() + action: () => $$('.search')?.focus() }, { icon: '📜', @@ -1334,8 +1334,8 @@ input.value = ''; } - $('#command-btn')?.addEventListener('click', openCommandPalette); - $('.command-backdrop')?.addEventListener('click', closeCommandPalette); + $$('#command-btn')?.addEventListener('click', openCommandPalette); + $$('.command-backdrop')?.addEventListener('click', closeCommandPalette); input?.addEventListener('input', (e) => { renderCommands(e.target.value); @@ -1345,7 +1345,7 @@ if (e.key === 'Escape') { closeCommandPalette(); } else if (e.key === 'Enter') { - const selected = $('.command-item.selected'); + const selected = $$('.command-item.selected'); if (selected) selected.click(); } }); @@ -1355,8 +1355,8 @@ // ========== Statistics ========== function calculateStats() { - const groups = $$('.group'); - const items = $$('.item'); + const groups = $$$$('.group'); + const items = $$$$('.item'); state.stats.totalGroups = groups.length; state.stats.totalItems = items.length; @@ -1373,7 +1373,7 @@ } function showStats() { - const dashboard = $('#stats-dashboard'); + const dashboard = $$('#stats-dashboard'); if (dashboard.children.length > 0) { dashboard.style.display = dashboard.style.display === 'none' ? 'grid' : 'none'; return; @@ -1428,7 +1428,7 @@ // ========== Charts ========== function showCharts() { - const container = $('#chart-container'); + const container = $$('#chart-container'); container.style.display = container.style.display === 'none' ? 'block' : 'none'; if (container.style.display === 'block') { @@ -1443,7 +1443,7 @@ const data = { generated: new Date().toISOString(), stats: state.stats, - groups: Array.from($$('.group')).map(g => ({ + groups: Array.from($$$$('.group')).map(g => ({ title: g.querySelector('.group-title')?.textContent, items: g.querySelectorAll('.item').length })) @@ -1467,14 +1467,14 @@ // ========== Group Controls ========== function expandAll() { - $$('.items').forEach(b => b.style.display = ''); - $$('[data-toggle-group]').forEach(c => c.style.transform = 'rotate(0deg)'); + $$$$('.items').forEach(b => b.style.display = ''); + $$$$('[data-toggle-group]').forEach(c => c.style.transform = 'rotate(0deg)'); showToast('All groups expanded', 'info'); } function collapseAll() { - $$('.items').forEach(b => b.style.display = 'none'); - $$('[data-toggle-group]').forEach(c => c.style.transform = 'rotate(-90deg)'); + $$$$('.items').forEach(b => b.style.display = 'none'); + $$$$('[data-toggle-group]').forEach(c => c.style.transform = 'rotate(-90deg)'); showToast('All groups collapsed', 'info'); } @@ -1493,7 +1493,7 @@ // / - Focus search if (e.key === '/') { e.preventDefault(); - $('.search')?.focus(); + $$('.search')?.focus(); } // T - Toggle theme @@ -1517,11 +1517,11 @@ // Escape - Close modals if (e.key === 'Escape') { if (state.commandPaletteOpen) { - const palette = $('#command-palette'); + const palette = $$('#command-palette'); palette?.classList.remove('show'); state.commandPaletteOpen = false; } - const search = $('.search'); + const search = $$('.search'); if (search && search.value) { search.value = ''; search.dispatchEvent(new Event('input', { bubbles: true })); @@ -1530,7 +1530,7 @@ }); // ========== Group Toggle ========== - $$('.group-head').forEach((head) => { + $$$$('.group-head').forEach((head) => { head.addEventListener('click', (e) => { if (e.target.closest('button')) return; const btn = head.querySelector('[data-toggle-group]'); @@ -1538,11 +1538,11 @@ }); }); - $$('[data-toggle-group]').forEach((btn) => { + $$$$('[data-toggle-group]').forEach((btn) => { btn.addEventListener('click', (e) => { e.stopPropagation(); const id = btn.getAttribute('data-toggle-group'); - const body = $('#group-body-' + id); + const body = $$('#group-body-' + id); if (!body) return; const isHidden = body.style.display === 'none'; @@ -1553,19 +1553,19 @@ // ========== Section Management ========== function initSection(sectionId) { - const section = $('section[data-section="' + sectionId + '"]'); + const section = $$('section[data-section="' + sectionId + '"]'); if (!section) return; - const groups = Array.from($$('.group[data-group="' + sectionId + '"]')); - const searchInput = $('#search-' + sectionId); - const btnPrev = $('[data-prev="' + sectionId + '"]'); - const btnNext = $('[data-next="' + sectionId + '"]'); - const meta = $('[data-page-meta="' + sectionId + '"]'); - const selPageSize = $('[data-pagesize="' + sectionId + '"]'); - const btnClear = $('[data-clear="' + sectionId + '"]'); - const btnCollapseAll = $('[data-collapse-all="' + sectionId + '"]'); - const btnExpandAll = $('[data-expand-all="' + sectionId + '"]'); - const pill = $('[data-count-pill="' + sectionId + '"]'); + const groups = Array.from($$$$('.group[data-group="' + sectionId + '"]')); + const searchInput = $$('#search-' + sectionId); + const btnPrev = $$('[data-prev="' + sectionId + '"]'); + const btnNext = $$('[data-next="' + sectionId + '"]'); + const meta = $$('[data-page-meta="' + sectionId + '"]'); + const selPageSize = $$('[data-pagesize="' + sectionId + '"]'); + const btnClear = $$('[data-clear="' + sectionId + '"]'); + const btnCollapseAll = $$('[data-collapse-all="' + sectionId + '"]'); + const btnExpandAll = $$('[data-expand-all="' + sectionId + '"]'); + const pill = $$('[data-count-pill="' + sectionId + '"]'); const sectionState = { q: '', @@ -1669,8 +1669,8 @@ } // ========== Event Listeners ========== - $('#theme-toggle')?.addEventListener('click', toggleTheme); - $('#export-btn')?.addEventListener('click', () => exportReport('json')); + $$('#theme-toggle')?.addEventListener('click', toggleTheme); + $$('#export-btn')?.addEventListener('click', () => exportReport('json')); // ========== Initialize ========== initTheme(); @@ -1681,7 +1681,7 @@ // Welcome message setTimeout(() => { - const groupCount = $$('.group').length; + const groupCount = $$$$('.group').length; if (groupCount > 0) { showToast(groupCount + ' clone groups loaded', 'success'); } From d66a3b76cd69c341cf8c794d98bfb0b205662339 Mon Sep 17 00:00:00 2001 From: Den Rozhnovskiy Date: Thu, 5 Feb 2026 17:52:11 +0500 Subject: [PATCH 03/28] chore(UI): update html report --- codeclone/templates.py | 191 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 172 insertions(+), 19 deletions(-) diff --git a/codeclone/templates.py b/codeclone/templates.py index 6a17fdb..de6993d 100644 --- a/codeclone/templates.py +++ b/codeclone/templates.py @@ -1125,6 +1125,159 @@ // ========== Utilities ========== const $$ = (sel) => document.querySelector(sel); const $$$$ = (sel) => document.querySelectorAll(sel); + const svg = (parts) => parts.join(''); + const ICONS = { + info: svg([ + '' + ]), + success: svg([ + '' + ]), + warning: svg([ + '' + ]), + error: svg([ + '' + ]), + exportJson: svg([ + '' + ]), + exportPdf: svg([ + '' + ]), + stats: svg([ + '' + ]), + charts: svg([ + '' + ]), + refresh: svg([ + '' + ]), + search: svg([ + '' + ]), + scrollTop: svg([ + '' + ]), + scrollBottom: svg([ + '' + ]), + theme: svg([ + '' + ]), + expand: svg([ + '' + ]), + collapse: svg([ + '' + ]), + cloneGroups: svg([ + '' + ]), + totalClones: svg([ + '' + ]), + avgGroup: svg([ + '' + ]), + largestGroup: svg([ + '' + ]) + }; // ========== State Management ========== const state = { @@ -1159,10 +1312,10 @@ // ========== Toast Notifications ========== function showToast(message, type = 'info') { const icons = { - info: 'i', - success: '✅', - warning: '⚠️', - error: '❌' + info: ICONS.info, + success: ICONS.success, + warning: ICONS.warning, + error: ICONS.error }; const toast = document.createElement('div'); @@ -1201,31 +1354,31 @@ section: 'Actions', items: [ { - icon: '📥', + icon: ICONS.exportJson, label: 'Export as JSON', shortcut: '⌘E', action: () => exportReport('json') }, { - icon: '📄', + icon: ICONS.exportPdf, label: 'Export as PDF', shortcut: null, action: () => exportReport('pdf') }, { - icon: '📊', + icon: ICONS.stats, label: 'Show Statistics', shortcut: '⌘S', action: () => showStats() }, { - icon: '📈', + icon: ICONS.charts, label: 'Show Charts', shortcut: null, action: () => showCharts() }, { - icon: '🔄', + icon: ICONS.refresh, label: 'Refresh View', shortcut: '⌘R', action: () => location.reload() @@ -1236,19 +1389,19 @@ section: 'Navigation', items: [ { - icon: '🔍', + icon: ICONS.search, label: 'Focus Search', shortcut: '/', action: () => $$('.search')?.focus() }, { - icon: '📜', + icon: ICONS.scrollTop, label: 'Scroll to Top', shortcut: null, action: () => window.scrollTo(0, 0) }, { - icon: '⬇️', + icon: ICONS.scrollBottom, label: 'Scroll to Bottom', shortcut: null, action: () => window.scrollTo(0, document.body.scrollHeight) @@ -1259,19 +1412,19 @@ section: 'View', items: [ { - icon: '🌓', + icon: ICONS.theme, label: 'Toggle Theme', shortcut: 'T', action: () => toggleTheme() }, { - icon: '📖', + icon: ICONS.expand, label: 'Expand All', shortcut: null, action: () => expandAll() }, { - icon: '📕', + icon: ICONS.collapse, label: 'Collapse All', shortcut: null, action: () => collapseAll() @@ -1383,25 +1536,25 @@ const stats = [ { - icon: '📊', + icon: ICONS.cloneGroups, value: state.stats.totalGroups, label: 'Clone Groups', trend: null }, { - icon: '📦', + icon: ICONS.totalClones, value: state.stats.totalItems, label: 'Total Clones', trend: null }, { - icon: '📈', + icon: ICONS.avgGroup, value: state.stats.avgGroupSize, label: 'Avg Group Size', trend: null }, { - icon: '🔝', + icon: ICONS.largestGroup, value: state.stats.largestGroup, label: 'Largest Group', trend: null From c5aea24df742e90d65fbc94f522c16224d22c294 Mon Sep 17 00:00:00 2001 From: Den Rozhnovskiy Date: Thu, 5 Feb 2026 18:57:08 +0500 Subject: [PATCH 04/28] feat(core): improves clone-detection precision and explainability with deterministic normalization and CFG upgrades, adds segment-level internal clone reporting, refreshes the HTML report UI, and introduces baseline versioning. This is a breaking change for CI workflows that rely on existing baselines. --- CHANGELOG.md | 91 ++++++++++++++++++++++ CONTRIBUTING.md | 12 +++ README.md | 17 +++- SECURITY.md | 8 +- codeclone.baseline.json | 4 +- codeclone/baseline.py | 41 +++++++++- codeclone/blocks.py | 57 ++++++++++++++ codeclone/cache.py | 23 +++++- codeclone/cfg.py | 80 ++++++++++++++++--- codeclone/cli.py | 58 ++++++++++++-- codeclone/extractor.py | 19 ++++- codeclone/html_report.py | 13 +++- codeclone/normalize.py | 72 +++++++++++++++++ codeclone/report.py | 37 ++++++++- codeclone/templates.py | 9 +++ docs/architecture.md | 34 +++++++- docs/cfg.md | 14 +++- tests/test_baseline.py | 18 ++++- tests/test_cache.py | 21 ++++- tests/test_cfg.py | 149 ++++++++++++++++++++++++++++++++++-- tests/test_cli_inprocess.py | 143 ++++++++++++++++++++++++++++++++-- tests/test_cli_unit.py | 10 +++ tests/test_extractor.py | 18 +++-- tests/test_html_report.py | 74 +++++++++++++++++- tests/test_normalize.py | 94 +++++++++++++++++++++++ tests/test_report.py | 79 ++++++++++++++++++- tests/test_security.py | 36 +++++++++ tests/test_segments.py | 105 +++++++++++++++++++++++++ 28 files changed, 1270 insertions(+), 66 deletions(-) create mode 100644 tests/test_segments.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e86fef2..92c7c8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,96 @@ # Changelog +## [1.3.0] - 2026-02-05 + +### Overview + +This release improves clone-detection precision and explainability with deterministic +normalization and CFG upgrades, adds segment-level internal clone reporting, refreshes +the HTML report UI, and introduces baseline versioning. This is a breaking change for CI +workflows that rely on existing baselines. + +### Clone Detection Accuracy + +- **Commutative normalization** + Canonicalized operand order for `+`, `*`, `|`, `&`, `^` when operands are free of side + effects, enabling safe detection of reordered expressions. + +- **Local logical equivalence** + Normalized `not (x in y)` to `x not in y` and `not (x is y)` to `x is not y` without + De Morgan transformations or broader boolean rewrites. + +### CFG Precision + +- **Short‑circuit modeling** + Represented `and`/`or` as micro‑CFGs with explicit branch splits after each operand. + +- **Exception linking** + Linked `try/except` only to statements that may raise (calls, attribute access, indexing, + `await`, `yield from`, `raise`) instead of blanket connections. + +### Segment‑Level Detection + +- **Window fingerprints** + Added deterministic segment windows inside functions for internal clone discovery. + +- **Candidate generation** + Used an order‑insensitive signature for candidate grouping and a strict segment hash for + final confirmation; segment matches do not affect baseline or CI failure logic. + +### Baseline & CI + +- Baselines are now **versioned** and include a schema version. +- Mismatched baseline versions **fail fast** and require regeneration. + +**Breaking (CI):** baseline version mismatch now fails hard; CI requires baseline regeneration on upgrade. + +Update the baseline: + +```bash +codeclone . --update-baseline +``` + +### HTML Report UI + +- **Visual refresh** + Introduced a redesigned, modern HTML report layout with a sticky top bar and improved + typography and spacing. + +- **Interactive tooling** + Added a command palette, keyboard shortcuts, toast notifications, and quick actions for + common tasks (export, stats, charts, navigation). + +- **Reporting widgets** + Added a stats dashboard and chart container to surface high‑level clone metrics directly + in the report. + +- **Icon system** + Replaced emoji glyphs with inline SVG icons for consistent rendering and a fully + self‑contained UI. + +- **Segment reporting** + Added a dedicated “Segment clones” section and summary metric in HTML/TXT/JSON outputs. + +### Cache & Internals + +- Extended cache schema to store segment fingerprints (cache version bump). + +### Packaging + +- Removed an invalid PyPI classifier from the package metadata. + +### Documentation + +- Updated architecture and CFG documentation to reflect new normalization, CFG, and + segment‑level detection behavior. +- Updated README, SECURITY, and CONTRIBUTING guidance for 1.3.0. + +### Testing & Security + +- Expanded security tests (HTML escaping and safety checks). + +--- + ## [1.2.1] - 2026-02-02 ### Overview diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7987e01..8ed5d91 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,6 +30,7 @@ We especially welcome contributions in the following areas: - Control Flow Graph (CFG) construction and semantics - AST normalization improvements +- Segment-level clone detection and reporting - False-positive reduction - HTML report UX improvements - Performance optimizations @@ -83,6 +84,14 @@ Such changes often require design-level discussion and may be staged across vers --- +## Security & Safety Expectations + +- Assume **untrusted input** (paths and source code). +- Add **negative tests** for any normalization or CFG change. +- Changes must preserve determinism and avoid new false positives. + +--- + ## Development Setup ```bash @@ -128,6 +137,9 @@ CodeClone follows **semantic versioning**: - **MINOR**: new detection capabilities (for example, CFG improvements) - **PATCH**: bug fixes, performance improvements, and UI/UX polish +Baselines are versioned. Any change to detection behavior must include documentation +and tests, and may require baseline regeneration. + --- ## License diff --git a/README.md b/README.md index 21b7cc9..65d2815 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,12 @@ Typical use cases: - no `__init__` noise, - size and statement-count thresholds. +### Segment-level internal clone detection + +- Detects repeated **segment windows** inside the same function. +- Uses a two‑step deterministic match (candidate signature → strict hash). +- Included in reports for explainability, **not** in baseline/CI failure logic. + ### Control-Flow Awareness (CFG v1) - Each function is converted into a **Control Flow Graph**. @@ -74,6 +80,8 @@ Typical use cases: - `with` / `async with` - `match` / `case` (Python 3.10+) - Current CFG semantics (v1): + - `and` / `or` are modeled as short‑circuit micro‑CFG branches, + - `try/except` links only from statements that may raise, - `break` and `continue` are treated as statements (no jump targets), - after-blocks are explicit and always present, - focus is on **structural similarity**, not precise runtime semantics. @@ -86,6 +94,7 @@ This design keeps clone detection **stable, deterministic, and low-noise**. - Conservative defaults tuned for real-world Python projects. - Explicit thresholds for size and statement count. - No probabilistic scoring or heuristic similarity thresholds. +- Safe commutative normalization and local logical equivalences only. - Focus on *architectural duplication*, not micro-similarities. ### CI-friendly baseline mode @@ -149,6 +158,9 @@ codeclone . --update-baseline Commit the generated baseline file to the repository. +Baselines are **versioned**. If CodeClone is upgraded, regenerate the baseline to keep +CI deterministic and explainable. + ### 2. Use in CI ```bash @@ -215,8 +227,9 @@ repos: 2. Normalize AST (names, constants, attributes, annotations). 3. Build a **Control Flow Graph (CFG)** per function. 4. Compute stable CFG fingerprints. -5. Detect function-level and block-level clones. -6. Apply conservative filters to suppress noise. +5. Extract segment windows for internal clone discovery. +6. Detect function-level, block-level, and segment-level clones. +7. Apply conservative filters to suppress noise. See the architectural overview: diff --git a/SECURITY.md b/SECURITY.md index 533843a..5e84183 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -9,7 +9,8 @@ The following versions currently receive security updates: | Version | Supported | |---------|-----------| -| 1.2.x | Yes | +| 1.3.x | Yes | +| 1.2.x | No | | 1.1.x | No | | 1.0.x | No | @@ -33,6 +34,11 @@ Potential risk areas include: These areas are explicitly tested and hardened, but are still the primary focus of ongoing security review. +Additional safeguards: + +- HTML report content is escaped to prevent script injection. +- Reports are static and do not execute analyzed code. + --- ## Reporting a Vulnerability diff --git a/codeclone.baseline.json b/codeclone.baseline.json index ce9a169..dcc2fc8 100644 --- a/codeclone.baseline.json +++ b/codeclone.baseline.json @@ -4,5 +4,7 @@ "7d573fa56fb11050f1642f18ca4bb3225e11e194|0-19" ], "blocks": [], - "python_version": "3.13" + "python_version": "3.13", + "baseline_version": "1.3.0", + "schema_version": 1 } \ No newline at end of file diff --git a/codeclone/baseline.py b/codeclone/baseline.py index 74f2030..e73ff7f 100644 --- a/codeclone/baseline.py +++ b/codeclone/baseline.py @@ -13,15 +13,28 @@ from pathlib import Path from typing import Any +from . import __version__ + +BASELINE_SCHEMA_VERSION = 1 + class Baseline: - __slots__ = ("blocks", "functions", "path", "python_version") + __slots__ = ( + "baseline_version", + "blocks", + "functions", + "path", + "python_version", + "schema_version", + ) def __init__(self, path: str | Path): self.path = Path(path) self.functions: set[str] = set() self.blocks: set[str] = set() self.python_version: str | None = None + self.baseline_version: str | None = None + self.schema_version: int | None = None def load(self) -> None: if not self.path.exists(): @@ -35,6 +48,14 @@ def load(self) -> None: self.python_version = ( python_version if isinstance(python_version, str) else None ) + baseline_version = data.get("baseline_version") + self.baseline_version = ( + baseline_version if isinstance(baseline_version, str) else None + ) + schema_version = data.get("schema_version") + self.schema_version = ( + schema_version if isinstance(schema_version, int) else None + ) except json.JSONDecodeError as e: raise ValueError(f"Corrupted baseline file at {self.path}: {e}") from e @@ -42,7 +63,13 @@ def save(self) -> None: self.path.parent.mkdir(parents=True, exist_ok=True) self.path.write_text( json.dumps( - _baseline_payload(self.functions, self.blocks, self.python_version), + _baseline_payload( + self.functions, + self.blocks, + self.python_version, + self.baseline_version, + self.schema_version, + ), indent=2, ensure_ascii=False, ), @@ -55,11 +82,15 @@ def from_groups( block_groups: Mapping[str, object], path: str | Path = "", python_version: str | None = None, + baseline_version: str | None = None, + schema_version: int | None = None, ) -> Baseline: bl = Baseline(path) bl.functions = set(func_groups.keys()) bl.blocks = set(block_groups.keys()) bl.python_version = python_version + bl.baseline_version = baseline_version + bl.schema_version = schema_version return bl def diff( @@ -74,6 +105,8 @@ def _baseline_payload( functions: set[str], blocks: set[str], python_version: str | None, + baseline_version: str | None, + schema_version: int | None, ) -> dict[str, Any]: payload: dict[str, Any] = { "functions": sorted(functions), @@ -81,4 +114,8 @@ def _baseline_payload( } if python_version: payload["python_version"] = python_version + payload["baseline_version"] = baseline_version or __version__ + payload["schema_version"] = ( + schema_version if schema_version is not None else BASELINE_SCHEMA_VERSION + ) return payload diff --git a/codeclone/blocks.py b/codeclone/blocks.py index 551d243..3469361 100644 --- a/codeclone/blocks.py +++ b/codeclone/blocks.py @@ -12,6 +12,7 @@ from dataclasses import dataclass from .blockhash import stmt_hash +from .fingerprint import sha1 from .normalize import NormalizationConfig @@ -25,6 +26,17 @@ class BlockUnit: size: int +@dataclass(frozen=True, slots=True) +class SegmentUnit: + segment_hash: str + segment_sig: str + filepath: str + qualname: str + start_line: int + end_line: int + size: int + + def extract_blocks( func_node: ast.AST, *, @@ -72,3 +84,48 @@ def extract_blocks( break return blocks + + +def extract_segments( + func_node: ast.AST, + *, + filepath: str, + qualname: str, + cfg: NormalizationConfig, + window_size: int, + max_segments: int, +) -> list[SegmentUnit]: + body = getattr(func_node, "body", None) + if not isinstance(body, list) or len(body) < window_size: + return [] + + stmt_hashes = [stmt_hash(stmt, cfg) for stmt in body] + + segments: list[SegmentUnit] = [] + + for i in range(len(stmt_hashes) - window_size + 1): + start = getattr(body[i], "lineno", None) + end = getattr(body[i + window_size - 1], "end_lineno", None) + if not start or not end: + continue + + window = stmt_hashes[i : i + window_size] + segment_hash = sha1("|".join(window)) + segment_sig = sha1("|".join(sorted(window))) + + segments.append( + SegmentUnit( + segment_hash=segment_hash, + segment_sig=segment_sig, + filepath=filepath, + qualname=qualname, + start_line=start, + end_line=end, + size=window_size, + ) + ) + + if len(segments) >= max_segments: + break + + return segments diff --git a/codeclone/cache.py b/codeclone/cache.py index f652d17..8045596 100644 --- a/codeclone/cache.py +++ b/codeclone/cache.py @@ -19,7 +19,7 @@ from typing import TYPE_CHECKING, Any, TypedDict, cast if TYPE_CHECKING: - from .blocks import BlockUnit + from .blocks import BlockUnit, SegmentUnit from .extractor import Unit from .errors import CacheError @@ -50,10 +50,21 @@ class BlockDict(TypedDict): size: int +class SegmentDict(TypedDict): + segment_hash: str + segment_sig: str + filepath: str + qualname: str + start_line: int + end_line: int + size: int + + class CacheEntry(TypedDict): stat: FileStat units: list[UnitDict] blocks: list[BlockDict] + segments: list[SegmentDict] class CacheData(TypedDict): @@ -63,7 +74,7 @@ class CacheData(TypedDict): class Cache: __slots__ = ("data", "load_warning", "path", "secret") - CACHE_VERSION = "1.0" + CACHE_VERSION = "1.1" def __init__(self, path: str | Path): self.path = Path(path) @@ -129,7 +140,7 @@ def load(self) -> None: self.data = {"version": self.CACHE_VERSION, "files": {}} return - self.data = cast(CacheData, data) + self.data = cast(CacheData, cast(object, data)) self.load_warning = None except (json.JSONDecodeError, ValueError): @@ -159,7 +170,7 @@ def get_file_entry(self, filepath: str) -> CacheEntry | None: if not isinstance(entry, dict): return None - required = {"stat", "units", "blocks"} + required = {"stat", "units", "blocks", "segments"} if not required.issubset(entry.keys()): return None @@ -171,11 +182,15 @@ def put_file_entry( stat_sig: FileStat, units: list[Unit], blocks: list[BlockUnit], + segments: list[SegmentUnit], ) -> None: self.data["files"][filepath] = { "stat": stat_sig, "units": cast(list[UnitDict], cast(object, [asdict(u) for u in units])), "blocks": cast(list[BlockDict], cast(object, [asdict(b) for b in blocks])), + "segments": cast( + list[SegmentDict], cast(object, [asdict(s) for s in segments]) + ), } diff --git a/codeclone/cfg.py b/codeclone/cfg.py index 9235a7f..ffa7aa8 100644 --- a/codeclone/cfg.py +++ b/codeclone/cfg.py @@ -102,14 +102,11 @@ def _visit(self, stmt: ast.stmt) -> None: # ---------- Control Flow ---------- def _visit_if(self, stmt: ast.If) -> None: - self.current.statements.append(ast.Expr(value=stmt.test)) - then_block = self.cfg.create_block() else_block = self.cfg.create_block() after_block = self.cfg.create_block() - self.current.add_successor(then_block) - self.current.add_successor(else_block) + self._emit_condition(stmt.test, then_block, else_block) self.current = then_block self._visit_statements(stmt.body) @@ -131,9 +128,7 @@ def _visit_while(self, stmt: ast.While) -> None: self.current.add_successor(cond_block) self.current = cond_block - self.current.statements.append(ast.Expr(value=stmt.test)) - self.current.add_successor(body_block) - self.current.add_successor(after_block) + self._emit_condition(stmt.test, body_block, after_block) self.current = body_block self._visit_statements(stmt.body) @@ -198,14 +193,14 @@ def _visit_try(self, stmt: _TryLike) -> None: final_block = self.cfg.create_block() # Process each statement in try body - # Link each to exception handlers + # Link only statements that can raise to exception handlers for stmt_node in stmt.body: if self.current.is_terminated: break - # Current statement could raise exception - for h_block in handlers_blocks: - self.current.add_successor(h_block) + if _stmt_can_raise(stmt_node): + for h_block in handlers_blocks: + self.current.add_successor(h_block) self._visit(stmt_node) @@ -261,3 +256,66 @@ def _visit_match(self, stmt: ast.Match) -> None: self.current.add_successor(after_block) self.current = after_block + + def _emit_condition( + self, test: ast.expr, true_block: Block, false_block: Block + ) -> None: + if isinstance(test, ast.BoolOp) and isinstance(test.op, (ast.And, ast.Or)): + self._emit_boolop(test, true_block, false_block) + return + + self.current.statements.append(ast.Expr(value=test)) + self.current.add_successor(true_block) + self.current.add_successor(false_block) + + def _emit_boolop( + self, test: ast.BoolOp, true_block: Block, false_block: Block + ) -> None: + values = test.values + op = test.op + current = self.current + + for idx, value in enumerate(values): + current.statements.append(ast.Expr(value=value)) + is_last = idx == len(values) - 1 + + if isinstance(op, ast.And): + if is_last: + current.add_successor(true_block) + current.add_successor(false_block) + else: + next_block = self.cfg.create_block() + current.add_successor(next_block) + current.add_successor(false_block) + current = next_block + else: + if is_last: + current.add_successor(true_block) + current.add_successor(false_block) + else: + next_block = self.cfg.create_block() + current.add_successor(true_block) + current.add_successor(next_block) + current = next_block + + self.current = current + + +def _stmt_can_raise(stmt: ast.stmt) -> bool: + if isinstance(stmt, ast.Raise): + return True + + for node in ast.walk(stmt): + if isinstance( + node, + ( + ast.Call, + ast.Attribute, + ast.Subscript, + ast.Await, + ast.YieldFrom, + ), + ): + return True + + return False diff --git a/codeclone/cli.py b/codeclone/cli.py index 677dbf8..6e52791 100644 --- a/codeclone/cli.py +++ b/codeclone/cli.py @@ -20,13 +20,20 @@ from rich.table import Table from rich.theme import Theme -from .baseline import Baseline +from . import __version__ +from .baseline import BASELINE_SCHEMA_VERSION, Baseline from .cache import Cache, CacheEntry, FileStat, file_stat_signature from .errors import CacheError from .extractor import extract_units_from_source from .html_report import build_html_report from .normalize import NormalizationConfig -from .report import build_block_groups, build_groups, to_json_report, to_text +from .report import ( + build_block_groups, + build_groups, + build_segment_groups, + to_json_report, + to_text, +) from .scanner import iter_py_files, module_name_from_path # Custom theme for Rich @@ -54,6 +61,7 @@ class ProcessingResult: error: str | None = None units: list[Any] | None = None blocks: list[Any] | None = None + segments: list[Any] | None = None stat: FileStat | None = None @@ -108,7 +116,7 @@ def process_file( stat = file_stat_signature(filepath) module_name = module_name_from_path(root, filepath) - units, blocks = extract_units_from_source( + units, blocks, segments = extract_units_from_source( source=source, filepath=filepath, module_name=module_name, @@ -122,6 +130,7 @@ def process_file( success=True, units=units, blocks=blocks, + segments=segments, stat=stat, ) @@ -262,6 +271,7 @@ def main() -> None: all_units: list[dict[str, Any]] = [] all_blocks: list[dict[str, Any]] = [] + all_segments: list[dict[str, Any]] = [] changed_files_count = 0 files_to_process: list[str] = [] @@ -315,6 +325,12 @@ def _safe_future_result(future: Any) -> tuple[ProcessingResult | None, str | Non cast(object, cached.get("blocks", [])), ) ) + all_segments.extend( + cast( + list[dict[str, Any]], + cast(object, cached.get("segments", [])), + ) + ) else: files_to_process.append(fp) except Exception as e: @@ -335,12 +351,15 @@ def handle_result(result: ProcessingResult) -> None: result.stat, result.units or [], result.blocks or [], + result.segments or [], ) changed_files_count += 1 if result.units: all_units.extend([asdict(u) for u in result.units]) if result.blocks: all_blocks.extend([asdict(b) for b in result.blocks]) + if result.segments: + all_segments.extend([asdict(s) for s in result.segments]) else: failed_files.append(f"{result.filepath}: {result.error}") @@ -460,6 +479,7 @@ def process_sequential(with_progress: bool) -> None: with console.status("[bold green]Grouping clones...", spinner="dots"): func_groups = build_groups(all_units) block_groups = build_block_groups(all_blocks) + segment_groups = build_segment_groups(all_segments) try: cache.save() except CacheError as e: @@ -468,6 +488,7 @@ def process_sequential(with_progress: bool) -> None: # Reporting func_clones_count = len(func_groups) block_clones_count = len(block_groups) + segment_clones_count = len(segment_groups) # Baseline Logic baseline_path = Path(args.baseline).expanduser().resolve() @@ -480,6 +501,27 @@ def process_sequential(with_progress: bool) -> None: if baseline_exists: baseline.load() + if not args.update_baseline: + if baseline.baseline_version != __version__: + console.print( + "[error]Baseline version mismatch.[/error]\n" + "Baseline was generated with CodeClone " + f"{baseline.baseline_version or 'unknown'}.\n" + f"Current version: {__version__}.\n" + "Please regenerate the baseline with --update-baseline." + ) + sys.exit(2) + if ( + baseline.schema_version is not None + and baseline.schema_version != BASELINE_SCHEMA_VERSION + ): + console.print( + "[error]Baseline schema version mismatch.[/error]\n" + f"Baseline schema: {baseline.schema_version}. " + f"Current schema: {BASELINE_SCHEMA_VERSION}.\n" + "Please regenerate the baseline with --update-baseline." + ) + sys.exit(2) if not args.update_baseline and baseline.python_version: current_version = f"{sys.version_info.major}.{sys.version_info.minor}" if baseline.python_version != current_version: @@ -511,6 +553,8 @@ def process_sequential(with_progress: bool) -> None: block_groups, path=baseline_path, python_version=f"{sys.version_info.major}.{sys.version_info.minor}", + baseline_version=__version__, + schema_version=BASELINE_SCHEMA_VERSION, ) new_baseline.save() console.print(f"[success]✔ Baseline updated:[/success] {baseline_path}") @@ -529,6 +573,7 @@ def process_sequential(with_progress: bool) -> None: table.add_row("Files Processed", str(changed_files_count)) table.add_row("Total Function Clones", str(func_clones_count)) table.add_row("Total Block Clones", str(block_clones_count)) + table.add_row("Total Segment Clones", str(segment_clones_count)) if baseline_exists: style = "error" if new_clones_count > 0 else "success" @@ -546,6 +591,7 @@ def process_sequential(with_progress: bool) -> None: build_html_report( func_groups=func_groups, block_groups=block_groups, + segment_groups=segment_groups, title="CodeClone Report", context_lines=3, max_snippet_lines=220, @@ -558,7 +604,7 @@ def process_sequential(with_progress: bool) -> None: out = Path(args.json_out).expanduser().resolve() out.parent.mkdir(parents=True, exist_ok=True) out.write_text( - to_json_report(func_groups, block_groups), + to_json_report(func_groups, block_groups, segment_groups), "utf-8", ) console.print(f"[info]JSON report saved:[/info] {out}") @@ -570,7 +616,9 @@ def process_sequential(with_progress: bool) -> None: "FUNCTION CLONES\n" + to_text(func_groups) + "\nBLOCK CLONES\n" - + to_text(block_groups), + + to_text(block_groups) + + "\nSEGMENT CLONES\n" + + to_text(segment_groups), "utf-8", ) console.print(f"[info]Text report saved:[/info] {out}") diff --git a/codeclone/extractor.py b/codeclone/extractor.py index 02f9730..8ffe0e3 100644 --- a/codeclone/extractor.py +++ b/codeclone/extractor.py @@ -15,7 +15,7 @@ from contextlib import contextmanager from dataclasses import dataclass -from .blocks import BlockUnit, extract_blocks +from .blocks import BlockUnit, SegmentUnit, extract_blocks, extract_segments from .cfg import CFGBuilder from .errors import ParseError from .fingerprint import bucket_loc, sha1 @@ -189,7 +189,7 @@ def extract_units_from_source( cfg: NormalizationConfig, min_loc: int, min_stmt: int, -) -> tuple[list[Unit], list[BlockUnit]]: +) -> tuple[list[Unit], list[BlockUnit], list[SegmentUnit]]: try: tree = _parse_with_limits(source, PARSE_TIMEOUT_SECONDS) except SyntaxError as e: @@ -200,6 +200,7 @@ def extract_units_from_source( units: list[Unit] = [] block_units: list[BlockUnit] = [] + segment_units: list[SegmentUnit] = [] for local_name, node in qb.units: start = getattr(node, "lineno", None) @@ -243,4 +244,16 @@ def extract_units_from_source( ) block_units.extend(blocks) - return units, block_units + # Segment-level units (windows within functions, for internal clones) + if loc >= 30 and stmt_count >= 12: + segments = extract_segments( + node, + filepath=filepath, + qualname=qualname, + cfg=cfg, + window_size=6, + max_segments=60, + ) + segment_units.extend(segments) + + return units, block_units, segment_units diff --git a/codeclone/html_report.py b/codeclone/html_report.py index eb23f87..0d8c9c1 100644 --- a/codeclone/html_report.py +++ b/codeclone/html_report.py @@ -239,6 +239,7 @@ def build_html_report( *, func_groups: dict[str, list[dict[str, Any]]], block_groups: dict[str, list[dict[str, Any]]], + segment_groups: dict[str, list[dict[str, Any]]], title: str = "CodeClone Report", context_lines: int = 3, max_snippet_lines: int = 220, @@ -247,8 +248,11 @@ def build_html_report( func_sorted = sorted(func_groups.items(), key=lambda kv: _group_sort_key(kv[1])) block_sorted = sorted(block_groups.items(), key=lambda kv: _group_sort_key(kv[1])) + segment_sorted = sorted( + segment_groups.items(), key=lambda kv: _group_sort_key(kv[1]) + ) - has_any = bool(func_sorted) or bool(block_sorted) + has_any = bool(func_sorted) or bool(block_sorted) or bool(segment_sorted) # Pygments CSS (scoped). Use modern GitHub-like styles when available. # We scope per theme to support toggle without reloading. @@ -467,7 +471,8 @@ def render_section(
{ICON_CHECK}

No code clones detected

- No structural or block-level duplication was found above configured thresholds. + No structural, block-level, or segment-level duplication was found above + configured thresholds.

This usually indicates healthy abstraction boundaries.

@@ -478,6 +483,9 @@ def render_section( "functions", "Function clones", func_sorted, "pill-func" ) block_section = render_section("blocks", "Block clones", block_sorted, "pill-block") + segment_section = render_section( + "segments", "Segment clones", segment_sorted, "pill-segment" + ) return REPORT_TEMPLATE.substitute( title=_escape(title), @@ -487,6 +495,7 @@ def render_section( empty_state_html=empty_state_html, func_section=func_section, block_section=block_section, + segment_section=segment_section, icon_theme=ICON_THEME, font_css_url=FONT_CSS_URL, ) diff --git a/codeclone/normalize.py b/codeclone/normalize.py index d78f0e0..8dbcd99 100644 --- a/codeclone/normalize.py +++ b/codeclone/normalize.py @@ -109,6 +109,78 @@ def visit_AugAssign(self, node: ast.AugAssign) -> AST: ) return self.generic_visit(new_node) + def visit_UnaryOp(self, node: ast.UnaryOp) -> ast.AST: + new_node = self.generic_visit(node) + assert isinstance(new_node, ast.UnaryOp) + + if isinstance(new_node.op, ast.Not): + operand = new_node.operand + if ( + isinstance(operand, ast.Compare) + and len(operand.ops) == 1 + and len(operand.comparators) == 1 + ): + op = operand.ops[0] + if isinstance(op, ast.In): + cmp = ast.Compare( + left=operand.left, + ops=[ast.NotIn()], + comparators=operand.comparators, + ) + return ast.copy_location(cmp, new_node) + if isinstance(op, ast.Is): + cmp = ast.Compare( + left=operand.left, + ops=[ast.IsNot()], + comparators=operand.comparators, + ) + return ast.copy_location(cmp, new_node) + return new_node + + def visit_BinOp(self, node: ast.BinOp) -> ast.AST: + new_node = self.generic_visit(node) + assert isinstance(new_node, ast.BinOp) + + if not isinstance( + new_node.op, (ast.Add, ast.Mult, ast.BitOr, ast.BitAnd, ast.BitXor) + ): + return new_node + + if not ( + _is_safe_commutative_operand(new_node.left) + and _is_safe_commutative_operand(new_node.right) + ): + return new_node + + left_key = _expr_sort_key(new_node.left) + right_key = _expr_sort_key(new_node.right) + if right_key < left_key: + new_node.left, new_node.right = new_node.right, new_node.left + return new_node + + +def _expr_sort_key(node: ast.AST) -> str: + return ast.dump(node, annotate_fields=True, include_attributes=False) + + +def _is_safe_commutative_operand(node: ast.AST) -> bool: + disallowed = ( + ast.Call, + ast.Attribute, + ast.Subscript, + ast.Await, + ast.Yield, + ast.YieldFrom, + ast.Lambda, + ast.NamedExpr, + ast.ListComp, + ast.SetComp, + ast.DictComp, + ast.GeneratorExp, + ) + + return all(not isinstance(child, disallowed) for child in ast.walk(node)) + def normalized_ast_dump(func_node: ast.AST, cfg: NormalizationConfig) -> str: """ diff --git a/codeclone/report.py b/codeclone/report.py index 04a83bd..8189dbd 100644 --- a/codeclone/report.py +++ b/codeclone/report.py @@ -37,6 +37,37 @@ def build_block_groups(blocks: list[GroupItem], min_functions: int = 2) -> Group return filtered +def build_segment_groups( + segments: list[GroupItem], min_occurrences: int = 2 +) -> GroupMap: + sig_groups: GroupMap = {} + for s in segments: + sig_groups.setdefault(s["segment_sig"], []).append(s) + + confirmed: GroupMap = {} + for items in sig_groups.values(): + if len(items) < min_occurrences: + continue + + hash_groups: GroupMap = {} + for item in items: + hash_groups.setdefault(item["segment_hash"], []).append(item) + + for segment_hash, hash_items in hash_groups.items(): + if len(hash_items) < min_occurrences: + continue + + by_func: GroupMap = {} + for it in hash_items: + by_func.setdefault(it["qualname"], []).append(it) + + for qualname, q_items in by_func.items(): + if len(q_items) >= min_occurrences: + confirmed[f"{segment_hash}|{qualname}"] = q_items + + return confirmed + + def to_json(groups: GroupMap) -> str: return json.dumps( { @@ -53,9 +84,11 @@ def to_json(groups: GroupMap) -> str: ) -def to_json_report(func_groups: GroupMap, block_groups: GroupMap) -> str: +def to_json_report( + func_groups: GroupMap, block_groups: GroupMap, segment_groups: GroupMap +) -> str: return json.dumps( - {"functions": func_groups, "blocks": block_groups}, + {"functions": func_groups, "blocks": block_groups, "segments": segment_groups}, ensure_ascii=False, indent=2, ) diff --git a/codeclone/templates.py b/codeclone/templates.py index de6993d..4d1227d 100644 --- a/codeclone/templates.py +++ b/codeclone/templates.py @@ -458,6 +458,13 @@ opacity: 0.9; } +.pill-segment { + color: var(--warning); + background: var(--warning-subtle); + border: 1px solid var(--warning); + opacity: 0.9; +} + /* Groups */ .group { margin-bottom: 16px; @@ -1109,6 +1116,7 @@ ${func_section} ${block_section} +${segment_section}
Generated by CodeClone v${version} • @@ -1830,6 +1838,7 @@ initCommandPalette(); initSection('functions'); initSection('blocks'); + initSection('segments'); calculateStats(); // Welcome message diff --git a/docs/architecture.md b/docs/architecture.md index c87d4a7..cc93f61 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -13,8 +13,9 @@ CodeClone processes Python projects in the following stages: 3. **AST normalization** 4. **CFG construction** 5. **Fingerprinting** -6. **Clone grouping** -7. **Reporting / CI decision** +6. **Segment window extraction** +7. **Clone grouping** +8. **Reporting / CI decision** --- @@ -40,6 +41,8 @@ Normalization removes non-structural noise: - constants → `_CONST_` - attributes → `_ATTR_` - syntactic sugar (e.g. `x += 1` → `x = x + 1`) +- commutative operand canonicalization (`+`, `*`, `|`, `&`, `^`) when side‑effect free +- local logical equivalence (`not (x in y)` → `x not in y`, `not (x is y)` → `x is not y`) - docstrings removed - type annotations removed @@ -52,6 +55,8 @@ This ensures structural stability across refactors. - Built per-function using `CFGBuilder`. - Produces deterministic basic blocks. - Captures structural control flow (`if`, `for`, `while`, `try`, `with`, `match`). +- Models short‑circuit `and`/`or` as micro‑CFG branches. +- Links `try/except` only from statements that may raise. 📄 See [docs/cfg.md](cfg.md) for full semantics. @@ -65,7 +70,17 @@ This fingerprint is used to group structurally identical functions. --- -## 6. Clone Detection +## 6. Segment Windows + +Large functions are also scanned with **segment windows** (sliding windows over normalized +statements). These are used to detect **internal clones** inside the same function. + +Segment windows are **never** used as a final equivalence signal; they are candidate +generators with strict hash confirmation. + +--- + +## 7. Clone Detection Two clone types are detected: @@ -86,7 +101,15 @@ Noise filters applied: --- -## 7. Reporting +### Segment clones (internal) + +- Detected only **inside the same function**. +- Used for internal copy‑paste discovery and report explainability. +- Not included in baseline or CI failure logic. + +--- + +## 8. Reporting Detected clone groups can be: @@ -101,6 +124,9 @@ Detected clone groups can be: Baseline comparison allows CI to fail **only on new clones**, enabling gradual architectural improvement. +Baseline files are **versioned**. The baseline stores the CodeClone version and schema +version used to generate it. Mismatches result in a hard stop and require regeneration. + ## Python Version Consistency for Baseline Checks Due to inherent differences in Python’s AST between interpreter versions, baseline diff --git a/docs/cfg.md b/docs/cfg.md index 1106771..9197dad 100644 --- a/docs/cfg.md +++ b/docs/cfg.md @@ -82,6 +82,9 @@ An `if` statement creates: Both branches always reconverge at an explicit after-block. +If the condition is a short‑circuit boolean (`and`/`or`), it is expanded into a +**micro‑CFG** with one block per operand and explicit branch edges between them. + --- ### `while` loops @@ -170,12 +173,15 @@ is more important than their exact runtime behavior. --- -### No exception flow +### Limited exception flow -- `try / except / finally` blocks are represented only structurally. -- No exceptional control-flow edges are created. +- `try / except / finally` blocks are represented structurally. +- `try/except` edges are created **only** from statements that may raise: + function calls, attribute access, indexing, `await`, `yield from`, and `raise`. +- No interprocedural exception propagation is modeled. -This avoids CFG explosion and keeps fingerprints stable. +This keeps CFGs deterministic while reducing false differences between safe and +potentially‑raising code. --- diff --git a/tests/test_baseline.py b/tests/test_baseline.py index 6247183..1fe66ea 100644 --- a/tests/test_baseline.py +++ b/tests/test_baseline.py @@ -32,11 +32,15 @@ def test_baseline_io(tmp_path: Path) -> None: assert content["functions"] == ["f1", "f2"] assert content["blocks"] == ["b1"] assert "python_version" not in content + assert "baseline_version" in content + assert "schema_version" in content bl2 = Baseline(f) bl2.load() assert bl2.functions == {"f1", "f2"} assert bl2.blocks == {"b1"} + assert isinstance(bl2.baseline_version, str) + assert bl2.schema_version == 1 def test_baseline_load_missing(tmp_path: Path) -> None: @@ -58,11 +62,19 @@ def test_baseline_load_corrupted(tmp_path: Path) -> None: def test_baseline_from_groups() -> None: func_groups: dict[str, object] = {"f1": [], "f2": []} block_groups: dict[str, object] = {"b1": []} - bl = Baseline.from_groups(func_groups, block_groups, path="custom.json") + bl = Baseline.from_groups( + func_groups, + block_groups, + path="custom.json", + baseline_version="1.3.0", + schema_version=1, + ) assert bl.functions == {"f1", "f2"} assert bl.blocks == {"b1"} assert bl.path == Path("custom.json") + assert bl.baseline_version == "1.3.0" + assert bl.schema_version == 1 def test_baseline_python_version_roundtrip(tmp_path: Path) -> None: @@ -75,7 +87,11 @@ def test_baseline_python_version_roundtrip(tmp_path: Path) -> None: content = json.loads(f.read_text("utf-8")) assert content["python_version"] == "3.13" + assert "baseline_version" in content + assert content["schema_version"] == 1 bl2 = Baseline(f) bl2.load() assert bl2.python_version == "3.13" + assert isinstance(bl2.baseline_version, str) + assert bl2.schema_version == 1 diff --git a/tests/test_cache.py b/tests/test_cache.py index f0bca78..6e0d00a 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -7,7 +7,7 @@ import pytest -from codeclone.blocks import BlockUnit +from codeclone.blocks import BlockUnit, SegmentUnit from codeclone.cache import Cache from codeclone.errors import CacheError from codeclone.extractor import Unit @@ -37,12 +37,27 @@ def _make_block(filepath: str) -> BlockUnit: ) +def _make_segment(filepath: str) -> SegmentUnit: + return SegmentUnit( + segment_hash="s1", + segment_sig="sig1", + filepath=filepath, + qualname="mod:func", + start_line=1, + end_line=6, + size=6, + ) + + def test_cache_roundtrip(tmp_path: Path) -> None: cache_path = tmp_path / "cache.json" cache = Cache(cache_path) unit = _make_unit("x.py") block = _make_block("x.py") - cache.put_file_entry("x.py", {"mtime_ns": 1, "size": 10}, [unit], [block]) + segment = _make_segment("x.py") + cache.put_file_entry( + "x.py", {"mtime_ns": 1, "size": 10}, [unit], [block], [segment] + ) cache.save() loaded = Cache(cache_path) @@ -56,7 +71,7 @@ def test_cache_roundtrip(tmp_path: Path) -> None: def test_cache_signature_mismatch_warns(tmp_path: Path) -> None: cache_path = tmp_path / "cache.json" cache = Cache(cache_path) - cache.put_file_entry("x.py", {"mtime_ns": 1, "size": 10}, [], []) + cache.put_file_entry("x.py", {"mtime_ns": 1, "size": 10}, [], [], []) cache.save() data = json.loads(cache_path.read_text("utf-8")) diff --git a/tests/test_cfg.py b/tests/test_cfg.py index abe945d..0f10e51 100644 --- a/tests/test_cfg.py +++ b/tests/test_cfg.py @@ -56,6 +56,51 @@ def f(a): assert cfg_str.strip() == dedent(expected).strip() +def test_cfg_if_with_boolop_and() -> None: + source = """ + def f(a, b): + if a and b: + x = 1 + else: + x = 2 + """ + cfg_str = cfg_to_str(build_cfg_from_source(source)) + expected = """ +Block 0 -> [3, 5] + Expr(value=Name(id='a', ctx=Load())) +Block 1 -> [] +Block 2 -> [4] + Assign(targets=[Name(id='x', ctx=Store())], value=Constant(value=1)) +Block 3 -> [4] + Assign(targets=[Name(id='x', ctx=Store())], value=Constant(value=2)) +Block 4 -> [1] +Block 5 -> [2, 3] + Expr(value=Name(id='b', ctx=Load())) +""" + assert cfg_str.strip() == dedent(expected).strip() + + +def test_cfg_while_with_boolop_or() -> None: + source = """ + def f(a, b): + while a or b: + x = 1 + """ + cfg_str = cfg_to_str(build_cfg_from_source(source)) + expected = """ +Block 0 -> [2] +Block 1 -> [] +Block 2 -> [3, 5] + Expr(value=Name(id='a', ctx=Load())) +Block 3 -> [2] + Assign(targets=[Name(id='x', ctx=Store())], value=Constant(value=1)) +Block 4 -> [1] +Block 5 -> [3, 4] + Expr(value=Name(id='b', ctx=Load())) +""" + assert cfg_str.strip() == dedent(expected).strip() + + def test_cfg_while_loop() -> None: source = """ def f(): @@ -328,12 +373,12 @@ def f(x): def test_cfg_try_handler_linking() -> None: - """Test that statements inside try block are linked to handlers.""" + """Test that only potentially raising statements inside try link to handlers.""" code = """ def f(): try: x = 1 - y = 2 + y = risky() except ValueError: pass """ @@ -358,13 +403,105 @@ def f(): predecessors = [b for b in cfg.blocks if handler_block in b.successors] - has_assignment = False + has_call = False for pred in predecessors: for stmt in pred.statements: - if isinstance(stmt, ast.Assign): - has_assignment = True + if ( + isinstance(stmt, ast.Assign) + and isinstance(stmt.value, ast.Call) + and isinstance(stmt.value.func, ast.Name) + and stmt.value.func.id == "risky" + ): + has_call = True + + assert has_call, "Handler should be reachable from potentially raising block" + + +def test_cfg_try_handler_linking_skips_safe_statements() -> None: + code = """ + def f(): + try: + x = 1 + y = 2 + except ValueError: + pass + """ + func = ast.parse(dedent(code)).body[0] + assert isinstance(func, (ast.FunctionDef, ast.AsyncFunctionDef)) + cfg = CFGBuilder().build("f", func) + + handler_blocks = [ + b + for b in cfg.blocks + if any( + isinstance(s, ast.Expr) + and isinstance(s.value, ast.Name) + and s.value.id == "ValueError" + for s in b.statements + ) + ] - assert has_assignment, "Handler should be reachable from assignment block" + assert len(handler_blocks) == 1 + handler_block = handler_blocks[0] + + predecessors = [b for b in cfg.blocks if handler_block in b.successors] + + has_assign_only = any( + any(isinstance(stmt, ast.Assign) for stmt in pred.statements) + for pred in predecessors + ) + + assert not has_assign_only, "Safe assignments should not link to handlers" + + +def test_cfg_try_body_breaks_after_termination() -> None: + code = """ + def f(): + try: + return 1 + x = 2 + except ValueError: + pass + """ + func = ast.parse(dedent(code)).body[0] + assert isinstance(func, (ast.FunctionDef, ast.AsyncFunctionDef)) + cfg = CFGBuilder().build("f", func) + assert any( + any(isinstance(stmt, ast.Return) for stmt in block.statements) + for block in cfg.blocks + ) + + +def test_cfg_try_handler_linking_for_raise() -> None: + code = """ + def f(): + try: + raise ValueError("x") + except ValueError: + pass + """ + func = ast.parse(dedent(code)).body[0] + assert isinstance(func, (ast.FunctionDef, ast.AsyncFunctionDef)) + cfg = CFGBuilder().build("f", func) + + handler_blocks = [ + b + for b in cfg.blocks + if any( + isinstance(s, ast.Expr) + and isinstance(s.value, ast.Name) + and s.value.id == "ValueError" + for s in b.statements + ) + ] + assert len(handler_blocks) == 1 + handler_block = handler_blocks[0] + + predecessors = [b for b in cfg.blocks if handler_block in b.successors] + assert any( + any(isinstance(stmt, ast.Raise) for stmt in pred.statements) + for pred in predecessors + ) def test_cfg_try_star() -> None: diff --git a/tests/test_cli_inprocess.py b/tests/test_cli_inprocess.py index f9c7a47..03f89c3 100644 --- a/tests/test_cli_inprocess.py +++ b/tests/test_cli_inprocess.py @@ -10,7 +10,8 @@ import pytest import codeclone.baseline as baseline -from codeclone import cli +from codeclone import __version__, cli +from codeclone.baseline import BASELINE_SCHEMA_VERSION from codeclone.cache import Cache from codeclone.errors import CacheError @@ -117,6 +118,19 @@ def test_cli_main_progress_fallback( assert "falling back to sequential" in out +def test_cli_main_no_progress_fallback( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + src = tmp_path / "a.py" + src.write_text("def f():\n return 1\n", "utf-8") + monkeypatch.setattr(cli, "ProcessPoolExecutor", _FailingExecutor) + _run_main(monkeypatch, [str(tmp_path), "--no-progress"]) + out = capsys.readouterr().out + assert "falling back to sequential" in out + + def test_cli_main_progress_path( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -260,7 +274,17 @@ def f2(): "utf-8", ) baseline = tmp_path / "baseline.json" - baseline.write_text('{"functions": [], "blocks": []}', "utf-8") + baseline.write_text( + json.dumps( + { + "functions": [], + "blocks": [], + "baseline_version": __version__, + "schema_version": BASELINE_SCHEMA_VERSION, + } + ), + "utf-8", + ) _patch_parallel(monkeypatch) _run_main( monkeypatch, @@ -288,7 +312,16 @@ def test_cli_baseline_python_version_mismatch_warns( src.write_text("def f():\n return 1\n", "utf-8") baseline = tmp_path / "baseline.json" baseline.write_text( - '{"functions": [], "blocks": [], "python_version": "0.0"}', "utf-8" + json.dumps( + { + "functions": [], + "blocks": [], + "python_version": "0.0", + "baseline_version": __version__, + "schema_version": BASELINE_SCHEMA_VERSION, + } + ), + "utf-8", ) _patch_parallel(monkeypatch) _run_main( @@ -305,6 +338,42 @@ def test_cli_baseline_python_version_mismatch_warns( assert "Current interpreter: Python" in out +def test_cli_baseline_version_mismatch_fails( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + src = tmp_path / "a.py" + src.write_text("def f():\n return 1\n", "utf-8") + baseline_path = tmp_path / "baseline.json" + baseline_path.write_text( + json.dumps( + { + "functions": [], + "blocks": [], + "python_version": f"{sys.version_info.major}.{sys.version_info.minor}", + "baseline_version": "0.0.0", + "schema_version": BASELINE_SCHEMA_VERSION, + } + ), + "utf-8", + ) + _patch_parallel(monkeypatch) + with pytest.raises(SystemExit) as exc: + _run_main( + monkeypatch, + [ + str(tmp_path), + "--baseline", + str(baseline_path), + "--no-progress", + ], + ) + assert exc.value.code == 2 + out = capsys.readouterr().out + assert "Baseline version mismatch" in out + + def test_cli_baseline_python_version_mismatch_fails( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -314,7 +383,16 @@ def test_cli_baseline_python_version_mismatch_fails( src.write_text("def f():\n return 1\n", "utf-8") baseline = tmp_path / "baseline.json" baseline.write_text( - '{"functions": [], "blocks": [], "python_version": "0.0"}', "utf-8" + json.dumps( + { + "functions": [], + "blocks": [], + "python_version": "0.0", + "baseline_version": __version__, + "schema_version": BASELINE_SCHEMA_VERSION, + } + ), + "utf-8", ) _patch_parallel(monkeypatch) with pytest.raises(SystemExit) as exc: @@ -384,6 +462,55 @@ def f2(): "functions": [], "blocks": [], "python_version": f"{sys.version_info.major}.{sys.version_info.minor}", + "baseline_version": __version__, + "schema_version": BASELINE_SCHEMA_VERSION, + } + ), + "utf-8", + ) + _patch_parallel(monkeypatch) + with pytest.raises(SystemExit) as exc: + _run_main( + monkeypatch, + [ + str(tmp_path), + "--baseline", + str(baseline), + "--min-loc", + "1", + "--min-stmt", + "1", + "--fail-on-new", + "--no-progress", + ], + ) + assert exc.value.code == 3 + + +def test_cli_main_fail_on_new_includes_blocks( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + body = "\n".join([f" x{i} = {i}" for i in range(50)]) + src = tmp_path / "a.py" + src.write_text( + f""" +def f1(): +{body} + +def f2(): +{body} +""", + "utf-8", + ) + baseline = tmp_path / "baseline.json" + baseline.write_text( + json.dumps( + { + "functions": [], + "blocks": [], + "python_version": f"{sys.version_info.major}.{sys.version_info.minor}", + "baseline_version": __version__, + "schema_version": BASELINE_SCHEMA_VERSION, } ), "utf-8", @@ -405,6 +532,9 @@ def f2(): ], ) assert exc.value.code == 3 + out = capsys.readouterr().out + assert "New Functions" in out + assert "New Blocks" in out def test_cli_blocks_processing(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: @@ -427,7 +557,7 @@ def test_cli_cache_warning( src.write_text("def f():\n return 1\n", "utf-8") cache_path = tmp_path / "cache.json" cache = Cache(cache_path) - cache.put_file_entry("x.py", {"mtime_ns": 1, "size": 1}, [], []) + cache.put_file_entry("x.py", {"mtime_ns": 1, "size": 1}, [], [], []) cache.save() data = json.loads(cache_path.read_text("utf-8")) data["_signature"] = "bad" @@ -501,6 +631,7 @@ def test_cli_discovery_cache_hit( } ], "blocks": [], + "segments": [], } cache.save() @@ -694,6 +825,8 @@ def _diff( "functions": [], "blocks": [], "python_version": f"{sys.version_info.major}.{sys.version_info.minor}", + "baseline_version": __version__, + "schema_version": BASELINE_SCHEMA_VERSION, } ), "utf-8", diff --git a/tests/test_cli_unit.py b/tests/test_cli_unit.py index 85bbf31..aa8b82e 100644 --- a/tests/test_cli_unit.py +++ b/tests/test_cli_unit.py @@ -1,4 +1,6 @@ import os +import runpy +import sys from pathlib import Path import pytest @@ -67,3 +69,11 @@ def test_process_file_success(tmp_path: Path) -> None: result = process_file(str(src), str(tmp_path), NormalizationConfig(), 1, 1) assert result.success is True assert result.stat is not None + + +def test_cli_module_main_guard(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delitem(sys.modules, "codeclone.cli", raising=False) + monkeypatch.setattr(sys, "argv", ["codeclone", "--help"]) + with pytest.raises(SystemExit) as exc: + runpy.run_module("codeclone.cli", run_name="__main__") + assert exc.value.code == 0 diff --git a/tests/test_extractor.py b/tests/test_extractor.py index 6eeaebe..0a08a53 100644 --- a/tests/test_extractor.py +++ b/tests/test_extractor.py @@ -22,7 +22,7 @@ def foo(): return a + b """ - units, blocks = extract_units_from_source( + units, blocks, segments = extract_units_from_source( source=src, filepath="x.py", module_name="mod", @@ -36,6 +36,7 @@ def foo(): assert u.qualname == "mod:foo" assert u.loc >= 3 assert blocks == [] + assert segments == [] def test_init_function_is_ignored_for_blocks() -> None: @@ -48,7 +49,7 @@ def __init__(self): w = 4 """ - units, blocks = extract_units_from_source( + units, blocks, segments = extract_units_from_source( source=src, filepath="x.py", module_name="mod", @@ -59,6 +60,7 @@ def __init__(self): assert len(units) == 1 assert blocks == [] + assert segments == [] def test_parse_timeout_raises(monkeypatch: pytest.MonkeyPatch) -> None: @@ -122,7 +124,7 @@ def test_extract_respects_min_loc_min_stmt() -> None: def f(): x = 1 """ - units, blocks = extract_units_from_source( + units, blocks, segments = extract_units_from_source( source=src, filepath="x.py", module_name="mod", @@ -132,6 +134,7 @@ def f(): ) assert units == [] assert blocks == [] + assert segments == [] def test_extract_block_units_generated() -> None: @@ -141,7 +144,7 @@ def test_extract_block_units_generated() -> None: def f(): {body_lines} """ - units, blocks = extract_units_from_source( + units, blocks, segments = extract_units_from_source( source=src, filepath="x.py", module_name="mod", @@ -151,6 +154,7 @@ def f(): ) assert units assert blocks + assert segments def test_extract_async_function() -> None: @@ -158,7 +162,7 @@ def test_extract_async_function() -> None: async def af(): return 1 """ - units, blocks = extract_units_from_source( + units, blocks, segments = extract_units_from_source( source=src, filepath="x.py", module_name="mod", @@ -168,6 +172,7 @@ async def af(): ) assert len(units) == 1 assert blocks == [] + assert segments == [] def test_extract_skips_invalid_positions(monkeypatch: pytest.MonkeyPatch) -> None: @@ -185,7 +190,7 @@ def _fake_parse(_source: str, _timeout_s: int) -> ast.AST: return tree monkeypatch.setattr(extractor, "_parse_with_limits", _fake_parse) - units, blocks = extract_units_from_source( + units, blocks, segments = extract_units_from_source( source="def f():\n return 1\n", filepath="x.py", module_name="mod", @@ -195,6 +200,7 @@ def _fake_parse(_source: str, _timeout_s: int) -> ast.AST: ) assert units == [] assert blocks == [] + assert segments == [] def test_parse_limits_triggers_timeout(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/test_html_report.py b/tests/test_html_report.py index 19cde1f..17dcbb5 100644 --- a/tests/test_html_report.py +++ b/tests/test_html_report.py @@ -14,7 +14,9 @@ def test_html_report_empty() -> None: - html = build_html_report(func_groups={}, block_groups={}, title="Empty Report") + html = build_html_report( + func_groups={}, block_groups={}, segment_groups={}, title="Empty Report" + ) assert "" in html assert "Empty Report" in html assert "No code clones detected" in html @@ -36,6 +38,7 @@ def test_html_report_generation(tmp_path: Path) -> None: html = build_html_report( func_groups=func_groups, block_groups={}, + segment_groups={}, title="Test Report", context_lines=1, max_snippet_lines=10, @@ -102,6 +105,7 @@ def test_render_code_block_truncate(tmp_path: Path) -> None: ] }, block_groups={}, + segment_groups={}, title="Truncate", context_lines=10, max_snippet_lines=5, @@ -117,6 +121,12 @@ def test_prefix_css() -> None: assert "/* c */" in prefixed +def test_prefix_css_empty_selector_passthrough() -> None: + css = " { color: red; }\n" + prefixed = _prefix_css(css, ".wrap") + assert "{ color: red; }" in prefixed + + def test_pygments_css() -> None: css = _pygments_css("default") assert ".codebox" in css or css == "" @@ -172,7 +182,12 @@ def test_html_report_with_blocks(tmp_path: Path) -> None: }, ] } - html = build_html_report(func_groups={}, block_groups=block_groups, title="Blocks") + html = build_html_report( + func_groups={}, + block_groups=block_groups, + segment_groups={}, + title="Blocks", + ) assert "Block clones" in html @@ -185,10 +200,63 @@ def _fake_css(name: str) -> str: return "x" monkeypatch.setattr(hr, "_pygments_css", _fake_css) - html = build_html_report(func_groups={}, block_groups={}, title="Pygments") + html = build_html_report( + func_groups={}, block_groups={}, segment_groups={}, title="Pygments" + ) assert "Pygments" in html +def test_html_report_segments_section(tmp_path: Path) -> None: + f = tmp_path / "a.py" + f.write_text("def f():\n x = 1\n y = 2\n", "utf-8") + segment_groups = { + "s1|mod:f": [ + { + "qualname": "mod:f", + "filepath": str(f), + "start_line": 1, + "end_line": 2, + "size": 2, + }, + { + "qualname": "mod:f", + "filepath": str(f), + "start_line": 2, + "end_line": 3, + "size": 2, + }, + ] + } + html = build_html_report( + func_groups={}, + block_groups={}, + segment_groups=segment_groups, + title="Segments", + ) + assert "Segment clones" in html + + +def test_render_code_block_truncates_and_fallback( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + f = tmp_path / "a.py" + f.write_text("\n".join([f"line{i}" for i in range(1, 30)]), "utf-8") + + import codeclone.html_report as hr + + monkeypatch.setattr(hr, "_try_pygments", lambda _text: None) + cache = _FileCache(maxsize=2) + snippet = hr._render_code_block( + filepath=str(f), + start_line=1, + end_line=20, + file_cache=cache, + context=5, + max_lines=5, + ) + assert "codebox" in snippet.code_html + + def test_pygments_css_get_style_defs_error(monkeypatch: pytest.MonkeyPatch) -> None: class _Fmt: def __init__(self, *args: object, **kwargs: object) -> None: diff --git a/tests/test_normalize.py b/tests/test_normalize.py index 00cf715..845be7d 100644 --- a/tests/test_normalize.py +++ b/tests/test_normalize.py @@ -85,6 +85,100 @@ def f(): assert normalized_ast_dump(a1, cfg) == normalized_ast_dump(a2, cfg) +def test_normalization_commutative_binop_reorders() -> None: + src1 = """ +def f(): + return a + b +""" + src2 = """ +def f(): + return b + a +""" + cfg = NormalizationConfig( + normalize_names=False, + normalize_attributes=False, + normalize_constants=False, + ) + a1 = ast.parse(src1).body[0] + a2 = ast.parse(src2).body[0] + assert normalized_ast_dump(a1, cfg) == normalized_ast_dump(a2, cfg) + + +def test_normalization_commutative_binop_side_effects_not_reordered() -> None: + src1 = """ +def f(): + return foo() + bar() +""" + src2 = """ +def f(): + return bar() + foo() +""" + cfg = NormalizationConfig( + normalize_names=False, + normalize_attributes=False, + normalize_constants=False, + ) + a1 = ast.parse(src1).body[0] + a2 = ast.parse(src2).body[0] + assert normalized_ast_dump(a1, cfg) != normalized_ast_dump(a2, cfg) + + +def test_normalization_non_commutative_binop_not_reordered() -> None: + src1 = """ +def f(): + return a - b +""" + src2 = """ +def f(): + return b - a +""" + cfg = NormalizationConfig(normalize_names=False) + a1 = ast.parse(src1).body[0] + a2 = ast.parse(src2).body[0] + assert normalized_ast_dump(a1, cfg) != normalized_ast_dump(a2, cfg) + + +def test_normalization_not_in_and_is_not_equivalence() -> None: + src1 = """ +def f(x, y): + return not (x in y) +""" + src2 = """ +def f(x, y): + return x not in y +""" + src3 = """ +def f(x, y): + return not (x is y) +""" + src4 = """ +def f(x, y): + return x is not y +""" + cfg = NormalizationConfig(normalize_names=False) + a1 = ast.parse(src1).body[0] + a2 = ast.parse(src2).body[0] + a3 = ast.parse(src3).body[0] + a4 = ast.parse(src4).body[0] + assert normalized_ast_dump(a1, cfg) == normalized_ast_dump(a2, cfg) + assert normalized_ast_dump(a3, cfg) == normalized_ast_dump(a4, cfg) + + +def test_normalization_no_demorgan() -> None: + src1 = """ +def f(x, y): + return not (x == y) +""" + src2 = """ +def f(x, y): + return x != y +""" + cfg = NormalizationConfig(normalize_names=False) + a1 = ast.parse(src1).body[0] + a2 = ast.parse(src2).body[0] + assert normalized_ast_dump(a1, cfg) != normalized_ast_dump(a2, cfg) + + def test_normalization_flags_false_preserve_details() -> None: src = """ def f(x: int, /, y: int, *, z: int, **k: int) -> int: diff --git a/tests/test_report.py b/tests/test_report.py index dec2aea..f059e91 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -1,6 +1,7 @@ from codeclone.report import ( build_block_groups, build_groups, + build_segment_groups, to_json, to_json_report, to_text, @@ -59,9 +60,85 @@ def test_report_output_formats() -> None: ], } json_out = to_json(groups) - report_out = to_json_report(groups, groups) + report_out = to_json_report(groups, groups, {}) text_out = to_text(groups) assert "group_count" in json_out assert '"functions"' in report_out assert "Clone group #1" in text_out + + +def test_segment_groups_internal_only() -> None: + segments = [ + { + "segment_sig": "sig1", + "segment_hash": "h1", + "qualname": "mod:f", + "filepath": "a.py", + "start_line": 1, + "end_line": 4, + "size": 4, + }, + { + "segment_sig": "sig1", + "segment_hash": "h1", + "qualname": "mod:f", + "filepath": "a.py", + "start_line": 10, + "end_line": 13, + "size": 4, + }, + { + "segment_sig": "sig1", + "segment_hash": "h1", + "qualname": "mod:g", + "filepath": "b.py", + "start_line": 1, + "end_line": 4, + "size": 4, + }, + ] + + groups = build_segment_groups(segments) + assert len(groups) == 1 + group_items = next(iter(groups.values())) + assert all(item["qualname"] == "mod:f" for item in group_items) + + +def test_segment_groups_filters_small_candidates() -> None: + segments = [ + { + "segment_sig": "sig1", + "segment_hash": "h1", + "qualname": "mod:f", + "filepath": "a.py", + "start_line": 1, + "end_line": 2, + "size": 2, + } + ] + groups = build_segment_groups(segments) + assert groups == {} + + segments = [ + { + "segment_sig": "sig2", + "segment_hash": "h1", + "qualname": "mod:f", + "filepath": "a.py", + "start_line": 1, + "end_line": 2, + "size": 2, + }, + { + "segment_sig": "sig2", + "segment_hash": "h2", + "qualname": "mod:f", + "filepath": "a.py", + "start_line": 3, + "end_line": 4, + "size": 2, + }, + ] + groups = build_segment_groups(segments) + assert groups == {} diff --git a/tests/test_security.py b/tests/test_security.py index a702fe2..99bac8e 100644 --- a/tests/test_security.py +++ b/tests/test_security.py @@ -1,11 +1,13 @@ import os import tempfile +from pathlib import Path from unittest.mock import patch import pytest from codeclone.cli import MAX_FILE_SIZE, process_file from codeclone.errors import ValidationError +from codeclone.html_report import build_html_report from codeclone.normalize import NormalizationConfig from codeclone.scanner import iter_py_files @@ -42,3 +44,37 @@ def test_process_file_size_limit() -> None: finally: os.remove(tmp_path) + + +def test_html_report_escapes_user_content(tmp_path: Path) -> None: + bad_path = tmp_path / 'x" onmouseover="alert(1).py' + good_path = tmp_path / "y.py" + bad_path.write_text("def f():\n return 1\n", "utf-8") + good_path.write_text("def g():\n return 2\n", "utf-8") + func_groups = { + "k": [ + { + "qualname": "", + "filepath": str(bad_path), + "start_line": 1, + "end_line": 2, + "loc": 2, + }, + { + "qualname": "ok", + "filepath": str(good_path), + "start_line": 3, + "end_line": 4, + "loc": 2, + }, + ] + } + html = build_html_report( + func_groups=func_groups, + block_groups={}, + segment_groups={}, + title="Security", + ) + assert "" not in html + assert "<script>alert(1)</script>" in html + assert 'onmouseover="alert(1)' not in html diff --git a/tests/test_segments.py b/tests/test_segments.py new file mode 100644 index 0000000..6e2ab63 --- /dev/null +++ b/tests/test_segments.py @@ -0,0 +1,105 @@ +import ast + +from codeclone.blocks import extract_segments +from codeclone.normalize import NormalizationConfig + + +def test_extract_segments_windows() -> None: + src = """ +def f(): + a = 1 + b = 2 + c = 3 +""" + func = ast.parse(src).body[0] + assert isinstance(func, ast.FunctionDef) + segments = extract_segments( + func, + filepath="x.py", + qualname="mod:f", + cfg=NormalizationConfig(), + window_size=2, + max_segments=10, + ) + assert len(segments) == 2 + assert segments[0].size == 2 + + +def test_extract_segments_short_function() -> None: + src = """ +def f(): + a = 1 +""" + func = ast.parse(src).body[0] + assert isinstance(func, ast.FunctionDef) + segments = extract_segments( + func, + filepath="x.py", + qualname="mod:f", + cfg=NormalizationConfig(), + window_size=3, + max_segments=10, + ) + assert segments == [] + + +def test_extract_segments_missing_lineno_and_max_limit() -> None: + src = """ +def f(): + a = 1 + b = 2 + c = 3 + d = 4 +""" + func = ast.parse(src).body[0] + assert isinstance(func, ast.FunctionDef) + func.body[0].lineno = 0 + segments = extract_segments( + func, + filepath="x.py", + qualname="mod:f", + cfg=NormalizationConfig(), + window_size=2, + max_segments=1, + ) + assert len(segments) == 1 + + +def test_extract_segments_signature_orderless() -> None: + src1 = """ +def f(): + a = 1 + b = 2 +""" + src2 = """ +def f(): + b = 2 + a = 1 +""" + func1 = ast.parse(src1).body[0] + func2 = ast.parse(src2).body[0] + assert isinstance(func1, ast.FunctionDef) + assert isinstance(func2, ast.FunctionDef) + cfg = NormalizationConfig( + normalize_names=False, + normalize_attributes=False, + normalize_constants=False, + ) + seg1 = extract_segments( + func1, + filepath="x.py", + qualname="mod:f", + cfg=cfg, + window_size=2, + max_segments=10, + )[0] + seg2 = extract_segments( + func2, + filepath="x.py", + qualname="mod:f", + cfg=cfg, + window_size=2, + max_segments=10, + )[0] + assert seg1.segment_sig == seg2.segment_sig + assert seg1.segment_hash != seg2.segment_hash From b86822956faaa048b93723d1096086ed007ee307 Mon Sep 17 00:00:00 2001 From: Den Rozhnovskiy Date: Thu, 5 Feb 2026 20:13:08 +0500 Subject: [PATCH 05/28] feat(core): Added --ci preset (--fail-on-new --no-color --quiet). Improved --fail-on-new output with aggregated counts and clear next steps. --- .github/workflows/tests.yml | 2 +- CHANGELOG.md | 6 + README.md | 20 +- codeclone.baseline.json | 5 +- codeclone/cache.py | 4 +- codeclone/cfg.py | 2 +- codeclone/cli.py | 222 ++++++-- codeclone/html_report.py | 171 +++--- tests/test_cache.py | 11 + tests/test_cfg.py | 17 + tests/test_cli_inprocess.py | 811 ++++++++++++++++++++++------- tests/test_cli_main_guard_runpy.py | 12 + tests/test_cli_unit.py | 4 +- tests/test_normalize.py | 81 ++- 14 files changed, 1011 insertions(+), 357 deletions(-) create mode 100644 tests/test_cli_main_guard_runpy.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 70c76d2..91c57c2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -46,7 +46,7 @@ jobs: - name: Check for new clones vs baseline if: ${{ matrix.python-version == '3.13' }} - run: uv run codeclone . --fail-on-new --no-progress + run: uv run codeclone . --ci lint: runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 92c7c8b..4d37500 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,12 @@ Update the baseline: codeclone . --update-baseline ``` +### CLI UX (CI) + +- Added `--ci` preset (`--fail-on-new --no-color --quiet`). +- Improved `--fail-on-new` output with aggregated counts and clear next steps. +- Validate report output extensions (`.html`, `.json`, `.txt`) and fail fast on mismatches. + ### HTML Report UI - **Visual refresh** diff --git a/README.md b/README.md index 65d2815..3a52ddb 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,12 @@ # CodeClone -[![PyPI](https://img.shields.io/pypi/v/codeclone.svg)](https://pypi.org/project/codeclone/) -[![Downloads](https://img.shields.io/pypi/dm/codeclone.svg)](https://pypi.org/project/codeclone/) -[![tests](https://github.com/orenlab/codeclone/actions/workflows/tests.yml/badge.svg?branch=main)](https://github.com/orenlab/codeclone/actions/workflows/tests.yml) -[![Python](https://img.shields.io/pypi/pyversions/codeclone.svg)](https://pypi.org/project/codeclone/) -[![License](https://img.shields.io/pypi/l/codeclone.svg)](LICENSE) +[![PyPI](https://img.shields.io/pypi/v/codeclone.svg?style=flat-square)](https://pypi.org/project/codeclone/) +[![Downloads](https://img.shields.io/pypi/dm/codeclone.svg?style=flat-square)](https://pypi.org/project/codeclone/) +[![tests](https://github.com/orenlab/codeclone/actions/workflows/tests.yml/badge.svg?branch=main&style=flat-square)](https://github.com/orenlab/codeclone/actions/workflows/tests.yml) +[![Python](https://img.shields.io/pypi/pyversions/codeclone.svg?style=flat-square)](https://pypi.org/project/codeclone/) +![CI First](https://img.shields.io/badge/CI-first-green?style=flat-square) +![Baseline](https://img.shields.io/badge/baseline-versioned-green?style=flat-square) +[![License](https://img.shields.io/pypi/l/codeclone.svg?style=flat-square)](LICENSE) **CodeClone** is a Python code clone detector based on **normalized Python AST and Control Flow Graphs (CFG)**. It helps teams discover architectural duplication and prevent new copy-paste from entering the codebase via CI. @@ -144,6 +146,14 @@ Generate an HTML report: codeclone . --html .cache/codeclone/report.html ``` +Run in CI mode: + +```bash +codeclone . --ci --html .cache/codeclone/report.html +``` + +`--ci` is equivalent to `--fail-on-new --no-color --quiet`. + --- ## Baseline Workflow (Recommended) diff --git a/codeclone.baseline.json b/codeclone.baseline.json index dcc2fc8..ba7e83f 100644 --- a/codeclone.baseline.json +++ b/codeclone.baseline.json @@ -1,8 +1,5 @@ { - "functions": [ - "23353998d062bbdf37c345cbe5256b3f5686d956|0-19", - "7d573fa56fb11050f1642f18ca4bb3225e11e194|0-19" - ], + "functions": [], "blocks": [], "python_version": "3.13", "baseline_version": "1.3.0", diff --git a/codeclone/cache.py b/codeclone/cache.py index 8045596..c3bd024 100644 --- a/codeclone/cache.py +++ b/codeclone/cache.py @@ -24,6 +24,8 @@ from .errors import CacheError +OS_NAME = os.name + class FileStat(TypedDict): mtime_ns: int @@ -96,7 +98,7 @@ def _load_secret(self) -> bytes: self.path.parent.mkdir(parents=True, exist_ok=True) secret_path.write_bytes(secret) # Set restrictive permissions on secret file (Unix only) - if os.name == "posix": + if OS_NAME == "posix": secret_path.chmod(0o600) except OSError: pass diff --git a/codeclone/cfg.py b/codeclone/cfg.py index ffa7aa8..e10fed3 100644 --- a/codeclone/cfg.py +++ b/codeclone/cfg.py @@ -88,7 +88,7 @@ def _visit(self, stmt: ast.stmt) -> None: case ast.Try(): self._visit_try(cast(_TryLike, stmt)) case _ if TryStar is not None and isinstance(stmt, TryStar): - self._visit_try(cast(_TryLike, stmt)) + self._visit_try(cast(_TryLike, cast(object, stmt))) case ast.With() | ast.AsyncWith(): self._visit_with(stmt) diff --git a/codeclone/cli.py b/codeclone/cli.py index 6e52791..b67bcf0 100644 --- a/codeclone/cli.py +++ b/codeclone/cli.py @@ -46,7 +46,13 @@ "dim": "dim", } ) -console = Console(theme=custom_theme, width=200) + + +def _make_console(*, no_color: bool) -> Console: + return Console(theme=custom_theme, width=200, no_color=no_color) + + +console = _make_console(no_color=False) MAX_FILE_SIZE = 10 * 1024 * 1024 # 10MB BATCH_SIZE = 100 @@ -145,7 +151,7 @@ def process_file( def print_banner() -> None: console.print( Panel.fit( - "[bold white]CodeClone[/bold white] [dim]v1.2.1[/dim]\n" + f"[bold white]CodeClone[/bold white] [dim]v{__version__}[/dim]\n" "[italic]Architectural duplication detector[/italic]", border_style="blue", padding=(0, 2), @@ -153,6 +159,17 @@ def print_banner() -> None: ) +def _validate_output_path(path: str, *, expected_suffix: str, label: str) -> Path: + out = Path(path).expanduser() + if out.suffix.lower() != expected_suffix: + console.print( + f"[error]Invalid {label} output extension: {out} " + f"(expected {expected_suffix}).[/error]" + ) + sys.exit(2) + return out.resolve() + + def main() -> None: ap = argparse.ArgumentParser( prog="codeclone", @@ -219,6 +236,11 @@ def main() -> None: metavar="MAX_CLONES", help="Exit with error if total clone groups exceed this number.", ) + ci_group.add_argument( + "--ci", + action="store_true", + help="CI preset: --fail-on-new --no-color --quiet.", + ) # Output out_group = ap.add_argument_group("Reporting") @@ -245,10 +267,37 @@ def main() -> None: action="store_true", help="Disable the progress bar (recommended for CI logs).", ) + out_group.add_argument( + "--no-color", + action="store_true", + help="Disable ANSI colors in output.", + ) + out_group.add_argument( + "--quiet", + action="store_true", + help="Minimize output (still shows warnings and errors).", + ) + out_group.add_argument( + "--verbose", + action="store_true", + help="Print detailed hash identifiers for new clones.", + ) args = ap.parse_args() - print_banner() + if args.ci: + args.fail_on_new = True + args.no_color = True + args.quiet = True + + if args.quiet: + args.no_progress = True + + global console + console = _make_console(no_color=args.no_color) + + if not args.quiet: + print_banner() try: root_path = Path(args.root).resolve() @@ -259,7 +308,24 @@ def main() -> None: console.print(f"[error]Invalid root path: {e}[/error]") sys.exit(1) - console.print(f"[info]Scanning root:[/info] {root_path}") + if not args.quiet: + console.print(f"[info]Scanning root:[/info] {root_path}") + + html_out_path: Path | None = None + json_out_path: Path | None = None + text_out_path: Path | None = None + if args.html_out: + html_out_path = _validate_output_path( + args.html_out, expected_suffix=".html", label="HTML" + ) + if args.json_out: + json_out_path = _validate_output_path( + args.json_out, expected_suffix=".json", label="JSON" + ) + if args.text_out: + text_out_path = _validate_output_path( + args.text_out, expected_suffix=".txt", label="text" + ) # Initialize Cache cfg = NormalizationConfig() @@ -305,8 +371,8 @@ def _safe_future_result(future: Any) -> tuple[ProcessingResult | None, str | Non return None, str(e) # Discovery phase - with console.status("[bold green]Discovering Python files...", spinner="dots"): - try: + try: + if args.quiet: for fp in iter_py_files(str(root_path)): stat, cached, warn = _get_cached_entry(fp) if warn: @@ -333,9 +399,39 @@ def _safe_future_result(future: Any) -> tuple[ProcessingResult | None, str | Non ) else: files_to_process.append(fp) - except Exception as e: - console.print(f"[error]Scan failed: {e}[/error]") - sys.exit(1) + else: + with console.status( + "[bold green]Discovering Python files...", spinner="dots" + ): + for fp in iter_py_files(str(root_path)): + stat, cached, warn = _get_cached_entry(fp) + if warn: + console.print(warn) + continue + if cached and cached.get("stat") == stat: + all_units.extend( + cast( + list[dict[str, Any]], + cast(object, cached.get("units", [])), + ) + ) + all_blocks.extend( + cast( + list[dict[str, Any]], + cast(object, cached.get("blocks", [])), + ) + ) + all_segments.extend( + cast( + list[dict[str, Any]], + cast(object, cached.get("segments", [])), + ) + ) + else: + files_to_process.append(fp) + except Exception as e: + console.print(f"[error]Scan failed: {e}[/error]") + sys.exit(1) total_files = len(files_to_process) failed_files = [] @@ -382,7 +478,10 @@ def process_sequential(with_progress: bool) -> None: handle_result(result) progress.advance(task) else: - console.print(f"[info]Processing {total_files} changed files...[/info]") + if not args.quiet: + console.print( + f"[info]Processing {total_files} changed files...[/info]" + ) for fp in files_to_process: result = _safe_process_file(fp) if result is not None: @@ -391,9 +490,10 @@ def process_sequential(with_progress: bool) -> None: try: with ProcessPoolExecutor(max_workers=args.processes) as executor: if args.no_progress: - console.print( - f"[info]Processing {total_files} changed files...[/info]" - ) + if not args.quiet: + console.print( + f"[info]Processing {total_files} changed files...[/info]" + ) # Process in batches to manage memory for i in range(0, total_files, BATCH_SIZE): @@ -476,7 +576,7 @@ def process_sequential(with_progress: bool) -> None: console.print(f" ... and {len(failed_files) - 10} more") # Analysis phase - with console.status("[bold green]Grouping clones...", spinner="dots"): + if args.quiet: func_groups = build_groups(all_units) block_groups = build_block_groups(all_blocks) segment_groups = build_segment_groups(all_segments) @@ -484,6 +584,15 @@ def process_sequential(with_progress: bool) -> None: cache.save() except CacheError as e: console.print(f"[warning]Failed to save cache: {e}[/warning]") + else: + with console.status("[bold green]Grouping clones...", spinner="dots"): + func_groups = build_groups(all_units) + block_groups = build_block_groups(all_blocks) + segment_groups = build_segment_groups(all_segments) + try: + cache.save() + except CacheError as e: + console.print(f"[warning]Failed to save cache: {e}[/warning]") # Reporting func_clones_count = len(func_groups) @@ -493,8 +602,7 @@ def process_sequential(with_progress: bool) -> None: # Baseline Logic baseline_path = Path(args.baseline).expanduser().resolve() - # If user didn't specify path and default logic applies, baseline_path - # is now ./codeclone_baseline.json + # If user didn't specify path, the default is ./codeclone.baseline.json. baseline = Baseline(baseline_path) baseline_exists = baseline_path.exists() @@ -566,26 +674,28 @@ def process_sequential(with_progress: bool) -> None: new_clones_count = len(new_func) + len(new_block) # Summary Table - table = Table(title="Analysis Summary", border_style="blue") - table.add_column("Metric", style="cyan") - table.add_column("Value", style="bold white") - - table.add_row("Files Processed", str(changed_files_count)) - table.add_row("Total Function Clones", str(func_clones_count)) - table.add_row("Total Block Clones", str(block_clones_count)) - table.add_row("Total Segment Clones", str(segment_clones_count)) - - if baseline_exists: - style = "error" if new_clones_count > 0 else "success" - table.add_row( - "New Clones (vs Baseline)", f"[{style}]{new_clones_count}[/{style}]" - ) + if not args.quiet: + table = Table(title="Analysis Summary", border_style="blue") + table.add_column("Metric", style="cyan") + table.add_column("Value", style="bold white") + + table.add_row("Files Processed", str(changed_files_count)) + table.add_row("Total Function Clones", str(func_clones_count)) + table.add_row("Total Block Clones", str(block_clones_count)) + table.add_row("Total Segment Clones", str(segment_clones_count)) + + if baseline_exists: + style = "error" if new_clones_count > 0 else "success" + table.add_row( + "New Clones (vs Baseline)", f"[{style}]{new_clones_count}[/{style}]" + ) - console.print(table) + console.print(table) # Outputs - if args.html_out: - out = Path(args.html_out).expanduser().resolve() + html_report_path: str | None = None + if html_out_path: + out = html_out_path out.parent.mkdir(parents=True, exist_ok=True) out.write_text( build_html_report( @@ -598,19 +708,22 @@ def process_sequential(with_progress: bool) -> None: ), "utf-8", ) - console.print(f"[info]HTML report saved:[/info] {out}") + html_report_path = str(out) + if not args.quiet: + console.print(f"[info]HTML report saved:[/info] {out}") - if args.json_out: - out = Path(args.json_out).expanduser().resolve() + if json_out_path: + out = json_out_path out.parent.mkdir(parents=True, exist_ok=True) out.write_text( to_json_report(func_groups, block_groups, segment_groups), "utf-8", ) - console.print(f"[info]JSON report saved:[/info] {out}") + if not args.quiet: + console.print(f"[info]JSON report saved:[/info] {out}") - if args.text_out: - out = Path(args.text_out).expanduser().resolve() + if text_out_path: + out = text_out_path out.parent.mkdir(parents=True, exist_ok=True) out.write_text( "FUNCTION CLONES\n" @@ -621,15 +734,34 @@ def process_sequential(with_progress: bool) -> None: + to_text(segment_groups), "utf-8", ) - console.print(f"[info]Text report saved:[/info] {out}") + if not args.quiet: + console.print(f"[info]Text report saved:[/info] {out}") # Exit Codes if args.fail_on_new and (new_func or new_block): - console.print("\n[error]❌ FAILED: New code clones detected![/error]") - if new_func: - console.print(f" New Functions: {', '.join(sorted(new_func))}") - if new_block: - console.print(f" New Blocks: {', '.join(sorted(new_block))}") + default_report = Path(".cache/codeclone/report.html") + if html_report_path is None and default_report.exists(): + html_report_path = str(default_report) + + console.print("\n[error]FAILED: New code clones detected.[/error]") + console.print("\nSummary:") + console.print(f"- New function clone groups: {len(new_func)}") + console.print(f"- New block clone groups: {len(new_block)}") + if html_report_path: + console.print("\nSee detailed report:") + console.print(f" {html_report_path}") + console.print("\nTo accept these clones as technical debt, run:") + console.print(" codeclone . --update-baseline") + + if args.verbose: + if new_func: + console.print("\nDetails (function clone hashes):") + for h in sorted(new_func): + console.print(f"- {h}") + if new_block: + console.print("\nDetails (block clone hashes):") + for h in sorted(new_block): + console.print(f"- {h}") sys.exit(3) if 0 <= args.fail_threshold < (func_clones_count + block_clones_count): diff --git a/codeclone/html_report.py b/codeclone/html_report.py index 0d8c9c1..446f7d3 100644 --- a/codeclone/html_report.py +++ b/codeclone/html_report.py @@ -64,8 +64,9 @@ def __init__(self, maxsize: int = 128) -> None: self._get_lines_impl = lru_cache(maxsize=maxsize)(self._read_file_range) + @staticmethod def _read_file_range( - self, filepath: str, start_line: int, end_line: int + filepath: str, start_line: int, end_line: int ) -> tuple[str, ...]: if start_line < 1: start_line = 1 @@ -93,7 +94,7 @@ def _read_with_errors(errors: str) -> tuple[str, ...]: raise FileProcessingError(f"Cannot read {filepath}: {e}") from e def get_lines_range( - self, filepath: str, start_line: int, end_line: int + self, filepath: str, start_line: int, end_line: int ) -> tuple[str, ...]: return self._get_lines_impl(filepath, start_line, end_line) @@ -112,7 +113,7 @@ def _try_pygments(code: str) -> str | None: pygments = importlib.import_module("pygments") formatters = importlib.import_module("pygments.formatters") lexers = importlib.import_module("pygments.lexers") - except Exception: + except ImportError: return None highlight = pygments.highlight @@ -129,7 +130,7 @@ def _pygments_css(style_name: str) -> str: """ try: formatters = importlib.import_module("pygments.formatters") - except Exception: + except ImportError: return "" try: @@ -178,13 +179,13 @@ def _prefix_css(css: str, prefix: str) -> str: def _render_code_block( - *, - filepath: str, - start_line: int, - end_line: int, - file_cache: _FileCache, - context: int, - max_lines: int, + *, + filepath: str, + start_line: int, + end_line: int, + file_cache: _FileCache, + context: int, + max_lines: int, ) -> _Snippet: s = max(1, start_line - context) e = end_line + context @@ -236,13 +237,13 @@ def _group_sort_key(items: list[dict[str, Any]]) -> tuple[int, int]: def build_html_report( - *, - func_groups: dict[str, list[dict[str, Any]]], - block_groups: dict[str, list[dict[str, Any]]], - segment_groups: dict[str, list[dict[str, Any]]], - title: str = "CodeClone Report", - context_lines: int = 3, - max_snippet_lines: int = 220, + *, + func_groups: dict[str, list[dict[str, Any]]], + block_groups: dict[str, list[dict[str, Any]]], + segment_groups: dict[str, list[dict[str, Any]]], + title: str = "CodeClone Report", + context_lines: int = 3, + max_snippet_lines: int = 220, ) -> str: file_cache = _FileCache() @@ -269,74 +270,70 @@ def build_html_report( # ============================ # Icons (Inline SVG) # ============================ - ICON_SEARCH = ( - '' - '' - '' - "" - ) - ICON_X = ( - '' - '' - '' - "" - ) - ICON_CHEV_DOWN = ( - '' - '' - "" - ) - # ICON_CHEV_RIGHT = ( - # '' - # '' - # "" - # ) - ICON_THEME = ( - '' - '' - "" - ) - ICON_CHECK = ( - '' - '' - "" - ) - ICON_PREV = ( - '' - '' - "" - ) - ICON_NEXT = ( - '' - '' - "" - ) + def _svg_icon(size: int, stroke_width: str, body: str) -> str: + return ( + f'' + f"{body}" + ) + + ICONS = { + "search": _svg_icon( + 16, + "2.5", + '' + '', + ), + "clear": _svg_icon( + 16, + "2.5", + '' + '', + ), + "chev_down": _svg_icon( + 16, + "2.5", + '', + ), + # ICON_CHEV_RIGHT = ( + # '' + # '' + # "" + # ) + "theme": _svg_icon( + 16, + "2", + '', + ), + "check": _svg_icon( + 48, + "2", + '', + ), + "prev": _svg_icon( + 16, + "2", + '', + ), + "next": _svg_icon( + 16, + "2", + '', + ), + } # ---------------------------- # Section renderer # ---------------------------- def render_section( - section_id: str, - section_title: str, - groups: list[tuple[str, list[dict[str, Any]]]], - pill_cls: str, + section_id: str, + section_title: str, + groups: list[tuple[str, list[dict[str, Any]]]], + pill_cls: str, ) -> str: if not groups: return "" @@ -354,7 +351,7 @@ def render_section( aria-label="{_escape(section_title)} controls">
- {ICON_SEARCH} + {ICONS["search"]} {ICON_X} + title="Clear search">{ICONS["clear"]}
+ data-prev="{section_id}">{ICONS["prev"]} Page 1 + data-next="{section_id}">{ICONS["next"]}
-
+
@@ -1107,14 +1231,16 @@
+${report_meta_html} + -
+ @@ -1297,6 +1423,7 @@ const state = { theme: 'dark', commandPaletteOpen: false, + chartVisible: false, stats: { totalGroups: 0, totalItems: 0, @@ -1305,6 +1432,14 @@ } }; + function getPrimarySearchInput() { + const inputs = Array.from($$$$('.search')); + for (const input of inputs) { + if (input.offsetParent !== null) return input; + } + return inputs[0] || null; + } + // ========== Theme Management ========== function initTheme() { const stored = localStorage.getItem('codeclone_theme'); @@ -1320,6 +1455,7 @@ state.theme = state.theme === 'dark' ? 'light' : 'dark'; document.documentElement.setAttribute('data-theme', state.theme); localStorage.setItem('codeclone_theme', state.theme); + if (state.chartVisible) renderComplexityChart(); showToast('Theme switched to ' + state.theme, 'info'); } @@ -1362,6 +1498,8 @@ const palette = $$('#command-palette'); const input = $$('#command-input'); const results = $$('#command-results'); + if (!palette || !input || !results) return; + let selectedIndex = -1; const commands = [ { @@ -1381,13 +1519,13 @@ }, { icon: ICONS.stats, - label: 'Show Statistics', + label: 'Toggle Statistics', shortcut: '⌘S', action: () => showStats() }, { icon: ICONS.charts, - label: 'Show Charts', + label: 'Toggle Charts', shortcut: null, action: () => showCharts() }, @@ -1406,7 +1544,15 @@ icon: ICONS.search, label: 'Focus Search', shortcut: '/', - action: () => $$('.search')?.focus() + action: () => { + const search = getPrimarySearchInput(); + if (!search) { + showToast('Search is not available in this report', 'warning'); + return; + } + search.focus(); + if (typeof search.select === 'function') search.select(); + } }, { icon: ICONS.scrollTop, @@ -1447,9 +1593,29 @@ } ]; + function getVisibleCommandItems() { + return Array.from(results.querySelectorAll('.command-item')); + } + + function setSelected(index) { + const items = getVisibleCommandItems(); + if (!items.length) { + selectedIndex = -1; + return; + } + selectedIndex = (index + items.length) % items.length; + items.forEach((item, idx) => { + const isSelected = idx === selectedIndex; + item.classList.toggle('selected', isSelected); + item.setAttribute('aria-selected', isSelected ? 'true' : 'false'); + }); + items[selectedIndex].scrollIntoView({ block: 'nearest' }); + } + function renderCommands(filter = '') { const f = filter.toLowerCase(); results.innerHTML = ''; + let rendered = 0; commands.forEach(section => { const filtered = section.items.filter(item => @@ -1465,9 +1631,12 @@ section.section + '
'; - filtered.forEach((item, idx) => { + filtered.forEach((item) => { const btn = document.createElement('button'); - btn.className = 'command-item' + (idx === 0 && f === '' ? ' selected' : ''); + btn.className = 'command-item'; + btn.type = 'button'; + btn.setAttribute('role', 'option'); + btn.setAttribute('aria-selected', 'false'); btn.innerHTML = '
' + '' + item.icon + '' + @@ -1480,24 +1649,40 @@ item.action(); closeCommandPalette(); }); + btn.addEventListener('mouseenter', () => { + const items = getVisibleCommandItems(); + const hoverIndex = items.indexOf(btn); + if (hoverIndex >= 0) setSelected(hoverIndex); + }); sectionEl.appendChild(btn); + rendered += 1; }); results.appendChild(sectionEl); }); + + if (rendered === 0) { + const empty = document.createElement('div'); + empty.className = 'command-empty'; + empty.textContent = 'No matching commands'; + results.appendChild(empty); + } + setSelected(0); } function openCommandPalette() { state.commandPaletteOpen = true; palette.classList.add('show'); + palette.setAttribute('aria-hidden', 'false'); input.value = ''; - input.focus(); renderCommands(); + input.focus(); } function closeCommandPalette() { state.commandPaletteOpen = false; palette.classList.remove('show'); + palette.setAttribute('aria-hidden', 'true'); input.value = ''; } @@ -1510,14 +1695,23 @@ input?.addEventListener('keydown', (e) => { if (e.key === 'Escape') { + e.preventDefault(); closeCommandPalette(); + } else if (e.key === 'ArrowDown') { + e.preventDefault(); + setSelected(selectedIndex + 1); + } else if (e.key === 'ArrowUp') { + e.preventDefault(); + setSelected(selectedIndex - 1); } else if (e.key === 'Enter') { + e.preventDefault(); const selected = $$('.command-item.selected'); if (selected) selected.click(); } }); window.openCommandPalette = openCommandPalette; + window.closeCommandPalette = closeCommandPalette; } // ========== Statistics ========== @@ -1541,8 +1735,12 @@ function showStats() { const dashboard = $$('#stats-dashboard'); - if (dashboard.children.length > 0) { - dashboard.style.display = dashboard.style.display === 'none' ? 'grid' : 'none'; + if (!dashboard) return; + + const isVisible = dashboard.style.display !== 'none'; + if (isVisible) { + dashboard.style.display = 'none'; + showToast('Statistics hidden', 'info'); return; } @@ -1593,27 +1791,176 @@ showToast('Statistics displayed', 'success'); } + function collectSectionMetrics() { + const sections = [ + { id: 'functions', label: 'Function', colorVar: '--accent-primary' }, + { id: 'blocks', label: 'Block', colorVar: '--success' }, + { id: 'segments', label: 'Segment', colorVar: '--warning' } + ]; + return sections.map((section) => { + const groups = Array.from($$$$('.group[data-group="' + section.id + '"]')); + return { + label: section.label, + colorVar: section.colorVar, + value: groups.length + }; + }); + } + + function renderComplexityChart() { + const canvas = $$('#complexity-chart'); + if (!canvas) return; + const ctx = canvas.getContext('2d'); + if (!ctx) return; + + const rect = canvas.getBoundingClientRect(); + const width = Math.max(320, Math.floor(rect.width || 320)); + const height = Math.max(220, Math.floor(rect.height || 300)); + const ratio = Math.max(1, window.devicePixelRatio || 1); + + canvas.width = Math.floor(width * ratio); + canvas.height = Math.floor(height * ratio); + ctx.setTransform(ratio, 0, 0, ratio, 0, 0); + ctx.clearRect(0, 0, width, height); + + const styles = getComputedStyle(document.documentElement); + const textPrimary = styles.getPropertyValue('--text-primary').trim() || '#d1d5db'; + const textMuted = styles.getPropertyValue('--text-muted').trim() || '#9ca3af'; + const gridColor = styles.getPropertyValue('--border-subtle').trim() || '#334155'; + const sansFont = + 'ui-sans-serif, -apple-system, BlinkMacSystemFont, Segoe UI, sans-serif'; + + const metrics = collectSectionMetrics(); + const maxValue = Math.max(1, ...metrics.map((m) => m.value)); + const left = 52; + const right = 18; + const top = 16; + const bottom = 40; + const chartWidth = width - left - right; + const chartHeight = height - top - bottom; + + ctx.strokeStyle = gridColor; + ctx.lineWidth = 1; + for (let step = 0; step <= 4; step += 1) { + const y = top + (chartHeight * step) / 4; + ctx.beginPath(); + ctx.moveTo(left, y); + ctx.lineTo(left + chartWidth, y); + ctx.stroke(); + + const value = Math.round(maxValue - (maxValue * step) / 4); + ctx.fillStyle = textMuted; + ctx.font = '12px ' + sansFont; + ctx.textAlign = 'right'; + ctx.fillText(String(value), left - 8, y + 4); + } + + const slot = chartWidth / metrics.length; + const barWidth = Math.max(40, Math.min(90, slot * 0.56)); + metrics.forEach((metric, idx) => { + const x = left + slot * idx + (slot - barWidth) / 2; + const barHeight = (metric.value / maxValue) * chartHeight; + const y = top + chartHeight - barHeight; + const color = styles.getPropertyValue(metric.colorVar).trim() || '#4f46e5'; + + ctx.fillStyle = color; + ctx.fillRect(x, y, barWidth, barHeight); + + ctx.fillStyle = textPrimary; + ctx.font = '600 12px ' + sansFont; + ctx.textAlign = 'center'; + ctx.fillText(String(metric.value), x + barWidth / 2, y - 6); + + ctx.fillStyle = textMuted; + ctx.font = '12px ' + sansFont; + ctx.fillText(metric.label, x + barWidth / 2, top + chartHeight + 18); + }); + + if (metrics.every((metric) => metric.value === 0)) { + ctx.fillStyle = textMuted; + ctx.font = '14px ' + sansFont; + ctx.textAlign = 'center'; + ctx.fillText( + 'No clone groups to visualize', + left + chartWidth / 2, + top + chartHeight / 2 + ); + } + } + // ========== Charts ========== function showCharts() { const container = $$('#chart-container'); - container.style.display = container.style.display === 'none' ? 'block' : 'none'; - - if (container.style.display === 'block') { - // Simple ASCII chart for demonstration - showToast('Chart feature coming soon', 'info'); + if (!container) return; + const visible = container.style.display !== 'none'; + if (visible) { + container.style.display = 'none'; + state.chartVisible = false; + showToast('Charts hidden', 'info'); + return; } + container.style.display = 'block'; + state.chartVisible = true; + renderComplexityChart(); + showToast('Charts displayed', 'success'); + } + + function readReportMetaFromDom() { + const metaEl = $$('#report-meta'); + if (!metaEl) return {}; + + const boolAttr = (name) => { + const value = (metaEl.getAttribute(name) || '').toLowerCase(); + if (value === 'true') return true; + if (value === 'false') return false; + return null; + }; + const textAttr = (name) => { + const value = (metaEl.getAttribute(name) || '').trim(); + return value || null; + }; + const intAttr = (name) => { + const value = textAttr(name); + if (value === null) return null; + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : null; + }; + + return { + codeclone_version: textAttr('data-codeclone-version'), + python_version: textAttr('data-python-version'), + baseline_path: textAttr('data-baseline-path'), + baseline_version: textAttr('data-baseline-version'), + baseline_schema_version: intAttr('data-baseline-schema-version'), + baseline_python_version: textAttr('data-baseline-python-version'), + baseline_loaded: boolAttr('data-baseline-loaded'), + baseline_status: textAttr('data-baseline-status'), + cache_path: textAttr('data-cache-path'), + cache_used: boolAttr('data-cache-used') + }; } // ========== Export ========== function exportReport(format) { + calculateStats(); if (format === 'json') { + const groups = Array.from($$$$('.group')).map((groupEl) => ({ + section: groupEl.getAttribute('data-group') || '', + group_index: Number(groupEl.getAttribute('data-group-index') || '0'), + group_key: groupEl.getAttribute('data-group-key') || '', + items: Array.from(groupEl.querySelectorAll('.item')).map((itemEl) => ({ + qualname: itemEl.getAttribute('data-qualname') || '', + filepath: itemEl.getAttribute('data-filepath') || '', + start_line: Number(itemEl.getAttribute('data-start-line') || '0'), + end_line: Number(itemEl.getAttribute('data-end-line') || '0') + })) + })); const data = { generated: new Date().toISOString(), + source: 'CodeClone HTML report', + meta: readReportMetaFromDom(), stats: state.stats, - groups: Array.from($$$$('.group')).map(g => ({ - title: g.querySelector('.group-title')?.textContent, - items: g.querySelectorAll('.item').length - })) + groups }; const blob = new Blob([JSON.stringify(data, null, 2)], { @@ -1627,9 +1974,16 @@ URL.revokeObjectURL(url); showToast('Report exported as JSON', 'success'); - } else { - showToast('Export to ' + format.toUpperCase() + ' coming soon', 'info'); + return; } + + if (format === 'pdf') { + showToast('Opening print dialog for PDF export', 'info'); + window.print(); + return; + } + + showToast('Unsupported export format: ' + format, 'warning'); } // ========== Group Controls ========== @@ -1647,48 +2001,72 @@ // ========== Keyboard Shortcuts ========== document.addEventListener('keydown', (e) => { + const key = String(e.key || '').toLowerCase(); + // Command Palette: ⌘K or Ctrl+K - if ((e.metaKey || e.ctrlKey) && e.key === 'k') { + if ((e.metaKey || e.ctrlKey) && key === 'k') { e.preventDefault(); - window.openCommandPalette(); + if (state.commandPaletteOpen) { + window.closeCommandPalette?.(); + } else { + window.openCommandPalette?.(); + } + return; + } + + if (state.commandPaletteOpen) { + if (key === 'escape') { + e.preventDefault(); + window.closeCommandPalette?.(); + } return; } // Don't trigger if typing in input - if (e.target.matches('input, textarea')) return; + const target = e.target; + if ( + target && + typeof target.matches === 'function' && + target.matches('input, textarea, [contenteditable="true"]') + ) { + return; + } // / - Focus search - if (e.key === '/') { + if (key === '/') { e.preventDefault(); - $$('.search')?.focus(); + const search = getPrimarySearchInput(); + search?.focus(); + if (search && typeof search.select === 'function') search.select(); } // T - Toggle theme - if (e.key === 't' || e.key === 'T') { + if (key === 't') { e.preventDefault(); toggleTheme(); } // S - Show stats - if ((e.metaKey || e.ctrlKey) && e.key === 's') { + if ((e.metaKey || e.ctrlKey) && key === 's') { e.preventDefault(); showStats(); } // E - Export - if ((e.metaKey || e.ctrlKey) && e.key === 'e') { + if ((e.metaKey || e.ctrlKey) && key === 'e') { e.preventDefault(); exportReport('json'); } + // R - Refresh view + if ((e.metaKey || e.ctrlKey) && key === 'r') { + e.preventDefault(); + location.reload(); + } + // Escape - Close modals - if (e.key === 'Escape') { - if (state.commandPaletteOpen) { - const palette = $$('#command-palette'); - palette?.classList.remove('show'); - state.commandPaletteOpen = false; - } - const search = $$('.search'); + if (key === 'escape') { + const search = getPrimarySearchInput(); if (search && search.value) { search.value = ''; search.dispatchEvent(new Event('input', { bubbles: true })); @@ -1838,6 +2216,9 @@ // ========== Event Listeners ========== $$('#theme-toggle')?.addEventListener('click', toggleTheme); $$('#export-btn')?.addEventListener('click', () => exportReport('json')); + window.addEventListener('resize', () => { + if (state.chartVisible) renderComplexityChart(); + }); // ========== Initialize ========== initTheme(); diff --git a/docs/architecture.md b/docs/architecture.md index 03731d2..64de067 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -122,6 +122,17 @@ Detected clone groups can be: - exported as JSON, - rendered as an interactive HTML report. +All report formats include provenance metadata: + +- `codeclone_version` +- `python_version` +- `baseline_path` +- `baseline_version` +- `baseline_schema_version` +- `baseline_python_version` +- `baseline_loaded` +- `baseline_status` (`ok | missing | mismatch | invalid`) + --- ## CI Integration diff --git a/tests/test_cli_inprocess.py b/tests/test_cli_inprocess.py index 77e9cd7..2c07b51 100644 --- a/tests/test_cli_inprocess.py +++ b/tests/test_cli_inprocess.py @@ -619,6 +619,187 @@ def test_cli_main_outputs( assert "Text report saved" in out +def test_cli_reports_include_audit_metadata_ok( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + src = tmp_path / "a.py" + src.write_text("def f():\n return 1\n", "utf-8") + baseline_path = _write_baseline( + tmp_path / "baseline.json", + python_version=f"{sys.version_info.major}.{sys.version_info.minor}", + ) + html_out = tmp_path / "report.html" + json_out = tmp_path / "report.json" + text_out = tmp_path / "report.txt" + _patch_parallel(monkeypatch) + _run_main( + monkeypatch, + [ + str(tmp_path), + "--baseline", + str(baseline_path), + "--html", + str(html_out), + "--json", + str(json_out), + "--text", + str(text_out), + "--no-progress", + ], + ) + + payload = json.loads(json_out.read_text("utf-8")) + meta = payload["meta"] + assert meta["baseline_status"] == "ok" + assert meta["baseline_loaded"] is True + assert meta["baseline_version"] == __version__ + assert meta["baseline_schema_version"] == BASELINE_SCHEMA_VERSION + assert meta["baseline_path"] == str(baseline_path.resolve()) + assert "function_clones" in payload + assert "block_clones" in payload + assert "segment_clones" in payload + + text = text_out.read_text("utf-8") + assert "REPORT METADATA" in text + assert "Baseline status: ok" in text + assert f"Baseline schema version: {BASELINE_SCHEMA_VERSION}" in text + + html = html_out.read_text("utf-8") + assert "Report Provenance" in html + assert 'data-baseline-status="ok"' in html + assert "Baseline schema" in html + + +def test_cli_reports_include_audit_metadata_missing_baseline( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + src = tmp_path / "a.py" + src.write_text("def f():\n return 1\n", "utf-8") + json_out = tmp_path / "report.json" + _patch_parallel(monkeypatch) + _run_main( + monkeypatch, + [ + str(tmp_path), + "--baseline", + str(tmp_path / "missing-baseline.json"), + "--json", + str(json_out), + "--no-progress", + ], + ) + payload = json.loads(json_out.read_text("utf-8")) + meta = payload["meta"] + assert meta["baseline_status"] == "missing" + assert meta["baseline_loaded"] is False + assert meta["baseline_version"] is None + assert meta["baseline_schema_version"] is None + + +def test_cli_reports_include_audit_metadata_version_mismatch( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + src = tmp_path / "a.py" + src.write_text("def f():\n return 1\n", "utf-8") + baseline_path = _write_baseline( + tmp_path / "baseline.json", + python_version=f"{sys.version_info.major}.{sys.version_info.minor}", + baseline_version="0.0.0", + ) + json_out = tmp_path / "report.json" + _patch_parallel(monkeypatch) + with pytest.raises(SystemExit) as exc: + _run_main( + monkeypatch, + [ + str(tmp_path), + "--baseline", + str(baseline_path), + "--json", + str(json_out), + "--no-progress", + ], + ) + assert exc.value.code == 2 + out = capsys.readouterr().out + assert "Baseline version mismatch" in out + payload = json.loads(json_out.read_text("utf-8")) + meta = payload["meta"] + assert meta["baseline_status"] == "mismatch" + assert meta["baseline_loaded"] is True + assert meta["baseline_version"] == "0.0.0" + + +def test_cli_reports_include_audit_metadata_schema_mismatch( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + src = tmp_path / "a.py" + src.write_text("def f():\n return 1\n", "utf-8") + baseline_path = _write_baseline( + tmp_path / "baseline.json", + python_version=f"{sys.version_info.major}.{sys.version_info.minor}", + schema_version=999, + ) + json_out = tmp_path / "report.json" + _patch_parallel(monkeypatch) + with pytest.raises(SystemExit) as exc: + _run_main( + monkeypatch, + [ + str(tmp_path), + "--baseline", + str(baseline_path), + "--json", + str(json_out), + "--no-progress", + ], + ) + assert exc.value.code == 2 + out = capsys.readouterr().out + assert "Baseline schema version mismatch" in out + payload = json.loads(json_out.read_text("utf-8")) + meta = payload["meta"] + assert meta["baseline_status"] == "mismatch" + assert meta["baseline_loaded"] is True + assert meta["baseline_schema_version"] == 999 + + +def test_cli_reports_include_audit_metadata_invalid_baseline( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + src = tmp_path / "a.py" + src.write_text("def f():\n return 1\n", "utf-8") + baseline_path = tmp_path / "baseline.json" + baseline_path.write_text("{broken json", "utf-8") + json_out = tmp_path / "report.json" + _patch_parallel(monkeypatch) + with pytest.raises(SystemExit) as exc: + _run_main( + monkeypatch, + [ + str(tmp_path), + "--baseline", + str(baseline_path), + "--json", + str(json_out), + "--no-progress", + ], + ) + assert exc.value.code == 2 + out = capsys.readouterr().out + assert "Invalid baseline file" in out + payload = json.loads(json_out.read_text("utf-8")) + meta = payload["meta"] + assert meta["baseline_status"] == "invalid" + assert meta["baseline_loaded"] is False + + @pytest.mark.parametrize( ("flag", "bad_name", "label", "expected"), [ @@ -755,6 +936,34 @@ def f2(): assert baseline.exists() +def test_cli_update_baseline_with_invalid_existing_file( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + src = tmp_path / "a.py" + src.write_text("def f():\n return 1\n", "utf-8") + baseline_path = tmp_path / "baseline.json" + baseline_path.write_text("{broken json", "utf-8") + _patch_parallel(monkeypatch) + _run_main( + monkeypatch, + [ + str(tmp_path), + "--baseline", + str(baseline_path), + "--update-baseline", + "--no-progress", + ], + ) + out = capsys.readouterr().out + assert "Baseline updated" in out + assert "Invalid baseline file" not in out + payload = json.loads(baseline_path.read_text("utf-8")) + assert payload.get("baseline_version") == __version__ + assert payload.get("schema_version") == BASELINE_SCHEMA_VERSION + + def test_cli_baseline_missing_warning( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/test_html_report.py b/tests/test_html_report.py index 92ef697..0039364 100644 --- a/tests/test_html_report.py +++ b/tests/test_html_report.py @@ -51,6 +51,81 @@ def test_html_report_generation(tmp_path: Path) -> None: assert "codebox" in html +def test_html_report_group_and_item_metadata_attrs(tmp_path: Path) -> None: + f = tmp_path / "a.py" + f.write_text("def f():\n return 1\n", "utf-8") + html = build_html_report( + func_groups={ + "hash1": [ + { + "qualname": "pkg.mod:f", + "filepath": str(f), + "start_line": 1, + "end_line": 2, + } + ] + }, + block_groups={}, + segment_groups={}, + title="Attrs", + ) + assert 'data-group-key="hash1"' in html + assert 'hash1' in html + assert 'data-qualname="pkg.mod:f"' in html + assert 'data-filepath="' in html + assert 'data-start-line="1"' in html + assert 'data-end-line="2"' in html + + +def test_html_report_command_palette_full_actions_present() -> None: + html = build_html_report(func_groups={}, block_groups={}, segment_groups={}) + assert "Export as PDF" in html + assert "window.print();" in html + assert "No matching commands" in html + assert "ArrowDown" in html + assert "ArrowUp" in html + assert "Chart feature coming soon" not in html + assert "Clone Group Distribution" in html + assert 'id="stats-dashboard" style="display: none;"' in html + + +def test_html_report_includes_provenance_metadata(tmp_path: Path) -> None: + f = tmp_path / "a.py" + f.write_text("def f():\n return 1\n", "utf-8") + html = build_html_report( + func_groups={ + "h1": [ + { + "qualname": "f", + "filepath": str(f), + "start_line": 1, + "end_line": 2, + } + ] + }, + block_groups={}, + segment_groups={}, + report_meta={ + "codeclone_version": "1.3.0", + "python_version": "3.13", + "baseline_path": "/repo/codeclone.baseline.json", + "baseline_version": "1.3.0", + "baseline_schema_version": 1, + "baseline_python_version": "3.13", + "baseline_loaded": True, + "baseline_status": "ok", + "cache_path": "/repo/.cache/codeclone/cache.json", + "cache_used": True, + }, + ) + assert "Report Provenance" in html + assert "CodeClone" in html + assert "Baseline schema" in html + assert 'data-baseline-status="ok"' in html + assert "/repo/codeclone.baseline.json" in html + assert 'data-cache-used="true"' in html + + def test_file_cache_reads_ranges(tmp_path: Path) -> None: f = tmp_path / "sample.py" f.write_text("\n".join([f"line{i}" for i in range(1, 21)]), "utf-8") diff --git a/tests/test_report.py b/tests/test_report.py index 3f9d12b..5b725fa 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -12,7 +12,7 @@ prepare_segment_report_groups, to_json, to_json_report, - to_text, + to_text_report, ) @@ -67,15 +67,52 @@ def test_report_output_formats() -> None: }, ], } + meta = { + "codeclone_version": "1.3.0", + "python_version": "3.13", + "baseline_path": "/tmp/codeclone.baseline.json", + "baseline_version": "1.3.0", + "baseline_schema_version": 1, + "baseline_python_version": "3.13", + "baseline_loaded": True, + "baseline_status": "ok", + "cache_path": "/tmp/cache.json", + "cache_used": True, + } json_out = to_json(groups) - report_out = to_json_report(groups, groups, {}) - text_out = to_text(groups) + report_out = to_json_report(groups, groups, {}, meta) + text_out = to_text_report( + meta=meta, + func_groups=groups, + block_groups=groups, + segment_groups={}, + ) assert "group_count" in json_out - assert '"functions"' in report_out + assert '"meta"' in report_out + assert '"function_clones"' in report_out + assert '"baseline_schema_version": 1' in report_out + assert "REPORT METADATA" in text_out + assert "Baseline schema version: 1" in text_out assert "Clone group #1" in text_out +def test_to_text_report_handles_missing_meta_fields() -> None: + text_out = to_text_report( + meta={}, + func_groups={}, + block_groups={}, + segment_groups={}, + ) + assert "CodeClone version: n/a" in text_out + assert "Baseline status: n/a" in text_out + assert "Cache path:" not in text_out + assert "Cache used:" not in text_out + assert "FUNCTION CLONES\n(none)" in text_out + assert "BLOCK CLONES\n(none)" in text_out + assert "SEGMENT CLONES\n(none)" in text_out + + def test_segment_groups_internal_only() -> None: segments = [ { diff --git a/tests/test_security.py b/tests/test_security.py index 99bac8e..56d0b00 100644 --- a/tests/test_security.py +++ b/tests/test_security.py @@ -78,3 +78,5 @@ def test_html_report_escapes_user_content(tmp_path: Path) -> None: assert "" not in html assert "<script>alert(1)</script>" in html assert 'onmouseover="alert(1)' not in html + assert 'data-qualname="<script>alert(1)</script>"' in html + assert "" onmouseover="alert(1).py" in html From 96e80a078fdafce107c23742a6480eafec4d7832 Mon Sep 17 00:00:00 2001 From: Den Rozhnovskiy Date: Fri, 6 Feb 2026 20:54:13 +0500 Subject: [PATCH 16/28] fix(ui): prevent long clone paths from breaking report cards --- codeclone/html_report.py | 8 +++++- codeclone/templates.py | 55 +++++++++++++++++++++++++++++++--------- 2 files changed, 50 insertions(+), 13 deletions(-) diff --git a/codeclone/html_report.py b/codeclone/html_report.py index 81877f4..2a63d1e 100644 --- a/codeclone/html_report.py +++ b/codeclone/html_report.py @@ -536,9 +536,15 @@ def render_section( f'data-cache-used="{_escape(_meta_display(meta.get("cache_used")))}"', ] ) + + def _meta_row_class(label: str) -> str: + if label in {"Baseline", "Cache path"}: + return "meta-row meta-row-wide" + return "meta-row" + meta_rows_html = "".join( ( - '
' + f'
' f"
{_escape(label)}
" f"
{_escape(_meta_display(value))}
" "
" diff --git a/codeclone/templates.py b/codeclone/templates.py index a1c9252..66fecc8 100644 --- a/codeclone/templates.py +++ b/codeclone/templates.py @@ -133,6 +133,14 @@ padding: 0; } +.icon { + width: 1em; + height: 1em; + display: inline-block; + vertical-align: middle; + flex-shrink: 0; +} + html { scroll-behavior: smooth; } @@ -327,7 +335,7 @@ .meta-grid { margin: 0; display: grid; - grid-template-columns: repeat(auto-fit, minmax(250px, 1fr)); + grid-template-columns: repeat(auto-fit, minmax(320px, 1fr)); gap: 8px 16px; } @@ -352,7 +360,12 @@ color: var(--text-secondary); font-family: var(--font-mono); font-size: var(--text-xs); - word-break: break-all; + word-break: normal; + overflow-wrap: anywhere; +} + +.meta-row-wide { + grid-column: 1 / -1; } .section-head { @@ -1014,30 +1027,47 @@ } .stat-card { - padding: 20px; + padding: 16px 18px; background: var(--surface-1); border: 1px solid var(--border-subtle); border-radius: var(--radius-lg); - text-align: center; + text-align: left; + display: grid; + grid-template-columns: auto 1fr; + grid-template-rows: auto auto; + gap: 2px 12px; + align-items: center; } .stat-icon { - font-size: 32px; - margin-bottom: 8px; - opacity: 0.8; + grid-column: 1; + grid-row: 1 / span 2; + width: 38px; + height: 38px; + color: var(--text-tertiary); + opacity: 0.92; +} + +.stat-icon .icon { + width: 100%; + height: 100%; } .stat-value { + grid-column: 2; + grid-row: 1; font-size: var(--text-2xl); font-weight: 700; color: var(--text-primary); - margin-bottom: 4px; + line-height: 1.1; } .stat-label { + grid-column: 2; + grid-row: 2; font-size: var(--text-sm); - color: var(--text-tertiary); - margin-bottom: 8px; + color: var(--text-muted); + margin: 0; } .stat-trend { @@ -1832,6 +1862,7 @@ const metrics = collectSectionMetrics(); const maxValue = Math.max(1, ...metrics.map((m) => m.value)); + const yMax = Math.max(4, Math.ceil(maxValue / 4) * 4); const left = 52; const right = 18; const top = 16; @@ -1848,7 +1879,7 @@ ctx.lineTo(left + chartWidth, y); ctx.stroke(); - const value = Math.round(maxValue - (maxValue * step) / 4); + const value = Math.round(yMax - (yMax * step) / 4); ctx.fillStyle = textMuted; ctx.font = '12px ' + sansFont; ctx.textAlign = 'right'; @@ -1859,7 +1890,7 @@ const barWidth = Math.max(40, Math.min(90, slot * 0.56)); metrics.forEach((metric, idx) => { const x = left + slot * idx + (slot - barWidth) / 2; - const barHeight = (metric.value / maxValue) * chartHeight; + const barHeight = (metric.value / yMax) * chartHeight; const y = top + chartHeight - barHeight; const color = styles.getPropertyValue(metric.colorVar).trim() || '#4f46e5'; From 2d194a0ef00d88069619b2ea59ea568232ee1128 Mon Sep 17 00:00:00 2001 From: Den Rozhnovskiy Date: Fri, 6 Feb 2026 21:07:05 +0500 Subject: [PATCH 17/28] fix(ui): prevent long clone paths from breaking report cards --- codeclone/html_report.py | 60 +++++++++++- codeclone/templates.py | 207 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 253 insertions(+), 14 deletions(-) diff --git a/codeclone/html_report.py b/codeclone/html_report.py index 2a63d1e..c4badd4 100644 --- a/codeclone/html_report.py +++ b/codeclone/html_report.py @@ -542,19 +542,75 @@ def _meta_row_class(label: str) -> str: return "meta-row meta-row-wide" return "meta-row" + def _is_path_field(label: str) -> bool: + """Check if field contains a file path.""" + return label in {"Baseline", "Cache path"} + + def _is_bool_field(label: str) -> bool: + """Check if field contains a boolean value.""" + return label in {"Baseline loaded", "Cache used"} + + def _format_meta_value(label: str, value: Any) -> str: + """Format meta value with appropriate styling.""" + display_val = _meta_display(value) + + # Boolean fields with badge styling + if _is_bool_field(label): + if isinstance(value, bool): + badge_class = "meta-bool-true" if value else "meta-bool-false" + badge_text = "true" if value else "false" + return f'{badge_text}' + else: + return 'n/a' + + # Path fields with tooltip on hover + if _is_path_field(label) and display_val != "n/a": + escaped_path = _escape(display_val) + return ( + f'' + f"{escaped_path}" + f'{escaped_path}' + f"" + ) + + # Regular fields + return _escape(display_val) + meta_rows_html = "".join( ( f'
' f"
{_escape(label)}
" - f"
{_escape(_meta_display(value))}
" + f"
{_format_meta_value(label, value)}
" "
" ) for label, value in meta_rows ) + + # Chevron icon for toggle + chevron_icon = ( + '' + '' + "" + ) + + # Count non-n/a fields for badge + non_na_count = sum(1 for _, val in meta_rows if _meta_display(val) != "n/a") + report_meta_html = ( f'
' - '
Report Provenance
' + '
' + '
' + f'
{chevron_icon}
' + '

Report Provenance

' + f'{non_na_count} fields' + "
" + "
" + '
' + '
' f'
{meta_rows_html}
' + "
" + "
" "
" ) diff --git a/codeclone/templates.py b/codeclone/templates.py index 66fecc8..3880e1a 100644 --- a/codeclone/templates.py +++ b/codeclone/templates.py @@ -316,56 +316,210 @@ margin-top: 40px; } +/* Meta Panel - Collapsible Pro Design 2025 */ .meta-panel { margin-top: 22px; margin-bottom: 20px; - padding: 14px 16px; background: var(--surface-1); border: 1px solid var(--border-subtle); border-radius: var(--radius-lg); + overflow: hidden; + transition: all var(--transition-base); +} + +.meta-panel:hover { + border-color: var(--border-default); +} + +.meta-header { + display: flex; + align-items: center; + justify-content: space-between; + padding: 14px 16px; + cursor: pointer; + user-select: none; + transition: background var(--transition-fast); +} + +.meta-header:hover { + background: var(--surface-2); +} + +.meta-header-left { + display: flex; + align-items: center; + gap: 12px; +} + +.meta-toggle { + width: 18px; + height: 18px; + display: flex; + align-items: center; + justify-content: center; + transition: transform var(--transition-base); + color: var(--text-tertiary); + flex-shrink: 0; +} + +.meta-toggle.collapsed { + transform: rotate(-90deg); } .meta-title { font-size: var(--text-sm); font-weight: 700; color: var(--text-primary); - margin-bottom: 10px; + margin: 0; +} + +.meta-badge { + font-size: 11px; + font-weight: 500; + color: var(--text-tertiary); + background: var(--surface-2); + padding: 2px 8px; + border-radius: var(--radius-sm); + border: 1px solid var(--border-subtle); +} + +.meta-content { + overflow: hidden; + transition: max-height var(--transition-slow), opacity var(--transition-base); + max-height: 2000px; + opacity: 1; +} + +.meta-content.collapsed { + max-height: 0; + opacity: 0; +} + +.meta-body { + padding: 14px 16px 16px; + border-top: 1px solid var(--border-subtle); } .meta-grid { margin: 0; display: grid; - grid-template-columns: repeat(auto-fit, minmax(320px, 1fr)); - gap: 8px 16px; + grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); + gap: 12px; + row-gap: 10px; } .meta-row { - display: grid; - grid-template-columns: 160px 1fr; - gap: 8px; + display: flex; + flex-direction: column; + gap: 6px; + padding: 10px 12px; + background: var(--surface-0); + border: 1px solid var(--border-subtle); + border-radius: var(--radius); + transition: all var(--transition-fast); margin: 0; min-width: 0; } +.meta-row:hover { + background: var(--surface-2); + border-color: var(--border-default); +} + +.meta-row-wide { + grid-column: 1 / -1; +} + .meta-row dt { color: var(--text-muted); font-size: var(--text-xs); font-weight: 600; text-transform: uppercase; letter-spacing: 0.02em; + display: flex; + align-items: center; + gap: 6px; } .meta-row dd { margin: 0; - color: var(--text-secondary); + color: var(--text-primary); font-family: var(--font-mono); font-size: var(--text-xs); - word-break: normal; - overflow-wrap: anywhere; + line-height: 1.4; + position: relative; + overflow: hidden; } -.meta-row-wide { - grid-column: 1 / -1; +/* Path truncation with hover tooltip */ +.meta-path { + display: block; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + max-width: 100%; + cursor: help; + position: relative; + padding: 2px 0; +} + +.meta-path:hover { + color: var(--accent-primary); +} + +.meta-path-tooltip { + position: absolute; + left: 0; + bottom: calc(100% + 8px); + background: var(--surface-3); + color: var(--text-primary); + padding: 8px 12px; + border-radius: var(--radius); + font-size: var(--text-xs); + white-space: normal; + word-break: break-all; + border: 1px solid var(--border-default); + box-shadow: var(--elevation-3); + opacity: 0; + pointer-events: none; + transition: opacity var(--transition-fast); + z-index: 1000; + max-width: 600px; + min-width: 200px; +} + +.meta-path:hover .meta-path-tooltip { + opacity: 1; +} + +/* Boolean value badges */ +.meta-bool { + display: inline-flex; + align-items: center; + padding: 3px 8px; + border-radius: var(--radius-sm); + font-size: 11px; + font-weight: 500; + font-family: var(--font-sans); +} + +.meta-bool-true { + background: var(--success-subtle); + color: var(--success); + border: 1px solid rgba(16, 185, 129, 0.3); +} + +.meta-bool-false { + background: var(--surface-2); + color: var(--text-muted); + border: 1px solid var(--border-default); +} + +.meta-bool-na { + background: var(--surface-2); + color: var(--text-tertiary); + border: 1px solid var(--border-subtle); + font-style: italic; } .section-head { @@ -2251,9 +2405,38 @@ if (state.chartVisible) renderComplexityChart(); }); + // ========== Meta Panel Toggle ========== + function initMetaPanel() { + const header = $$('.meta-header'); + const toggle = $$('.meta-toggle'); + const content = $$('.meta-content'); + + if (!header || !toggle || !content) return; + + // Start collapsed by default to save space + const startCollapsed = true; + if (startCollapsed) { + toggle.classList.add('collapsed'); + content.classList.add('collapsed'); + } + + header.addEventListener('click', (e) => { + const isCollapsed = toggle.classList.contains('collapsed'); + + if (isCollapsed) { + toggle.classList.remove('collapsed'); + content.classList.remove('collapsed'); + } else { + toggle.classList.add('collapsed'); + content.classList.add('collapsed'); + } + }); + } + // ========== Initialize ========== initTheme(); initCommandPalette(); + initMetaPanel(); initSection('functions'); initSection('blocks'); initSection('segments'); From e99dc578fd6cca900d360e0cfdb68ca9b4bbe249 Mon Sep 17 00:00:00 2001 From: Den Rozhnovskiy Date: Fri, 6 Feb 2026 22:41:26 +0500 Subject: [PATCH 18/28] fix(core): fix detection integrity --- codeclone/meta_markers.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 codeclone/meta_markers.py diff --git a/codeclone/meta_markers.py b/codeclone/meta_markers.py new file mode 100644 index 0000000..cc9197f --- /dev/null +++ b/codeclone/meta_markers.py @@ -0,0 +1,13 @@ +""" +CodeClone — AST and CFG-based code clone detector for Python +focused on architectural duplication. + +Copyright (c) 2026 Den Rozhnovskiy +Licensed under the MIT License. +""" + +from __future__ import annotations + +# Contains `:` characters, so it cannot be produced by valid Python identifiers +# from parsed source code. It is only emitted programmatically by CFG builder. +CFG_META_PREFIX = "__CC_META__::" From f2cb95ec1f3046114cb57491bf5bdfe4b00a1c5b Mon Sep 17 00:00:00 2001 From: Den Rozhnovskiy Date: Fri, 6 Feb 2026 22:45:52 +0500 Subject: [PATCH 19/28] fix(core): fix detection integrity --- CHANGELOG.md | 22 ++- codeclone.baseline.json | 9 +- codeclone/cfg.py | 127 ++++++++++++--- codeclone/cli.py | 156 +++++++++++++++--- codeclone/normalize.py | 67 +++++--- codeclone/scanner.py | 2 +- tests/test_cfg.py | 317 ++++++++++++++++++++++++++++++++++-- tests/test_cli_inprocess.py | 89 +++++++++- tests/test_cli_smoke.py | 5 +- tests/test_cli_unit.py | 33 ++++ tests/test_extractor.py | 20 +++ tests/test_normalize.py | 136 +++++++++++++++- tests/test_scanner_extra.py | 12 ++ 13 files changed, 909 insertions(+), 86 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6898eec..118545c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,13 +12,17 @@ workflows that reuse old baselines. ### Clone Detection Accuracy - **Commutative normalization** - Canonicalized operand order for `+`, `*`, `|`, `&`, `^` when operands are free of side - effects, enabling safe detection of reordered expressions. + Canonicalized operand order for `+`, `*`, `|`, `&`, `^` only for provably safe constant + domains. Symbolic operands are no longer reordered. - **Local logical equivalence** Normalized `not (x in y)` to `x not in y` and `not (x is y)` to `x is not y` without De Morgan transformations or broader boolean rewrites. +- **Call-target preservation** + Kept symbolic call targets during normalization to avoid conflating different APIs + (for example, `load_user(...)` vs `delete_user(...)`). + ### CFG Precision - **Short‑circuit modeling** @@ -28,6 +32,19 @@ workflows that reuse old baselines. Linked `try/except` only to statements that may raise (calls, attribute access, indexing, `await`, `yield from`, `raise`) instead of blanket links. +### Detection Integrity + +- **Internal CFG marker hardening** + Switched CFG metadata markers to an internal namespace (`__CC_META__::...`) emitted as + synthetic AST names, preventing collisions with user string literals. + +- **Ordered control-flow semantics** + Modeled `break`/`continue` as terminating loop transitions, added correct `for/while ... else` + semantics, preserved `match case` evaluation order, and preserved `except` handler order. + +- **Deterministic traversal order** + Sorted Python file discovery to stabilize processing and report ordering across runs/platforms. + ### Segment‑Level Detection - **Window fingerprints** @@ -36,6 +53,7 @@ workflows that reuse old baselines. - **Candidate generation** Used an order‑insensitive signature for candidate grouping and a strict segment hash for final confirmation. Segment matches do not affect baseline or CI failure logic. + - **Noise reduction (report‑only)** Merged overlapping segment windows into a single span per function and suppressed boilerplate-only groups (attribute assignment wiring) with deterministic AST criteria. diff --git a/codeclone.baseline.json b/codeclone.baseline.json index ba7e83f..0692509 100644 --- a/codeclone.baseline.json +++ b/codeclone.baseline.json @@ -1,5 +1,12 @@ { - "functions": [], + "functions": [ + "056967a5e5569762522015b120810353dc84986a|20-49", + "2482e4ffa6c3c349be9626d4a95abbbb57193345|20-49", + "54b7b79d1ff3384fb96b3dd97944d7b67990b3f3|0-19", + "554bfacfa9bf1565bd2f5ea36e89b6efaee29c2d|0-19", + "b0070927d98fa8274982caef45107f4e6b4d6fef|0-19", + "c8e7da40a2dc106d1aa092044f88f01d9abae054|0-19" + ], "blocks": [], "python_version": "3.13", "baseline_version": "1.3.0", diff --git a/codeclone/cfg.py b/codeclone/cfg.py index e10fed3..625f1f8 100644 --- a/codeclone/cfg.py +++ b/codeclone/cfg.py @@ -10,9 +10,11 @@ import ast from collections.abc import Iterable +from dataclasses import dataclass from typing import Protocol, cast from .cfg_model import CFG, Block +from .meta_markers import CFG_META_PREFIX __all__ = ["CFG", "CFGBuilder"] @@ -26,17 +28,28 @@ class _TryLike(Protocol): finalbody: list[ast.stmt] +@dataclass(slots=True) +class _LoopContext: + continue_target: Block + break_target: Block + + +def _meta_expr(value: str) -> ast.Expr: + return ast.Expr(value=ast.Name(id=f"{CFG_META_PREFIX}{value}", ctx=ast.Load())) + + # ========================= # CFG Builder # ========================= class CFGBuilder: - __slots__ = ("cfg", "current") + __slots__ = ("_loop_stack", "cfg", "current") def __init__(self) -> None: self.cfg: CFG self.current: Block + self._loop_stack: list[_LoopContext] = [] def build( self, @@ -73,6 +86,12 @@ def _visit(self, stmt: ast.stmt) -> None: self.current.is_terminated = True self.current.add_successor(self.cfg.exit) + case ast.Break(): + self._visit_break(stmt) + + case ast.Continue(): + self._visit_continue(stmt) + case ast.If(): self._visit_if(stmt) @@ -123,23 +142,36 @@ def _visit_if(self, stmt: ast.If) -> None: def _visit_while(self, stmt: ast.While) -> None: cond_block = self.cfg.create_block() body_block = self.cfg.create_block() + else_block = self.cfg.create_block() if stmt.orelse else None after_block = self.cfg.create_block() self.current.add_successor(cond_block) self.current = cond_block - self._emit_condition(stmt.test, body_block, after_block) + false_target = else_block if else_block is not None else after_block + self._emit_condition(stmt.test, body_block, false_target) + self._loop_stack.append( + _LoopContext(continue_target=cond_block, break_target=after_block) + ) self.current = body_block self._visit_statements(stmt.body) if not self.current.is_terminated: self.current.add_successor(cond_block) + self._loop_stack.pop() + + if else_block is not None: + self.current = else_block + self._visit_statements(stmt.orelse) + if not self.current.is_terminated: + self.current.add_successor(after_block) self.current = after_block def _visit_for(self, stmt: ast.For | ast.AsyncFor) -> None: iter_block = self.cfg.create_block() body_block = self.cfg.create_block() + else_block = self.cfg.create_block() if stmt.orelse else None after_block = self.cfg.create_block() self.current.add_successor(iter_block) @@ -147,12 +179,24 @@ def _visit_for(self, stmt: ast.For | ast.AsyncFor) -> None: self.current = iter_block self.current.statements.append(ast.Expr(value=stmt.iter)) self.current.add_successor(body_block) - self.current.add_successor(after_block) + self.current.add_successor( + else_block if else_block is not None else after_block + ) + self._loop_stack.append( + _LoopContext(continue_target=iter_block, break_target=after_block) + ) self.current = body_block self._visit_statements(stmt.body) if not self.current.is_terminated: self.current.add_successor(iter_block) + self._loop_stack.pop() + + if else_block is not None: + self.current = else_block + self._visit_statements(stmt.orelse) + if not self.current.is_terminated: + self.current.add_successor(after_block) self.current = after_block @@ -188,19 +232,36 @@ def _visit_try(self, stmt: _TryLike) -> None: self.current.add_successor(try_entry) self.current = try_entry - handlers_blocks = [self.cfg.create_block() for _ in stmt.handlers] + handler_test_blocks = [self.cfg.create_block() for _ in stmt.handlers] + handler_body_blocks = [self.cfg.create_block() for _ in stmt.handlers] else_block = self.cfg.create_block() if stmt.orelse else None final_block = self.cfg.create_block() + for idx, (handler, test_block, body_block) in enumerate( + zip(stmt.handlers, handler_test_blocks, handler_body_blocks, strict=True) + ): + test_block.statements.append(_meta_expr(f"TRY_HANDLER_INDEX:{idx}")) + if handler.type is not None: + type_repr = ast.dump(handler.type, annotate_fields=False) + test_block.statements.append( + _meta_expr(f"TRY_HANDLER_TYPE:{type_repr}") + ) + else: + test_block.statements.append(_meta_expr("TRY_HANDLER_TYPE:BARE")) + test_block.add_successor(body_block) + if idx + 1 < len(handler_test_blocks): + test_block.add_successor(handler_test_blocks[idx + 1]) + else: + test_block.add_successor(final_block) + # Process each statement in try body # Link only statements that can raise to exception handlers for stmt_node in stmt.body: if self.current.is_terminated: break - if _stmt_can_raise(stmt_node): - for h_block in handlers_blocks: - self.current.add_successor(h_block) + if _stmt_can_raise(stmt_node) and handler_test_blocks: + self.current.add_successor(handler_test_blocks[0]) self._visit(stmt_node) @@ -212,11 +273,8 @@ def _visit_try(self, stmt: _TryLike) -> None: self.current.add_successor(final_block) # Process handlers - for handler, h_block in zip(stmt.handlers, handlers_blocks, strict=True): - self.current = h_block - if handler.type: - self.current.statements.append(ast.Expr(value=handler.type)) - + for handler, body_block in zip(stmt.handlers, handler_body_blocks, strict=True): + self.current = body_block self._visit_statements(handler.body) if not self.current.is_terminated: self.current.add_successor(final_block) @@ -236,25 +294,40 @@ def _visit_try(self, stmt: _TryLike) -> None: def _visit_match(self, stmt: ast.Match) -> None: self.current.statements.append(ast.Expr(value=stmt.subject)) - subject_block = self.current + previous_test_block: Block | None = None after_block = self.cfg.create_block() - for case_ in stmt.cases: - case_block = self.cfg.create_block() - subject_block.add_successor(case_block) + for idx, case_ in enumerate(stmt.cases): + case_test_block = self.cfg.create_block() + case_body_block = self.cfg.create_block() + + if previous_test_block is None: + self.current.add_successor(case_test_block) + else: + previous_test_block.add_successor(case_test_block) - self.current = case_block + case_test_block.statements.append(_meta_expr(f"MATCH_CASE_INDEX:{idx}")) # Record pattern structure pattern_repr = ast.dump(case_.pattern, annotate_fields=False) - self.current.statements.append( - ast.Expr(value=ast.Constant(value=f"PATTERN:{pattern_repr}")) + case_test_block.statements.append( + _meta_expr(f"MATCH_PATTERN:{pattern_repr}") ) + if case_.guard is not None: + case_test_block.statements.append(ast.Expr(value=case_.guard)) + + case_test_block.add_successor(case_body_block) + self.current = case_body_block self._visit_statements(case_.body) if not self.current.is_terminated: self.current.add_successor(after_block) + previous_test_block = case_test_block + + if previous_test_block is not None: + previous_test_block.add_successor(after_block) + self.current = after_block def _emit_condition( @@ -300,6 +373,22 @@ def _emit_boolop( self.current = current + def _visit_break(self, stmt: ast.Break) -> None: + self.current.statements.append(stmt) + self.current.is_terminated = True + if self._loop_stack: + self.current.add_successor(self._loop_stack[-1].break_target) + return + self.current.add_successor(self.cfg.exit) + + def _visit_continue(self, stmt: ast.Continue) -> None: + self.current.statements.append(stmt) + self.current.is_terminated = True + if self._loop_stack: + self.current.add_successor(self._loop_stack[-1].continue_target) + return + self.current.add_successor(self.cfg.exit) + def _stmt_can_raise(stmt: ast.stmt) -> bool: if isinstance(stmt, ast.Raise): diff --git a/codeclone/cli.py b/codeclone/cli.py index 0946102..47caa1f 100644 --- a/codeclone/cli.py +++ b/codeclone/cli.py @@ -208,6 +208,84 @@ def _build_report_meta( } +def _aligned_summary_lines(rows: list[tuple[str, int]]) -> list[str]: + if not rows: + return [] + label_width = max(len(label) for label, _ in rows) + value_width = max(len(str(value)) for _, value in rows) + return [ + f" {label + ':':<{label_width + 1}} {value:>{value_width}}" + for label, value in rows + ] + + +def _print_summary( + *, + quiet: bool, + files_found: int, + files_parsed: int, + cache_hits: int, + files_skipped: int, + func_clones_count: int, + block_clones_count: int, + segment_clones_count: int, + suppressed_segment_groups: int, + new_clones_count: int, +) -> None: + invariant_ok = files_found == (files_parsed + cache_hits + files_skipped) + input_rows = [ + ("Files found", files_found), + ("Files parsed", files_parsed), + ("Cache hits", cache_hits), + ("Files skipped", files_skipped), + ] + clone_rows = [ + ("Function", func_clones_count), + ("Block", block_clones_count), + ("Segment", segment_clones_count), + ("Suppressed", suppressed_segment_groups), + ("New (baseline)", new_clones_count), + ] + + if quiet: + console.print("Analysis Summary") + console.print( + "Input: " + f"found={files_found} parsed={files_parsed} " + f"cache_hits={cache_hits} skipped={files_skipped}" + ) + console.print( + "Clone groups: " + f"function={func_clones_count} block={block_clones_count} " + f"segment={segment_clones_count} suppressed={suppressed_segment_groups} " + f"new={new_clones_count}" + ) + else: + input_table = Table(title="Input", show_header=False) + input_table.add_column("Metric", style="dim") + input_table.add_column("Value", justify="right") + for label, value in input_rows: + input_table.add_row(label, str(value)) + + clone_table = Table(title="Clone groups", show_header=False) + clone_table.add_column("Metric", style="dim") + clone_table.add_column("Value", justify="right") + for label, value in clone_rows: + clone_table.add_row(label, str(value)) + + console.print("[bold]Analysis Summary[/bold]") + console.print("") + console.print(input_table) + console.print("") + console.print(clone_table) + + if not invariant_ok: + console.print( + "[warning]Summary accounting mismatch: " + "files_found != files_parsed + cache_hits + files_skipped[/warning]" + ) + + def main() -> None: ap = argparse.ArgumentParser( prog="codeclone", @@ -416,7 +494,10 @@ def main() -> None: all_units: list[dict[str, Any]] = [] all_blocks: list[dict[str, Any]] = [] all_segments: list[dict[str, Any]] = [] - changed_files_count = 0 + files_found = 0 + files_parsed = 0 + cache_hits = 0 + files_skipped = 0 files_to_process: list[str] = [] def _get_cached_entry( @@ -452,11 +533,14 @@ def _safe_future_result(future: Any) -> tuple[ProcessingResult | None, str | Non try: if args.quiet: for fp in iter_py_files(str(root_path)): + files_found += 1 stat, cached, warn = _get_cached_entry(fp) if warn: console.print(warn) + files_skipped += 1 continue if cached and cached.get("stat") == stat: + cache_hits += 1 all_units.extend( cast( list[dict[str, Any]], @@ -482,11 +566,14 @@ def _safe_future_result(future: Any) -> tuple[ProcessingResult | None, str | Non "[bold green]Discovering Python files...", spinner="dots" ): for fp in iter_py_files(str(root_path)): + files_found += 1 stat, cached, warn = _get_cached_entry(fp) if warn: console.print(warn) + files_skipped += 1 continue if cached and cached.get("stat") == stat: + cache_hits += 1 all_units.extend( cast( list[dict[str, Any]], @@ -518,7 +605,7 @@ def _safe_future_result(future: Any) -> tuple[ProcessingResult | None, str | Non if total_files > 0: def handle_result(result: ProcessingResult) -> None: - nonlocal changed_files_count + nonlocal files_parsed, files_skipped if result.success and result.stat: cache.put_file_entry( result.filepath, @@ -527,7 +614,7 @@ def handle_result(result: ProcessingResult) -> None: result.blocks or [], result.segments or [], ) - changed_files_count += 1 + files_parsed += 1 if result.units: all_units.extend([asdict(u) for u in result.units]) if result.blocks: @@ -535,9 +622,11 @@ def handle_result(result: ProcessingResult) -> None: if result.segments: all_segments.extend([asdict(s) for s in result.segments]) else: + files_skipped += 1 failed_files.append(f"{result.filepath}: {result.error}") def process_sequential(with_progress: bool) -> None: + nonlocal files_skipped if with_progress: with Progress( SpinnerColumn(), @@ -554,6 +643,9 @@ def process_sequential(with_progress: bool) -> None: result = _safe_process_file(fp) if result is not None: handle_result(result) + else: + files_skipped += 1 + failed_files.append(f"{fp}: worker failed") progress.advance(task) else: if not args.quiet: @@ -564,6 +656,9 @@ def process_sequential(with_progress: bool) -> None: result = _safe_process_file(fp) if result is not None: handle_result(result) + else: + files_skipped += 1 + failed_files.append(f"{fp}: worker failed") try: with ProcessPoolExecutor(max_workers=args.processes) as executor: @@ -587,16 +682,25 @@ def process_sequential(with_progress: bool) -> None: ) for fp in batch ] + future_to_fp = { + id(fut): fp for fut, fp in zip(futures, batch, strict=True) + } for future in as_completed(futures): + fp = future_to_fp[id(future)] result, err = _safe_future_result(future) if result is not None: handle_result(result) elif err is not None: + files_skipped += 1 + reason = err + failed_files.append(f"{fp}: {reason}") console.print( "[warning]Failed to process batch item: " - f"{err}[/warning]" + f"{reason}[/warning]" ) + else: + files_skipped += 1 else: with Progress( @@ -625,17 +729,27 @@ def process_sequential(with_progress: bool) -> None: ) for fp in batch ] + future_to_fp = { + id(fut): fp + for fut, fp in zip(futures, batch, strict=True) + } for future in as_completed(futures): + fp = future_to_fp[id(future)] result, err = _safe_future_result(future) if result is not None: handle_result(result) elif err is not None: + files_skipped += 1 + reason = err + failed_files.append(f"{fp}: {reason}") # Should rarely happen due to try/except # in process_file. console.print( - f"[warning]Worker failed: {err}[/warning]" + f"[warning]Worker failed: {reason}[/warning]" ) + else: + files_skipped += 1 progress.advance(task) except (OSError, RuntimeError, PermissionError) as e: console.print( @@ -796,26 +910,18 @@ def process_sequential(with_progress: bool) -> None: new_func, new_block = baseline.diff(func_groups, block_groups) new_clones_count = len(new_func) + len(new_block) - # Summary Table - if not args.quiet: - table = Table(title="Analysis Summary", border_style="blue") - table.add_column("Metric", style="cyan") - table.add_column("Value", style="bold white") - - table.add_row("Files Processed", str(changed_files_count)) - table.add_row("Total Function Clones", str(func_clones_count)) - table.add_row("Total Block Clones", str(block_clones_count)) - table.add_row("Total Segment Clones", str(segment_clones_count)) - if suppressed_segment_groups > 0: - table.add_row("Suppressed Segment Groups", str(suppressed_segment_groups)) - - if baseline_exists: - style = "error" if new_clones_count > 0 else "success" - table.add_row( - "New Clones (vs Baseline)", f"[{style}]{new_clones_count}[/{style}]" - ) - - console.print(table) + _print_summary( + quiet=args.quiet, + files_found=files_found, + files_parsed=files_parsed, + cache_hits=cache_hits, + files_skipped=files_skipped, + func_clones_count=func_clones_count, + block_clones_count=block_clones_count, + segment_clones_count=segment_clones_count, + suppressed_segment_groups=suppressed_segment_groups, + new_clones_count=new_clones_count, + ) # Outputs html_report_path: str | None = None diff --git a/codeclone/normalize.py b/codeclone/normalize.py index 8dbcd99..67e5eef 100644 --- a/codeclone/normalize.py +++ b/codeclone/normalize.py @@ -13,6 +13,9 @@ from ast import AST from collections.abc import Sequence from dataclasses import dataclass +from typing import cast + +from .meta_markers import CFG_META_PREFIX @dataclass(frozen=True, slots=True) @@ -71,7 +74,8 @@ def visit_arg(self, node: ast.arg) -> ast.arg: return node def visit_Name(self, node: ast.Name) -> ast.Name: - if self.cfg.normalize_names: + # Call targets are handled in visit_Call to avoid erasing callee symbols. + if self.cfg.normalize_names and not _is_semantic_marker_name(node.id): node.id = "_VAR_" return node @@ -87,6 +91,26 @@ def visit_Constant(self, node: ast.Constant) -> ast.Constant: node.value = "_CONST_" return node + def visit_Call(self, node: ast.Call) -> ast.Call: + node.func = self._visit_call_target(node.func) + node.args = [cast(ast.expr, self.visit(arg)) for arg in node.args] + for kw in node.keywords: + kw.value = cast(ast.expr, self.visit(kw.value)) + return node + + def _visit_call_target(self, node: ast.expr) -> ast.expr: + # Keep symbolic call targets intact to avoid conflating different APIs. + if isinstance(node, ast.Name): + return node + if isinstance(node, ast.Attribute): + value = node.value + if isinstance(value, (ast.Name, ast.Attribute)): + node.value = self._visit_call_target(value) + else: + node.value = cast(ast.expr, self.visit(value)) + return node + return cast(ast.expr, self.visit(node)) + def visit_AugAssign(self, node: ast.AugAssign) -> AST: # Normalize x += 1 to x = x + 1 # This allows detecting clones where one uses += and another uses = + @@ -147,8 +171,8 @@ def visit_BinOp(self, node: ast.BinOp) -> ast.AST: return new_node if not ( - _is_safe_commutative_operand(new_node.left) - and _is_safe_commutative_operand(new_node.right) + _is_proven_commutative_operand(new_node.left, new_node.op) + and _is_proven_commutative_operand(new_node.right, new_node.op) ): return new_node @@ -163,23 +187,26 @@ def _expr_sort_key(node: ast.AST) -> str: return ast.dump(node, annotate_fields=True, include_attributes=False) -def _is_safe_commutative_operand(node: ast.AST) -> bool: - disallowed = ( - ast.Call, - ast.Attribute, - ast.Subscript, - ast.Await, - ast.Yield, - ast.YieldFrom, - ast.Lambda, - ast.NamedExpr, - ast.ListComp, - ast.SetComp, - ast.DictComp, - ast.GeneratorExp, - ) - - return all(not isinstance(child, disallowed) for child in ast.walk(node)) +def _is_semantic_marker_name(name: str) -> bool: + return name.startswith(CFG_META_PREFIX) + + +def _is_proven_commutative_operand(node: ast.AST, op: ast.operator) -> bool: + if isinstance(node, ast.Constant): + return _is_proven_commutative_constant(node.value, op) + if isinstance(node, ast.BinOp) and type(node.op) is type(op): + return _is_proven_commutative_operand( + node.left, op + ) and _is_proven_commutative_operand(node.right, op) + return False + + +def _is_proven_commutative_constant(value: object, op: ast.operator) -> bool: + if isinstance(op, (ast.BitOr, ast.BitAnd, ast.BitXor)): + return isinstance(value, int) and not isinstance(value, bool) + if isinstance(op, (ast.Add, ast.Mult)): + return isinstance(value, (int, float, complex)) and not isinstance(value, bool) + return False def normalized_ast_dump(func_node: ast.AST, cfg: NormalizationConfig) -> str: diff --git a/codeclone/scanner.py b/codeclone/scanner.py index db34202..b2421bf 100644 --- a/codeclone/scanner.py +++ b/codeclone/scanner.py @@ -78,7 +78,7 @@ def iter_py_files( raise ValidationError(f"Cannot scan under sensitive directory: {root}") file_count = 0 - for p in rootp.rglob("*.py"): + for p in sorted(rootp.rglob("*.py"), key=lambda path: str(path)): # Verify path is actually under root (prevent symlink attacks) try: p.resolve().relative_to(rootp) diff --git a/tests/test_cfg.py b/tests/test_cfg.py index 03e0039..e58bae2 100644 --- a/tests/test_cfg.py +++ b/tests/test_cfg.py @@ -4,6 +4,10 @@ import pytest from codeclone.cfg import CFG, CFGBuilder +from codeclone.cfg_model import CFG as CFGModel +from codeclone.extractor import get_cfg_fingerprint +from codeclone.meta_markers import CFG_META_PREFIX +from codeclone.normalize import NormalizationConfig def build_cfg_from_source(source: str) -> CFG: @@ -30,6 +34,16 @@ def cfg_to_str(cfg: CFG) -> str: return "\n".join(lines) +def _const_meta_value(stmt: ast.stmt) -> str | None: + if not isinstance(stmt, ast.Expr): + return None + if not isinstance(stmt.value, ast.Name): + return None + if not isinstance(stmt.value.id, str): + return None + return stmt.value.id + + def test_cfg_if_else() -> None: source = """ def f(a): @@ -408,9 +422,8 @@ def f(): b for b in cfg.blocks if any( - isinstance(s, ast.Expr) - and isinstance(s.value, ast.Name) - and s.value.id == "ValueError" + (meta := _const_meta_value(s)) is not None + and meta.startswith(f"{CFG_META_PREFIX}TRY_HANDLER_TYPE:") for s in b.statements ) ] @@ -451,9 +464,8 @@ def f(): b for b in cfg.blocks if any( - isinstance(s, ast.Expr) - and isinstance(s.value, ast.Name) - and s.value.id == "ValueError" + (meta := _const_meta_value(s)) is not None + and meta.startswith(f"{CFG_META_PREFIX}TRY_HANDLER_TYPE:") for s in b.statements ) ] @@ -505,9 +517,8 @@ def f(): b for b in cfg.blocks if any( - isinstance(s, ast.Expr) - and isinstance(s.value, ast.Name) - and s.value.id == "ValueError" + (meta := _const_meta_value(s)) is not None + and meta.startswith(f"{CFG_META_PREFIX}TRY_HANDLER_TYPE:") for s in b.statements ) ] @@ -561,11 +572,291 @@ def f(x): patterns_found = [] for block in cfg.blocks: for stmt in block.statements: - if isinstance(stmt, ast.Expr) and isinstance(stmt.value, ast.Constant): - val = stmt.value.value - if isinstance(val, str) and val.startswith("PATTERN:"): - patterns_found.append(val) + meta = _const_meta_value(stmt) + if meta and meta.startswith(f"{CFG_META_PREFIX}MATCH_PATTERN:"): + patterns_found.append(meta) assert len(patterns_found) == 2 assert "MatchSequence" in patterns_found[0] assert "MatchMapping" in patterns_found[1] + + +def test_cfg_match_guard_affects_fingerprint() -> None: + code_with_guard = """ + def f(x): + match x: + case 1 if cond(): + return 1 + case _: + return 2 + """ + code_without_guard = """ + def f(x): + match x: + case 1: + return 1 + case _: + return 2 + """ + try: + func_with_guard = ast.parse(dedent(code_with_guard)).body[0] + func_without_guard = ast.parse(dedent(code_without_guard)).body[0] + except SyntaxError: + pytest.skip("Match syntax is unavailable") + + assert isinstance(func_with_guard, (ast.FunctionDef, ast.AsyncFunctionDef)) + assert isinstance(func_without_guard, (ast.FunctionDef, ast.AsyncFunctionDef)) + cfg = NormalizationConfig() + fp_with_guard = get_cfg_fingerprint(func_with_guard, cfg, "m:f") + fp_without_guard = get_cfg_fingerprint(func_without_guard, cfg, "m:g") + assert fp_with_guard != fp_without_guard + + +def test_cfg_match_case_order_affects_fingerprint() -> None: + source_a = """ + def f(x): + match x: + case 1: + return 1 + case _: + return 2 + """ + source_b = """ + def g(x): + match x: + case _: + return 2 + case 1: + return 1 + """ + try: + func_a = ast.parse(dedent(source_a)).body[0] + func_b = ast.parse(dedent(source_b)).body[0] + except SyntaxError: + pytest.skip("Match syntax is unavailable") + assert isinstance(func_a, (ast.FunctionDef, ast.AsyncFunctionDef)) + assert isinstance(func_b, (ast.FunctionDef, ast.AsyncFunctionDef)) + cfg = NormalizationConfig() + fp_a = get_cfg_fingerprint(func_a, cfg, "m:f") + fp_b = get_cfg_fingerprint(func_b, cfg, "m:g") + assert fp_a != fp_b + + +def test_cfg_try_handler_order_affects_fingerprint() -> None: + source_a = """ + def f(x): + try: + return risky(x) + except ValueError: + return 1 + except Exception: + return 2 + """ + source_b = """ + def g(x): + try: + return risky(x) + except Exception: + return 2 + except ValueError: + return 1 + """ + func_a = ast.parse(dedent(source_a)).body[0] + func_b = ast.parse(dedent(source_b)).body[0] + assert isinstance(func_a, (ast.FunctionDef, ast.AsyncFunctionDef)) + assert isinstance(func_b, (ast.FunctionDef, ast.AsyncFunctionDef)) + cfg = NormalizationConfig() + fp_a = get_cfg_fingerprint(func_a, cfg, "m:f") + fp_b = get_cfg_fingerprint(func_b, cfg, "m:g") + assert fp_a != fp_b + + +def test_cfg_for_else_affects_fingerprint() -> None: + with_else = """ + def f(xs): + for x in xs: + pass + else: + y = 1 + """ + without_else = """ + def f(xs): + for x in xs: + pass + """ + func_with_else = ast.parse(dedent(with_else)).body[0] + func_without_else = ast.parse(dedent(without_else)).body[0] + assert isinstance(func_with_else, (ast.FunctionDef, ast.AsyncFunctionDef)) + assert isinstance(func_without_else, (ast.FunctionDef, ast.AsyncFunctionDef)) + cfg = NormalizationConfig() + fp_with_else = get_cfg_fingerprint(func_with_else, cfg, "m:f") + fp_without_else = get_cfg_fingerprint(func_without_else, cfg, "m:g") + assert fp_with_else != fp_without_else + + +def test_cfg_while_else_affects_fingerprint() -> None: + with_else = """ + def f(flag): + while flag: + flag = False + else: + x = 1 + """ + without_else = """ + def f(flag): + while flag: + flag = False + """ + func_with_else = ast.parse(dedent(with_else)).body[0] + func_without_else = ast.parse(dedent(without_else)).body[0] + assert isinstance(func_with_else, (ast.FunctionDef, ast.AsyncFunctionDef)) + assert isinstance(func_without_else, (ast.FunctionDef, ast.AsyncFunctionDef)) + cfg = NormalizationConfig() + fp_with_else = get_cfg_fingerprint(func_with_else, cfg, "m:f") + fp_without_else = get_cfg_fingerprint(func_without_else, cfg, "m:g") + assert fp_with_else != fp_without_else + + +def test_cfg_break_terminates_block() -> None: + source = """ + def f(xs): + for x in xs: + break + y = 1 + """ + cfg = build_cfg_from_source(source) + break_blocks = [ + block + for block in cfg.blocks + if any(isinstance(stmt, ast.Break) for stmt in block.statements) + ] + assert len(break_blocks) == 1 + break_block = break_blocks[0] + assert break_block.is_terminated is True + assert all(not isinstance(stmt, ast.Assign) for stmt in break_block.statements) + + +def test_cfg_continue_terminates_block() -> None: + source = """ + def f(xs): + for x in xs: + continue + y = 1 + """ + cfg = build_cfg_from_source(source) + continue_blocks = [ + block + for block in cfg.blocks + if any(isinstance(stmt, ast.Continue) for stmt in block.statements) + ] + assert len(continue_blocks) == 1 + continue_block = continue_blocks[0] + assert continue_block.is_terminated is True + assert all(not isinstance(stmt, ast.Assign) for stmt in continue_block.statements) + + +def test_cfg_break_skips_for_else_block() -> None: + source = """ + def f(xs): + for x in xs: + break + else: + y = 1 + """ + cfg = build_cfg_from_source(source) + break_blocks = [ + b + for b in cfg.blocks + if any(isinstance(stmt, ast.Break) for stmt in b.statements) + ] + else_blocks = [ + b + for b in cfg.blocks + if any( + isinstance(stmt, ast.Assign) + and any(isinstance(t, ast.Name) and t.id == "y" for t in stmt.targets) + for stmt in b.statements + ) + ] + assert len(break_blocks) == 1 + assert len(else_blocks) == 1 + assert else_blocks[0] not in break_blocks[0].successors + + +def test_cfg_while_else_terminated_branch() -> None: + source = """ + def f(flag): + while flag: + flag = False + else: + return 1 + """ + cfg = build_cfg_from_source(source) + return_blocks = [ + block + for block in cfg.blocks + if any(isinstance(stmt, ast.Return) for stmt in block.statements) + ] + assert len(return_blocks) == 1 + assert return_blocks[0].is_terminated is True + assert cfg.exit in return_blocks[0].successors + + +def test_cfg_for_else_terminated_branch() -> None: + source = """ + def f(xs): + for x in xs: + pass + else: + return 1 + """ + cfg = build_cfg_from_source(source) + return_blocks = [ + block + for block in cfg.blocks + if any(isinstance(stmt, ast.Return) for stmt in block.statements) + ] + assert len(return_blocks) == 1 + assert return_blocks[0].is_terminated is True + assert cfg.exit in return_blocks[0].successors + + +def test_cfg_break_outside_loop_falls_back_to_exit() -> None: + builder = CFGBuilder() + builder.cfg = CFGModel("m:f") + builder.current = builder.cfg.entry + builder._visit_break(ast.Break()) + assert builder.current.is_terminated is True + assert builder.cfg.exit in builder.current.successors + + +def test_cfg_continue_outside_loop_falls_back_to_exit() -> None: + builder = CFGBuilder() + builder.cfg = CFGModel("m:f") + builder.current = builder.cfg.entry + builder._visit_continue(ast.Continue()) + assert builder.current.is_terminated is True + assert builder.cfg.exit in builder.current.successors + + +def test_cfg_match_with_empty_cases_ast() -> None: + # Defensive coverage for the fallback branch when Match.cases is empty. + match_stmt = ast.Match(subject=ast.Name(id="x", ctx=ast.Load()), cases=[]) + fn = ast.FunctionDef( + name="f", + args=ast.arguments( + posonlyargs=[], + args=[ast.arg(arg="x")], + kwonlyargs=[], + kw_defaults=[], + defaults=[], + ), + body=[match_stmt], + decorator_list=[], + ) + module = ast.Module(body=[fn], type_ignores=[]) + module = ast.fix_missing_locations(module) + func = module.body[0] + assert isinstance(func, ast.FunctionDef) + cfg = CFGBuilder().build("f", func) + assert len(cfg.blocks) >= 3 diff --git a/tests/test_cli_inprocess.py b/tests/test_cli_inprocess.py index 2c07b51..5ae2244 100644 --- a/tests/test_cli_inprocess.py +++ b/tests/test_cli_inprocess.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import re import sys from collections.abc import Callable, Iterable from dataclasses import dataclass @@ -178,6 +179,21 @@ def _assert_fail_on_new_summary(out: str, *, include_blocks: bool = True) -> Non assert "codeclone . --update-baseline" in out +def _summary_metric(out: str, label: str) -> int: + match = re.search(rf"{re.escape(label)}:\s+(\d+)", out) + if match: + return int(match.group(1)) + match = re.search(rf"{re.escape(label)}\s+[│|]\s+(\d+)", out) + assert match, f"summary label not found: {label}\n{out}" + return int(match.group(1)) + + +def _compact_summary_metric(out: str, key: str) -> int: + match = re.search(rf"{re.escape(key)}=(\d+)", out) + assert match, f"compact summary key not found: {key}\n{out}" + return int(match.group(1)) + + def test_cli_main_no_progress_parallel( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -209,7 +225,8 @@ def f2(): ], ) out = capsys.readouterr().out - assert "Total Function Clones" in out + assert "Clone groups" in out + assert "Function" in out def test_cli_default_cache_dir_uses_root( @@ -1454,7 +1471,14 @@ def test_cli_discovery_cache_hit( ], ) out = capsys.readouterr().out - assert "Files Processed" in out + files_found = _summary_metric(out, "Files found") + files_parsed = _summary_metric(out, "Files parsed") + cache_hits = _summary_metric(out, "Cache hits") + files_skipped = _summary_metric(out, "Files skipped") + assert files_found > 0 + assert cache_hits == files_found + assert files_parsed == 0 + assert files_found == files_parsed + cache_hits + files_skipped @pytest.mark.parametrize("extra_args", [["--no-progress"], ["--ci"]]) @@ -1482,6 +1506,17 @@ def _bad_stat(_path: str) -> dict[str, int]: _run_main(monkeypatch, args) out = capsys.readouterr().out assert "Skipping file" in out + if "--ci" in extra_args: + files_found = _compact_summary_metric(out, "found") + files_parsed = _compact_summary_metric(out, "parsed") + cache_hits = _compact_summary_metric(out, "cache_hits") + files_skipped = _compact_summary_metric(out, "skipped") + else: + files_found = _summary_metric(out, "Files found") + files_parsed = _summary_metric(out, "Files parsed") + cache_hits = _summary_metric(out, "Cache hits") + files_skipped = _summary_metric(out, "Files skipped") + assert files_found == files_parsed + cache_hits + files_skipped def test_cli_ci_discovery_cache_hit( @@ -1521,6 +1556,56 @@ def test_cli_ci_discovery_cache_hit( ) out = capsys.readouterr().out assert "CodeClone v" not in out + assert "Analysis Summary" in out + assert _compact_summary_metric(out, "found") == 1 + assert _compact_summary_metric(out, "parsed") == 0 + assert _compact_summary_metric(out, "cache_hits") == 1 + assert _compact_summary_metric(out, "skipped") == 0 + + +def test_cli_summary_cache_miss_metrics( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + src = tmp_path / "a.py" + src.write_text("def f():\n return 1\n", "utf-8") + _patch_parallel(monkeypatch) + _run_main(monkeypatch, [str(tmp_path), "--no-progress"]) + out = capsys.readouterr().out + files_found = _summary_metric(out, "Files found") + files_parsed = _summary_metric(out, "Files parsed") + cache_hits = _summary_metric(out, "Cache hits") + files_skipped = _summary_metric(out, "Files skipped") + assert files_found > 0 + assert files_parsed == files_found + assert cache_hits == 0 + assert files_skipped == 0 + assert files_found == files_parsed + cache_hits + files_skipped + + +def test_cli_summary_format_stable( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + src = tmp_path / "a.py" + src.write_text("def f():\n return 1\n", "utf-8") + _patch_parallel(monkeypatch) + _run_main(monkeypatch, [str(tmp_path), "--no-progress"]) + out = capsys.readouterr().out + assert "Analysis Summary" in out + assert "Input" in out + assert "Clone groups" in out + assert _summary_metric(out, "Files found") >= 0 + assert _summary_metric(out, "Files parsed") >= 0 + assert _summary_metric(out, "Cache hits") >= 0 + assert _summary_metric(out, "Files skipped") >= 0 + assert _summary_metric(out, "Function") >= 0 + assert _summary_metric(out, "Block") >= 0 + assert _summary_metric(out, "Segment") >= 0 + assert _summary_metric(out, "Suppressed") >= 0 + assert _summary_metric(out, "New (baseline)") >= 0 def test_cli_scan_failed(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/test_cli_smoke.py b/tests/test_cli_smoke.py index 603ea77..04ed73f 100644 --- a/tests/test_cli_smoke.py +++ b/tests/test_cli_smoke.py @@ -39,7 +39,8 @@ def f(): result = run_cli([str(tmp_path)], cwd=tmp_path) assert result.returncode == 0 - assert "Total Function Clones" in result.stdout + assert "Analysis Summary" in result.stdout + assert "Function" in result.stdout def test_cli_baseline_missing_warning(tmp_path: Path) -> None: @@ -107,4 +108,4 @@ def f2(): ] ) assert result2.returncode == 0 - assert "New Clones (vs Baseline)" in result2.stdout + assert "New (baseline)" in result2.stdout diff --git a/tests/test_cli_unit.py b/tests/test_cli_unit.py index dde4733..0cbdd91 100644 --- a/tests/test_cli_unit.py +++ b/tests/test_cli_unit.py @@ -111,3 +111,36 @@ def test_cli_help_text_consistency( assert "CI preset: --fail-on-new --no-color --quiet." in out assert "total clone groups (function +" in out assert "block) exceed this number" in out + + +def test_aligned_summary_lines_empty() -> None: + assert cli._aligned_summary_lines([]) == [] + + +def test_aligned_summary_lines_non_empty() -> None: + lines = cli._aligned_summary_lines([("Files found", 12), ("Cache hits", 3)]) + assert len(lines) == 2 + assert "Files found:" in lines[0] + assert "12" in lines[0] + assert "Cache hits:" in lines[1] + assert "3" in lines[1] + + +def test_print_summary_invariant_warning( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + monkeypatch.setattr(cli, "console", cli._make_console(no_color=True)) + cli._print_summary( + quiet=False, + files_found=1, + files_parsed=0, + cache_hits=0, + files_skipped=0, + func_clones_count=0, + block_clones_count=0, + segment_clones_count=0, + suppressed_segment_groups=0, + new_clones_count=0, + ) + out = capsys.readouterr().out + assert "Summary accounting mismatch" in out diff --git a/tests/test_extractor.py b/tests/test_extractor.py index 0a08a53..5ed5fea 100644 --- a/tests/test_extractor.py +++ b/tests/test_extractor.py @@ -203,6 +203,26 @@ def _fake_parse(_source: str, _timeout_s: int) -> ast.AST: assert segments == [] +def test_extract_distinguishes_call_targets() -> None: + src = """ +def load(x): + return load_user(x) + +def delete(x): + return delete_user(x) +""" + units, _, _ = extract_units_from_source( + source=src, + filepath="x.py", + module_name="mod", + cfg=NormalizationConfig(), + min_loc=1, + min_stmt=1, + ) + fps = {u.qualname: u.fingerprint for u in units} + assert fps["mod:load"] != fps["mod:delete"] + + def test_parse_limits_triggers_timeout(monkeypatch: pytest.MonkeyPatch) -> None: def _fake_signal(_sig: int, handler: Callable[[int, object], None] | None) -> None: if callable(handler): diff --git a/tests/test_normalize.py b/tests/test_normalize.py index 58ac6e8..0325468 100644 --- a/tests/test_normalize.py +++ b/tests/test_normalize.py @@ -3,6 +3,8 @@ import pytest +import codeclone.normalize as normalize_mod +from codeclone.meta_markers import CFG_META_PREFIX from codeclone.normalize import ( NormalizationConfig, normalized_ast_dump, @@ -130,6 +132,21 @@ def f(x): def test_normalization_commutative_binop_reorders() -> None: src1 = """ +def f(): + return 1 + 2 +""" + src2 = """ +def f(): + return 2 + 1 +""" + cfg = NormalizationConfig(normalize_constants=False) + a1 = ast.parse(src1).body[0] + a2 = ast.parse(src2).body[0] + assert normalized_ast_dump(a1, cfg) == normalized_ast_dump(a2, cfg) + + +def test_normalization_commutative_name_operands_not_reordered() -> None: + src1 = """ def f(): return a + b """ @@ -144,7 +161,7 @@ def f(): ) a1 = ast.parse(src1).body[0] a2 = ast.parse(src2).body[0] - assert normalized_ast_dump(a1, cfg) == normalized_ast_dump(a2, cfg) + assert normalized_ast_dump(a1, cfg) != normalized_ast_dump(a2, cfg) def test_normalization_commutative_binop_side_effects_not_reordered() -> None: @@ -166,6 +183,123 @@ def f(): assert normalized_ast_dump(a1, cfg) != normalized_ast_dump(a2, cfg) +def test_normalization_preserves_call_target_names() -> None: + src1 = """ +def f(x): + return load_user(x) +""" + src2 = """ +def f(x): + return delete_user(x) +""" + cfg = NormalizationConfig() + a1 = ast.parse(src1).body[0] + a2 = ast.parse(src2).body[0] + assert normalized_ast_dump(a1, cfg) != normalized_ast_dump(a2, cfg) + + +def test_normalization_preserves_call_target_attributes() -> None: + src1 = """ +def f(): + return svc.load_user() +""" + src2 = """ +def f(): + return svc.delete_user() +""" + cfg = NormalizationConfig() + a1 = ast.parse(src1).body[0] + a2 = ast.parse(src2).body[0] + assert normalized_ast_dump(a1, cfg) != normalized_ast_dump(a2, cfg) + + +def test_normalization_call_keyword_values_are_visited() -> None: + src1 = """ +def f(): + x = 1 + return process(payload=x) +""" + src2 = """ +def f(): + y = 2 + return process(payload=y) +""" + cfg = NormalizationConfig() + a1 = ast.parse(src1).body[0] + a2 = ast.parse(src2).body[0] + assert normalized_ast_dump(a1, cfg) == normalized_ast_dump(a2, cfg) + + +def test_normalization_preserves_attribute_call_target_with_call_value() -> None: + src1 = """ +def f(): + return factory_a().run() +""" + src2 = """ +def f(): + return factory_b().run() +""" + cfg = NormalizationConfig() + a1 = ast.parse(src1).body[0] + a2 = ast.parse(src2).body[0] + assert normalized_ast_dump(a1, cfg) != normalized_ast_dump(a2, cfg) + + +def test_normalization_non_name_non_attribute_call_target() -> None: + src1 = """ +def f(): + handlers = [run] + return handlers[0]() +""" + src2 = """ +def f(): + callbacks = [run] + return callbacks[0]() +""" + cfg = NormalizationConfig() + a1 = ast.parse(src1).body[0] + a2 = ast.parse(src2).body[0] + assert normalized_ast_dump(a1, cfg) == normalized_ast_dump(a2, cfg) + + +def test_commutative_operand_recursive_and_constant_guards() -> None: + nested = ast.parse("(1 + 2) + 3", mode="eval").body + assert isinstance(nested, ast.BinOp) + assert normalize_mod._is_proven_commutative_operand(nested, ast.Add()) + assert not normalize_mod._is_proven_commutative_constant(True, ast.BitOr()) + assert not normalize_mod._is_proven_commutative_constant("x", ast.Add()) + assert not normalize_mod._is_proven_commutative_constant(1, ast.Sub()) + + +def test_normalization_preserves_semantic_marker_names() -> None: + fn = ast.FunctionDef( + name="f", + args=ast.arguments( + posonlyargs=[], + args=[], + kwonlyargs=[], + kw_defaults=[], + defaults=[], + ), + body=[ + ast.Expr( + value=ast.Name( + id=f"{CFG_META_PREFIX}MATCH_PATTERN:MatchValue(Constant(value=1))", + ctx=ast.Load(), + ) + ) + ], + decorator_list=[], + ) + module = ast.Module(body=[fn], type_ignores=[]) + module = ast.fix_missing_locations(module) + node = module.body[0] + assert isinstance(node, ast.FunctionDef) + cfg = NormalizationConfig() + dump = normalized_ast_dump(node, cfg) + assert f"{CFG_META_PREFIX}MATCH_PATTERN:MatchValue(Constant(value=1))" in dump + + def test_normalization_non_commutative_binop_not_reordered() -> None: src1 = """ def f(): diff --git a/tests/test_scanner_extra.py b/tests/test_scanner_extra.py index 80ec544..0b15539 100644 --- a/tests/test_scanner_extra.py +++ b/tests/test_scanner_extra.py @@ -32,6 +32,18 @@ def test_iter_py_files_excludes(tmp_path: Path) -> None: assert str(skip) not in files +def test_iter_py_files_deterministic_sorted_order(tmp_path: Path) -> None: + z_file = tmp_path / "z.py" + z_file.write_text("z = 1\n", "utf-8") + a_dir = tmp_path / "pkg" + a_dir.mkdir() + a_file = a_dir / "a.py" + a_file.write_text("a = 1\n", "utf-8") + + files = list(iter_py_files(str(tmp_path))) + assert files == sorted(files) + + def test_module_name_from_path(tmp_path: Path) -> None: pkg = tmp_path / "pkg" pkg.mkdir() From e658f0bbfb3a9508db166cda7fc7ba99d93f50e2 Mon Sep 17 00:00:00 2001 From: Den Rozhnovskiy Date: Fri, 6 Feb 2026 23:02:28 +0500 Subject: [PATCH 20/28] refactor(cli): unify summary into one cache-aware table and centralize UI text contracts in ui_messages --- CHANGELOG.md | 5 + codeclone/cli.py | 344 +++++++++++++++++------------------- codeclone/ui_messages.py | 251 ++++++++++++++++++++++++++ tests/test_cli_inprocess.py | 65 ++++--- tests/test_cli_smoke.py | 4 +- tests/test_cli_unit.py | 48 +++-- 6 files changed, 501 insertions(+), 216 deletions(-) create mode 100644 codeclone/ui_messages.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 118545c..701d6cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,6 +78,11 @@ codeclone . --update-baseline - Added `--ci` preset (`--fail-on-new --no-color --quiet`). - Improved `--fail-on-new` output with aggregated counts and clear next steps. - Added strict report output extension validation (`.html`, `.json`, `.txt`). +- Centralized user-facing CLI strings in `codeclone/ui_messages.py` to keep text contracts + consistent and maintainable. +- Refined Summary output: a single compact table with deterministic metric order and + explicit `Files analyzed` semantics (cache-aware), plus stable compact output for + `--quiet/--ci`. ### HTML Report UI diff --git a/codeclone/cli.py b/codeclone/cli.py index 47caa1f..2d633de 100644 --- a/codeclone/cli.py +++ b/codeclone/cli.py @@ -18,9 +18,11 @@ TimeElapsedColumn, ) from rich.table import Table +from rich.text import Text from rich.theme import Theme from . import __version__ +from . import ui_messages as ui from .baseline import BASELINE_SCHEMA_VERSION, Baseline from .cache import Cache, CacheEntry, FileStat, file_stat_signature from .errors import CacheError @@ -162,8 +164,7 @@ def process_file( def print_banner() -> None: console.print( Panel.fit( - f"[bold white]CodeClone[/bold white] [dim]v{__version__}[/dim]\n" - "[italic]Architectural duplication detector[/italic]", + ui.banner_title(__version__), border_style="blue", padding=(0, 2), ) @@ -174,8 +175,9 @@ def _validate_output_path(path: str, *, expected_suffix: str, label: str) -> Pat out = Path(path).expanduser() if out.suffix.lower() != expected_suffix: console.print( - f"[error]Invalid {label} output extension: {out} " - f"(expected {expected_suffix}).[/error]" + ui.fmt_invalid_output_extension( + label=label, path=out, expected_suffix=expected_suffix + ) ) sys.exit(2) return out.resolve() @@ -208,22 +210,58 @@ def _build_report_meta( } -def _aligned_summary_lines(rows: list[tuple[str, int]]) -> list[str]: - if not rows: - return [] - label_width = max(len(label) for label, _ in rows) - value_width = max(len(str(value)) for _, value in rows) +def _summary_value_style(*, label: str, value: int) -> str: + if value == 0: + return "dim" + if label == ui.SUMMARY_LABEL_NEW_BASELINE: + return "bold red" + if label == ui.SUMMARY_LABEL_SUPPRESSED: + return "yellow" + return "bold green" + + +def _build_summary_rows( + *, + files_found: int, + files_analyzed: int, + cache_hits: int, + files_skipped: int, + func_clones_count: int, + block_clones_count: int, + segment_clones_count: int, + suppressed_segment_groups: int, + new_clones_count: int, +) -> list[tuple[str, int]]: return [ - f" {label + ':':<{label_width + 1}} {value:>{value_width}}" - for label, value in rows + (ui.SUMMARY_LABEL_FILES_FOUND, files_found), + (ui.SUMMARY_LABEL_FILES_ANALYZED, files_analyzed), + (ui.SUMMARY_LABEL_CACHE_HITS, cache_hits), + (ui.SUMMARY_LABEL_FILES_SKIPPED, files_skipped), + (ui.SUMMARY_LABEL_FUNCTION, func_clones_count), + (ui.SUMMARY_LABEL_BLOCK, block_clones_count), + (ui.SUMMARY_LABEL_SEGMENT, segment_clones_count), + (ui.SUMMARY_LABEL_SUPPRESSED, suppressed_segment_groups), + (ui.SUMMARY_LABEL_NEW_BASELINE, new_clones_count), ] +def _build_summary_table(rows: list[tuple[str, int]]) -> Table: + summary_table = Table(title=ui.SUMMARY_TITLE, show_header=True) + summary_table.add_column("Metric") + summary_table.add_column("Value", justify="right") + for label, value in rows: + summary_table.add_row( + label, + Text(str(value), style=_summary_value_style(label=label, value=value)), + ) + return summary_table + + def _print_summary( *, quiet: bool, files_found: int, - files_parsed: int, + files_analyzed: int, cache_hits: int, files_skipped: int, func_clones_count: int, @@ -232,58 +270,43 @@ def _print_summary( suppressed_segment_groups: int, new_clones_count: int, ) -> None: - invariant_ok = files_found == (files_parsed + cache_hits + files_skipped) - input_rows = [ - ("Files found", files_found), - ("Files parsed", files_parsed), - ("Cache hits", cache_hits), - ("Files skipped", files_skipped), - ] - clone_rows = [ - ("Function", func_clones_count), - ("Block", block_clones_count), - ("Segment", segment_clones_count), - ("Suppressed", suppressed_segment_groups), - ("New (baseline)", new_clones_count), - ] + invariant_ok = files_found == (files_analyzed + cache_hits + files_skipped) + rows = _build_summary_rows( + files_found=files_found, + files_analyzed=files_analyzed, + cache_hits=cache_hits, + files_skipped=files_skipped, + func_clones_count=func_clones_count, + block_clones_count=block_clones_count, + segment_clones_count=segment_clones_count, + suppressed_segment_groups=suppressed_segment_groups, + new_clones_count=new_clones_count, + ) if quiet: - console.print("Analysis Summary") + console.print(ui.SUMMARY_TITLE) console.print( - "Input: " - f"found={files_found} parsed={files_parsed} " - f"cache_hits={cache_hits} skipped={files_skipped}" + ui.fmt_summary_compact_input( + found=files_found, + analyzed=files_analyzed, + cache_hits=cache_hits, + skipped=files_skipped, + ) ) console.print( - "Clone groups: " - f"function={func_clones_count} block={block_clones_count} " - f"segment={segment_clones_count} suppressed={suppressed_segment_groups} " - f"new={new_clones_count}" + ui.fmt_summary_compact_clones( + function=func_clones_count, + block=block_clones_count, + segment=segment_clones_count, + suppressed=suppressed_segment_groups, + new=new_clones_count, + ) ) else: - input_table = Table(title="Input", show_header=False) - input_table.add_column("Metric", style="dim") - input_table.add_column("Value", justify="right") - for label, value in input_rows: - input_table.add_row(label, str(value)) - - clone_table = Table(title="Clone groups", show_header=False) - clone_table.add_column("Metric", style="dim") - clone_table.add_column("Value", justify="right") - for label, value in clone_rows: - clone_table.add_row(label, str(value)) - - console.print("[bold]Analysis Summary[/bold]") - console.print("") - console.print(input_table) - console.print("") - console.print(clone_table) + console.print(_build_summary_table(rows)) if not invariant_ok: - console.print( - "[warning]Summary accounting mismatch: " - "files_found != files_parsed + cache_hits + files_skipped[/warning]" - ) + console.print(f"[warning]{ui.WARN_SUMMARY_ACCOUNTING_MISMATCH}[/warning]") def main() -> None: @@ -295,8 +318,8 @@ def main() -> None: ap.add_argument( "--version", action="version", - version=f"CodeClone {__version__}", - help="Print the CodeClone version and exit.", + version=ui.version_output(__version__), + help=ui.HELP_VERSION, ) # Core Arguments @@ -305,7 +328,7 @@ def main() -> None: "root", nargs="?", default=".", - help="Project root directory to scan.", + help=ui.HELP_ROOT, ) # Tuning @@ -314,33 +337,33 @@ def main() -> None: "--min-loc", type=int, default=15, - help="Minimum Lines of Code (LOC) to consider.", + help=ui.HELP_MIN_LOC, ) tune_group.add_argument( "--min-stmt", type=int, default=6, - help="Minimum AST statements to consider.", + help=ui.HELP_MIN_STMT, ) tune_group.add_argument( "--processes", type=int, default=4, - help="Number of parallel worker processes.", + help=ui.HELP_PROCESSES, ) tune_group.add_argument( "--cache-path", dest="cache_path", metavar="FILE", default=None, - help="Path to the cache file. Default: /.cache/codeclone/cache.json.", + help=ui.HELP_CACHE_PATH, ) tune_group.add_argument( "--cache-dir", dest="cache_path", metavar="FILE", default=None, - help="Legacy alias for --cache-path.", + help=ui.HELP_CACHE_DIR_LEGACY, ) # Baseline & CI @@ -348,32 +371,29 @@ def main() -> None: ci_group.add_argument( "--baseline", default="codeclone.baseline.json", - help="Path to the baseline file (stored in repo).", + help=ui.HELP_BASELINE, ) ci_group.add_argument( "--update-baseline", action="store_true", - help="Overwrite the baseline file with current results.", + help=ui.HELP_UPDATE_BASELINE, ) ci_group.add_argument( "--fail-on-new", action="store_true", - help="Exit with error if NEW clones (not in baseline) are detected.", + help=ui.HELP_FAIL_ON_NEW, ) ci_group.add_argument( "--fail-threshold", type=int, default=-1, metavar="MAX_CLONES", - help=( - "Exit with error if total clone groups (function + block) " - "exceed this number." - ), + help=ui.HELP_FAIL_THRESHOLD, ) ci_group.add_argument( "--ci", action="store_true", - help="CI preset: --fail-on-new --no-color --quiet.", + help=ui.HELP_CI, ) # Output @@ -382,39 +402,39 @@ def main() -> None: "--html", dest="html_out", metavar="FILE", - help="Generate an HTML report to FILE.", + help=ui.HELP_HTML, ) out_group.add_argument( "--json", dest="json_out", metavar="FILE", - help="Generate a JSON report to FILE.", + help=ui.HELP_JSON, ) out_group.add_argument( "--text", dest="text_out", metavar="FILE", - help="Generate a text report to FILE.", + help=ui.HELP_TEXT, ) out_group.add_argument( "--no-progress", action="store_true", - help="Disable the progress bar (recommended for CI logs).", + help=ui.HELP_NO_PROGRESS, ) out_group.add_argument( "--no-color", action="store_true", - help="Disable ANSI colors in output.", + help=ui.HELP_NO_COLOR, ) out_group.add_argument( "--quiet", action="store_true", - help="Minimize output (still shows warnings and errors).", + help=ui.HELP_QUIET, ) out_group.add_argument( "--verbose", action="store_true", - help="Print detailed hash identifiers for new clones.", + help=ui.HELP_VERBOSE, ) cache_path_from_args = any( @@ -441,14 +461,14 @@ def main() -> None: try: root_path = Path(args.root).resolve() if not root_path.exists(): - console.print(f"[error]Root path does not exist: {root_path}[/error]") + console.print(ui.ERR_ROOT_NOT_FOUND.format(path=root_path)) sys.exit(1) except Exception as e: - console.print(f"[error]Invalid root path: {e}[/error]") + console.print(ui.ERR_INVALID_ROOT_PATH.format(error=e)) sys.exit(1) if not args.quiet: - console.print(f"[info]Scanning root:[/info] {root_path}") + console.print(ui.fmt_scanning_root(root_path)) html_out_path: Path | None = None json_out_path: Path | None = None @@ -479,12 +499,9 @@ def main() -> None: legacy_resolved = LEGACY_CACHE_PATH if legacy_resolved != cache_path: console.print( - "[warning]Legacy cache file found at: " - f"{legacy_resolved}.[/warning]\n" - "[warning]Cache is now stored per-project at: " - f"{cache_path}.[/warning]\n" - "[warning]Please delete the legacy cache file and add " - ".cache/ to .gitignore.[/warning]" + ui.fmt_legacy_cache_warning( + legacy_path=legacy_resolved, new_path=cache_path + ) ) cache = Cache(cache_path) cache.load() @@ -495,7 +512,7 @@ def main() -> None: all_blocks: list[dict[str, Any]] = [] all_segments: list[dict[str, Any]] = [] files_found = 0 - files_parsed = 0 + files_analyzed = 0 cache_hits = 0 files_skipped = 0 files_to_process: list[str] = [] @@ -506,7 +523,7 @@ def _get_cached_entry( try: stat = file_stat_signature(fp) except OSError as e: - return None, None, f"[warning]Skipping file {fp}: {e}[/warning]" + return None, None, ui.fmt_skipping_file(fp, e) cached = cache.get_file_entry(fp) return stat, cached, None @@ -520,7 +537,7 @@ def _safe_process_file(fp: str) -> ProcessingResult | None: args.min_stmt, ) except Exception as e: - console.print(f"[warning]Worker failed: {e}[/warning]") + console.print(ui.fmt_worker_failed(e)) return None def _safe_future_result(future: Any) -> tuple[ProcessingResult | None, str | None]: @@ -562,9 +579,7 @@ def _safe_future_result(future: Any) -> tuple[ProcessingResult | None, str | Non else: files_to_process.append(fp) else: - with console.status( - "[bold green]Discovering Python files...", spinner="dots" - ): + with console.status(ui.STATUS_DISCOVERING, spinner="dots"): for fp in iter_py_files(str(root_path)): files_found += 1 stat, cached, warn = _get_cached_entry(fp) @@ -595,7 +610,7 @@ def _safe_future_result(future: Any) -> tuple[ProcessingResult | None, str | Non else: files_to_process.append(fp) except Exception as e: - console.print(f"[error]Scan failed: {e}[/error]") + console.print(ui.ERR_SCAN_FAILED.format(error=e)) sys.exit(1) total_files = len(files_to_process) @@ -605,7 +620,7 @@ def _safe_future_result(future: Any) -> tuple[ProcessingResult | None, str | Non if total_files > 0: def handle_result(result: ProcessingResult) -> None: - nonlocal files_parsed, files_skipped + nonlocal files_analyzed, files_skipped if result.success and result.stat: cache.put_file_entry( result.filepath, @@ -614,7 +629,7 @@ def handle_result(result: ProcessingResult) -> None: result.blocks or [], result.segments or [], ) - files_parsed += 1 + files_analyzed += 1 if result.units: all_units.extend([asdict(u) for u in result.units]) if result.blocks: @@ -649,9 +664,7 @@ def process_sequential(with_progress: bool) -> None: progress.advance(task) else: if not args.quiet: - console.print( - f"[info]Processing {total_files} changed files...[/info]" - ) + console.print(ui.fmt_processing_changed(total_files)) for fp in files_to_process: result = _safe_process_file(fp) if result is not None: @@ -664,9 +677,7 @@ def process_sequential(with_progress: bool) -> None: with ProcessPoolExecutor(max_workers=args.processes) as executor: if args.no_progress: if not args.quiet: - console.print( - f"[info]Processing {total_files} changed files...[/info]" - ) + console.print(ui.fmt_processing_changed(total_files)) # Process in batches to manage memory for i in range(0, total_files, BATCH_SIZE): @@ -695,10 +706,7 @@ def process_sequential(with_progress: bool) -> None: files_skipped += 1 reason = err failed_files.append(f"{fp}: {reason}") - console.print( - "[warning]Failed to process batch item: " - f"{reason}[/warning]" - ) + console.print(ui.fmt_batch_item_failed(reason)) else: files_skipped += 1 @@ -745,23 +753,16 @@ def process_sequential(with_progress: bool) -> None: failed_files.append(f"{fp}: {reason}") # Should rarely happen due to try/except # in process_file. - console.print( - f"[warning]Worker failed: {reason}[/warning]" - ) + console.print(ui.fmt_worker_failed(reason)) else: files_skipped += 1 progress.advance(task) except (OSError, RuntimeError, PermissionError) as e: - console.print( - "[warning]Parallel processing unavailable, " - f"falling back to sequential: {e}[/warning]" - ) + console.print(ui.fmt_parallel_fallback(e)) process_sequential(with_progress=not args.no_progress) if failed_files: - console.print( - f"\n[warning]⚠ {len(failed_files)} files failed to process:[/warning]" - ) + console.print(ui.fmt_failed_files_header(len(failed_files))) for failure in failed_files[:10]: console.print(f" • {failure}") if len(failed_files) > 10: @@ -779,9 +780,9 @@ def process_sequential(with_progress: bool) -> None: try: cache.save() except CacheError as e: - console.print(f"[warning]Failed to save cache: {e}[/warning]") + console.print(ui.fmt_cache_save_failed(e)) else: - with console.status("[bold green]Grouping clones...", spinner="dots"): + with console.status(ui.STATUS_GROUPING, spinner="dots"): func_groups = build_groups(all_units) block_groups = build_block_groups(all_blocks) segment_groups = build_segment_groups(all_segments) @@ -791,7 +792,7 @@ def process_sequential(with_progress: bool) -> None: try: cache.save() except CacheError as e: - console.print(f"[warning]Failed to save cache: {e}[/warning]") + console.print(ui.fmt_cache_save_failed(e)) # Reporting func_clones_count = len(func_groups) @@ -815,11 +816,7 @@ def process_sequential(with_progress: bool) -> None: except ValueError as e: baseline_status = "invalid" if not args.update_baseline: - console.print( - "[error]Invalid baseline file.[/error]\n" - f"{e}\n" - "Please regenerate the baseline with --update-baseline." - ) + console.print(ui.fmt_invalid_baseline(e)) baseline_failure_code = 2 else: baseline_loaded = True @@ -828,19 +825,13 @@ def process_sequential(with_progress: bool) -> None: if baseline.baseline_version != __version__: baseline_status = "mismatch" if baseline.baseline_version is None: - console.print( - "[error]Baseline version mismatch.[/error]\n" - "Baseline version missing (legacy baseline format).\n" - f"Current version: {__version__}.\n" - "Please regenerate the baseline with --update-baseline." - ) + console.print(ui.fmt_baseline_version_missing(__version__)) else: console.print( - "[error]Baseline version mismatch.[/error]\n" - "Baseline was generated with CodeClone " - f"{baseline.baseline_version}.\n" - f"Current version: {__version__}.\n" - "Please regenerate the baseline with --update-baseline." + ui.fmt_baseline_version_mismatch( + baseline_version=baseline.baseline_version, + current_version=__version__, + ) ) baseline_failure_code = 2 if ( @@ -849,10 +840,10 @@ def process_sequential(with_progress: bool) -> None: ): baseline_status = "mismatch" console.print( - "[error]Baseline schema version mismatch.[/error]\n" - f"Baseline schema: {baseline.schema_version}. " - f"Current schema: {BASELINE_SCHEMA_VERSION}.\n" - "Please regenerate the baseline with --update-baseline." + ui.fmt_baseline_schema_mismatch( + baseline_schema=baseline.schema_version, + current_schema=BASELINE_SCHEMA_VERSION, + ) ) baseline_failure_code = 2 if not args.update_baseline and baseline.python_version: @@ -860,28 +851,17 @@ def process_sequential(with_progress: bool) -> None: if baseline.python_version != current_version: baseline_status = "mismatch" console.print( - "[warning]Baseline Python version mismatch.[/warning]\n" - "Baseline was generated with Python " - f"{baseline.python_version}.\n" - f"Current interpreter: Python {current_version}." + ui.fmt_baseline_python_mismatch( + baseline_python=baseline.python_version, + current_python=current_version, + ) ) if args.fail_on_new: - console.print( - "[error]Baseline checks require the same Python version to " - "ensure deterministic results. Please regenerate the " - "baseline " - "using the current interpreter.[/error]" - ) + console.print(ui.ERR_BASELINE_SAME_PYTHON_REQUIRED) baseline_failure_code = 2 else: if not args.update_baseline: - console.print( - "[warning]Baseline file not found at: [bold]" - f"{baseline_path}" - "[/bold][/warning]\n" - "[dim]Comparing against an empty baseline. " - "Use --update-baseline to create it.[/dim]" - ) + console.print(ui.fmt_path(ui.WARN_BASELINE_MISSING, baseline_path)) if args.update_baseline: new_baseline = Baseline.from_groups( @@ -893,7 +873,7 @@ def process_sequential(with_progress: bool) -> None: schema_version=BASELINE_SCHEMA_VERSION, ) new_baseline.save() - console.print(f"[success]✔ Baseline updated:[/success] {baseline_path}") + console.print(ui.fmt_path(ui.SUCCESS_BASELINE_UPDATED, baseline_path)) # When updating, we don't fail on new, we just saved the new state. # But we might still want to print the summary. @@ -913,7 +893,7 @@ def process_sequential(with_progress: bool) -> None: _print_summary( quiet=args.quiet, files_found=files_found, - files_parsed=files_parsed, + files_analyzed=files_analyzed, cache_hits=cache_hits, files_skipped=files_skipped, func_clones_count=func_clones_count, @@ -925,6 +905,17 @@ def process_sequential(with_progress: bool) -> None: # Outputs html_report_path: str | None = None + output_notice_printed = False + + def _print_output_notice(message: str) -> None: + nonlocal output_notice_printed + if args.quiet: + return + if not output_notice_printed: + console.print("") + output_notice_printed = True + console.print(message) + if html_out_path: out = html_out_path out.parent.mkdir(parents=True, exist_ok=True) @@ -941,8 +932,7 @@ def process_sequential(with_progress: bool) -> None: "utf-8", ) html_report_path = str(out) - if not args.quiet: - console.print(f"[info]HTML report saved:[/info] {out}") + _print_output_notice(ui.fmt_path(ui.INFO_HTML_REPORT_SAVED, out)) if json_out_path: out = json_out_path @@ -951,8 +941,7 @@ def process_sequential(with_progress: bool) -> None: to_json_report(func_groups, block_groups, segment_groups, report_meta), "utf-8", ) - if not args.quiet: - console.print(f"[info]JSON report saved:[/info] {out}") + _print_output_notice(ui.fmt_path(ui.INFO_JSON_REPORT_SAVED, out)) if text_out_path: out = text_out_path @@ -966,8 +955,7 @@ def process_sequential(with_progress: bool) -> None: ), "utf-8", ) - if not args.quiet: - console.print(f"[info]Text report saved:[/info] {out}") + _print_output_notice(ui.fmt_path(ui.INFO_TEXT_REPORT_SAVED, out)) if baseline_failure_code is not None: sys.exit(baseline_failure_code) @@ -978,40 +966,34 @@ def process_sequential(with_progress: bool) -> None: if html_report_path is None and default_report.exists(): html_report_path = str(default_report) - console.print("\n[error]FAILED: New code clones detected.[/error]") - console.print("\nSummary:") - console.print(f"- New function clone groups: {len(new_func)}") - console.print(f"- New block clone groups: {len(new_block)}") + console.print(f"\n{ui.FAIL_NEW_TITLE}") + console.print(f"\n{ui.FAIL_NEW_SUMMARY_TITLE}") + console.print(ui.FAIL_NEW_FUNCTION.format(count=len(new_func))) + console.print(ui.FAIL_NEW_BLOCK.format(count=len(new_block))) if html_report_path: - console.print("\nSee detailed report:") + console.print(f"\n{ui.FAIL_NEW_REPORT_TITLE}") console.print(f" {html_report_path}") - console.print("\nTo accept these clones as technical debt, run:") - console.print(" codeclone . --update-baseline") + console.print(f"\n{ui.FAIL_NEW_ACCEPT_TITLE}") + console.print(ui.FAIL_NEW_ACCEPT_COMMAND) if args.verbose: if new_func: - console.print("\nDetails (function clone hashes):") + console.print(f"\n{ui.FAIL_NEW_DETAIL_FUNCTION}") for h in sorted(new_func): console.print(f"- {h}") if new_block: - console.print("\nDetails (block clone hashes):") + console.print(f"\n{ui.FAIL_NEW_DETAIL_BLOCK}") for h in sorted(new_block): console.print(f"- {h}") sys.exit(3) if 0 <= args.fail_threshold < (func_clones_count + block_clones_count): total = func_clones_count + block_clones_count - console.print( - f"\n[error]❌ FAILED: Total clones ({total}) " - f"exceed threshold ({args.fail_threshold})![/error]" - ) + console.print(ui.fmt_fail_threshold(total=total, threshold=args.fail_threshold)) sys.exit(2) if not args.update_baseline and not args.fail_on_new and new_clones_count > 0: - console.print( - "\n[warning]New clones detected but --fail-on-new not set.[/warning]\n" - "Run with --update-baseline to accept them as technical debt." - ) + console.print(ui.WARN_NEW_CLONES_WITHOUT_FAIL) if __name__ == "__main__": diff --git a/codeclone/ui_messages.py b/codeclone/ui_messages.py new file mode 100644 index 0000000..8c0fa64 --- /dev/null +++ b/codeclone/ui_messages.py @@ -0,0 +1,251 @@ +from __future__ import annotations + +from pathlib import Path + +BANNER_SUBTITLE = "[italic]Architectural duplication detector[/italic]" + +HELP_VERSION = "Print the CodeClone version and exit." +HELP_ROOT = "Project root directory to scan." +HELP_MIN_LOC = "Minimum Lines of Code (LOC) to consider." +HELP_MIN_STMT = "Minimum AST statements to consider." +HELP_PROCESSES = "Number of parallel worker processes." +HELP_CACHE_PATH = "Path to the cache file. Default: /.cache/codeclone/cache.json." +HELP_CACHE_DIR_LEGACY = "Legacy alias for --cache-path." +HELP_BASELINE = "Path to the baseline file (stored in repo)." +HELP_UPDATE_BASELINE = "Overwrite the baseline file with current results." +HELP_FAIL_ON_NEW = "Exit with error if NEW clones (not in baseline) are detected." +HELP_FAIL_THRESHOLD = ( + "Exit with error if total clone groups (function + block) exceed this number." +) +HELP_CI = "CI preset: --fail-on-new --no-color --quiet." +HELP_HTML = "Generate an HTML report to FILE." +HELP_JSON = "Generate a JSON report to FILE." +HELP_TEXT = "Generate a text report to FILE." +HELP_NO_PROGRESS = "Disable the progress bar (recommended for CI logs)." +HELP_NO_COLOR = "Disable ANSI colors in output." +HELP_QUIET = "Minimize output (still shows warnings and errors)." +HELP_VERBOSE = "Print detailed hash identifiers for new clones." + +SUMMARY_TITLE = "Analysis Summary" +SUMMARY_LABEL_FILES_FOUND = "Files found" +SUMMARY_LABEL_FILES_ANALYZED = "Files analyzed" +SUMMARY_LABEL_CACHE_HITS = "Cache hits" +SUMMARY_LABEL_FILES_SKIPPED = "Files skipped" +SUMMARY_LABEL_FUNCTION = "Function clone groups" +SUMMARY_LABEL_BLOCK = "Block clone groups" +SUMMARY_LABEL_SEGMENT = "Segment clone groups" +SUMMARY_LABEL_SUPPRESSED = "Suppressed segment groups" +SUMMARY_LABEL_NEW_BASELINE = "New vs baseline" +SUMMARY_COMPACT_INPUT = ( + "Input: found={found} analyzed={analyzed} cache_hits={cache_hits} skipped={skipped}" +) +SUMMARY_COMPACT_CLONES = ( + "Clone groups: function={function} block={block} " + "segment={segment} suppressed={suppressed} new_vs_baseline={new}" +) +WARN_SUMMARY_ACCOUNTING_MISMATCH = ( + "Summary accounting mismatch: " + "files_found != files_analyzed + cache_hits + files_skipped" +) + +STATUS_DISCOVERING = "[bold green]Discovering Python files..." +STATUS_GROUPING = "[bold green]Grouping clones..." + +INFO_SCANNING_ROOT = "[info]Scanning root:[/info] {root}" +INFO_PROCESSING_CHANGED = "[info]Processing {count} changed files...[/info]" +INFO_HTML_REPORT_SAVED = "[info]HTML report saved:[/info] {path}" +INFO_JSON_REPORT_SAVED = "[info]JSON report saved:[/info] {path}" +INFO_TEXT_REPORT_SAVED = "[info]Text report saved:[/info] {path}" + +WARN_SKIPPING_FILE = "[warning]Skipping file {path}: {error}[/warning]" +WARN_WORKER_FAILED = "[warning]Worker failed: {error}[/warning]" +WARN_BATCH_ITEM_FAILED = "[warning]Failed to process batch item: {error}[/warning]" +WARN_PARALLEL_FALLBACK = ( + "[warning]Parallel processing unavailable, " + "falling back to sequential: {error}[/warning]" +) +WARN_FAILED_FILES_HEADER = "\n[warning]{count} files failed to process:[/warning]" +WARN_CACHE_SAVE_FAILED = "[warning]Failed to save cache: {error}[/warning]" + +ERR_INVALID_OUTPUT_EXT = ( + "[error]Invalid {label} output extension: {path} " + "(expected {expected_suffix}).[/error]" +) +ERR_ROOT_NOT_FOUND = "[error]Root path does not exist: {path}[/error]" +ERR_INVALID_ROOT_PATH = "[error]Invalid root path: {error}[/error]" +ERR_SCAN_FAILED = "[error]Scan failed: {error}[/error]" + +WARN_LEGACY_CACHE = ( + "[warning]Legacy cache file found at: {legacy_path}.[/warning]\n" + "[warning]Cache is now stored per-project at: {new_path}.[/warning]\n" + "[warning]Please delete the legacy cache file and add " + ".cache/ to .gitignore.[/warning]" +) + +ERR_INVALID_BASELINE = ( + "[error]Invalid baseline file.[/error]\n" + "{error}\n" + "Please regenerate the baseline with --update-baseline." +) +ERR_BASELINE_VERSION_MISMATCH = "[error]Baseline version mismatch.[/error]" +ERR_BASELINE_SCHEMA_MISMATCH = "[error]Baseline schema version mismatch.[/error]" +WARN_BASELINE_PYTHON_MISMATCH = "[warning]Baseline Python version mismatch.[/warning]" +ERR_BASELINE_SAME_PYTHON_REQUIRED = ( + "[error]Baseline checks require the same Python version to " + "ensure deterministic results. Please regenerate the " + "baseline using the current interpreter.[/error]" +) +WARN_BASELINE_MISSING = ( + "[warning]Baseline file not found at: [bold]{path}[/bold][/warning]\n" + "[dim]Comparing against an empty baseline. " + "Use --update-baseline to create it.[/dim]" +) +SUCCESS_BASELINE_UPDATED = "✔ Baseline updated: {path}" + +FAIL_NEW_TITLE = "[error]FAILED: New code clones detected.[/error]" +FAIL_NEW_SUMMARY_TITLE = "Summary:" +FAIL_NEW_FUNCTION = "- New function clone groups: {count}" +FAIL_NEW_BLOCK = "- New block clone groups: {count}" +FAIL_NEW_REPORT_TITLE = "See detailed report:" +FAIL_NEW_ACCEPT_TITLE = "To accept these clones as technical debt, run:" +FAIL_NEW_ACCEPT_COMMAND = " codeclone . --update-baseline" +FAIL_NEW_DETAIL_FUNCTION = "Details (function clone hashes):" +FAIL_NEW_DETAIL_BLOCK = "Details (block clone hashes):" + +ERR_FAIL_THRESHOLD = ( + "\n[error]❌ FAILED: Total clones ({total}) exceed threshold ({threshold})![/error]" +) +WARN_NEW_CLONES_WITHOUT_FAIL = ( + "\n[warning]New clones detected but --fail-on-new not set.[/warning]\n" + "Run with --update-baseline to accept them as technical debt." +) + + +def version_output(version: str) -> str: + return f"CodeClone {version}" + + +def banner_title(version: str) -> str: + return ( + f"[bold white]CodeClone[/bold white] [dim]v{version}[/dim]\n{BANNER_SUBTITLE}" + ) + + +def fmt_invalid_output_extension( + *, label: str, path: Path, expected_suffix: str +) -> str: + return ERR_INVALID_OUTPUT_EXT.format( + label=label, path=path, expected_suffix=expected_suffix + ) + + +def fmt_scanning_root(root: Path) -> str: + return INFO_SCANNING_ROOT.format(root=root) + + +def fmt_processing_changed(count: int) -> str: + return INFO_PROCESSING_CHANGED.format(count=count) + + +def fmt_skipping_file(path: str, error: object) -> str: + return WARN_SKIPPING_FILE.format(path=path, error=error) + + +def fmt_worker_failed(error: object) -> str: + return WARN_WORKER_FAILED.format(error=error) + + +def fmt_batch_item_failed(error: object) -> str: + return WARN_BATCH_ITEM_FAILED.format(error=error) + + +def fmt_parallel_fallback(error: object) -> str: + return WARN_PARALLEL_FALLBACK.format(error=error) + + +def fmt_failed_files_header(count: int) -> str: + return WARN_FAILED_FILES_HEADER.format(count=count) + + +def fmt_cache_save_failed(error: object) -> str: + return WARN_CACHE_SAVE_FAILED.format(error=error) + + +def fmt_legacy_cache_warning(*, legacy_path: Path, new_path: Path) -> str: + return WARN_LEGACY_CACHE.format(legacy_path=legacy_path, new_path=new_path) + + +def fmt_invalid_baseline(error: object) -> str: + return ERR_INVALID_BASELINE.format(error=error) + + +def fmt_baseline_version_missing(current_version: str) -> str: + return ( + f"{ERR_BASELINE_VERSION_MISMATCH}\n" + "Baseline version missing (legacy baseline format).\n" + f"Current version: {current_version}.\n" + "Please regenerate the baseline with --update-baseline." + ) + + +def fmt_baseline_version_mismatch( + *, baseline_version: str, current_version: str +) -> str: + return ( + f"{ERR_BASELINE_VERSION_MISMATCH}\n" + "Baseline was generated with CodeClone " + f"{baseline_version}.\n" + f"Current version: {current_version}.\n" + "Please regenerate the baseline with --update-baseline." + ) + + +def fmt_baseline_schema_mismatch(*, baseline_schema: int, current_schema: int) -> str: + return ( + f"{ERR_BASELINE_SCHEMA_MISMATCH}\n" + f"Baseline schema: {baseline_schema}. " + f"Current schema: {current_schema}.\n" + "Please regenerate the baseline with --update-baseline." + ) + + +def fmt_baseline_python_mismatch(*, baseline_python: str, current_python: str) -> str: + return ( + f"{WARN_BASELINE_PYTHON_MISMATCH}\n" + "Baseline was generated with Python " + f"{baseline_python}.\n" + f"Current interpreter: Python {current_python}." + ) + + +def fmt_path(template: str, path: Path) -> str: + return template.format(path=path) + + +def fmt_summary_compact_input( + *, found: int, analyzed: int, cache_hits: int, skipped: int +) -> str: + return SUMMARY_COMPACT_INPUT.format( + found=found, analyzed=analyzed, cache_hits=cache_hits, skipped=skipped + ) + + +def fmt_summary_compact_clones( + *, + function: int, + block: int, + segment: int, + suppressed: int, + new: int, +) -> str: + return SUMMARY_COMPACT_CLONES.format( + function=function, + block=block, + segment=segment, + suppressed=suppressed, + new=new, + ) + + +def fmt_fail_threshold(*, total: int, threshold: int) -> str: + return ERR_FAIL_THRESHOLD.format(total=total, threshold=threshold) diff --git a/tests/test_cli_inprocess.py b/tests/test_cli_inprocess.py index 5ae2244..96daafe 100644 --- a/tests/test_cli_inprocess.py +++ b/tests/test_cli_inprocess.py @@ -225,8 +225,8 @@ def f2(): ], ) out = capsys.readouterr().out - assert "Clone groups" in out - assert "Function" in out + assert "Analysis Summary" in out + assert "Function clone groups" in out def test_cli_default_cache_dir_uses_root( @@ -631,9 +631,10 @@ def test_cli_main_outputs( assert json_out.exists() assert text_out.exists() out = capsys.readouterr().out - assert "HTML report saved" in out - assert "JSON report saved" in out - assert "Text report saved" in out + assert "HTML report saved:" in out + assert "JSON report saved:" in out + assert "Text report saved:" in out + assert out.index("Analysis Summary") < out.index("HTML report saved:") def test_cli_reports_include_audit_metadata_ok( @@ -1472,13 +1473,13 @@ def test_cli_discovery_cache_hit( ) out = capsys.readouterr().out files_found = _summary_metric(out, "Files found") - files_parsed = _summary_metric(out, "Files parsed") + files_analyzed = _summary_metric(out, "Files analyzed") cache_hits = _summary_metric(out, "Cache hits") files_skipped = _summary_metric(out, "Files skipped") assert files_found > 0 assert cache_hits == files_found - assert files_parsed == 0 - assert files_found == files_parsed + cache_hits + files_skipped + assert files_analyzed == 0 + assert files_found == files_analyzed + cache_hits + files_skipped @pytest.mark.parametrize("extra_args", [["--no-progress"], ["--ci"]]) @@ -1508,15 +1509,15 @@ def _bad_stat(_path: str) -> dict[str, int]: assert "Skipping file" in out if "--ci" in extra_args: files_found = _compact_summary_metric(out, "found") - files_parsed = _compact_summary_metric(out, "parsed") + files_analyzed = _compact_summary_metric(out, "analyzed") cache_hits = _compact_summary_metric(out, "cache_hits") files_skipped = _compact_summary_metric(out, "skipped") else: files_found = _summary_metric(out, "Files found") - files_parsed = _summary_metric(out, "Files parsed") + files_analyzed = _summary_metric(out, "Files analyzed") cache_hits = _summary_metric(out, "Cache hits") files_skipped = _summary_metric(out, "Files skipped") - assert files_found == files_parsed + cache_hits + files_skipped + assert files_found == files_analyzed + cache_hits + files_skipped def test_cli_ci_discovery_cache_hit( @@ -1557,8 +1558,10 @@ def test_cli_ci_discovery_cache_hit( out = capsys.readouterr().out assert "CodeClone v" not in out assert "Analysis Summary" in out + assert "Analyzing" not in out + assert "\x1b[" not in out assert _compact_summary_metric(out, "found") == 1 - assert _compact_summary_metric(out, "parsed") == 0 + assert _compact_summary_metric(out, "analyzed") == 0 assert _compact_summary_metric(out, "cache_hits") == 1 assert _compact_summary_metric(out, "skipped") == 0 @@ -1574,14 +1577,14 @@ def test_cli_summary_cache_miss_metrics( _run_main(monkeypatch, [str(tmp_path), "--no-progress"]) out = capsys.readouterr().out files_found = _summary_metric(out, "Files found") - files_parsed = _summary_metric(out, "Files parsed") + files_analyzed = _summary_metric(out, "Files analyzed") cache_hits = _summary_metric(out, "Cache hits") files_skipped = _summary_metric(out, "Files skipped") assert files_found > 0 - assert files_parsed == files_found + assert files_analyzed == files_found assert cache_hits == 0 assert files_skipped == 0 - assert files_found == files_parsed + cache_hits + files_skipped + assert files_found == files_analyzed + cache_hits + files_skipped def test_cli_summary_format_stable( @@ -1595,17 +1598,33 @@ def test_cli_summary_format_stable( _run_main(monkeypatch, [str(tmp_path), "--no-progress"]) out = capsys.readouterr().out assert "Analysis Summary" in out - assert "Input" in out - assert "Clone groups" in out + assert out.count("Analysis Summary") == 1 + assert out.count("Metric") == 1 + assert out.count("Value") == 1 + assert "Files parsed" not in out + assert "Input" not in out assert _summary_metric(out, "Files found") >= 0 - assert _summary_metric(out, "Files parsed") >= 0 + assert _summary_metric(out, "Files analyzed") >= 0 assert _summary_metric(out, "Cache hits") >= 0 assert _summary_metric(out, "Files skipped") >= 0 - assert _summary_metric(out, "Function") >= 0 - assert _summary_metric(out, "Block") >= 0 - assert _summary_metric(out, "Segment") >= 0 - assert _summary_metric(out, "Suppressed") >= 0 - assert _summary_metric(out, "New (baseline)") >= 0 + assert _summary_metric(out, "Function clone groups") >= 0 + assert _summary_metric(out, "Block clone groups") >= 0 + assert _summary_metric(out, "Segment clone groups") >= 0 + assert _summary_metric(out, "Suppressed segment groups") >= 0 + assert _summary_metric(out, "New vs baseline") >= 0 + + +def test_cli_summary_no_color_has_no_ansi( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + src = tmp_path / "a.py" + src.write_text("def f():\n return 1\n", "utf-8") + _patch_parallel(monkeypatch) + _run_main(monkeypatch, [str(tmp_path), "--no-progress", "--no-color"]) + out = capsys.readouterr().out + assert "\x1b[" not in out def test_cli_scan_failed(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/test_cli_smoke.py b/tests/test_cli_smoke.py index 04ed73f..3218ebb 100644 --- a/tests/test_cli_smoke.py +++ b/tests/test_cli_smoke.py @@ -40,7 +40,7 @@ def f(): assert result.returncode == 0 assert "Analysis Summary" in result.stdout - assert "Function" in result.stdout + assert "Function clone groups" in result.stdout def test_cli_baseline_missing_warning(tmp_path: Path) -> None: @@ -108,4 +108,4 @@ def f2(): ] ) assert result2.returncode == 0 - assert "New (baseline)" in result2.stdout + assert "New vs baseline" in result2.stdout diff --git a/tests/test_cli_unit.py b/tests/test_cli_unit.py index 0cbdd91..e9d3e91 100644 --- a/tests/test_cli_unit.py +++ b/tests/test_cli_unit.py @@ -1,11 +1,14 @@ import os import sys from pathlib import Path +from typing import cast import pytest +from rich.text import Text import codeclone.cli as cli from codeclone import __version__ +from codeclone import ui_messages as ui from codeclone.cli import expand_path, process_file from codeclone.normalize import NormalizationConfig @@ -113,17 +116,42 @@ def test_cli_help_text_consistency( assert "block) exceed this number" in out -def test_aligned_summary_lines_empty() -> None: - assert cli._aligned_summary_lines([]) == [] +def test_summary_value_style_mapping() -> None: + assert cli._summary_value_style(label=ui.SUMMARY_LABEL_FUNCTION, value=0) == "dim" + assert ( + cli._summary_value_style(label=ui.SUMMARY_LABEL_FUNCTION, value=2) + == "bold green" + ) + assert ( + cli._summary_value_style(label=ui.SUMMARY_LABEL_SUPPRESSED, value=1) == "yellow" + ) + assert ( + cli._summary_value_style(label=ui.SUMMARY_LABEL_NEW_BASELINE, value=3) + == "bold red" + ) -def test_aligned_summary_lines_non_empty() -> None: - lines = cli._aligned_summary_lines([("Files found", 12), ("Cache hits", 3)]) - assert len(lines) == 2 - assert "Files found:" in lines[0] - assert "12" in lines[0] - assert "Cache hits:" in lines[1] - assert "3" in lines[1] +def test_build_summary_table_rows_and_styles() -> None: + rows = cli._build_summary_rows( + files_found=2, + files_analyzed=0, + cache_hits=2, + files_skipped=0, + func_clones_count=1, + block_clones_count=0, + segment_clones_count=0, + suppressed_segment_groups=1, + new_clones_count=1, + ) + table = cli._build_summary_table(rows) + assert table.title == ui.SUMMARY_TITLE + assert table.columns[0]._cells == [label for label, _ in rows] + value_cells = table.columns[1]._cells + assert isinstance(value_cells[0], Text) + assert str(value_cells[0]) == "2" + assert cast(Text, value_cells[1]).style == "dim" + assert cast(Text, value_cells[7]).style == "yellow" + assert cast(Text, value_cells[8]).style == "bold red" def test_print_summary_invariant_warning( @@ -133,7 +161,7 @@ def test_print_summary_invariant_warning( cli._print_summary( quiet=False, files_found=1, - files_parsed=0, + files_analyzed=0, cache_hits=0, files_skipped=0, func_clones_count=0, From eefa10da166018f8c9938a12f93a5a18b9c20f7e Mon Sep 17 00:00:00 2001 From: Den Rozhnovskiy Date: Fri, 6 Feb 2026 23:11:01 +0500 Subject: [PATCH 21/28] feat(changelog): polish CHANGELOG.md --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 701d6cb..864b93d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,9 @@ This release improves clone-detection precision and explainability with deterministic normalization and CFG upgrades, adds segment-level internal clone reporting, refreshes -the HTML report UI, and introduces baseline versioning. It is a breaking CI change for -workflows that reuse old baselines. +the HTML report UI, and introduces baseline versioning. + +**Breaking change:** CI workflows that reuse old baselines must regenerate them. ### Clone Detection Accuracy From 03fcb6ffccd85a82f687a45a3dbac186f76a60a0 Mon Sep 17 00:00:00 2001 From: Den Rozhnovskiy Date: Sun, 8 Feb 2026 12:23:51 +0500 Subject: [PATCH 22/28] fix(clone): remove internal test clone groups with shared helpers and parametrized CFG/normalize checks --- codeclone.baseline.json | 9 +- pyproject.toml | 8 +- tests/test_cfg.py | 260 +++++++++++++++++++--------------------- tests/test_normalize.py | 111 +++++++++-------- uv.lock | 51 ++++---- 5 files changed, 213 insertions(+), 226 deletions(-) diff --git a/codeclone.baseline.json b/codeclone.baseline.json index 0692509..ba7e83f 100644 --- a/codeclone.baseline.json +++ b/codeclone.baseline.json @@ -1,12 +1,5 @@ { - "functions": [ - "056967a5e5569762522015b120810353dc84986a|20-49", - "2482e4ffa6c3c349be9626d4a95abbbb57193345|20-49", - "54b7b79d1ff3384fb96b3dd97944d7b67990b3f3|0-19", - "554bfacfa9bf1565bd2f5ea36e89b6efaee29c2d|0-19", - "b0070927d98fa8274982caef45107f4e6b4d6fef|0-19", - "c8e7da40a2dc106d1aa092044f88f01d9abae054|0-19" - ], + "functions": [], "blocks": [], "python_version": "3.13", "baseline_version": "1.3.0", diff --git a/pyproject.toml b/pyproject.toml index 738faae..4de9503 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,7 +65,8 @@ dev = [ "build>=1.2.0", "twine>=5.0.0", "mypy>=1.19.1", - "ruff>=0.12.0", + "ruff>=0.15.0", + "pre-commit>=4.5.1", ] [project.scripts] @@ -107,8 +108,3 @@ select = ["E", "F", "W", "I", "B", "UP", "SIM", "C4", "PIE", "PERF", "RUF"] quote-style = "double" indent-style = "space" line-ending = "auto" - -[dependency-groups] -dev = [ - "pre-commit>=4.5.1", -] diff --git a/tests/test_cfg.py b/tests/test_cfg.py index e58bae2..71e25ee 100644 --- a/tests/test_cfg.py +++ b/tests/test_cfg.py @@ -5,6 +5,7 @@ from codeclone.cfg import CFG, CFGBuilder from codeclone.cfg_model import CFG as CFGModel +from codeclone.cfg_model import Block from codeclone.extractor import get_cfg_fingerprint from codeclone.meta_markers import CFG_META_PREFIX from codeclone.normalize import NormalizationConfig @@ -44,6 +45,47 @@ def _const_meta_value(stmt: ast.stmt) -> str | None: return stmt.value.id +def _parse_function( + source: str, *, skip_reason: str | None = None +) -> ast.FunctionDef | ast.AsyncFunctionDef: + try: + module = ast.parse(dedent(source)) + except SyntaxError: + if skip_reason: + pytest.skip(skip_reason) + raise + for node in ast.walk(module): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + return node + raise AssertionError("Expected at least one function in source") + + +def _cfg_fingerprint( + source: str, qualname: str, *, skip_reason: str | None = None +) -> str: + func = _parse_function(source, skip_reason=skip_reason) + cfg = NormalizationConfig() + return get_cfg_fingerprint(func, cfg, qualname) + + +def _assert_fingerprint_diff( + source_a: str, source_b: str, *, skip_reason: str | None = None +) -> None: + fp_a = _cfg_fingerprint(source_a, "m:f", skip_reason=skip_reason) + fp_b = _cfg_fingerprint(source_b, "m:g", skip_reason=skip_reason) + assert fp_a != fp_b + + +def _single_return_block(cfg: CFG) -> Block: + return_blocks = [ + block + for block in cfg.blocks + if any(isinstance(stmt, ast.Return) for stmt in block.statements) + ] + assert len(return_blocks) == 1 + return return_blocks[0] + + def test_cfg_if_else() -> None: source = """ def f(a): @@ -581,69 +623,49 @@ def f(x): assert "MatchMapping" in patterns_found[1] -def test_cfg_match_guard_affects_fingerprint() -> None: - code_with_guard = """ +@pytest.mark.parametrize( + ("source_a", "source_b", "skip_reason"), + [ + ( + """ def f(x): match x: case 1 if cond(): return 1 case _: return 2 - """ - code_without_guard = """ + """, + """ def f(x): match x: case 1: return 1 case _: return 2 - """ - try: - func_with_guard = ast.parse(dedent(code_with_guard)).body[0] - func_without_guard = ast.parse(dedent(code_without_guard)).body[0] - except SyntaxError: - pytest.skip("Match syntax is unavailable") - - assert isinstance(func_with_guard, (ast.FunctionDef, ast.AsyncFunctionDef)) - assert isinstance(func_without_guard, (ast.FunctionDef, ast.AsyncFunctionDef)) - cfg = NormalizationConfig() - fp_with_guard = get_cfg_fingerprint(func_with_guard, cfg, "m:f") - fp_without_guard = get_cfg_fingerprint(func_without_guard, cfg, "m:g") - assert fp_with_guard != fp_without_guard - - -def test_cfg_match_case_order_affects_fingerprint() -> None: - source_a = """ + """, + "Match syntax is unavailable", + ), + ( + """ def f(x): match x: case 1: return 1 case _: return 2 - """ - source_b = """ + """, + """ def g(x): match x: case _: return 2 case 1: return 1 - """ - try: - func_a = ast.parse(dedent(source_a)).body[0] - func_b = ast.parse(dedent(source_b)).body[0] - except SyntaxError: - pytest.skip("Match syntax is unavailable") - assert isinstance(func_a, (ast.FunctionDef, ast.AsyncFunctionDef)) - assert isinstance(func_b, (ast.FunctionDef, ast.AsyncFunctionDef)) - cfg = NormalizationConfig() - fp_a = get_cfg_fingerprint(func_a, cfg, "m:f") - fp_b = get_cfg_fingerprint(func_b, cfg, "m:g") - assert fp_a != fp_b - - -def test_cfg_try_handler_order_affects_fingerprint() -> None: - source_a = """ + """, + "Match syntax is unavailable", + ), + ( + """ def f(x): try: return risky(x) @@ -651,8 +673,8 @@ def f(x): return 1 except Exception: return 2 - """ - source_b = """ + """, + """ def g(x): try: return risky(x) @@ -660,99 +682,78 @@ def g(x): return 2 except ValueError: return 1 - """ - func_a = ast.parse(dedent(source_a)).body[0] - func_b = ast.parse(dedent(source_b)).body[0] - assert isinstance(func_a, (ast.FunctionDef, ast.AsyncFunctionDef)) - assert isinstance(func_b, (ast.FunctionDef, ast.AsyncFunctionDef)) - cfg = NormalizationConfig() - fp_a = get_cfg_fingerprint(func_a, cfg, "m:f") - fp_b = get_cfg_fingerprint(func_b, cfg, "m:g") - assert fp_a != fp_b - - -def test_cfg_for_else_affects_fingerprint() -> None: - with_else = """ + """, + None, + ), + ( + """ def f(xs): for x in xs: pass else: y = 1 - """ - without_else = """ + """, + """ def f(xs): for x in xs: pass - """ - func_with_else = ast.parse(dedent(with_else)).body[0] - func_without_else = ast.parse(dedent(without_else)).body[0] - assert isinstance(func_with_else, (ast.FunctionDef, ast.AsyncFunctionDef)) - assert isinstance(func_without_else, (ast.FunctionDef, ast.AsyncFunctionDef)) - cfg = NormalizationConfig() - fp_with_else = get_cfg_fingerprint(func_with_else, cfg, "m:f") - fp_without_else = get_cfg_fingerprint(func_without_else, cfg, "m:g") - assert fp_with_else != fp_without_else - - -def test_cfg_while_else_affects_fingerprint() -> None: - with_else = """ + """, + None, + ), + ( + """ def f(flag): while flag: flag = False else: x = 1 - """ - without_else = """ + """, + """ def f(flag): while flag: flag = False - """ - func_with_else = ast.parse(dedent(with_else)).body[0] - func_without_else = ast.parse(dedent(without_else)).body[0] - assert isinstance(func_with_else, (ast.FunctionDef, ast.AsyncFunctionDef)) - assert isinstance(func_without_else, (ast.FunctionDef, ast.AsyncFunctionDef)) - cfg = NormalizationConfig() - fp_with_else = get_cfg_fingerprint(func_with_else, cfg, "m:f") - fp_without_else = get_cfg_fingerprint(func_without_else, cfg, "m:g") - assert fp_with_else != fp_without_else - - -def test_cfg_break_terminates_block() -> None: - source = """ - def f(xs): - for x in xs: - break - y = 1 - """ - cfg = build_cfg_from_source(source) - break_blocks = [ - block - for block in cfg.blocks - if any(isinstance(stmt, ast.Break) for stmt in block.statements) - ] - assert len(break_blocks) == 1 - break_block = break_blocks[0] - assert break_block.is_terminated is True - assert all(not isinstance(stmt, ast.Assign) for stmt in break_block.statements) - - -def test_cfg_continue_terminates_block() -> None: - source = """ + """, + None, + ), + ], + ids=[ + "match_guard", + "match_case_order", + "try_handler_order", + "for_else", + "while_else", + ], +) +def test_cfg_fingerprint_variants( + source_a: str, source_b: str, skip_reason: str | None +) -> None: + _assert_fingerprint_diff(source_a, source_b, skip_reason=skip_reason) + + +@pytest.mark.parametrize( + ("keyword", "stmt_type"), + [("break", ast.Break), ("continue", ast.Continue)], + ids=["break", "continue"], +) +def test_cfg_loop_control_terminates_block( + keyword: str, stmt_type: type[ast.stmt] +) -> None: + source = f""" def f(xs): for x in xs: - continue + {keyword} y = 1 """ cfg = build_cfg_from_source(source) - continue_blocks = [ + control_blocks = [ block for block in cfg.blocks - if any(isinstance(stmt, ast.Continue) for stmt in block.statements) + if any(isinstance(stmt, stmt_type) for stmt in block.statements) ] - assert len(continue_blocks) == 1 - continue_block = continue_blocks[0] - assert continue_block.is_terminated is True - assert all(not isinstance(stmt, ast.Assign) for stmt in continue_block.statements) + assert len(control_blocks) == 1 + control_block = control_blocks[0] + assert control_block.is_terminated is True + assert all(not isinstance(stmt, ast.Assign) for stmt in control_block.statements) def test_cfg_break_skips_for_else_block() -> None: @@ -783,42 +784,31 @@ def f(xs): assert else_blocks[0] not in break_blocks[0].successors -def test_cfg_while_else_terminated_branch() -> None: - source = """ +@pytest.mark.parametrize( + "source", + [ + """ def f(flag): while flag: flag = False else: return 1 - """ - cfg = build_cfg_from_source(source) - return_blocks = [ - block - for block in cfg.blocks - if any(isinstance(stmt, ast.Return) for stmt in block.statements) - ] - assert len(return_blocks) == 1 - assert return_blocks[0].is_terminated is True - assert cfg.exit in return_blocks[0].successors - - -def test_cfg_for_else_terminated_branch() -> None: - source = """ + """, + """ def f(xs): for x in xs: pass else: return 1 - """ + """, + ], + ids=["while_else", "for_else"], +) +def test_cfg_loop_else_terminated_branch(source: str) -> None: cfg = build_cfg_from_source(source) - return_blocks = [ - block - for block in cfg.blocks - if any(isinstance(stmt, ast.Return) for stmt in block.statements) - ] - assert len(return_blocks) == 1 - assert return_blocks[0].is_terminated is True - assert cfg.exit in return_blocks[0].successors + return_block = _single_return_block(cfg) + assert return_block.is_terminated is True + assert cfg.exit in return_block.successors def test_cfg_break_outside_loop_falls_back_to_exit() -> None: diff --git a/tests/test_normalize.py b/tests/test_normalize.py index 0325468..1310571 100644 --- a/tests/test_normalize.py +++ b/tests/test_normalize.py @@ -12,6 +12,23 @@ ) +def _normalized_dump(source: str, cfg: NormalizationConfig) -> str: + node = ast.parse(source).body[0] + return normalized_ast_dump(node, cfg) + + +def _assert_normalized_equal( + source_a: str, source_b: str, cfg: NormalizationConfig +) -> None: + assert _normalized_dump(source_a, cfg) == _normalized_dump(source_b, cfg) + + +def _assert_normalized_not_equal( + source_a: str, source_b: str, cfg: NormalizationConfig +) -> None: + assert _normalized_dump(source_a, cfg) != _normalized_dump(source_b, cfg) + + @pytest.mark.parametrize( ("src1", "src2"), [ @@ -145,42 +162,39 @@ def f(): assert normalized_ast_dump(a1, cfg) == normalized_ast_dump(a2, cfg) -def test_normalization_commutative_name_operands_not_reordered() -> None: - src1 = """ +@pytest.mark.parametrize( + ("src1", "src2"), + [ + ( + """ def f(): return a + b -""" - src2 = """ +""", + """ def f(): return b + a -""" - cfg = NormalizationConfig( - normalize_names=False, - normalize_attributes=False, - normalize_constants=False, - ) - a1 = ast.parse(src1).body[0] - a2 = ast.parse(src2).body[0] - assert normalized_ast_dump(a1, cfg) != normalized_ast_dump(a2, cfg) - - -def test_normalization_commutative_binop_side_effects_not_reordered() -> None: - src1 = """ +""", + ), + ( + """ def f(): return foo() + bar() -""" - src2 = """ +""", + """ def f(): return bar() + foo() -""" +""", + ), + ], + ids=["name_operands", "call_operands"], +) +def test_normalization_commutative_binop_not_reordered(src1: str, src2: str) -> None: cfg = NormalizationConfig( normalize_names=False, normalize_attributes=False, normalize_constants=False, ) - a1 = ast.parse(src1).body[0] - a2 = ast.parse(src2).body[0] - assert normalized_ast_dump(a1, cfg) != normalized_ast_dump(a2, cfg) + _assert_normalized_not_equal(src1, src2, cfg) def test_normalization_preserves_call_target_names() -> None: @@ -213,21 +227,39 @@ def f(): assert normalized_ast_dump(a1, cfg) != normalized_ast_dump(a2, cfg) -def test_normalization_call_keyword_values_are_visited() -> None: - src1 = """ +@pytest.mark.parametrize( + ("src1", "src2"), + [ + ( + """ def f(): x = 1 return process(payload=x) -""" - src2 = """ +""", + """ def f(): y = 2 return process(payload=y) -""" +""", + ), + ( + """ +def f(): + handlers = [run] + return handlers[0]() +""", + """ +def f(): + callbacks = [run] + return callbacks[0]() +""", + ), + ], + ids=["call_keyword_values", "non_name_call_target"], +) +def test_normalization_call_values_normalize(src1: str, src2: str) -> None: cfg = NormalizationConfig() - a1 = ast.parse(src1).body[0] - a2 = ast.parse(src2).body[0] - assert normalized_ast_dump(a1, cfg) == normalized_ast_dump(a2, cfg) + _assert_normalized_equal(src1, src2, cfg) def test_normalization_preserves_attribute_call_target_with_call_value() -> None: @@ -245,23 +277,6 @@ def f(): assert normalized_ast_dump(a1, cfg) != normalized_ast_dump(a2, cfg) -def test_normalization_non_name_non_attribute_call_target() -> None: - src1 = """ -def f(): - handlers = [run] - return handlers[0]() -""" - src2 = """ -def f(): - callbacks = [run] - return callbacks[0]() -""" - cfg = NormalizationConfig() - a1 = ast.parse(src1).body[0] - a2 = ast.parse(src2).body[0] - assert normalized_ast_dump(a1, cfg) == normalized_ast_dump(a2, cfg) - - def test_commutative_operand_recursive_and_constant_guards() -> None: nested = ast.parse("(1 + 2) + 3", mode="eval").body assert isinstance(nested, ast.BinOp) diff --git a/uv.lock b/uv.lock index a237116..611e943 100644 --- a/uv.lock +++ b/uv.lock @@ -200,33 +200,27 @@ dependencies = [ dev = [ { name = "build" }, { name = "mypy" }, + { name = "pre-commit" }, { name = "pytest" }, { name = "pytest-cov" }, { name = "ruff" }, { name = "twine" }, ] -[package.dev-dependencies] -dev = [ - { name = "pre-commit" }, -] - [package.metadata] requires-dist = [ { name = "build", marker = "extra == 'dev'", specifier = ">=1.2.0" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.19.1" }, + { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=4.5.1" }, { name = "pygments", specifier = ">=2.19.2" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0.0" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=6.1.0" }, { name = "rich", specifier = ">=14.3.2" }, - { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.12.0" }, + { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.15.0" }, { name = "twine", marker = "extra == 'dev'", specifier = ">=5.0.0" }, ] provides-extras = ["dev"] -[package.metadata.requires-dev] -dev = [{ name = "pre-commit", specifier = ">=4.5.1" }] - [[package]] name = "colorama" version = "0.4.6" @@ -991,28 +985,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.14.14" +version = "0.15.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2e/06/f71e3a86b2df0dfa2d2f72195941cd09b44f87711cb7fa5193732cb9a5fc/ruff-0.14.14.tar.gz", hash = "sha256:2d0f819c9a90205f3a867dbbd0be083bee9912e170fd7d9704cc8ae45824896b", size = 4515732, upload-time = "2026-01-22T22:30:17.527Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c8/39/5cee96809fbca590abea6b46c6d1c586b49663d1d2830a751cc8fc42c666/ruff-0.15.0.tar.gz", hash = "sha256:6bdea47cdbea30d40f8f8d7d69c0854ba7c15420ec75a26f463290949d7f7e9a", size = 4524893, upload-time = "2026-02-03T17:53:35.357Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d2/89/20a12e97bc6b9f9f68343952da08a8099c57237aef953a56b82711d55edd/ruff-0.14.14-py3-none-linux_armv6l.whl", hash = "sha256:7cfe36b56e8489dee8fbc777c61959f60ec0f1f11817e8f2415f429552846aed", size = 10467650, upload-time = "2026-01-22T22:30:08.578Z" }, - { url = "https://files.pythonhosted.org/packages/a3/b1/c5de3fd2d5a831fcae21beda5e3589c0ba67eec8202e992388e4b17a6040/ruff-0.14.14-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:6006a0082336e7920b9573ef8a7f52eec837add1265cc74e04ea8a4368cd704c", size = 10883245, upload-time = "2026-01-22T22:30:04.155Z" }, - { url = "https://files.pythonhosted.org/packages/b8/7c/3c1db59a10e7490f8f6f8559d1db8636cbb13dccebf18686f4e3c9d7c772/ruff-0.14.14-py3-none-macosx_11_0_arm64.whl", hash = "sha256:026c1d25996818f0bf498636686199d9bd0d9d6341c9c2c3b62e2a0198b758de", size = 10231273, upload-time = "2026-01-22T22:30:34.642Z" }, - { url = "https://files.pythonhosted.org/packages/a1/6e/5e0e0d9674be0f8581d1f5e0f0a04761203affce3232c1a1189d0e3b4dad/ruff-0.14.14-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f666445819d31210b71e0a6d1c01e24447a20b85458eea25a25fe8142210ae0e", size = 10585753, upload-time = "2026-01-22T22:30:31.781Z" }, - { url = "https://files.pythonhosted.org/packages/23/09/754ab09f46ff1884d422dc26d59ba18b4e5d355be147721bb2518aa2a014/ruff-0.14.14-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3c0f18b922c6d2ff9a5e6c3ee16259adc513ca775bcf82c67ebab7cbd9da5bc8", size = 10286052, upload-time = "2026-01-22T22:30:24.827Z" }, - { url = "https://files.pythonhosted.org/packages/c8/cc/e71f88dd2a12afb5f50733851729d6b571a7c3a35bfdb16c3035132675a0/ruff-0.14.14-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1629e67489c2dea43e8658c3dba659edbfd87361624b4040d1df04c9740ae906", size = 11043637, upload-time = "2026-01-22T22:30:13.239Z" }, - { url = "https://files.pythonhosted.org/packages/67/b2/397245026352494497dac935d7f00f1468c03a23a0c5db6ad8fc49ca3fb2/ruff-0.14.14-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:27493a2131ea0f899057d49d303e4292b2cae2bb57253c1ed1f256fbcd1da480", size = 12194761, upload-time = "2026-01-22T22:30:22.542Z" }, - { url = "https://files.pythonhosted.org/packages/5b/06/06ef271459f778323112c51b7587ce85230785cd64e91772034ddb88f200/ruff-0.14.14-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:01ff589aab3f5b539e35db38425da31a57521efd1e4ad1ae08fc34dbe30bd7df", size = 12005701, upload-time = "2026-01-22T22:30:20.499Z" }, - { url = "https://files.pythonhosted.org/packages/41/d6/99364514541cf811ccc5ac44362f88df66373e9fec1b9d1c4cc830593fe7/ruff-0.14.14-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1cc12d74eef0f29f51775f5b755913eb523546b88e2d733e1d701fe65144e89b", size = 11282455, upload-time = "2026-01-22T22:29:59.679Z" }, - { url = "https://files.pythonhosted.org/packages/ca/71/37daa46f89475f8582b7762ecd2722492df26421714a33e72ccc9a84d7a5/ruff-0.14.14-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb8481604b7a9e75eff53772496201690ce2687067e038b3cc31aaf16aa0b974", size = 11215882, upload-time = "2026-01-22T22:29:57.032Z" }, - { url = "https://files.pythonhosted.org/packages/2c/10/a31f86169ec91c0705e618443ee74ede0bdd94da0a57b28e72db68b2dbac/ruff-0.14.14-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:14649acb1cf7b5d2d283ebd2f58d56b75836ed8c6f329664fa91cdea19e76e66", size = 11180549, upload-time = "2026-01-22T22:30:27.175Z" }, - { url = "https://files.pythonhosted.org/packages/fd/1e/c723f20536b5163adf79bdd10c5f093414293cdf567eed9bdb7b83940f3f/ruff-0.14.14-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:e8058d2145566510790eab4e2fad186002e288dec5e0d343a92fe7b0bc1b3e13", size = 10543416, upload-time = "2026-01-22T22:30:01.964Z" }, - { url = "https://files.pythonhosted.org/packages/3e/34/8a84cea7e42c2d94ba5bde1d7a4fae164d6318f13f933d92da6d7c2041ff/ruff-0.14.14-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:e651e977a79e4c758eb807f0481d673a67ffe53cfa92209781dfa3a996cf8412", size = 10285491, upload-time = "2026-01-22T22:30:29.51Z" }, - { url = "https://files.pythonhosted.org/packages/55/ef/b7c5ea0be82518906c978e365e56a77f8de7678c8bb6651ccfbdc178c29f/ruff-0.14.14-py3-none-musllinux_1_2_i686.whl", hash = "sha256:cc8b22da8d9d6fdd844a68ae937e2a0adf9b16514e9a97cc60355e2d4b219fc3", size = 10733525, upload-time = "2026-01-22T22:30:06.499Z" }, - { url = "https://files.pythonhosted.org/packages/6a/5b/aaf1dfbcc53a2811f6cc0a1759de24e4b03e02ba8762daabd9b6bd8c59e3/ruff-0.14.14-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:16bc890fb4cc9781bb05beb5ab4cd51be9e7cb376bf1dd3580512b24eb3fda2b", size = 11315626, upload-time = "2026-01-22T22:30:36.848Z" }, - { url = "https://files.pythonhosted.org/packages/2c/aa/9f89c719c467dfaf8ad799b9bae0df494513fb21d31a6059cb5870e57e74/ruff-0.14.14-py3-none-win32.whl", hash = "sha256:b530c191970b143375b6a68e6f743800b2b786bbcf03a7965b06c4bf04568167", size = 10502442, upload-time = "2026-01-22T22:30:38.93Z" }, - { url = "https://files.pythonhosted.org/packages/87/44/90fa543014c45560cae1fffc63ea059fb3575ee6e1cb654562197e5d16fb/ruff-0.14.14-py3-none-win_amd64.whl", hash = "sha256:3dde1435e6b6fe5b66506c1dff67a421d0b7f6488d466f651c07f4cab3bf20fd", size = 11630486, upload-time = "2026-01-22T22:30:10.852Z" }, - { url = "https://files.pythonhosted.org/packages/9e/6a/40fee331a52339926a92e17ae748827270b288a35ef4a15c9c8f2ec54715/ruff-0.14.14-py3-none-win_arm64.whl", hash = "sha256:56e6981a98b13a32236a72a8da421d7839221fa308b223b9283312312e5ac76c", size = 10920448, upload-time = "2026-01-22T22:30:15.417Z" }, + { url = "https://files.pythonhosted.org/packages/bc/88/3fd1b0aa4b6330d6aaa63a285bc96c9f71970351579152d231ed90914586/ruff-0.15.0-py3-none-linux_armv6l.whl", hash = "sha256:aac4ebaa612a82b23d45964586f24ae9bc23ca101919f5590bdb368d74ad5455", size = 10354332, upload-time = "2026-02-03T17:52:54.892Z" }, + { url = "https://files.pythonhosted.org/packages/72/f6/62e173fbb7eb75cc29fe2576a1e20f0a46f671a2587b5f604bfb0eaf5f6f/ruff-0.15.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:dcd4be7cc75cfbbca24a98d04d0b9b36a270d0833241f776b788d59f4142b14d", size = 10767189, upload-time = "2026-02-03T17:53:19.778Z" }, + { url = "https://files.pythonhosted.org/packages/99/e4/968ae17b676d1d2ff101d56dc69cf333e3a4c985e1ec23803df84fc7bf9e/ruff-0.15.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:d747e3319b2bce179c7c1eaad3d884dc0a199b5f4d5187620530adf9105268ce", size = 10075384, upload-time = "2026-02-03T17:53:29.241Z" }, + { url = "https://files.pythonhosted.org/packages/a2/bf/9843c6044ab9e20af879c751487e61333ca79a2c8c3058b15722386b8cae/ruff-0.15.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:650bd9c56ae03102c51a5e4b554d74d825ff3abe4db22b90fd32d816c2e90621", size = 10481363, upload-time = "2026-02-03T17:52:43.332Z" }, + { url = "https://files.pythonhosted.org/packages/55/d9/4ada5ccf4cd1f532db1c8d44b6f664f2208d3d93acbeec18f82315e15193/ruff-0.15.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a6664b7eac559e3048223a2da77769c2f92b43a6dfd4720cef42654299a599c9", size = 10187736, upload-time = "2026-02-03T17:53:00.522Z" }, + { url = "https://files.pythonhosted.org/packages/86/e2/f25eaecd446af7bb132af0a1d5b135a62971a41f5366ff41d06d25e77a91/ruff-0.15.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f811f97b0f092b35320d1556f3353bf238763420ade5d9e62ebd2b73f2ff179", size = 10968415, upload-time = "2026-02-03T17:53:15.705Z" }, + { url = "https://files.pythonhosted.org/packages/e7/dc/f06a8558d06333bf79b497d29a50c3a673d9251214e0d7ec78f90b30aa79/ruff-0.15.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:761ec0a66680fab6454236635a39abaf14198818c8cdf691e036f4bc0f406b2d", size = 11809643, upload-time = "2026-02-03T17:53:23.031Z" }, + { url = "https://files.pythonhosted.org/packages/dd/45/0ece8db2c474ad7df13af3a6d50f76e22a09d078af63078f005057ca59eb/ruff-0.15.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:940f11c2604d317e797b289f4f9f3fa5555ffe4fb574b55ed006c3d9b6f0eb78", size = 11234787, upload-time = "2026-02-03T17:52:46.432Z" }, + { url = "https://files.pythonhosted.org/packages/8a/d9/0e3a81467a120fd265658d127db648e4d3acfe3e4f6f5d4ea79fac47e587/ruff-0.15.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bcbca3d40558789126da91d7ef9a7c87772ee107033db7191edefa34e2c7f1b4", size = 11112797, upload-time = "2026-02-03T17:52:49.274Z" }, + { url = "https://files.pythonhosted.org/packages/b2/cb/8c0b3b0c692683f8ff31351dfb6241047fa873a4481a76df4335a8bff716/ruff-0.15.0-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:9a121a96db1d75fa3eb39c4539e607f628920dd72ff1f7c5ee4f1b768ac62d6e", size = 11033133, upload-time = "2026-02-03T17:53:33.105Z" }, + { url = "https://files.pythonhosted.org/packages/f8/5e/23b87370cf0f9081a8c89a753e69a4e8778805b8802ccfe175cc410e50b9/ruff-0.15.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:5298d518e493061f2eabd4abd067c7e4fb89e2f63291c94332e35631c07c3662", size = 10442646, upload-time = "2026-02-03T17:53:06.278Z" }, + { url = "https://files.pythonhosted.org/packages/e1/9a/3c94de5ce642830167e6d00b5c75aacd73e6347b4c7fc6828699b150a5ee/ruff-0.15.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:afb6e603d6375ff0d6b0cee563fa21ab570fd15e65c852cb24922cef25050cf1", size = 10195750, upload-time = "2026-02-03T17:53:26.084Z" }, + { url = "https://files.pythonhosted.org/packages/30/15/e396325080d600b436acc970848d69df9c13977942fb62bb8722d729bee8/ruff-0.15.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:77e515f6b15f828b94dc17d2b4ace334c9ddb7d9468c54b2f9ed2b9c1593ef16", size = 10676120, upload-time = "2026-02-03T17:53:09.363Z" }, + { url = "https://files.pythonhosted.org/packages/8d/c9/229a23d52a2983de1ad0fb0ee37d36e0257e6f28bfd6b498ee2c76361874/ruff-0.15.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:6f6e80850a01eb13b3e42ee0ebdf6e4497151b48c35051aab51c101266d187a3", size = 11201636, upload-time = "2026-02-03T17:52:57.281Z" }, + { url = "https://files.pythonhosted.org/packages/6f/b0/69adf22f4e24f3677208adb715c578266842e6e6a3cc77483f48dd999ede/ruff-0.15.0-py3-none-win32.whl", hash = "sha256:238a717ef803e501b6d51e0bdd0d2c6e8513fe9eec14002445134d3907cd46c3", size = 10465945, upload-time = "2026-02-03T17:53:12.591Z" }, + { url = "https://files.pythonhosted.org/packages/51/ad/f813b6e2c97e9b4598be25e94a9147b9af7e60523b0cb5d94d307c15229d/ruff-0.15.0-py3-none-win_amd64.whl", hash = "sha256:dd5e4d3301dc01de614da3cdffc33d4b1b96fb89e45721f1598e5532ccf78b18", size = 11564657, upload-time = "2026-02-03T17:52:51.893Z" }, + { url = "https://files.pythonhosted.org/packages/f6/b0/2d823f6e77ebe560f4e397d078487e8d52c1516b331e3521bc75db4272ca/ruff-0.15.0-py3-none-win_arm64.whl", hash = "sha256:c480d632cc0ca3f0727acac8b7d053542d9e114a462a145d0b00e7cd658c515a", size = 10865753, upload-time = "2026-02-03T17:53:03.014Z" }, ] [[package]] From 985e5c2b2bfaf850995e3899964bfe4b9b3aa3b9 Mon Sep 17 00:00:00 2001 From: Den Rozhnovskiy Date: Sun, 8 Feb 2026 14:18:56 +0500 Subject: [PATCH 23/28] feat(baseline): tamper-evident baseline contract --- CHANGELOG.md | 20 +- README.md | 13 + SECURITY.md | 6 +- codeclone.baseline.json | 9 +- codeclone/_cli_args.py | 161 +++++++++++ codeclone/_cli_meta.py | 43 +++ codeclone/_cli_paths.py | 36 +++ codeclone/_cli_summary.py | 115 ++++++++ codeclone/_html_escape.py | 35 +++ codeclone/_html_snippets.py | 208 ++++++++++++++ codeclone/_report_grouping.py | 64 +++++ codeclone/_report_segments.py | 247 ++++++++++++++++ codeclone/_report_serialize.py | 160 +++++++++++ codeclone/_report_types.py | 14 + codeclone/baseline.py | 164 +++++++++-- codeclone/cache.py | 87 +++++- codeclone/cli.py | 370 ++++++++---------------- codeclone/errors.py | 14 + codeclone/html_report.py | 315 ++++----------------- codeclone/report.py | 442 +++-------------------------- codeclone/ui_messages.py | 2 + docs/architecture.md | 13 +- tests/test_baseline.py | 201 ++++++++++++- tests/test_cache.py | 213 ++++++++++++++ tests/test_cli_inprocess.py | 495 ++++++++++++++++++++++++++++++++- tests/test_cli_unit.py | 28 ++ tests/test_extractor.py | 19 ++ tests/test_html_report.py | 208 ++++++++++++++ tests/test_report.py | 132 +++++++++ tests/test_scanner_extra.py | 42 ++- 30 files changed, 2931 insertions(+), 945 deletions(-) create mode 100644 codeclone/_cli_args.py create mode 100644 codeclone/_cli_meta.py create mode 100644 codeclone/_cli_paths.py create mode 100644 codeclone/_cli_summary.py create mode 100644 codeclone/_html_escape.py create mode 100644 codeclone/_html_snippets.py create mode 100644 codeclone/_report_grouping.py create mode 100644 codeclone/_report_segments.py create mode 100644 codeclone/_report_serialize.py create mode 100644 codeclone/_report_types.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 864b93d..cff5537 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [1.3.0] - 2026-02-05 +## [1.3.0] - 2026-02-08 ### Overview @@ -63,6 +63,14 @@ the HTML report UI, and introduces baseline versioning. - Baselines are now **versioned** and include a schema version. - Mismatched baseline versions **fail fast** and require regeneration. +- Baseline loading is now strict: invalid schema/types or oversized baseline files + fail fast to preserve CI integrity. +- Added baseline tamper-evident integrity for v1.3+ files (`generator`, `payload_sha256`) + while keeping legacy baseline behavior as explicit regeneration-required fail-fast. +- Added configurable size guards (`--max-baseline-size-mb`, `--max-cache-size-mb`): + oversized baseline fails fast, oversized cache is ignored with warning. +- Behavioral hardening (CLI): baseline validation is now an explicit contract + (legacy/version/schema/python/integrity/size states) with deterministic fail-fast behavior. **Breaking (CI):** baseline version mismatch now fails hard; CI requires baseline regeneration on upgrade. @@ -104,12 +112,20 @@ codeclone . --update-baseline - **Segment reporting** Added a dedicated “Segment clones” section and summary metric in HTML/TXT/JSON outputs. +- **Escaping and snippet resilience** + Hardened HTML escaping for text and attribute contexts, and added a safe fallback when + source snippets are unavailable during report rendering. + ### Cache & Internals - Extended cache schema to store segment fingerprints (cache version bump). - Default cache location moved to `/.cache/codeclone/cache.json` (project‑local). - Added a legacy cache warning for `~/.cache/codeclone/cache.json` with guidance to delete it and add `.cache/` to `.gitignore`. +- Strengthened cache integrity handling with constant-time signature checks and explicit + warnings for oversized cache files. +- Added deterministic deep-schema cache entry validation (`stat/units/blocks/segments`); + invalid cache entries are ignored instead of affecting analysis results. ### Packaging @@ -124,6 +140,8 @@ codeclone . --update-baseline ### Testing & Security - Expanded security tests (HTML escaping and safety checks). +- Added regression tests for deterministic report ordering across HTML/TXT/JSON, + baseline/cache integrity edge cases, and symlink traversal/loop safety. --- diff --git a/README.md b/README.md index d2e3ec0..3653f8b 100644 --- a/README.md +++ b/README.md @@ -144,6 +144,9 @@ All report formats include provenance metadata for auditability: `codeclone_version`, `python_version`, `baseline_path`, `baseline_version`, `baseline_schema_version`, `baseline_python_version`, `baseline_loaded`, `baseline_status` (and cache metadata when available). +`baseline_status` values: `ok`, `missing`, `legacy`, `invalid`, +`mismatch_version`, `mismatch_schema`, `mismatch_python`, +`generator_mismatch`, `integrity_missing`, `integrity_failed`, `too_large`. Generate an HTML report: @@ -171,6 +174,9 @@ Commit the generated baseline file to the repository. Baselines are **versioned**. If CodeClone is upgraded, regenerate the baseline to keep CI deterministic and explainable. +Baseline format in 1.3+ is tamper-evident (`generator`, `payload_sha256`) and validated +before baseline comparison. +Invalid or oversized baseline files fail fast in CI mode and must be regenerated. ### 2. Use in CI @@ -207,6 +213,11 @@ You can override this path with `--cache-path` (`--cache-dir` is a legacy alias) If you used an older version of CodeClone, delete the legacy cache file at `~/.cache/codeclone/cache.json` and add `.cache/` to `.gitignore`. +Cache integrity checks are strict: signature mismatch or oversized cache files are ignored +with an explicit warning, then rebuilt from source. +Cache entries are validated against expected structure/types; invalid entries are ignored +deterministically. + ### Python Version Consistency for Baseline Checks Due to inherent differences in Python’s AST between interpreter versions, baseline @@ -294,7 +305,9 @@ See full design and semantics: | `--processes` | Number of worker processes | `4` | | `--cache-path FILE` | Cache file path | `/.cache/codeclone/cache.json` | | `--cache-dir FILE` | Legacy alias for `--cache-path` | - | +| `--max-cache-size-mb MB` | Max cache size before ignore + warning | `50` | | `--baseline FILE` | Baseline file path | `codeclone.baseline.json` | +| `--max-baseline-size-mb MB` | Max baseline size before fail-fast | `5` | | `--update-baseline` | Regenerate baseline from current results | `False` | | `--fail-on-new` | Fail if new function/block clone groups appear vs baseline | `False` | | `--fail-threshold MAX_CLONES` | Fail if total clone groups (`function + block`) exceed threshold | `-1` (disabled) | diff --git a/SECURITY.md b/SECURITY.md index 7647ea5..08c55a4 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -36,9 +36,11 @@ ongoing security review. Additional safeguards: -- HTML report content is escaped to prevent script injection. +- HTML report content is escaped in both text and attribute contexts to prevent script injection. - Reports are static and do not execute analyzed code. -- Cache files are HMAC-signed; signature mismatches are ignored. +- Scanner traversal is root-confined and prevents symlink-based path escape. +- Baseline files are schema/type validated with size limits; invalid baselines fail fast. +- Cache files are HMAC-signed (constant-time comparison), size-limited, and ignored on mismatch. - Cache secrets are stored next to the cache (`.cache_secret`) and must not be committed. --- diff --git a/codeclone.baseline.json b/codeclone.baseline.json index ba7e83f..507b6db 100644 --- a/codeclone.baseline.json +++ b/codeclone.baseline.json @@ -1,7 +1,12 @@ { - "functions": [], + "functions": [ + "15f730bde91427ef6cc7878fd98d8f9d2ca3b57e|20-49" + ], "blocks": [], "python_version": "3.13", "baseline_version": "1.3.0", - "schema_version": 1 + "schema_version": 1, + "generator": "codeclone", + "payload_sha256": "ab5bbbe4c098b6aae44202f69c7f26398d4d85ae759ca096418d6fc054571cf1", + "created_at": "2026-02-08T09:06:10+00:00" } \ No newline at end of file diff --git a/codeclone/_cli_args.py b/codeclone/_cli_args.py new file mode 100644 index 0000000..49c0ad5 --- /dev/null +++ b/codeclone/_cli_args.py @@ -0,0 +1,161 @@ +""" +CodeClone — AST and CFG-based code clone detector for Python +focused on architectural duplication. + +Copyright (c) 2026 Den Rozhnovskiy +Licensed under the MIT License. +""" + +from __future__ import annotations + +import argparse +from typing import cast + +from . import ui_messages as ui + + +class _HelpFormatter(argparse.ArgumentDefaultsHelpFormatter): + def _get_help_string(self, action: argparse.Action) -> str: + if action.dest == "cache_path": + return action.help or "" + return cast(str, super()._get_help_string(action)) + + +def build_parser(version: str) -> argparse.ArgumentParser: + ap = argparse.ArgumentParser( + prog="codeclone", + description="AST and CFG-based code clone detector for Python.", + formatter_class=_HelpFormatter, + ) + ap.add_argument( + "--version", + action="version", + version=ui.version_output(version), + help=ui.HELP_VERSION, + ) + + core_group = ap.add_argument_group("Target") + core_group.add_argument( + "root", + nargs="?", + default=".", + help=ui.HELP_ROOT, + ) + + tune_group = ap.add_argument_group("Analysis Tuning") + tune_group.add_argument( + "--min-loc", + type=int, + default=15, + help=ui.HELP_MIN_LOC, + ) + tune_group.add_argument( + "--min-stmt", + type=int, + default=6, + help=ui.HELP_MIN_STMT, + ) + tune_group.add_argument( + "--processes", + type=int, + default=4, + help=ui.HELP_PROCESSES, + ) + tune_group.add_argument( + "--cache-path", + dest="cache_path", + metavar="FILE", + default=None, + help=ui.HELP_CACHE_PATH, + ) + tune_group.add_argument( + "--cache-dir", + dest="cache_path", + metavar="FILE", + default=None, + help=ui.HELP_CACHE_DIR_LEGACY, + ) + tune_group.add_argument( + "--max-cache-size-mb", + type=int, + default=50, + metavar="MB", + help=ui.HELP_MAX_CACHE_SIZE_MB, + ) + + ci_group = ap.add_argument_group("Baseline & CI/CD") + ci_group.add_argument( + "--baseline", + default="codeclone.baseline.json", + help=ui.HELP_BASELINE, + ) + ci_group.add_argument( + "--max-baseline-size-mb", + type=int, + default=5, + metavar="MB", + help=ui.HELP_MAX_BASELINE_SIZE_MB, + ) + ci_group.add_argument( + "--update-baseline", + action="store_true", + help=ui.HELP_UPDATE_BASELINE, + ) + ci_group.add_argument( + "--fail-on-new", + action="store_true", + help=ui.HELP_FAIL_ON_NEW, + ) + ci_group.add_argument( + "--fail-threshold", + type=int, + default=-1, + metavar="MAX_CLONES", + help=ui.HELP_FAIL_THRESHOLD, + ) + ci_group.add_argument( + "--ci", + action="store_true", + help=ui.HELP_CI, + ) + + out_group = ap.add_argument_group("Reporting") + out_group.add_argument( + "--html", + dest="html_out", + metavar="FILE", + help=ui.HELP_HTML, + ) + out_group.add_argument( + "--json", + dest="json_out", + metavar="FILE", + help=ui.HELP_JSON, + ) + out_group.add_argument( + "--text", + dest="text_out", + metavar="FILE", + help=ui.HELP_TEXT, + ) + out_group.add_argument( + "--no-progress", + action="store_true", + help=ui.HELP_NO_PROGRESS, + ) + out_group.add_argument( + "--no-color", + action="store_true", + help=ui.HELP_NO_COLOR, + ) + out_group.add_argument( + "--quiet", + action="store_true", + help=ui.HELP_QUIET, + ) + out_group.add_argument( + "--verbose", + action="store_true", + help=ui.HELP_VERBOSE, + ) + return ap diff --git a/codeclone/_cli_meta.py b/codeclone/_cli_meta.py new file mode 100644 index 0000000..fe6a04e --- /dev/null +++ b/codeclone/_cli_meta.py @@ -0,0 +1,43 @@ +""" +CodeClone — AST and CFG-based code clone detector for Python +focused on architectural duplication. + +Copyright (c) 2026 Den Rozhnovskiy +Licensed under the MIT License. +""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Any + +from .baseline import Baseline + + +def _current_python_version() -> str: + return f"{sys.version_info.major}.{sys.version_info.minor}" + + +def _build_report_meta( + *, + codeclone_version: str, + baseline_path: Path, + baseline: Baseline, + baseline_loaded: bool, + baseline_status: str, + cache_path: Path, + cache_used: bool, +) -> dict[str, Any]: + return { + "codeclone_version": codeclone_version, + "python_version": _current_python_version(), + "baseline_path": str(baseline_path), + "baseline_version": baseline.baseline_version, + "baseline_schema_version": baseline.schema_version, + "baseline_python_version": baseline.python_version, + "baseline_loaded": baseline_loaded, + "baseline_status": baseline_status, + "cache_path": str(cache_path), + "cache_used": cache_used, + } diff --git a/codeclone/_cli_paths.py b/codeclone/_cli_paths.py new file mode 100644 index 0000000..4dcd72f --- /dev/null +++ b/codeclone/_cli_paths.py @@ -0,0 +1,36 @@ +""" +CodeClone — AST and CFG-based code clone detector for Python +focused on architectural duplication. + +Copyright (c) 2026 Den Rozhnovskiy +Licensed under the MIT License. +""" + +from __future__ import annotations + +import sys +from collections.abc import Callable +from pathlib import Path + +from rich.console import Console + + +def expand_path(p: str) -> Path: + return Path(p).expanduser().resolve() + + +def _validate_output_path( + path: str, + *, + expected_suffix: str, + label: str, + console: Console, + invalid_message: Callable[..., str], +) -> Path: + out = Path(path).expanduser() + if out.suffix.lower() != expected_suffix: + console.print( + invalid_message(label=label, path=out, expected_suffix=expected_suffix) + ) + sys.exit(2) + return out.resolve() diff --git a/codeclone/_cli_summary.py b/codeclone/_cli_summary.py new file mode 100644 index 0000000..40df388 --- /dev/null +++ b/codeclone/_cli_summary.py @@ -0,0 +1,115 @@ +""" +CodeClone — AST and CFG-based code clone detector for Python +focused on architectural duplication. + +Copyright (c) 2026 Den Rozhnovskiy +Licensed under the MIT License. +""" + +from __future__ import annotations + +from rich.console import Console +from rich.table import Table +from rich.text import Text + +from . import ui_messages as ui + + +def _summary_value_style(*, label: str, value: int) -> str: + if value == 0: + return "dim" + if label == ui.SUMMARY_LABEL_NEW_BASELINE: + return "bold red" + if label == ui.SUMMARY_LABEL_SUPPRESSED: + return "yellow" + return "bold green" + + +def _build_summary_rows( + *, + files_found: int, + files_analyzed: int, + cache_hits: int, + files_skipped: int, + func_clones_count: int, + block_clones_count: int, + segment_clones_count: int, + suppressed_segment_groups: int, + new_clones_count: int, +) -> list[tuple[str, int]]: + return [ + (ui.SUMMARY_LABEL_FILES_FOUND, files_found), + (ui.SUMMARY_LABEL_FILES_ANALYZED, files_analyzed), + (ui.SUMMARY_LABEL_CACHE_HITS, cache_hits), + (ui.SUMMARY_LABEL_FILES_SKIPPED, files_skipped), + (ui.SUMMARY_LABEL_FUNCTION, func_clones_count), + (ui.SUMMARY_LABEL_BLOCK, block_clones_count), + (ui.SUMMARY_LABEL_SEGMENT, segment_clones_count), + (ui.SUMMARY_LABEL_SUPPRESSED, suppressed_segment_groups), + (ui.SUMMARY_LABEL_NEW_BASELINE, new_clones_count), + ] + + +def _build_summary_table(rows: list[tuple[str, int]]) -> Table: + summary_table = Table(title=ui.SUMMARY_TITLE, show_header=True) + summary_table.add_column("Metric") + summary_table.add_column("Value", justify="right") + for label, value in rows: + summary_table.add_row( + label, + Text(str(value), style=_summary_value_style(label=label, value=value)), + ) + return summary_table + + +def _print_summary( + *, + console: Console, + quiet: bool, + files_found: int, + files_analyzed: int, + cache_hits: int, + files_skipped: int, + func_clones_count: int, + block_clones_count: int, + segment_clones_count: int, + suppressed_segment_groups: int, + new_clones_count: int, +) -> None: + invariant_ok = files_found == (files_analyzed + cache_hits + files_skipped) + rows = _build_summary_rows( + files_found=files_found, + files_analyzed=files_analyzed, + cache_hits=cache_hits, + files_skipped=files_skipped, + func_clones_count=func_clones_count, + block_clones_count=block_clones_count, + segment_clones_count=segment_clones_count, + suppressed_segment_groups=suppressed_segment_groups, + new_clones_count=new_clones_count, + ) + + if quiet: + console.print(ui.SUMMARY_TITLE) + console.print( + ui.fmt_summary_compact_input( + found=files_found, + analyzed=files_analyzed, + cache_hits=cache_hits, + skipped=files_skipped, + ) + ) + console.print( + ui.fmt_summary_compact_clones( + function=func_clones_count, + block=block_clones_count, + segment=segment_clones_count, + suppressed=suppressed_segment_groups, + new=new_clones_count, + ) + ) + else: + console.print(_build_summary_table(rows)) + + if not invariant_ok: + console.print(f"[warning]{ui.WARN_SUMMARY_ACCOUNTING_MISMATCH}[/warning]") diff --git a/codeclone/_html_escape.py b/codeclone/_html_escape.py new file mode 100644 index 0000000..16f7ae4 --- /dev/null +++ b/codeclone/_html_escape.py @@ -0,0 +1,35 @@ +""" +CodeClone — AST and CFG-based code clone detector for Python +focused on architectural duplication. + +Copyright (c) 2026 Den Rozhnovskiy +Licensed under the MIT License. +""" + +from __future__ import annotations + +import html +from typing import Any + + +def _escape_html(v: Any) -> str: + text = html.escape("" if v is None else str(v), quote=True) + text = text.replace("`", "`") + text = text.replace("\u2028", "
").replace("\u2029", "
") + return text + + +def _escape_attr(v: Any) -> str: + text = html.escape("" if v is None else str(v), quote=True) + text = text.replace("`", "`") + text = text.replace("\u2028", "
").replace("\u2029", "
") + return text + + +def _meta_display(v: Any) -> str: + if isinstance(v, bool): + return "true" if v else "false" + if v is None: + return "n/a" + text = str(v).strip() + return text if text else "n/a" diff --git a/codeclone/_html_snippets.py b/codeclone/_html_snippets.py new file mode 100644 index 0000000..915cb1d --- /dev/null +++ b/codeclone/_html_snippets.py @@ -0,0 +1,208 @@ +""" +CodeClone — AST and CFG-based code clone detector for Python +focused on architectural duplication. + +Copyright (c) 2026 Den Rozhnovskiy +Licensed under the MIT License. +""" + +from __future__ import annotations + +import html +import importlib +import itertools +from collections.abc import Iterable +from dataclasses import dataclass +from functools import lru_cache +from typing import Any, NamedTuple, cast + +from .errors import FileProcessingError + + +def pairwise(iterable: Iterable[Any]) -> Iterable[tuple[Any, Any]]: + a, b = itertools.tee(iterable) + next(b, None) + return zip(a, b, strict=False) + + +@dataclass(slots=True) +class _Snippet: + filepath: str + start_line: int + end_line: int + code_html: str + + +class _FileCache: + __slots__ = ("_get_lines_impl", "maxsize") + + def __init__(self, maxsize: int = 128) -> None: + self.maxsize = maxsize + self._get_lines_impl = lru_cache(maxsize=maxsize)(self._read_file_range) + + @staticmethod + def _read_file_range( + filepath: str, start_line: int, end_line: int + ) -> tuple[str, ...]: + if start_line < 1: + start_line = 1 + if end_line < start_line: + return () + + try: + + def _read_with_errors(errors: str) -> tuple[str, ...]: + lines: list[str] = [] + with open(filepath, encoding="utf-8", errors=errors) as f: + for lineno, line in enumerate(f, start=1): + if lineno < start_line: + continue + if lineno > end_line: + break + lines.append(line.rstrip("\n")) + return tuple(lines) + + try: + return _read_with_errors("strict") + except UnicodeDecodeError: + return _read_with_errors("replace") + except OSError as e: + raise FileProcessingError(f"Cannot read {filepath}: {e}") from e + + def get_lines_range( + self, filepath: str, start_line: int, end_line: int + ) -> tuple[str, ...]: + return self._get_lines_impl(filepath, start_line, end_line) + + class _CacheInfo(NamedTuple): + hits: int + misses: int + maxsize: int | None + currsize: int + + def cache_info(self) -> _CacheInfo: + return cast(_FileCache._CacheInfo, self._get_lines_impl.cache_info()) + + +def _try_pygments(code: str) -> str | None: + try: + pygments = importlib.import_module("pygments") + formatters = importlib.import_module("pygments.formatters") + lexers = importlib.import_module("pygments.lexers") + except ImportError: + return None + + highlight = pygments.highlight + formatter_cls = formatters.HtmlFormatter + lexer_cls = lexers.PythonLexer + result = highlight(code, lexer_cls(), formatter_cls(nowrap=True)) + return result if isinstance(result, str) else None + + +def _pygments_css(style_name: str) -> str: + """ + Returns CSS for pygments tokens. Scoped to `.codebox` to avoid leaking styles. + If Pygments is not available or style missing, returns "". + """ + try: + formatters = importlib.import_module("pygments.formatters") + except ImportError: + return "" + + try: + formatter_cls = formatters.HtmlFormatter + fmt = formatter_cls(style=style_name) + except Exception: + try: + fmt = formatter_cls() + except Exception: + return "" + + try: + css = fmt.get_style_defs(".codebox") + return css if isinstance(css, str) else "" + except Exception: + return "" + + +def _prefix_css(css: str, prefix: str) -> str: + """ + Prefix every selector block with `prefix `. + Safe enough for pygments CSS which is mostly selector blocks and comments. + """ + out_lines: list[str] = [] + for line in css.splitlines(): + stripped = line.strip() + if not stripped: + out_lines.append(line) + continue + if stripped.startswith(("/*", "*", "*/")): + out_lines.append(line) + continue + if "{" in line: + before, after = line.split("{", 1) + sel = before.strip() + if sel: + out_lines.append(f"{prefix} {sel} {{ {after}".rstrip()) + else: + out_lines.append(line) + else: + out_lines.append(line) + return "\n".join(out_lines) + + +def _render_code_block( + *, + filepath: str, + start_line: int, + end_line: int, + file_cache: _FileCache, + context: int, + max_lines: int, +) -> _Snippet: + s = max(1, start_line - context) + e = end_line + context + + if e - s + 1 > max_lines: + e = s + max_lines - 1 + + try: + lines = file_cache.get_lines_range(filepath, s, e) + except FileProcessingError: + missing = ( + '
'
+            '
Source file unavailable
'
+            "
" + ) + return _Snippet( + filepath=filepath, + start_line=start_line, + end_line=end_line, + code_html=missing, + ) + + numbered: list[tuple[bool, str]] = [] + for lineno, line in enumerate(lines, start=s): + hit = start_line <= lineno <= end_line + numbered.append((hit, f"{lineno:>5} | {line.rstrip()}")) + + raw = "\n".join(text for _, text in numbered) + highlighted = _try_pygments(raw) + + if highlighted is None: + rendered: list[str] = [] + for hit, text in numbered: + cls = "hitline" if hit else "line" + rendered.append( + f'
{html.escape(text, quote=False)}
' + ) + body = "\n".join(rendered) + else: + body = highlighted + + return _Snippet( + filepath=filepath, + start_line=start_line, + end_line=end_line, + code_html=f'
{body}
', + ) diff --git a/codeclone/_report_grouping.py b/codeclone/_report_grouping.py new file mode 100644 index 0000000..3ad44ab --- /dev/null +++ b/codeclone/_report_grouping.py @@ -0,0 +1,64 @@ +""" +CodeClone — AST and CFG-based code clone detector for Python +focused on architectural duplication. + +Copyright (c) 2026 Den Rozhnovskiy +Licensed under the MIT License. +""" + +from __future__ import annotations + +from ._report_types import GroupItem, GroupMap + + +def build_groups(units: list[GroupItem]) -> GroupMap: + groups: GroupMap = {} + for u in units: + key = f"{u['fingerprint']}|{u['loc_bucket']}" + groups.setdefault(key, []).append(u) + return {k: v for k, v in groups.items() if len(v) > 1} + + +def build_block_groups(blocks: list[GroupItem], min_functions: int = 2) -> GroupMap: + groups: GroupMap = {} + for b in blocks: + groups.setdefault(b["block_hash"], []).append(b) + + filtered: GroupMap = {} + for h, items in groups.items(): + functions = {i["qualname"] for i in items} + if len(functions) >= min_functions: + filtered[h] = items + + return filtered + + +def build_segment_groups( + segments: list[GroupItem], min_occurrences: int = 2 +) -> GroupMap: + sig_groups: GroupMap = {} + for s in segments: + sig_groups.setdefault(s["segment_sig"], []).append(s) + + confirmed: GroupMap = {} + for items in sig_groups.values(): + if len(items) < min_occurrences: + continue + + hash_groups: GroupMap = {} + for item in items: + hash_groups.setdefault(item["segment_hash"], []).append(item) + + for segment_hash, hash_items in hash_groups.items(): + if len(hash_items) < min_occurrences: + continue + + by_func: GroupMap = {} + for it in hash_items: + by_func.setdefault(it["qualname"], []).append(it) + + for qualname, q_items in by_func.items(): + if len(q_items) >= min_occurrences: + confirmed[f"{segment_hash}|{qualname}"] = q_items + + return confirmed diff --git a/codeclone/_report_segments.py b/codeclone/_report_segments.py new file mode 100644 index 0000000..bd985cb --- /dev/null +++ b/codeclone/_report_segments.py @@ -0,0 +1,247 @@ +""" +CodeClone — AST and CFG-based code clone detector for Python +focused on architectural duplication. + +Copyright (c) 2026 Den Rozhnovskiy +Licensed under the MIT License. +""" + +from __future__ import annotations + +import ast +from dataclasses import dataclass +from pathlib import Path + +from ._report_types import GroupItem, GroupMap + +SEGMENT_MIN_UNIQUE_STMT_TYPES = 2 + +_CONTROL_FLOW_STMTS = ( + ast.If, + ast.For, + ast.While, + ast.Try, + ast.With, + ast.Match, + ast.AsyncFor, + ast.AsyncWith, +) +_FORBIDDEN_STMTS = (ast.Return, ast.Raise, ast.Assert) + + +@dataclass(frozen=True, slots=True) +class _SegmentAnalysis: + unique_stmt_types: int + has_control_flow: bool + is_boilerplate: bool + + +class _QualnameCollector(ast.NodeVisitor): + __slots__ = ("funcs", "stack") + + def __init__(self) -> None: + self.stack: list[str] = [] + self.funcs: dict[str, ast.FunctionDef | ast.AsyncFunctionDef] = {} + + def visit_ClassDef(self, node: ast.ClassDef) -> None: + self.stack.append(node.name) + self.generic_visit(node) + self.stack.pop() + + def visit_FunctionDef(self, node: ast.FunctionDef) -> None: + name = ".".join([*self.stack, node.name]) if self.stack else node.name + self.funcs[name] = node + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None: + name = ".".join([*self.stack, node.name]) if self.stack else node.name + self.funcs[name] = node + + +def _merge_segment_items(items: list[GroupItem]) -> list[GroupItem]: + if not items: + return [] + + items_sorted = sorted( + items, + key=lambda i: ( + i.get("filepath", ""), + i.get("qualname", ""), + int(i.get("start_line", 0)), + int(i.get("end_line", 0)), + ), + ) + + merged: list[GroupItem] = [] + current: GroupItem | None = None + + for item in items_sorted: + start = int(item.get("start_line", 0)) + end = int(item.get("end_line", 0)) + if start <= 0 or end <= 0: + continue + + if current is None: + current = dict(item) + current["start_line"] = start + current["end_line"] = end + current["size"] = max(1, end - start + 1) + continue + + same_owner = current.get("filepath") == item.get("filepath") and current.get( + "qualname" + ) == item.get("qualname") + if same_owner and start <= int(current["end_line"]) + 1: + current["end_line"] = max(int(current["end_line"]), end) + current["size"] = max( + 1, int(current["end_line"]) - int(current["start_line"]) + 1 + ) + continue + + merged.append(current) + current = dict(item) + current["start_line"] = start + current["end_line"] = end + current["size"] = max(1, end - start + 1) + + if current is not None: + merged.append(current) + + return merged + + +def _collect_file_functions( + filepath: str, +) -> dict[str, ast.FunctionDef | ast.AsyncFunctionDef] | None: + try: + source = Path(filepath).read_text("utf-8") + except OSError: + return None + try: + tree = ast.parse(source) + except SyntaxError: + return None + + collector = _QualnameCollector() + collector.visit(tree) + return collector.funcs + + +def _segment_statements( + func_node: ast.FunctionDef | ast.AsyncFunctionDef, start_line: int, end_line: int +) -> list[ast.stmt]: + body = getattr(func_node, "body", None) + if not isinstance(body, list): + return [] + stmts: list[ast.stmt] = [] + for stmt in body: + lineno = getattr(stmt, "lineno", None) + end = getattr(stmt, "end_lineno", None) + if lineno is None or end is None: + continue + if lineno >= start_line and end <= end_line: + stmts.append(stmt) + return stmts + + +def _assign_targets_attribute_only(stmt: ast.stmt) -> bool: + if isinstance(stmt, ast.Assign): + return all(isinstance(t, ast.Attribute) for t in stmt.targets) + if isinstance(stmt, ast.AnnAssign): + return isinstance(stmt.target, ast.Attribute) + return False + + +def _analyze_segment_statements(stmts: list[ast.stmt]) -> _SegmentAnalysis | None: + if not stmts: + return None + + unique_types = {type(s) for s in stmts} + has_control_flow = any(isinstance(s, _CONTROL_FLOW_STMTS) for s in stmts) + has_forbidden = any(isinstance(s, _FORBIDDEN_STMTS) for s in stmts) + has_call_stmt = any( + isinstance(s, ast.Expr) and isinstance(s.value, ast.Call) for s in stmts + ) + + assign_stmts = [s for s in stmts if isinstance(s, (ast.Assign, ast.AnnAssign))] + assign_ratio = len(assign_stmts) / len(stmts) + assign_attr_only = all(_assign_targets_attribute_only(s) for s in assign_stmts) + + is_boilerplate = ( + assign_ratio >= 0.8 + and assign_attr_only + and not has_control_flow + and not has_forbidden + and not has_call_stmt + ) + + return _SegmentAnalysis( + unique_stmt_types=len(unique_types), + has_control_flow=has_control_flow, + is_boilerplate=is_boilerplate, + ) + + +def prepare_segment_report_groups( + segment_groups: GroupMap, +) -> tuple[GroupMap, int]: + """ + Merge overlapping segment windows and suppress low-value boilerplate groups + for reporting. Detection hashes remain unchanged. + """ + suppressed = 0 + filtered: GroupMap = {} + file_cache: dict[str, dict[str, ast.FunctionDef | ast.AsyncFunctionDef] | None] = {} + + for key, items in segment_groups.items(): + merged_items = _merge_segment_items(items) + if not merged_items: + continue + + analyses: list[_SegmentAnalysis] = [] + unknown = False + for item in merged_items: + filepath = str(item.get("filepath", "")) + qualname = str(item.get("qualname", "")) + start_line = int(item.get("start_line", 0)) + end_line = int(item.get("end_line", 0)) + if not filepath or not qualname or start_line <= 0 or end_line <= 0: + unknown = True + break + + if filepath not in file_cache: + file_cache[filepath] = _collect_file_functions(filepath) + funcs = file_cache[filepath] + if not funcs: + unknown = True + break + + local_name = qualname.split(":", 1)[1] if ":" in qualname else qualname + func_node = funcs.get(local_name) + if func_node is None: + unknown = True + break + + stmts = _segment_statements(func_node, start_line, end_line) + analysis = _analyze_segment_statements(stmts) + if analysis is None: + unknown = True + break + analyses.append(analysis) + + if unknown: + filtered[key] = merged_items + continue + + all_boilerplate = all(a.is_boilerplate for a in analyses) + all_too_simple = all( + (not a.has_control_flow) + and (a.unique_stmt_types < SEGMENT_MIN_UNIQUE_STMT_TYPES) + for a in analyses + ) + if all_boilerplate or all_too_simple: + suppressed += 1 + continue + + filtered[key] = merged_items + + return filtered, suppressed diff --git a/codeclone/_report_serialize.py b/codeclone/_report_serialize.py new file mode 100644 index 0000000..54dcef5 --- /dev/null +++ b/codeclone/_report_serialize.py @@ -0,0 +1,160 @@ +""" +CodeClone — AST and CFG-based code clone detector for Python +focused on architectural duplication. + +Copyright (c) 2026 Den Rozhnovskiy +Licensed under the MIT License. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from typing import Any + +from ._report_types import GroupItem, GroupMap + + +def to_json(groups: GroupMap) -> str: + def _sorted_items(items: list[GroupItem]) -> list[GroupItem]: + return sorted( + items, + key=lambda item: ( + str(item.get("filepath", "")), + int(item.get("start_line", 0)), + int(item.get("end_line", 0)), + str(item.get("qualname", "")), + ), + ) + + return json.dumps( + { + "group_count": len(groups), + "groups": [ + {"key": k, "count": len(v), "items": _sorted_items(v)} + for k, v in sorted( + groups.items(), + key=lambda kv: (-len(kv[1]), kv[0]), + ) + ], + }, + ensure_ascii=False, + indent=2, + ) + + +def to_json_report( + func_groups: GroupMap, + block_groups: GroupMap, + segment_groups: GroupMap, + meta: Mapping[str, Any] | None = None, +) -> str: + def _sorted_items(items: list[GroupItem]) -> list[GroupItem]: + return sorted( + items, + key=lambda item: ( + str(item.get("filepath", "")), + int(item.get("start_line", 0)), + int(item.get("end_line", 0)), + str(item.get("qualname", "")), + ), + ) + + def _sorted_group_map(groups: GroupMap) -> GroupMap: + return { + k: _sorted_items(v) + for k, v in sorted(groups.items(), key=lambda kv: (-len(kv[1]), kv[0])) + } + + meta_payload = dict(meta or {}) + func_sorted = _sorted_group_map(func_groups) + block_sorted = _sorted_group_map(block_groups) + segment_sorted = _sorted_group_map(segment_groups) + return json.dumps( + { + "meta": meta_payload, + "function_clones": func_sorted, + "block_clones": block_sorted, + "segment_clones": segment_sorted, + # Backward-compatible keys. + "functions": func_sorted, + "blocks": block_sorted, + "segments": segment_sorted, + }, + ensure_ascii=False, + indent=2, + ) + + +def to_text(groups: GroupMap) -> str: + lines: list[str] = [] + for i, (_, v) in enumerate( + sorted(groups.items(), key=lambda kv: (-len(kv[1]), kv[0])) + ): + items = sorted( + v, + key=lambda item: ( + str(item.get("filepath", "")), + int(item.get("start_line", 0)), + int(item.get("end_line", 0)), + str(item.get("qualname", "")), + ), + ) + lines.append(f"\n=== Clone group #{i + 1} (count={len(v)}) ===") + lines.extend( + [ + f"- {item['qualname']} " + f"{item['filepath']}:{item['start_line']}-{item['end_line']} " + f"loc={item.get('loc', item.get('size'))}" + for item in items + ] + ) + return "\n".join(lines).strip() + "\n" + + +def _format_meta_text_value(value: Any) -> str: + if isinstance(value, bool): + return "true" if value else "false" + if value is None: + return "n/a" + text = str(value).strip() + return text if text else "n/a" + + +def to_text_report( + *, + meta: Mapping[str, Any], + func_groups: GroupMap, + block_groups: GroupMap, + segment_groups: GroupMap, +) -> str: + lines = [ + "REPORT METADATA", + f"CodeClone version: {_format_meta_text_value(meta.get('codeclone_version'))}", + f"Python version: {_format_meta_text_value(meta.get('python_version'))}", + f"Baseline path: {_format_meta_text_value(meta.get('baseline_path'))}", + f"Baseline version: {_format_meta_text_value(meta.get('baseline_version'))}", + "Baseline schema version: " + f"{_format_meta_text_value(meta.get('baseline_schema_version'))}", + "Baseline Python version: " + f"{_format_meta_text_value(meta.get('baseline_python_version'))}", + f"Baseline loaded: {_format_meta_text_value(meta.get('baseline_loaded'))}", + f"Baseline status: {_format_meta_text_value(meta.get('baseline_status'))}", + ] + if "cache_path" in meta: + lines.append(f"Cache path: {_format_meta_text_value(meta.get('cache_path'))}") + if "cache_used" in meta: + lines.append(f"Cache used: {_format_meta_text_value(meta.get('cache_used'))}") + + sections = [ + ("FUNCTION CLONES", func_groups), + ("BLOCK CLONES", block_groups), + ("SEGMENT CLONES", segment_groups), + ] + for title, groups in sections: + lines.append("") + lines.append(title) + block = to_text(groups).rstrip() + lines.append(block if block else "(none)") + + return "\n".join(lines).rstrip() + "\n" diff --git a/codeclone/_report_types.py b/codeclone/_report_types.py new file mode 100644 index 0000000..6fbe632 --- /dev/null +++ b/codeclone/_report_types.py @@ -0,0 +1,14 @@ +""" +CodeClone — AST and CFG-based code clone detector for Python +focused on architectural duplication. + +Copyright (c) 2026 Den Rozhnovskiy +Licensed under the MIT License. +""" + +from __future__ import annotations + +from typing import Any + +GroupItem = dict[str, Any] +GroupMap = dict[str, list[GroupItem]] diff --git a/codeclone/baseline.py b/codeclone/baseline.py index e73ff7f..4e0894f 100644 --- a/codeclone/baseline.py +++ b/codeclone/baseline.py @@ -8,22 +8,31 @@ from __future__ import annotations +import hashlib +import hmac import json from collections.abc import Mapping +from datetime import datetime, timezone from pathlib import Path from typing import Any from . import __version__ +from .errors import BaselineValidationError BASELINE_SCHEMA_VERSION = 1 +MAX_BASELINE_SIZE_BYTES = 5 * 1024 * 1024 +BASELINE_GENERATOR = "codeclone" class Baseline: __slots__ = ( "baseline_version", "blocks", + "created_at", "functions", + "generator", "path", + "payload_sha256", "python_version", "schema_version", ) @@ -35,32 +44,63 @@ def __init__(self, path: str | Path): self.python_version: str | None = None self.baseline_version: str | None = None self.schema_version: int | None = None + self.generator: str | None = None + self.payload_sha256: str | None = None + self.created_at: str | None = None - def load(self) -> None: + def load(self, *, max_size_bytes: int | None = None) -> None: if not self.path.exists(): return + size_limit = ( + MAX_BASELINE_SIZE_BYTES if max_size_bytes is None else max_size_bytes + ) try: - data = json.loads(self.path.read_text("utf-8")) - self.functions = set(data.get("functions", [])) - self.blocks = set(data.get("blocks", [])) - python_version = data.get("python_version") - self.python_version = ( - python_version if isinstance(python_version, str) else None - ) - baseline_version = data.get("baseline_version") - self.baseline_version = ( - baseline_version if isinstance(baseline_version, str) else None - ) - schema_version = data.get("schema_version") - self.schema_version = ( - schema_version if isinstance(schema_version, int) else None + size = self.path.stat().st_size + except OSError as e: + raise BaselineValidationError( + f"Cannot stat baseline file at {self.path}: {e}" + ) from e + if size > size_limit: + raise BaselineValidationError( + "Baseline file is too large " + f"({size} bytes, max {size_limit} bytes) at {self.path}", + status="too_large", ) + + try: + data = json.loads(self.path.read_text("utf-8")) except json.JSONDecodeError as e: - raise ValueError(f"Corrupted baseline file at {self.path}: {e}") from e + raise BaselineValidationError( + f"Corrupted baseline file at {self.path}: {e}" + ) from e + + if not isinstance(data, dict): + raise BaselineValidationError( + f"Baseline payload must be an object at {self.path}" + ) + + functions = _require_str_list(data, "functions", path=self.path) + blocks = _require_str_list(data, "blocks", path=self.path) + python_version = _optional_str(data, "python_version", path=self.path) + baseline_version = _optional_str(data, "baseline_version", path=self.path) + schema_version = _optional_int(data, "schema_version", path=self.path) + generator = _optional_str_loose(data, "generator") + payload_sha256 = _optional_str_loose(data, "payload_sha256") + created_at = _optional_str(data, "created_at", path=self.path) + + self.functions = set(functions) + self.blocks = set(blocks) + self.python_version = python_version + self.baseline_version = baseline_version + self.schema_version = schema_version + self.generator = generator + self.payload_sha256 = payload_sha256 + self.created_at = created_at def save(self) -> None: self.path.parent.mkdir(parents=True, exist_ok=True) + now_utc = datetime.now(timezone.utc).replace(microsecond=0).isoformat() self.path.write_text( json.dumps( _baseline_payload( @@ -69,6 +109,8 @@ def save(self) -> None: self.python_version, self.baseline_version, self.schema_version, + self.generator, + now_utc, ), indent=2, ensure_ascii=False, @@ -76,6 +118,29 @@ def save(self) -> None: "utf-8", ) + def is_legacy_format(self) -> bool: + return self.baseline_version is None or self.schema_version is None + + def verify_integrity(self) -> None: + if self.is_legacy_format(): + return + if self.generator != BASELINE_GENERATOR: + raise BaselineValidationError( + "Baseline generator mismatch: expected 'codeclone'.", + status="generator_mismatch", + ) + if not isinstance(self.payload_sha256, str): + raise BaselineValidationError( + "Baseline integrity payload hash is missing.", + status="integrity_missing", + ) + expected = _compute_payload_sha256(self.functions, self.blocks) + if not hmac.compare_digest(self.payload_sha256, expected): + raise BaselineValidationError( + "Baseline integrity check failed: payload_sha256 mismatch.", + status="integrity_failed", + ) + @staticmethod def from_groups( func_groups: Mapping[str, object], @@ -91,6 +156,7 @@ def from_groups( bl.python_version = python_version bl.baseline_version = baseline_version bl.schema_version = schema_version + bl.generator = BASELINE_GENERATOR return bl def diff( @@ -107,15 +173,73 @@ def _baseline_payload( python_version: str | None, baseline_version: str | None, schema_version: int | None, + generator: str | None, + created_at: str | None, ) -> dict[str, Any]: - payload: dict[str, Any] = { - "functions": sorted(functions), - "blocks": sorted(blocks), - } + payload: dict[str, Any] = _canonical_payload(functions, blocks) if python_version: payload["python_version"] = python_version payload["baseline_version"] = baseline_version or __version__ payload["schema_version"] = ( schema_version if schema_version is not None else BASELINE_SCHEMA_VERSION ) + payload["generator"] = generator or BASELINE_GENERATOR + payload["payload_sha256"] = _compute_payload_sha256(functions, blocks) + if created_at: + payload["created_at"] = created_at return payload + + +def _canonical_payload(functions: set[str], blocks: set[str]) -> dict[str, list[str]]: + return { + "functions": sorted(functions), + "blocks": sorted(blocks), + } + + +def _compute_payload_sha256(functions: set[str], blocks: set[str]) -> str: + serialized = json.dumps( + _canonical_payload(functions, blocks), + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ) + return hashlib.sha256(serialized.encode("utf-8")).hexdigest() + + +def _require_str_list(data: dict[str, Any], key: str, *, path: Path) -> list[str]: + value = data.get(key) + if not isinstance(value, list) or not all(isinstance(v, str) for v in value): + raise BaselineValidationError( + f"Invalid baseline schema at {path}: '{key}' must be list[str]" + ) + return value + + +def _optional_str(data: dict[str, Any], key: str, *, path: Path) -> str | None: + value = data.get(key) + if value is None: + return None + if not isinstance(value, str): + raise BaselineValidationError( + f"Invalid baseline schema at {path}: '{key}' must be string" + ) + return value + + +def _optional_int(data: dict[str, Any], key: str, *, path: Path) -> int | None: + value = data.get(key) + if value is None: + return None + if not isinstance(value, int): + raise BaselineValidationError( + f"Invalid baseline schema at {path}: '{key}' must be integer" + ) + return value + + +def _optional_str_loose(data: dict[str, Any], key: str) -> str | None: + value = data.get(key) + if isinstance(value, str): + return value + return None diff --git a/codeclone/cache.py b/codeclone/cache.py index c3bd024..566e82e 100644 --- a/codeclone/cache.py +++ b/codeclone/cache.py @@ -25,6 +25,7 @@ from .errors import CacheError OS_NAME = os.name +MAX_CACHE_SIZE_BYTES = 50 * 1024 * 1024 class FileStat(TypedDict): @@ -75,14 +76,17 @@ class CacheData(TypedDict): class Cache: - __slots__ = ("data", "load_warning", "path", "secret") + __slots__ = ("data", "load_warning", "max_size_bytes", "path", "secret") CACHE_VERSION = "1.1" - def __init__(self, path: str | Path): + def __init__(self, path: str | Path, *, max_size_bytes: int | None = None): self.path = Path(path) self.data: CacheData = {"version": self.CACHE_VERSION, "files": {}} self.secret = self._load_secret() self.load_warning: str | None = None + self.max_size_bytes = ( + MAX_CACHE_SIZE_BYTES if max_size_bytes is None else max_size_bytes + ) def _load_secret(self) -> bytes: """Load or create cache signing secret.""" @@ -115,6 +119,15 @@ def load(self) -> None: return try: + size = self.path.stat().st_size + if size > self.max_size_bytes: + self.load_warning = ( + "Cache file too large " + f"({size} bytes, max {self.max_size_bytes}); ignoring cache." + ) + self.data = {"version": self.CACHE_VERSION, "files": {}} + return + raw = json.loads(self.path.read_text("utf-8")) stored_sig = raw.get("_signature") @@ -123,7 +136,10 @@ def load(self) -> None: # Verify signature expected_sig = self._sign_data(data) - if stored_sig != expected_sig: + if not ( + isinstance(stored_sig, str) + and hmac.compare_digest(stored_sig, expected_sig) + ): self.load_warning = "Cache signature mismatch; ignoring cache." self.data = {"version": self.CACHE_VERSION, "files": {}} return @@ -176,6 +192,18 @@ def get_file_entry(self, filepath: str) -> CacheEntry | None: if not required.issubset(entry.keys()): return None + stat = entry.get("stat") + units = entry.get("units") + blocks = entry.get("blocks") + segments = entry.get("segments") + if not ( + _is_file_stat_dict(stat) + and _is_unit_list(units) + and _is_block_list(blocks) + and _is_segment_list(segments) + ): + return None + return entry def put_file_entry( @@ -202,3 +230,56 @@ def file_stat_signature(path: str) -> FileStat: "mtime_ns": st.st_mtime_ns, "size": st.st_size, } + + +def _is_file_stat_dict(value: object) -> bool: + if not isinstance(value, dict): + return False + return isinstance(value.get("mtime_ns"), int) and isinstance(value.get("size"), int) + + +def _is_unit_dict(value: object) -> bool: + if not isinstance(value, dict): + return False + string_keys = ("qualname", "filepath", "fingerprint", "loc_bucket") + int_keys = ("start_line", "end_line", "loc", "stmt_count") + return _has_typed_fields(value, string_keys=string_keys, int_keys=int_keys) + + +def _is_block_dict(value: object) -> bool: + if not isinstance(value, dict): + return False + string_keys = ("block_hash", "filepath", "qualname") + int_keys = ("start_line", "end_line", "size") + return _has_typed_fields(value, string_keys=string_keys, int_keys=int_keys) + + +def _is_segment_dict(value: object) -> bool: + if not isinstance(value, dict): + return False + string_keys = ("segment_hash", "segment_sig", "filepath", "qualname") + int_keys = ("start_line", "end_line", "size") + return _has_typed_fields(value, string_keys=string_keys, int_keys=int_keys) + + +def _is_unit_list(value: object) -> bool: + return isinstance(value, list) and all(_is_unit_dict(item) for item in value) + + +def _is_block_list(value: object) -> bool: + return isinstance(value, list) and all(_is_block_dict(item) for item in value) + + +def _is_segment_list(value: object) -> bool: + return isinstance(value, list) and all(_is_segment_dict(item) for item in value) + + +def _has_typed_fields( + value: dict[str, object], + *, + string_keys: tuple[str, ...], + int_keys: tuple[str, ...], +) -> bool: + return all(isinstance(value.get(key), str) for key in string_keys) and all( + isinstance(value.get(key), int) for key in int_keys + ) diff --git a/codeclone/cli.py b/codeclone/cli.py index 2d633de..98a6dc0 100644 --- a/codeclone/cli.py +++ b/codeclone/cli.py @@ -1,6 +1,5 @@ from __future__ import annotations -import argparse import os import sys from concurrent.futures import ProcessPoolExecutor, as_completed @@ -17,15 +16,22 @@ TextColumn, TimeElapsedColumn, ) -from rich.table import Table -from rich.text import Text from rich.theme import Theme from . import __version__ from . import ui_messages as ui +from ._cli_args import build_parser +from ._cli_meta import _build_report_meta as _build_report_meta_impl +from ._cli_meta import _current_python_version as _current_python_version_impl +from ._cli_paths import _validate_output_path as _validate_output_path_impl +from ._cli_paths import expand_path as _expand_path_impl +from ._cli_summary import _build_summary_rows as _build_summary_rows_impl +from ._cli_summary import _build_summary_table as _build_summary_table_impl +from ._cli_summary import _print_summary as _print_summary_impl +from ._cli_summary import _summary_value_style as _summary_value_style_impl from .baseline import BASELINE_SCHEMA_VERSION, Baseline from .cache import Cache, CacheEntry, FileStat, file_stat_signature -from .errors import CacheError +from .errors import BaselineValidationError, CacheError from .extractor import extract_units_from_source from .html_report import build_html_report from .normalize import NormalizationConfig @@ -51,13 +57,6 @@ ) -class _HelpFormatter(argparse.ArgumentDefaultsHelpFormatter): - def _get_help_string(self, action: argparse.Action) -> str: - if action.dest == "cache_path": - return action.help or "" - return cast(str, super()._get_help_string(action)) - - LEGACY_CACHE_PATH = Path("~/.cache/codeclone/cache.json").expanduser() @@ -69,6 +68,19 @@ def _make_console(*, no_color: bool) -> Console: MAX_FILE_SIZE = 10 * 1024 * 1024 # 10MB BATCH_SIZE = 100 +_VALID_BASELINE_STATUSES = { + "ok", + "missing", + "legacy", + "invalid", + "mismatch_version", + "mismatch_schema", + "mismatch_python", + "generator_mismatch", + "integrity_missing", + "integrity_failed", + "too_large", +} @dataclass(slots=True) @@ -85,7 +97,7 @@ class ProcessingResult: def expand_path(p: str) -> Path: - return Path(p).expanduser().resolve() + return _expand_path_impl(p) def process_file( @@ -172,19 +184,17 @@ def print_banner() -> None: def _validate_output_path(path: str, *, expected_suffix: str, label: str) -> Path: - out = Path(path).expanduser() - if out.suffix.lower() != expected_suffix: - console.print( - ui.fmt_invalid_output_extension( - label=label, path=out, expected_suffix=expected_suffix - ) - ) - sys.exit(2) - return out.resolve() + return _validate_output_path_impl( + path, + expected_suffix=expected_suffix, + label=label, + console=console, + invalid_message=ui.fmt_invalid_output_extension, + ) def _current_python_version() -> str: - return f"{sys.version_info.major}.{sys.version_info.minor}" + return _current_python_version_impl() def _build_report_meta( @@ -196,28 +206,19 @@ def _build_report_meta( cache_path: Path, cache_used: bool, ) -> dict[str, Any]: - return { - "codeclone_version": __version__, - "python_version": _current_python_version(), - "baseline_path": str(baseline_path), - "baseline_version": baseline.baseline_version, - "baseline_schema_version": baseline.schema_version, - "baseline_python_version": baseline.python_version, - "baseline_loaded": baseline_loaded, - "baseline_status": baseline_status, - "cache_path": str(cache_path), - "cache_used": cache_used, - } + return _build_report_meta_impl( + codeclone_version=__version__, + baseline_path=baseline_path, + baseline=baseline, + baseline_loaded=baseline_loaded, + baseline_status=baseline_status, + cache_path=cache_path, + cache_used=cache_used, + ) def _summary_value_style(*, label: str, value: int) -> str: - if value == 0: - return "dim" - if label == ui.SUMMARY_LABEL_NEW_BASELINE: - return "bold red" - if label == ui.SUMMARY_LABEL_SUPPRESSED: - return "yellow" - return "bold green" + return _summary_value_style_impl(label=label, value=value) def _build_summary_rows( @@ -232,29 +233,21 @@ def _build_summary_rows( suppressed_segment_groups: int, new_clones_count: int, ) -> list[tuple[str, int]]: - return [ - (ui.SUMMARY_LABEL_FILES_FOUND, files_found), - (ui.SUMMARY_LABEL_FILES_ANALYZED, files_analyzed), - (ui.SUMMARY_LABEL_CACHE_HITS, cache_hits), - (ui.SUMMARY_LABEL_FILES_SKIPPED, files_skipped), - (ui.SUMMARY_LABEL_FUNCTION, func_clones_count), - (ui.SUMMARY_LABEL_BLOCK, block_clones_count), - (ui.SUMMARY_LABEL_SEGMENT, segment_clones_count), - (ui.SUMMARY_LABEL_SUPPRESSED, suppressed_segment_groups), - (ui.SUMMARY_LABEL_NEW_BASELINE, new_clones_count), - ] - - -def _build_summary_table(rows: list[tuple[str, int]]) -> Table: - summary_table = Table(title=ui.SUMMARY_TITLE, show_header=True) - summary_table.add_column("Metric") - summary_table.add_column("Value", justify="right") - for label, value in rows: - summary_table.add_row( - label, - Text(str(value), style=_summary_value_style(label=label, value=value)), - ) - return summary_table + return _build_summary_rows_impl( + files_found=files_found, + files_analyzed=files_analyzed, + cache_hits=cache_hits, + files_skipped=files_skipped, + func_clones_count=func_clones_count, + block_clones_count=block_clones_count, + segment_clones_count=segment_clones_count, + suppressed_segment_groups=suppressed_segment_groups, + new_clones_count=new_clones_count, + ) + + +def _build_summary_table(rows: list[tuple[str, int]]) -> Any: + return _build_summary_table_impl(rows) def _print_summary( @@ -270,8 +263,9 @@ def _print_summary( suppressed_segment_groups: int, new_clones_count: int, ) -> None: - invariant_ok = files_found == (files_analyzed + cache_hits + files_skipped) - rows = _build_summary_rows( + _print_summary_impl( + console=console, + quiet=quiet, files_found=files_found, files_analyzed=files_analyzed, cache_hits=cache_hits, @@ -283,159 +277,9 @@ def _print_summary( new_clones_count=new_clones_count, ) - if quiet: - console.print(ui.SUMMARY_TITLE) - console.print( - ui.fmt_summary_compact_input( - found=files_found, - analyzed=files_analyzed, - cache_hits=cache_hits, - skipped=files_skipped, - ) - ) - console.print( - ui.fmt_summary_compact_clones( - function=func_clones_count, - block=block_clones_count, - segment=segment_clones_count, - suppressed=suppressed_segment_groups, - new=new_clones_count, - ) - ) - else: - console.print(_build_summary_table(rows)) - - if not invariant_ok: - console.print(f"[warning]{ui.WARN_SUMMARY_ACCOUNTING_MISMATCH}[/warning]") - def main() -> None: - ap = argparse.ArgumentParser( - prog="codeclone", - description="AST and CFG-based code clone detector for Python.", - formatter_class=_HelpFormatter, - ) - ap.add_argument( - "--version", - action="version", - version=ui.version_output(__version__), - help=ui.HELP_VERSION, - ) - - # Core Arguments - core_group = ap.add_argument_group("Target") - core_group.add_argument( - "root", - nargs="?", - default=".", - help=ui.HELP_ROOT, - ) - - # Tuning - tune_group = ap.add_argument_group("Analysis Tuning") - tune_group.add_argument( - "--min-loc", - type=int, - default=15, - help=ui.HELP_MIN_LOC, - ) - tune_group.add_argument( - "--min-stmt", - type=int, - default=6, - help=ui.HELP_MIN_STMT, - ) - tune_group.add_argument( - "--processes", - type=int, - default=4, - help=ui.HELP_PROCESSES, - ) - tune_group.add_argument( - "--cache-path", - dest="cache_path", - metavar="FILE", - default=None, - help=ui.HELP_CACHE_PATH, - ) - tune_group.add_argument( - "--cache-dir", - dest="cache_path", - metavar="FILE", - default=None, - help=ui.HELP_CACHE_DIR_LEGACY, - ) - - # Baseline & CI - ci_group = ap.add_argument_group("Baseline & CI/CD") - ci_group.add_argument( - "--baseline", - default="codeclone.baseline.json", - help=ui.HELP_BASELINE, - ) - ci_group.add_argument( - "--update-baseline", - action="store_true", - help=ui.HELP_UPDATE_BASELINE, - ) - ci_group.add_argument( - "--fail-on-new", - action="store_true", - help=ui.HELP_FAIL_ON_NEW, - ) - ci_group.add_argument( - "--fail-threshold", - type=int, - default=-1, - metavar="MAX_CLONES", - help=ui.HELP_FAIL_THRESHOLD, - ) - ci_group.add_argument( - "--ci", - action="store_true", - help=ui.HELP_CI, - ) - - # Output - out_group = ap.add_argument_group("Reporting") - out_group.add_argument( - "--html", - dest="html_out", - metavar="FILE", - help=ui.HELP_HTML, - ) - out_group.add_argument( - "--json", - dest="json_out", - metavar="FILE", - help=ui.HELP_JSON, - ) - out_group.add_argument( - "--text", - dest="text_out", - metavar="FILE", - help=ui.HELP_TEXT, - ) - out_group.add_argument( - "--no-progress", - action="store_true", - help=ui.HELP_NO_PROGRESS, - ) - out_group.add_argument( - "--no-color", - action="store_true", - help=ui.HELP_NO_COLOR, - ) - out_group.add_argument( - "--quiet", - action="store_true", - help=ui.HELP_QUIET, - ) - out_group.add_argument( - "--verbose", - action="store_true", - help=ui.HELP_VERBOSE, - ) + ap = build_parser(__version__) cache_path_from_args = any( arg in {"--cache-dir", "--cache-path"} @@ -455,6 +299,10 @@ def main() -> None: global console console = _make_console(no_color=args.no_color) + if args.max_baseline_size_mb < 0 or args.max_cache_size_mb < 0: + console.print("[error]Size limits must be non-negative integers (MB).[/error]") + sys.exit(1) + if not args.quiet: print_banner() @@ -503,7 +351,7 @@ def main() -> None: legacy_path=legacy_resolved, new_path=cache_path ) ) - cache = Cache(cache_path) + cache = Cache(cache_path, max_size_bytes=args.max_cache_size_mb * 1024 * 1024) cache.load() if cache.load_warning: console.print(f"[warning]{cache.load_warning}[/warning]") @@ -812,53 +660,70 @@ def process_sequential(with_progress: bool) -> None: if baseline_exists: try: - baseline.load() - except ValueError as e: - baseline_status = "invalid" - if not args.update_baseline: + baseline.load(max_size_bytes=args.max_baseline_size_mb * 1024 * 1024) + except BaselineValidationError as e: + baseline_status = ( + e.status if e.status in _VALID_BASELINE_STATUSES else "invalid" + ) + if baseline_status == "too_large" or not args.update_baseline: console.print(ui.fmt_invalid_baseline(e)) baseline_failure_code = 2 else: baseline_loaded = True baseline_status = "ok" if not args.update_baseline: - if baseline.baseline_version != __version__: - baseline_status = "mismatch" - if baseline.baseline_version is None: - console.print(ui.fmt_baseline_version_missing(__version__)) - else: + if baseline.is_legacy_format(): + baseline_status = "legacy" + console.print(ui.fmt_baseline_version_missing(__version__)) + baseline_failure_code = 2 + else: + if baseline.baseline_version != __version__: + assert baseline.baseline_version is not None + baseline_status = "mismatch_version" console.print( ui.fmt_baseline_version_mismatch( baseline_version=baseline.baseline_version, current_version=__version__, ) ) - baseline_failure_code = 2 - if ( - baseline.schema_version is not None - and baseline.schema_version != BASELINE_SCHEMA_VERSION - ): - baseline_status = "mismatch" - console.print( - ui.fmt_baseline_schema_mismatch( - baseline_schema=baseline.schema_version, - current_schema=BASELINE_SCHEMA_VERSION, - ) - ) - baseline_failure_code = 2 - if not args.update_baseline and baseline.python_version: - current_version = _current_python_version() - if baseline.python_version != current_version: - baseline_status = "mismatch" - console.print( - ui.fmt_baseline_python_mismatch( - baseline_python=baseline.python_version, - current_python=current_version, + baseline_failure_code = 2 + if baseline.schema_version != BASELINE_SCHEMA_VERSION: + assert baseline.schema_version is not None + if baseline_status == "ok": + baseline_status = "mismatch_schema" + console.print( + ui.fmt_baseline_schema_mismatch( + baseline_schema=baseline.schema_version, + current_schema=BASELINE_SCHEMA_VERSION, + ) ) - ) - if args.fail_on_new: - console.print(ui.ERR_BASELINE_SAME_PYTHON_REQUIRED) baseline_failure_code = 2 + if baseline.python_version: + current_version = _current_python_version() + if baseline.python_version != current_version: + if baseline_status == "ok": + baseline_status = "mismatch_python" + console.print( + ui.fmt_baseline_python_mismatch( + baseline_python=baseline.python_version, + current_python=current_version, + ) + ) + if args.fail_on_new: + console.print(ui.ERR_BASELINE_SAME_PYTHON_REQUIRED) + baseline_failure_code = 2 + if baseline_status == "ok": + try: + baseline.verify_integrity() + except BaselineValidationError as e: + status = ( + e.status + if e.status in _VALID_BASELINE_STATUSES + else "invalid" + ) + baseline_status = status + console.print(ui.fmt_invalid_baseline(e)) + baseline_failure_code = 2 else: if not args.update_baseline: console.print(ui.fmt_path(ui.WARN_BASELINE_MISSING, baseline_path)) @@ -874,6 +739,9 @@ def process_sequential(with_progress: bool) -> None: ) new_baseline.save() console.print(ui.fmt_path(ui.SUCCESS_BASELINE_UPDATED, baseline_path)) + baseline = new_baseline + baseline_loaded = True + baseline_status = "ok" # When updating, we don't fail on new, we just saved the new state. # But we might still want to print the summary. diff --git a/codeclone/errors.py b/codeclone/errors.py index c2ab463..11e32b8 100644 --- a/codeclone/errors.py +++ b/codeclone/errors.py @@ -25,3 +25,17 @@ class ValidationError(CodeCloneError): class CacheError(CodeCloneError): """Cache operation failed.""" + + +class BaselineSchemaError(CodeCloneError): + """Baseline file structure is invalid.""" + + +class BaselineValidationError(BaselineSchemaError): + """Baseline validation error with machine-readable status.""" + + __slots__ = ("status",) + + def __init__(self, message: str, *, status: str = "invalid") -> None: + super().__init__(message) + self.status = status diff --git a/codeclone/html_report.py b/codeclone/html_report.py index c4badd4..c7ddf36 100644 --- a/codeclone/html_report.py +++ b/codeclone/html_report.py @@ -8,241 +8,37 @@ from __future__ import annotations -import html -import importlib -import itertools -from collections.abc import Iterable -from dataclasses import dataclass -from functools import lru_cache -from typing import Any, NamedTuple, cast - -from codeclone import __version__ -from codeclone.errors import FileProcessingError - +from typing import Any + +from . import __version__ +from ._html_escape import _escape_attr, _escape_html, _meta_display +from ._html_snippets import ( + _FileCache, + _prefix_css, + _pygments_css, + _render_code_block, + _try_pygments, + pairwise, +) from .templates import FONT_CSS_URL, REPORT_TEMPLATE -# ============================ -# Pairwise -# ============================ - - -def pairwise(iterable: Iterable[Any]) -> Iterable[tuple[Any, Any]]: - a, b = itertools.tee(iterable) - next(b, None) - return zip(a, b, strict=False) - - -# ============================ -# Code snippet infrastructure -# ============================ - - -@dataclass(slots=True) -class _Snippet: - filepath: str - start_line: int - end_line: int - code_html: str - - -class _FileCache: - __slots__ = ("_get_lines_impl", "maxsize") - - def __init__(self, maxsize: int = 128) -> None: - self.maxsize = maxsize - # Create a bound method with lru_cache - # We need to cache on the method to have instance-level caching if we wanted - # different caches per instance. But lru_cache on method actually caches - # on the function object (class level) if not careful, - # or we use a wrapper. - # However, for this script, we usually have one reporter. - # To be safe and cleaner, we can use a method that delegates to a cached - # function, OR just use lru_cache on a method (which requires 'self' to be - # hashable, which it is by default id). - # But 'self' changes if we create new instances. - # Let's use the audit's pattern: cache the implementation. - - self._get_lines_impl = lru_cache(maxsize=maxsize)(self._read_file_range) - - @staticmethod - def _read_file_range( - filepath: str, start_line: int, end_line: int - ) -> tuple[str, ...]: - if start_line < 1: - start_line = 1 - if end_line < start_line: - return () - - try: - - def _read_with_errors(errors: str) -> tuple[str, ...]: - lines: list[str] = [] - with open(filepath, encoding="utf-8", errors=errors) as f: - for lineno, line in enumerate(f, start=1): - if lineno < start_line: - continue - if lineno > end_line: - break - lines.append(line.rstrip("\n")) - return tuple(lines) - - try: - return _read_with_errors("strict") - except UnicodeDecodeError: - return _read_with_errors("replace") - except OSError as e: - raise FileProcessingError(f"Cannot read {filepath}: {e}") from e - - def get_lines_range( - self, filepath: str, start_line: int, end_line: int - ) -> tuple[str, ...]: - return self._get_lines_impl(filepath, start_line, end_line) - - class _CacheInfo(NamedTuple): - hits: int - misses: int - maxsize: int | None - currsize: int - - def cache_info(self) -> _CacheInfo: - return cast(_FileCache._CacheInfo, self._get_lines_impl.cache_info()) - - -def _try_pygments(code: str) -> str | None: - try: - pygments = importlib.import_module("pygments") - formatters = importlib.import_module("pygments.formatters") - lexers = importlib.import_module("pygments.lexers") - except ImportError: - return None - - highlight = pygments.highlight - formatter_cls = formatters.HtmlFormatter - lexer_cls = lexers.PythonLexer - result = highlight(code, lexer_cls(), formatter_cls(nowrap=True)) - return result if isinstance(result, str) else None - - -def _pygments_css(style_name: str) -> str: - """ - Returns CSS for pygments tokens. Scoped to `.codebox` to avoid leaking styles. - If Pygments is not available or style missing, returns "". - """ - try: - formatters = importlib.import_module("pygments.formatters") - except ImportError: - return "" - - try: - formatter_cls = formatters.HtmlFormatter - fmt = formatter_cls(style=style_name) - except Exception: - try: - fmt = formatter_cls() - except Exception: - return "" - - try: - # `.codebox` scope: pygments will emit selectors like `.codebox .k { ... }` - css = fmt.get_style_defs(".codebox") - return css if isinstance(css, str) else "" - except Exception: - return "" - - -def _prefix_css(css: str, prefix: str) -> str: - """ - Prefix every selector block with `prefix `. - Safe enough for pygments CSS which is mostly selector blocks and comments. - """ - out_lines: list[str] = [] - for line in css.splitlines(): - stripped = line.strip() - if not stripped: - out_lines.append(line) - continue - if stripped.startswith(("/*", "*", "*/")): - out_lines.append(line) - continue - # Selector lines usually end with `{ - if "{" in line: - # naive prefix: split at "{", prefix selector part - before, after = line.split("{", 1) - sel = before.strip() - if sel: - out_lines.append(f"{prefix} {sel} {{ {after}".rstrip()) - else: - out_lines.append(line) - else: - out_lines.append(line) - return "\n".join(out_lines) - - -def _render_code_block( - *, - filepath: str, - start_line: int, - end_line: int, - file_cache: _FileCache, - context: int, - max_lines: int, -) -> _Snippet: - s = max(1, start_line - context) - e = end_line + context - - if e - s + 1 > max_lines: - e = s + max_lines - 1 - - lines = file_cache.get_lines_range(filepath, s, e) - - numbered: list[tuple[bool, str]] = [] - for lineno, line in enumerate(lines, start=s): - hit = start_line <= lineno <= end_line - numbered.append((hit, f"{lineno:>5} | {line.rstrip()}")) - - raw = "\n".join(text for _, text in numbered) - highlighted = _try_pygments(raw) - - if highlighted is None: - rendered: list[str] = [] - for hit, text in numbered: - cls = "hitline" if hit else "line" - rendered.append(f'
{html.escape(text)}
') - body = "\n".join(rendered) - else: - body = highlighted - - return _Snippet( - filepath=filepath, - start_line=start_line, - end_line=end_line, - code_html=f'
{body}
', - ) - +__all__ = [ + "_FileCache", + "_prefix_css", + "_pygments_css", + "_render_code_block", + "_try_pygments", + "build_html_report", + "pairwise", +] # ============================ # HTML report builder # ============================ -def _escape(v: Any) -> str: - return html.escape("" if v is None else str(v)) - - -def _meta_display(v: Any) -> str: - if isinstance(v, bool): - return "true" if v else "false" - if v is None: - return "n/a" - text = str(v).strip() - return text if text else "n/a" - - -def _group_sort_key(items: list[dict[str, Any]]) -> tuple[int, int]: - return ( - -len(items), - -max(int(i.get("loc") or i.get("size") or 0) for i in items), - ) +def _group_sort_key(items: list[dict[str, Any]]) -> tuple[int]: + return (-len(items),) def build_html_report( @@ -257,10 +53,14 @@ def build_html_report( ) -> str: file_cache = _FileCache() - func_sorted = sorted(func_groups.items(), key=lambda kv: _group_sort_key(kv[1])) - block_sorted = sorted(block_groups.items(), key=lambda kv: _group_sort_key(kv[1])) + func_sorted = sorted( + func_groups.items(), key=lambda kv: (*_group_sort_key(kv[1]), kv[0]) + ) + block_sorted = sorted( + block_groups.items(), key=lambda kv: (*_group_sort_key(kv[1]), kv[0]) + ) segment_sorted = sorted( - segment_groups.items(), key=lambda kv: _group_sort_key(kv[1]) + segment_groups.items(), key=lambda kv: (*_group_sort_key(kv[1]), kv[0]) ) has_any = bool(func_sorted) or bool(block_sorted) or bool(segment_sorted) @@ -352,13 +152,13 @@ def render_section( out: list[str] = [ f'
', '
', - f"

{_escape(section_title)} " + f"

{_escape_html(section_title)} " f'' f"{len(groups)} groups

", f"""