diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml index 034156a..c91a979 100644 --- a/.github/dependabot.yaml +++ b/.github/dependabot.yaml @@ -12,9 +12,7 @@ updates: default-days: 7 - package-ecosystem: uv - directories: - - / - - packages/* + directory: / schedule: interval: monthly groups: diff --git a/.github/workflows/python-ci.yaml b/.github/workflows/python-ci.yaml index f0297dc..81069c8 100644 --- a/.github/workflows/python-ci.yaml +++ b/.github/workflows/python-ci.yaml @@ -9,9 +9,14 @@ on: permissions: contents: read +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: lint: runs-on: ubuntu-latest + timeout-minutes: 10 steps: - name: Checkout repository @@ -30,7 +35,9 @@ jobs: python-version-file: "pyproject.toml" - name: Sync dependencies - run: uv sync + # --locked: fail on a stale uv.lock instead of silently re-locking it, which would + # neutralize the uv-lock prek hook that runs right after. + run: uv sync --locked - name: Run prek run: uv run prek run --all-files diff --git a/.github/workflows/stdlib-introspect.yaml b/.github/workflows/stdlib-introspect.yaml index f7606db..548e1a0 100644 --- a/.github/workflows/stdlib-introspect.yaml +++ b/.github/workflows/stdlib-introspect.yaml @@ -31,7 +31,9 @@ jobs: introspect: name: introspect ${{ matrix.os }} / py${{ matrix.python-version }} runs-on: ${{ matrix.os }} - timeout-minutes: 2 + # The script itself takes ~1s; the budget is runner provisioning, and Windows prerelease + # setup-python alone can eat minutes. A timed-out cell silently vanishes from the union. + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -80,7 +82,7 @@ jobs: needs: [introspect] if: always() # union whatever cells succeeded, even if some cells failed runs-on: ubuntu-latest - timeout-minutes: 1 + timeout-minutes: 5 steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: @@ -120,14 +122,17 @@ jobs: with: persist-credentials: false - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + - name: Install uv + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: - python-version: "3.x" + enable-cache: true - # Needs egress to pypi.org/files.pythonhosted.org (pip) and docs.python.org + # Needs egress to pypi.org/files.pythonhosted.org (uv) and docs.python.org # (objects.inv). Allowlist those if egress is ever restricted. - - name: Install sphobjinv - run: pip install sphobjinv==2.4 + - name: Sync dependencies + # --locked so sphobjinv and its transitives come hash-pinned from uv.lock instead of + # floating from PyPI on every nightly run; --no-dev skips the lint toolchain. + run: uv sync --locked --no-dev - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -138,7 +143,7 @@ jobs: echo "running: tools/coverage_diff.py stdlib_api_union.jsonl --target-version $TARGET_VERSION" # Iterates every version in the union internally, fetching each one's inventory; # writes the $GITHUB_STEP_SUMMARY report of what the official docs are missing. - python tools/coverage_diff.py stdlib_api_union.jsonl --target-version "$TARGET_VERSION" + uv run tools/coverage_diff.py stdlib_api_union.jsonl --target-version "$TARGET_VERSION" - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: always() diff --git a/noxfile.py b/noxfile.py index bf42e3d..5a0c7c4 100644 --- a/noxfile.py +++ b/noxfile.py @@ -32,11 +32,9 @@ def lints(session: nox.Session) -> None: @nox.session def clean(_: nox.Session) -> None: """Clean cache, .pyc, .pyo, and test/build artifact files from project.""" - count = 0 for searchpath in CLEANABLE_TARGETS: for filepath in Path().glob(searchpath): if filepath.is_dir(): shutil.rmtree(filepath) else: filepath.unlink() - count += 1 diff --git a/tools/coverage_diff.py b/tools/coverage_diff.py index 8df47bc..f4ac7cd 100755 --- a/tools/coverage_diff.py +++ b/tools/coverage_diff.py @@ -16,9 +16,11 @@ """ import argparse +import functools import json import os import re +import sys import time import urllib.error import urllib.request @@ -33,6 +35,8 @@ INVENTORY_URL = "https://docs.python.org/{version}/objects.inv" DEV_INVENTORY_URL = "https://docs.python.org/dev/objects.inv" HTTP_NOT_FOUND = 404 +HTTP_TOO_MANY_REQUESTS = 429 +HTTP_SERVER_ERROR = 500 # Introspected prefix -> docs-canonical prefix. The docs index posix|nt as os.*, # posixpath|ntpath|genericpath as os.path.*, and builtins members unprefixed. @@ -54,7 +58,7 @@ "nt": "os", } -CELL_VERSION = re.compile(r"-py([0-9][^-]*)$") +CELL_VERSION = re.compile(r"-py([0-9].*)$") # version_label() canonicalizes hyphenated specs (3.15-dev) TOP_MODULES = 15 @@ -69,6 +73,7 @@ def version_label(version: str) -> str: return ".".join(str(part) for part in version_key(version)) +@functools.cache # hot path: called per cell per record per version pass; only ~18 distinct ids exist def cell_version(cell: str) -> str | None: """Extract the ``X.Y`` minor from a ``...-py3.14`` cell id, or ``None``.""" match = CELL_VERSION.search(cell) @@ -107,7 +112,7 @@ def load_union(path: str) -> list[Record]: def http_get(url: str, attempts: int = 4) -> bytes: - """GET ``url``, retrying transient URL errors with exponential backoff.""" + """GET ``url``, retrying transient URL/server errors (5xx, 429) with exponential backoff.""" if attempts < 1: message = "attempts must be >= 1" raise ValueError(message) @@ -117,8 +122,13 @@ def http_get(url: str, attempts: int = 4) -> bytes: with urllib.request.urlopen(request, timeout=30) as response: body: bytes = response.read() return body - except urllib.error.HTTPError: - raise # 4xx/5xx: caller decides (404 -> dev fallback) + except urllib.error.HTTPError as error: + # Client errors are the caller's to interpret (404 -> dev fallback); a transient + # 5xx/429 must not abort the whole nightly run, so those retry like URLErrors. + if (error.code < HTTP_SERVER_ERROR and error.code != HTTP_TOO_MANY_REQUESTS) or attempt == attempts - 1: + raise + error.close() # release the held response instead of leaving it to the GC + time.sleep(2**attempt) except urllib.error.URLError: if attempt == attempts - 1: raise @@ -249,6 +259,19 @@ def main() -> None: key=version_key, ) + # Resolve the target up front and refuse one with no cells in the union: silently writing + # an empty gap that reads as "0 undocumented" is exactly the failure mode to avoid. + target: str | None + if args.target_version: + target = version_label(args.target_version) + if target not in versions: + sys.exit( + f"--target-version {args.target_version!r} (normalized to {target!r}) is not in " + f"the union; versions present: {', '.join(versions) or 'none'}.", + ) + else: + target = default_target(versions) if versions else None + results: dict[str, Record] = {} surfaces: dict[str, dict[str, Record]] = {} gaps: dict[str, set[str]] = {} @@ -264,7 +287,6 @@ def main() -> None: f"({summary['coverage_pct']}%)" + flag, ) - target = args.target_version or (default_target(versions) if versions else None) target_surface = surfaces.get(target, {}) if target else {} target_missing = gaps.get(target, set()) if target else set() target_docs_only = docs_onlys.get(target, set()) if target else set() @@ -274,13 +296,20 @@ def main() -> None: json.dump(coverage, out_file, indent=2) out_file.write("\n") - gap_path = args.gap_out or f"official_docs_gap_{target}.jsonl" rows = gap_records(target_surface, target_missing) - with Path(gap_path).open("w", encoding="utf-8", newline="\n") as out_file: - out_file.writelines(json.dumps(row) + "\n" for row in rows) + gap_path = None + if target is not None: + gap_path = args.gap_out or f"official_docs_gap_{target}.jsonl" + with Path(gap_path).open("w", encoding="utf-8", newline="\n") as out_file: + out_file.writelines(json.dumps(row) + "\n" for row in rows) report(versions, results, target, rows, target_docs_only, args.output, gap_path, args) + # Gate last, after the outputs and summaries, mirroring stdlib_introspect: the report + # says "no versions" loudly, and the job must go red rather than publish nothing. + if target is None: + sys.exit("SANITY GATE: no versions found in the union -- nothing to diff.") + def report( versions: list[str], @@ -289,7 +318,7 @@ def report( gap_rows: list[Record], docs_only: set[str], output_path: str, - gap_path: str, + gap_path: str | None, args: argparse.Namespace, ) -> None: """Render the run as a Markdown report and a console summary.""" @@ -375,7 +404,7 @@ def report( print(f"official docs gap: {len(gap_rows)} undocumented ({data_entries} data)") else: print("no versions found in union") - print(f"wrote {output_path} and {gap_path}") + print(f"wrote {output_path}" + (f" and {gap_path}" if gap_path else "")) print("=" * 58) diff --git a/tools/merge_summary.py b/tools/merge_summary.py index 6140627..e79b2ee 100755 --- a/tools/merge_summary.py +++ b/tools/merge_summary.py @@ -12,8 +12,8 @@ Outputs: * stdlib_api_union.jsonl -- every qualname once, each record annotated with the set - of cells (os-family + python version) it appeared in, plus added_in/removed_in - derived from the OS-collapsed version presence. + of cells (os-family + python version) it appeared in, plus added_in/removed_in/ + presence_gaps derived from the OS-collapsed version presence. * a Markdown report to $GITHUB_STEP_SUMMARY (or --md-summary PATH): union size, per-cell counts, platform-exclusive API counts, and the per-adjacent-minor added/removed deltas (OS-collapsed) across the matrix. @@ -66,25 +66,28 @@ def format_version(version_tuple: VersionKey) -> str: return ".".join(str(part) for part in version_tuple) -def version_span(present_keys: set[VersionKey], matrix_keys: list[VersionKey]) -> tuple[str, str | None]: - """(added_in, removed_in) for one entity from its OS-collapsed version set. +def version_span(present_keys: set[VersionKey], matrix_keys: list[VersionKey]) -> tuple[str, str | None, list[str]]: + """(added_in, removed_in, presence_gaps) for one entity from its OS-collapsed version set. added_in is floored at the matrix minimum: present at the floor means it was added then or in some earlier, unobserved release, recorded as '<=X.Y'. - removed_in is precise -- the first matrix minor where it disappears. + removed_in is the matrix minor after the LAST observed presence (None when present + at the matrix end) -- a mid-matrix hole (a failed cell, or a remove-then-readd) is + NOT a removal; it is reported in presence_gaps instead. """ - floor = matrix_keys[0] - added_in = "<=" + format_version(floor) if floor in present_keys else format_version(min(present_keys)) - removed_in = None - for earlier, later in pairwise(matrix_keys): - if earlier in present_keys and later not in present_keys: - removed_in = format_version(later) - break - return added_in, removed_in + present = [key for key in matrix_keys if key in present_keys] + first, last = present[0], present[-1] + added_in = ("<=" if first == matrix_keys[0] else "") + format_version(first) + after_last = matrix_keys.index(last) + 1 + removed_in = format_version(matrix_keys[after_last]) if after_last < len(matrix_keys) else None + gaps = [ + format_version(key) for key in matrix_keys[matrix_keys.index(first) : after_last] if key not in present_keys + ] + return added_in, removed_in, gaps class Cell: - """One matrix cell's records, tagged with its OS family and Python version.""" + """One matrix cell's dump, tagged with its OS family and Python version.""" def __init__(self, path: Path, os_label: str, version: str) -> None: self.path = path @@ -93,11 +96,20 @@ def __init__(self, path: Path, os_label: str, version: str) -> None: self.version = version self.version_key = version_key(version) self.cell_id = f"{self.family}-py{version}" - self.records: dict[str, Record] = {} # qualname -> record (last wins within a cell) + self.entities = 0 self.malformed = 0 - def load(self) -> None: - """Parse the cell's JSONL file, tolerating malformed lines from a crashed cell.""" + @property + def sort_key(self) -> tuple[VersionKey, int]: + """Matrix order: Python minor first, then a stable OS-family rank.""" + return (self.version_key, FAMILY_ORDER.get(self.family, UNRANKED_FAMILY)) + + def load(self) -> dict[str, Record]: + """Parse and return the cell's records (last wins per qualname), keeping only the counts. + + Malformed lines from a crashed cell are tolerated and counted. + """ + records: dict[str, Record] = {} with self.path.open(encoding="utf-8") as source_file: for line in source_file: stripped = line.strip() @@ -105,56 +117,57 @@ def load(self) -> None: continue try: record = json.loads(stripped) - self.records[record["qualname"]] = record + records[record["qualname"]] = record except json.JSONDecodeError, KeyError, TypeError: self.malformed += 1 # tolerate a half-written dump from a crashed cell + self.entities = len(records) + return records def discover(cells_dir: str) -> list[Cell]: - """Load every recognized ``stdlib_api__py.jsonl`` cell under ``cells_dir``.""" + """Identify every ``stdlib_api__py.jsonl`` cell under ``cells_dir``.""" cells = [] for path in sorted(Path(cells_dir).glob("stdlib_api_*_py*.jsonl")): match = CELL_PATTERN.match(path.name) if not match: print(f" ! unrecognized file name, skipping: {path}", file=sys.stderr) continue - cell = Cell(path, match["os"], match["ver"]) - cell.load() - cells.append(cell) + cells.append(Cell(path, match["os"], match["ver"])) return cells def aggregate( cells: list[Cell], -) -> tuple[list[str], dict[str, Record], dict[str, set[str]], dict[str, set[VersionKey]]]: +) -> tuple[list[str], dict[str, Record], dict[str, set[str]], dict[str, set[VersionKey]], list[VersionKey]]: """Union the cells' records and annotate each with its cell/version presence.""" present_cells: defaultdict[str, set[str]] = defaultdict(set) # qualname -> {cell_id} present_families: defaultdict[str, set[str]] = defaultdict(set) # qualname -> {family} present_version_keys: defaultdict[str, set[VersionKey]] = defaultdict(set) # qualname -> {version_key} - for cell in cells: - for qualname in cell.records: + union_records: dict[str, Record] = {} + + # Stream cells oldest -> newest, letting the newer cell overwrite, so the union's base + # record carries the newest minor's signature/docstring -- and only one cell's parse is + # ever held in memory alongside the union. + for cell in sorted(cells, key=lambda cell: cell.sort_key): + for qualname, record in cell.load().items(): present_cells[qualname].add(cell.cell_id) present_families[qualname].add(cell.family) present_version_keys[qualname].add(cell.version_key) + union_records[qualname] = record union = sorted(present_cells) - - # Walk cells oldest -> newest and let the newer cell overwrite, so the union's base - # record carries the newest minor's signature/docstring. - union_records: dict[str, Record] = { - qualname: record - for cell in sorted(cells, key=lambda cell: (cell.version_key, FAMILY_ORDER.get(cell.family, UNRANKED_FAMILY))) - for qualname, record in cell.records.items() - } matrix_keys = sorted({cell.version_key for cell in cells}) for qualname in union: record = dict(union_records[qualname]) record["cells"] = sorted(present_cells[qualname]) record["n_cells"] = len(record["cells"]) - record["added_in"], record["removed_in"] = version_span(present_version_keys[qualname], matrix_keys) + record["added_in"], record["removed_in"], record["presence_gaps"] = version_span( + present_version_keys[qualname], + matrix_keys, + ) union_records[qualname] = record - return union, union_records, present_families, present_version_keys + return union, union_records, present_families, present_version_keys, matrix_keys def main() -> None: @@ -170,7 +183,7 @@ def main() -> None: args = parser.parse_args() cells = discover(args.cells_dir) - union, union_records, present_families, present_version_keys = aggregate(cells) + union, union_records, present_families, present_version_keys, matrix_keys = aggregate(cells) # Always write the union file (even empty) so the upload step has an artifact; # newline="\n" so the artifact is byte-identical regardless of which runner ran us. @@ -183,12 +196,11 @@ def main() -> None: for qualname in union: families_present = present_families[qualname] if len(families_present) == 1: - exclusive.setdefault(next(iter(families_present)), []).append(qualname) + exclusive[next(iter(families_present))].append(qualname) # Per-adjacent-minor deltas, OS-collapsed (present in a minor == present in any OS cell). - version_keys = sorted({cell.version_key for cell in cells}) transitions: list[Transition] = [] - for earlier, later in pairwise(version_keys): + for earlier, later in pairwise(matrix_keys): added = [ qualname for qualname in union @@ -201,7 +213,7 @@ def main() -> None: ] transitions.append((earlier, later, added, removed)) - report(cells, union, exclusive, families, transitions, version_keys, args) + report(cells, union, exclusive, families, transitions, matrix_keys, args) def report( @@ -214,7 +226,7 @@ def report( args: argparse.Namespace, ) -> None: """Render the union as a Markdown report and a console summary.""" - rows = sorted(cells, key=lambda cell: (cell.version_key, FAMILY_ORDER.get(cell.family, UNRANKED_FAMILY))) + rows = sorted(cells, key=lambda cell: cell.sort_key) lines = ["# stdlib introspection — cross-platform union", ""] if not cells: @@ -235,7 +247,7 @@ def report( lines += ["## Per-cell entity counts", "", "| cell | os | python | entities |", "| --- | --- | --- | ---: |"] for cell in rows: note = f" ⚠️ {cell.malformed} bad lines" if cell.malformed else "" - lines.append(f"| `{cell.cell_id}` | {cell.os_label} | {cell.version} | {len(cell.records)}{note} |") + lines.append(f"| `{cell.cell_id}` | {cell.os_label} | {cell.version} | {cell.entities}{note} |") lines.append("") lines += [ @@ -247,7 +259,7 @@ def report( "| --- | ---: |", ] for family in families: - lines.append(f"| {family} | {len(exclusive.get(family, []))} |") + lines.append(f"| {family} | {len(exclusive[family])} |") lines.append("") if transitions: @@ -285,9 +297,9 @@ def report( print(f"aggregated {len(cells)} cells -> {len(union)} unique qualnames") for cell in rows: extra = f" ({cell.malformed} malformed lines)" if cell.malformed else "" - print(f" {cell.cell_id:18s} {len(cell.records):6d} entities{extra}") + print(f" {cell.cell_id:18s} {cell.entities:6d} entities{extra}") for family in families: - print(f" {family}-exclusive APIs : {len(exclusive.get(family, []))}") + print(f" {family}-exclusive APIs : {len(exclusive[family])}") for earlier, later, added, removed in transitions: print(f" {format_version(earlier)} -> {format_version(later)}: +{len(added)} / -{len(removed)}") print(f"wrote {len(union)} records -> {args.output}") diff --git a/tools/stdlib_introspect.py b/tools/stdlib_introspect.py index 00049b4..bec11b2 100755 --- a/tools/stdlib_introspect.py +++ b/tools/stdlib_introspect.py @@ -42,6 +42,7 @@ "this", # prints the Zen of Python "__hello__", "__phello__", # print on import + "__main__", # importing it returns the running introspection script itself "test", # CPython's own test suite "idlelib", # the IDLE GUI app "turtledemo", # demo scripts @@ -91,7 +92,8 @@ def safe_import(name: str) -> ModuleType | None: def roster(include_private: bool) -> Iterator[tuple[str, ModuleType | None]]: """Yield (name, imported-module-or-None) for every stdlib module to document.""" seen: set[str] = set() - names = sorted(name for name in sys.stdlib_module_names if include_private or not name.startswith("_")) + # is_private, not startswith("_"), so documented dunder modules (__future__) stay in. + names = sorted(name for name in sys.stdlib_module_names if include_private or not is_private(name)) for top_level in names: yield from _emit(top_level, include_private, seen) @@ -127,7 +129,11 @@ def _emit(name: str, include_private: bool, seen: set[str]) -> Iterator[tuple[st SEEN: dict[int, str] = {} # id(obj) -> canonical qualname (first sighting) RECORDS: dict[str, Record] = {} -PENDING: dict[int, list[str]] = {} # id(obj) -> [alias qualnames seen before the canonical one] +PendingName: TypeAlias = tuple[str, str, str, str, bool] # (qualname, module, parent, short_name, in_class) +# id(obj) -> (entity, [names seen before any canonical record]). Attached as aliases once a +# canonical record exists; entities whose defining module is never walked (private C modules +# like _functools) are promoted to canonical records by flush_pending() after the walk. +PENDING: dict[int, tuple[Any, list[PendingName]]] = {} def kind_of(entity: object, in_class: bool) -> str: @@ -148,7 +154,9 @@ def kind_of(entity: object, in_class: bool) -> str: def get_signature(entity: object) -> tuple[str | None, str]: """Return (signature_text, source) where source is ``inspect``/``text_signature``/``none``.""" if callable(entity): - with contextlib.suppress(ValueError, TypeError): + # Exception, not just ValueError/TypeError: signature() evaluates __text_signature__ + # defaults, which can raise anything (select.epoll.register -> AttributeError). + with contextlib.suppress(Exception): return str(inspect.signature(entity)), "inspect" text_signature = getattr(entity, "__text_signature__", None) return (text_signature, "text_signature") if text_signature else (None, "none") @@ -156,7 +164,17 @@ def get_signature(entity: object) -> tuple[str | None, str]: def doc_info(entity: object) -> tuple[bool, bool, str]: """Return (has_own_doc, has_resolved_doc, first_line) for an entity.""" - has_own_doc = bool(getattr(entity, "__doc__", None)) + entity_doc = getattr(entity, "__doc__", None) + if ( + entity_doc is not None + and not isinstance(entity, type) + and not inspect.ismodule(entity) + and entity_doc == getattr(type(entity), "__doc__", None) + ): + # Plain values (math.pi, errno.E2BIG) reach their type's docstring through attribute + # lookup; that documents the type, not the entity. + return False, False, "" + has_own_doc = bool(entity_doc) resolved_doc = inspect.getdoc(entity) first_line = "" if resolved_doc and resolved_doc.strip(): @@ -171,19 +189,16 @@ def attach(canonical: str, alias: str) -> None: existing["aliases"].append(alias) -def note_alias(entity: object, qualname: str) -> None: - """Remember ``qualname`` as an alias, attaching it once the canonical record exists.""" - try: - object_id = id(entity) - except Exception: - return +def note_alias(entity: object, qualname: str, module: str, parent: str, short_name: str, in_class: bool) -> None: + """Remember a non-canonical name: attached as an alias now or later, or promoted at flush time.""" + object_id = id(entity) if object_id in SEEN: attach(SEEN[object_id], qualname) else: - PENDING.setdefault(object_id, []).append(qualname) + PENDING.setdefault(object_id, (entity, []))[1].append((qualname, module, parent, short_name, in_class)) -def record(qualname: str, kind: str, module: str, parent: str, entity: object, short_name: str) -> None: +def record(qualname: str, kind: str, module: str, parent: str | None, entity: object, short_name: str) -> None: """Build and store the JSON record for one entity.""" signature, signature_source = ( get_signature(entity) if kind in {"function", "method", "class", "exception"} else (None, "n/a") @@ -194,7 +209,8 @@ def record(qualname: str, kind: str, module: str, parent: str, entity: object, s "kind": kind, "module": module, "parent": parent, - "is_dunder": is_dunder(short_name), + # Dunder-named modules (__future__) are documented modules, not data-model dunders. + "is_dunder": kind != "module" and is_dunder(short_name), "signature": signature, "sig_source": signature_source, "doc_own": has_own_doc, @@ -215,6 +231,8 @@ def process( include_private: bool, ) -> None: """Record an entity (deduping re-exports by identity) and recurse into a class's members.""" + if isinstance(entity, (staticmethod, classmethod)): + entity = entity.__func__ # vars() yields the raw wrapper, which hides the signature/doc kind = kind_of(entity, in_class) if kind == "data": # Value-typed: identity is NOT meaningful (small ints / interned strings @@ -228,15 +246,29 @@ def process( return SEEN[object_id] = qualname record(qualname, kind, module_name, parent, entity, short_name) - for alias in PENDING.pop(object_id, []): + _, pending_names = PENDING.pop(object_id, (None, [])) + for alias, *_ in pending_names: attach(qualname, alias) # Recurse into a class's OWN members only (mirrors "document where defined"). if kind in {"class", "exception"}: + own_qualname = getattr(entity, "__qualname__", None) for child_name, child_value in sorted(vars(entity).items()): if (is_dunder(child_name) and not include_dunders) or (is_private(child_name) and not include_private): continue + child = child_value.__func__ if isinstance(child_value, (staticmethod, classmethod)) else child_value + child_qualname = getattr(child, "__qualname__", None) + if ( + own_qualname is not None + and child_qualname is not None + and (isinstance(child, type) or inspect.isroutine(child)) + and child_qualname != f"{own_qualname}.{child_name}" + ): + # A reference to something defined elsewhere (HTTPConnection.response_class + # -> HTTPResponse): alias it rather than let it steal canonical identity. + note_alias(child, f"{qualname}.{child_name}", module_name, qualname, child_name, True) + continue process( - child_value, + child, f"{qualname}.{child_name}", qualname, module_name, @@ -247,20 +279,48 @@ def process( ) -def walk_module(module: ModuleType, include_dunders: bool, include_private: bool) -> None: - """Record every own, non-imported member of a module.""" +def walk_module(module: ModuleType, include_dunders: bool, include_private: bool) -> list[str]: + """Record every own, non-imported member of a module; return the members that failed.""" module_name = module.__name__ + failures: list[str] = [] # vars() (not dir()+getattr) so we don't trigger lazy __getattr__ / property side effects. for name, value in sorted(vars(module).items()): if (is_dunder(name) and not include_dunders) or (is_private(name) and not include_private): continue if inspect.ismodule(value): # imported module ref (ast.sys, os.path); roster handles real modules continue - owner = getattr(value, "__module__", None) - if owner is not None and owner != module_name: - note_alias(value, f"{module_name}.{name}") # imported from elsewhere - continue - process(value, f"{module_name}.{name}", module_name, module_name, name, False, include_dunders, include_private) + try: + owner = getattr(value, "__module__", None) + # Identity-based aliasing is meaningless for value types (data), so a re-exported + # value is recorded under the re-exporting module instead of note_alias'd. + if owner is not None and owner != module_name and kind_of(value, False) != "data": + note_alias(value, f"{module_name}.{name}", module_name, module_name, name, False) + continue + process( + value, + f"{module_name}.{name}", + module_name, + module_name, + name, + False, + include_dunders, + include_private, + ) + except Exception as error: # one poisoned member must not sink its siblings + failures.append(f"{module_name}.{name} ({error!r})") + return failures + + +def flush_pending(include_dunders: bool, include_private: bool) -> None: + """Promote pending names whose canonical entity was never walked (private C modules).""" + # process() pops the flushed entry and attaches its remaining names as aliases; walking a + # promoted class can park new children in PENDING, so drain until stable. The extra pop is + # belt-and-braces so the loop always shrinks even if process() takes an unexpected path. + while PENDING: + object_id, (entity, names) = next(iter(PENDING.items())) + qualname, module_name, parent, short_name, in_class = names[0] + process(entity, qualname, parent, module_name, short_name, in_class, include_dunders, include_private) + PENDING.pop(object_id, None) def main() -> None: @@ -290,24 +350,14 @@ def main() -> None: if module is None: failed.append(name) continue - has_own_doc, has_resolved_doc, first_line = doc_info(module) - RECORDS[name] = { - "qualname": name, - "kind": "module", - "module": name, - "parent": name.rpartition(".")[0] or None, - "is_dunder": False, - "signature": None, - "sig_source": "n/a", - "doc_own": has_own_doc, - "doc_resolved": has_resolved_doc, - "doc_firstline": first_line, - "aliases": [], - } + record(name, "module", name, name.rpartition(".")[0] or None, module, name.rpartition(".")[-1]) try: - walk_module(module, args.include_dunders, args.include_private) + failed.extend(walk_module(module, args.include_dunders, args.include_private)) except Exception as error: failed.append(f"{name} (walk: {error!r})") + # Re-exports whose defining module was never walked (functools.reduce lives in the + # private _functools) would otherwise vanish: promote them to canonical records now. + flush_pending(args.include_dunders, args.include_private) records = sorted(RECORDS.values(), key=lambda entry: entry["qualname"]) # Default encoding is cp1252 on Windows (crashes on non-ASCII docstrings) and text