Fix data-corrupting introspection bugs, harden the pipeline and CI

.github/dependabot.yaml

@@ -12,9 +12,7 @@ updates:
       default-days: 7
 
   - package-ecosystem: uv
-    directories:
-      - /
-      - packages/*
+    directory: /
     schedule:
       interval: monthly
     groups:

.github/workflows/python-ci.yaml

@@ -9,9 +9,14 @@ on:
 permissions:
   contents: read
 
+concurrency:
+  group: ci-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   lint:
     runs-on: ubuntu-latest
+    timeout-minutes: 10
 
     steps:
       - name: Checkout repository
@@ -30,7 +35,9 @@ jobs:
           python-version-file: "pyproject.toml"
 
       - name: Sync dependencies
-        run: uv sync
+        # --locked: fail on a stale uv.lock instead of silently re-locking it, which would
+        # neutralize the uv-lock prek hook that runs right after.
+        run: uv sync --locked
 
       - name: Run prek
         run: uv run prek run --all-files

.github/workflows/stdlib-introspect.yaml

@@ -31,7 +31,9 @@ jobs:
   introspect:
     name: introspect ${{ matrix.os }} / py${{ matrix.python-version }}
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 2
+    # The script itself takes ~1s; the budget is runner provisioning, and Windows prerelease
+    # setup-python alone can eat minutes. A timed-out cell silently vanishes from the union.
+    timeout-minutes: 10
     strategy:
       fail-fast: false
       matrix:
@@ -80,7 +82,7 @@ jobs:
     needs: [introspect]
     if: always()   # union whatever cells succeeded, even if some cells failed
     runs-on: ubuntu-latest
-    timeout-minutes: 1
+    timeout-minutes: 5
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
         with:
@@ -120,14 +122,17 @@ jobs:
         with:
           persist-credentials: false
 
-      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+      - name: Install uv
+        uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
         with:
-          python-version: "3.x"
+          enable-cache: true
 
-      # Needs egress to pypi.org/files.pythonhosted.org (pip) and docs.python.org
+      # Needs egress to pypi.org/files.pythonhosted.org (uv) and docs.python.org
       # (objects.inv). Allowlist those if egress is ever restricted.
-      - name: Install sphobjinv
-        run: pip install sphobjinv==2.4
+      - name: Sync dependencies
+        # --locked so sphobjinv and its transitives come hash-pinned from uv.lock instead of
+        # floating from PyPI on every nightly run; --no-dev skips the lint toolchain.
+        run: uv sync --locked --no-dev
 
       - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
@@ -138,7 +143,7 @@ jobs:
           echo "running: tools/coverage_diff.py stdlib_api_union.jsonl --target-version $TARGET_VERSION"
           # Iterates every version in the union internally, fetching each one's inventory;
           # writes the $GITHUB_STEP_SUMMARY report of what the official docs are missing.
-          python tools/coverage_diff.py stdlib_api_union.jsonl --target-version "$TARGET_VERSION"
+          uv run tools/coverage_diff.py stdlib_api_union.jsonl --target-version "$TARGET_VERSION"
 
       - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         if: always()

noxfile.py

@@ -32,11 +32,9 @@ def lints(session: nox.Session) -> None:
 @nox.session
 def clean(_: nox.Session) -> None:
     """Clean cache, .pyc, .pyo, and test/build artifact files from project."""
-    count = 0
     for searchpath in CLEANABLE_TARGETS:
         for filepath in Path().glob(searchpath):
             if filepath.is_dir():
                 shutil.rmtree(filepath)
             else:
                 filepath.unlink()
-            count += 1

tools/coverage_diff.py

@@ -16,9 +16,11 @@
 """
 
 import argparse
+import functools
 import json
 import os
 import re
+import sys
 import time
 import urllib.error
 import urllib.request
@@ -33,6 +35,8 @@
 INVENTORY_URL = "https://docs.python.org/{version}/objects.inv"
 DEV_INVENTORY_URL = "https://docs.python.org/dev/objects.inv"
 HTTP_NOT_FOUND = 404
+HTTP_TOO_MANY_REQUESTS = 429
+HTTP_SERVER_ERROR = 500
 
 # Introspected prefix -> docs-canonical prefix. The docs index posix|nt as os.*,
 # posixpath|ntpath|genericpath as os.path.*, and builtins members unprefixed.
@@ -54,7 +58,7 @@
     "nt": "os",
 }
 
-CELL_VERSION = re.compile(r"-py([0-9][^-]*)$")
+CELL_VERSION = re.compile(r"-py([0-9].*)$")  # version_label() canonicalizes hyphenated specs (3.15-dev)
 TOP_MODULES = 15
 
 
@@ -69,6 +73,7 @@ def version_label(version: str) -> str:
     return ".".join(str(part) for part in version_key(version))
 
 
+@functools.cache  # hot path: called per cell per record per version pass; only ~18 distinct ids exist
 def cell_version(cell: str) -> str | None:
     """Extract the ``X.Y`` minor from a ``...-py3.14`` cell id, or ``None``."""
     match = CELL_VERSION.search(cell)
@@ -107,7 +112,7 @@ def load_union(path: str) -> list[Record]:
 
 
 def http_get(url: str, attempts: int = 4) -> bytes:
-    """GET ``url``, retrying transient URL errors with exponential backoff."""
+    """GET ``url``, retrying transient URL/server errors (5xx, 429) with exponential backoff."""
     if attempts < 1:
         message = "attempts must be >= 1"
         raise ValueError(message)
@@ -117,8 +122,13 @@ def http_get(url: str, attempts: int = 4) -> bytes:
             with urllib.request.urlopen(request, timeout=30) as response:
                 body: bytes = response.read()
                 return body
-        except urllib.error.HTTPError:
-            raise  # 4xx/5xx: caller decides (404 -> dev fallback)
+        except urllib.error.HTTPError as error:
+            # Client errors are the caller's to interpret (404 -> dev fallback); a transient
+            # 5xx/429 must not abort the whole nightly run, so those retry like URLErrors.
+            if (error.code < HTTP_SERVER_ERROR and error.code != HTTP_TOO_MANY_REQUESTS) or attempt == attempts - 1:
+                raise
+            error.close()  # release the held response instead of leaving it to the GC
+            time.sleep(2**attempt)
         except urllib.error.URLError:
             if attempt == attempts - 1:
                 raise
@@ -249,6 +259,19 @@ def main() -> None:
         key=version_key,
     )
 
+    # Resolve the target up front and refuse one with no cells in the union: silently writing
+    # an empty gap that reads as "0 undocumented" is exactly the failure mode to avoid.
+    target: str | None
+    if args.target_version:
+        target = version_label(args.target_version)
+        if target not in versions:
+            sys.exit(
+                f"--target-version {args.target_version!r} (normalized to {target!r}) is not in "
+                f"the union; versions present: {', '.join(versions) or 'none'}.",
+            )
+    else:
+        target = default_target(versions) if versions else None
+
     results: dict[str, Record] = {}
     surfaces: dict[str, dict[str, Record]] = {}
     gaps: dict[str, set[str]] = {}
@@ -264,7 +287,6 @@ def main() -> None:
             f"({summary['coverage_pct']}%)" + flag,
         )
 
-    target = args.target_version or (default_target(versions) if versions else None)
     target_surface = surfaces.get(target, {}) if target else {}
     target_missing = gaps.get(target, set()) if target else set()
     target_docs_only = docs_onlys.get(target, set()) if target else set()
@@ -274,13 +296,20 @@ def main() -> None:
         json.dump(coverage, out_file, indent=2)
         out_file.write("\n")
 
-    gap_path = args.gap_out or f"official_docs_gap_{target}.jsonl"
     rows = gap_records(target_surface, target_missing)
-    with Path(gap_path).open("w", encoding="utf-8", newline="\n") as out_file:
-        out_file.writelines(json.dumps(row) + "\n" for row in rows)
+    gap_path = None
+    if target is not None:
+        gap_path = args.gap_out or f"official_docs_gap_{target}.jsonl"
+        with Path(gap_path).open("w", encoding="utf-8", newline="\n") as out_file:
+            out_file.writelines(json.dumps(row) + "\n" for row in rows)
 
     report(versions, results, target, rows, target_docs_only, args.output, gap_path, args)
 
+    # Gate last, after the outputs and summaries, mirroring stdlib_introspect: the report
+    # says "no versions" loudly, and the job must go red rather than publish nothing.
+    if target is None:
+        sys.exit("SANITY GATE: no versions found in the union -- nothing to diff.")
+
 
 def report(
     versions: list[str],
@@ -289,7 +318,7 @@ def report(
     gap_rows: list[Record],
     docs_only: set[str],
     output_path: str,
-    gap_path: str,
+    gap_path: str | None,
     args: argparse.Namespace,
 ) -> None:
     """Render the run as a Markdown report and a console summary."""
@@ -375,7 +404,7 @@ def report(
         print(f"official docs gap: {len(gap_rows)} undocumented ({data_entries} data)")
     else:
         print("no versions found in union")
-    print(f"wrote {output_path} and {gap_path}")
+    print(f"wrote {output_path}" + (f" and {gap_path}" if gap_path else ""))
     print("=" * 58)