Regenerate Client #63
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Regenerate Client | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| title: | |
| description: PR title, auto-generated from the spec diff in www. | |
| required: false | |
| type: string | |
| summary: | |
| description: PR body summarizing the spec changes. | |
| required: false | |
| type: string | |
| jobs: | |
| regenerate: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Generate GitHub App token | |
| id: app-token | |
| uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 | |
| with: | |
| client-id: Iv23liKBX2RYMoZIYuKa | |
| private-key: ${{ secrets.HOTDATA_AUTOMATION_PRIVATE_KEY }} | |
| owner: hotdata-dev | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| token: ${{ steps.app-token.outputs.token }} | |
| fetch-depth: 0 | |
| fetch-tags: true | |
| - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 | |
| with: | |
| python-version: '3.12' | |
| - name: Fetch merged OpenAPI spec | |
| env: | |
| GH_TOKEN: ${{ steps.app-token.outputs.token }} | |
| run: | | |
| curl -sS -f -L \ | |
| -H "Accept: application/vnd.github.v3.raw" \ | |
| -H "Authorization: Bearer $GH_TOKEN" \ | |
| https://api.github.com/repos/hotdata-dev/www.hotdata.dev/contents/api/openapi.yaml \ | |
| -o openapi.yaml | |
| # Remove the generator-owned subtrees before regenerating so endpoints and | |
| # models dropped from the spec don't linger as orphaned files. The package | |
| # lives in hotdata/, so the previous `rm -rf src/` was a no-op and removed | |
| # APIs (e.g. the sandbox endpoints) kept shipping as dead modules. The | |
| # hand-written modules (hotdata/_auth.py, hotdata/arrow.py) live at the | |
| # package root and are untouched; the integration tests in tests/ (plural) | |
| # are distinct from the generated test/ (singular) unit tests. | |
| # | |
| # A few hand-written files DO live inside these subtrees (e.g. | |
| # test/test_api_client_close.py). They're listed in .openapi-generator-ignore | |
| # — the source of truth for "hand-maintained, don't touch" — so restore any | |
| # ignored path under a cleaned subtree after the wipe. The generator already | |
| # won't overwrite them; this makes the clean step honor the same list so a | |
| # regen never deletes hand-maintained code. | |
| - name: Clean generated source | |
| run: | | |
| rm -rf hotdata/api hotdata/models docs test | |
| grep -vE '^[[:space:]]*(#|$)' .openapi-generator-ignore | while IFS= read -r path; do | |
| case "$path" in | |
| hotdata/api/*|hotdata/models/*|docs/*|test/*) git checkout -- "$path" ;; | |
| esac | |
| done | |
| # The version bump is intentionally NOT done here. A regen is just a set of | |
| # changes; which release they ship in (and the bump kind) is decided later | |
| # by scripts/release.sh prepare. Seed the regen notes under [Unreleased] so | |
| # they accumulate there until a release rolls them into a version. This | |
| # avoids minting a dated version section per regen that may never publish. | |
| - name: Seed changelog entry under [Unreleased] | |
| env: | |
| TITLE: ${{ inputs.title }} | |
| run: | | |
| python3 - <<'PY' | |
| import os, pathlib, re | |
| title = (os.environ.get("TITLE") or "").strip() \ | |
| or "Regenerate the client from the updated Hotdata OpenAPI spec" | |
| bullet = f"- {title}" | |
| path = pathlib.Path("CHANGELOG.md") | |
| text = path.read_text() | |
| heading = re.search(r"^## \[Unreleased\][^\n]*\n", text, re.M) | |
| if not heading: | |
| raise SystemExit("CHANGELOG.md has no '## [Unreleased]' section to anchor the new entry") | |
| # Scope edits to the [Unreleased] body (up to the next '## [' or EOF). | |
| start = heading.end() | |
| nxt = re.search(r"^## \[", text[start:], re.M) | |
| end = start + nxt.start() if nxt else len(text) | |
| body = text[start:end] | |
| if any(line.strip() == bullet for line in body.splitlines()): | |
| print("CHANGELOG [Unreleased] already lists this entry; leaving it untouched.") | |
| raise SystemExit(0) | |
| changed = re.search(r"^### Changed[ \t]*\n", body, re.M) | |
| if changed: | |
| # Prepend the bullet to the existing ### Changed list. | |
| i = changed.end() | |
| while i < len(body) and body[i] == "\n": | |
| i += 1 | |
| body = body[:i] + bullet + "\n" + body[i:] | |
| else: | |
| # No ### Changed yet: open one right under the heading. | |
| body = "\n### Changed\n\n" + bullet + "\n\n" + body.lstrip("\n") | |
| path.write_text(text[:start] + body + text[end:]) | |
| print("Added regen entry under CHANGELOG [Unreleased].") | |
| PY | |
| # No packageVersion: pyproject.toml is hand-maintained and the generator | |
| # doesn't stamp it, while __version__ and the SDK version string both read | |
| # from installed package metadata — so the generator's packageVersion never | |
| # reaches committed output. | |
| - name: Generate client | |
| run: | | |
| npx @openapitools/openapi-generator-cli generate \ | |
| -i openapi.yaml \ | |
| -g python \ | |
| -o . \ | |
| -t .openapi-generator-templates \ | |
| --additional-properties=packageName=hotdata,projectName=hotdata,gitUserId=hotdata-dev,gitRepoId=sdk-python \ | |
| --skip-validate-spec | |
| # pyproject.toml/requirements*.txt are hand-maintained, so the generator no | |
| # longer writes them. Guard against silent drift: regenerate the generator's | |
| # pyproject into a throwaway dir and fail if it now requires a runtime | |
| # dependency we don't declare. Version floors are intentionally raised (e.g. | |
| # for security), so compare dependency NAMES only, never the specs. | |
| - name: Check for generator dependency drift | |
| run: | | |
| npx @openapitools/openapi-generator-cli generate \ | |
| -i openapi.yaml \ | |
| -g python \ | |
| -o /tmp/gencheck \ | |
| -t .openapi-generator-templates \ | |
| --additional-properties=packageName=hotdata,projectName=hotdata,gitUserId=hotdata-dev,gitRepoId=sdk-python \ | |
| --skip-validate-spec >/dev/null | |
| python3 - <<'PY' | |
| import re, sys, pathlib, tomllib | |
| def dep_names(path): | |
| data = tomllib.loads(pathlib.Path(path).read_text()) | |
| specs = list(data.get("project", {}).get("dependencies", []) or []) | |
| poetry = data.get("tool", {}).get("poetry", {}) | |
| specs += list((poetry.get("dependencies") or {}).keys()) | |
| names = set() | |
| for spec in specs: | |
| m = re.match(r"\s*([A-Za-z0-9._-]+)", spec) | |
| if m and m.group(1).lower() != "python": | |
| names.add(m.group(1).lower().replace("_", "-")) | |
| return names | |
| ours = dep_names("pyproject.toml") | |
| theirs = dep_names("/tmp/gencheck/pyproject.toml") | |
| missing = sorted(theirs - ours) | |
| extra = sorted(ours - theirs) | |
| if extra: | |
| print(f"::notice::pyproject declares runtime deps the generator does not: {extra} (intentional project additions)") | |
| if missing: | |
| print("::error::The generated client now requires runtime dependencies missing from the hand-maintained pyproject.toml:") | |
| for name in missing: | |
| print(f" - {name}") | |
| print("Add them to [project].dependencies (and requirements.txt), then re-run.") | |
| sys.exit(1) | |
| print(f"No runtime dependency drift. Generator runtime deps: {sorted(theirs)}") | |
| PY | |
| rm -rf /tmp/gencheck | |
| - name: Patch generated __version__ to read from package metadata | |
| run: | | |
| python3 - <<'PY' | |
| import re, pathlib, sys | |
| p = pathlib.Path("hotdata/__init__.py") | |
| src = p.read_text() | |
| replacement = ( | |
| 'from importlib.metadata import PackageNotFoundError, version as _pkg_version\n' | |
| '\n' | |
| 'try:\n' | |
| ' __version__ = _pkg_version("hotdata")\n' | |
| 'except PackageNotFoundError: # running from a source checkout without install\n' | |
| ' __version__ = "0.0.0+unknown"\n' | |
| ) | |
| new, n = re.subn(r'^__version__ = "[^"]*"\n', replacement, src, count=1, flags=re.MULTILINE) | |
| if n != 1: | |
| sys.exit("Failed to patch __version__ line in hotdata/__init__.py") | |
| p.write_text(new) | |
| PY | |
| - name: Patch ApiClient close lifecycle | |
| run: python3 scripts/patch_api_client_close.py | |
| - name: Patch default client exports (enhanced query/results) | |
| run: python3 scripts/patch_query_exports.py | |
| - name: Verify JWT-exchange code survived regeneration | |
| run: | | |
| python3 - <<'PY' | |
| import ast, pathlib, sys | |
| errors = [] | |
| # 1. The hand-written, regen-immune auth module must survive. | |
| if not pathlib.Path("hotdata/_auth.py").is_file(): | |
| errors.append("hotdata/_auth.py is missing (regen overwrote/dropped it)") | |
| config = pathlib.Path("hotdata/configuration.py") | |
| if not config.is_file(): | |
| errors.append("hotdata/configuration.py is missing") | |
| else: | |
| tree = ast.parse(config.read_text()) | |
| cls = next( | |
| (n for n in tree.body | |
| if isinstance(n, ast.ClassDef) and n.name == "Configuration"), | |
| None, | |
| ) | |
| if cls is None: | |
| errors.append("Configuration class not found in configuration.py") | |
| else: | |
| # 2. api_key must be a property (decorated getter), so every | |
| # request transparently exchanges for a fresh JWT. | |
| api_key_is_property = any( | |
| isinstance(n, ast.FunctionDef) | |
| and n.name == "api_key" | |
| and any( | |
| isinstance(d, ast.Name) and d.id == "property" | |
| for d in n.decorator_list | |
| ) | |
| for n in cls.body | |
| ) | |
| if not api_key_is_property: | |
| errors.append("Configuration.api_key is not a @property (template drift)") | |
| # 3. The token manager must be created eagerly in __init__ | |
| # (lazy creation has a concurrent-first-request race). | |
| init = next( | |
| (n for n in cls.body | |
| if isinstance(n, ast.FunctionDef) and n.name == "__init__"), | |
| None, | |
| ) | |
| init_src = ast.get_source_segment(config.read_text(), init) if init else "" | |
| if "self._token_manager = _TokenManager(" not in (init_src or ""): | |
| errors.append("eager self._token_manager assignment missing from __init__") | |
| # 4. __deepcopy__ must skip _token_manager (lock + PoolManager | |
| # are not deepcopy-able) and rebuild it. | |
| deepcopy = next( | |
| (n for n in cls.body | |
| if isinstance(n, ast.FunctionDef) and n.name == "__deepcopy__"), | |
| None, | |
| ) | |
| if deepcopy is None: | |
| errors.append("__deepcopy__ missing from Configuration") | |
| else: | |
| # Look for _token_manager as a real identifier/string in the | |
| # body (AST, so comments mentioning it don't count) — proves | |
| # the lock/PoolManager skip-and-rebuild actually survived. | |
| refs = any( | |
| (isinstance(n, ast.Constant) and n.value == "_token_manager") | |
| or (isinstance(n, ast.Attribute) and n.attr == "_token_manager") | |
| for n in ast.walk(deepcopy) | |
| ) | |
| if not refs: | |
| errors.append("__deepcopy__ does not skip/rebuild _token_manager") | |
| if errors: | |
| print("::error::JWT-exchange regen-safety check failed:") | |
| for e in errors: | |
| print(f" - {e}") | |
| sys.exit(1) | |
| print("JWT-exchange code survived regeneration: " | |
| "_auth.py present, api_key property, eager _token_manager, " | |
| "__deepcopy__ handling all intact.") | |
| PY | |
| - name: Clean up generated artifacts | |
| run: | | |
| rm -f openapi.yaml | |
| rm -f .github/workflows/python.yml | |
| # NOTE: regeneration deliberately does NOT build/import the package. A | |
| # failure here used to abort the job before "Create PR", so a regen that | |
| # produced valid-but-not-yet-wired output failed silently with no PR to act | |
| # on. The PR is the artifact we want, and breakage surfaces on it as red CI: | |
| # integration-tests.yml installs the package (`pip install -e .`) and runs | |
| # pytest, and check-release.yml builds + installs + imports the wheel on | |
| # every PR touching pyproject.toml/CHANGELOG.md (a regen seeds a [Unreleased] | |
| # changelog entry, so it always touches CHANGELOG.md). Auto-merge is gated on | |
| # those checks, so a broken regen can't merge. | |
| - name: Check integration test scenario parity | |
| env: | |
| GH_TOKEN: ${{ steps.app-token.outputs.token }} | |
| run: | | |
| curl -sS -f -L \ | |
| -H "Accept: application/vnd.github.v3.raw" \ | |
| -H "Authorization: Bearer $GH_TOKEN" \ | |
| https://api.github.com/repos/hotdata-dev/www.hotdata.dev/contents/api/test-scenarios.yaml \ | |
| -o test-scenarios.yaml | |
| pip install --quiet pyyaml | |
| python3 - <<'PY' | |
| import sys, pathlib, yaml | |
| scenarios = yaml.safe_load(pathlib.Path("test-scenarios.yaml").read_text())["scenarios"] | |
| missing = [] | |
| for s in scenarios: | |
| if "python" in (s.get("optional_for") or []): | |
| continue | |
| expected = pathlib.Path("tests/integration") / f"test_{s['name']}.py" | |
| if not expected.exists(): | |
| missing.append(str(expected)) | |
| if missing: | |
| print(f"::warning::sdk-python is missing tests for {len(missing)} scenarios after regen:") | |
| for m in missing: | |
| print(f" - {m}") | |
| else: | |
| print(f"All {len(scenarios)} scenarios have corresponding test files.") | |
| PY | |
| rm -f test-scenarios.yaml | |
| - name: Create PR | |
| id: cpr | |
| uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8 | |
| with: | |
| token: ${{ steps.app-token.outputs.token }} | |
| title: "${{ inputs.title || 'chore: regenerate client from updated OpenAPI spec' }}" | |
| branch: openapi-update-${{ github.run_id }} | |
| commit-message: "${{ inputs.title || 'chore: regenerate client from OpenAPI spec' }}" | |
| body: "${{ inputs.summary || 'Auto-generated from updated HotData OpenAPI spec.' }}" | |
| # Enable native auto-merge (squash). Branch protection on main gates the | |
| # merge on the test checks (scenario-parity, integration) plus the org | |
| # Claude review check and its approving review, so this only merges once | |
| # everything is green and Claude has approved. | |
| - name: Enable auto-merge | |
| if: steps.cpr.outputs.pull-request-number | |
| env: | |
| GH_TOKEN: ${{ steps.app-token.outputs.token }} | |
| run: | | |
| gh pr merge "${{ steps.cpr.outputs.pull-request-number }}" \ | |
| --repo "${{ github.repository }}" \ | |
| --squash --auto |