From 66f6dbe602df883803b1a0f60595b27efc289087 Mon Sep 17 00:00:00 2001 From: mldangelo Date: Sun, 11 Jan 2026 02:46:43 -0500 Subject: [PATCH] feat: add comprehensive type checking with mypy strict mode and pyright - Upgrade mypy to strict mode with additional error codes (ignore-without-code, redundant-cast, truthy-bool, truthy-iterable, unused-awaitable) - Add pyright as a second type checker for comprehensive coverage - Configure pyright in strict mode with practical exceptions for third-party libraries lacking full stubs - Update CI to run both type checkers in parallel via matrix strategy - Fix CompletedProcess generic type parameter in cli.py - Add type: ignore comments for untyped posthog methods in telemetry.py - Update AGENTS.md documentation with new type checking setup The dual type checker approach catches different categories of issues: - mypy: The standard Python type checker, excellent for protocol validation - pyright: Microsoft's type checker, catches additional edge cases Both run in strict mode to enforce rigorous typing standards. Co-Authored-By: Claude Opus 4.5 --- .github/workflows/test.yml | 10 ++++- AGENTS.md | 20 ++++++++-- pyproject.toml | 76 ++++++++++++++++++++++++++++++++++---- src/promptfoo/cli.py | 2 +- src/promptfoo/telemetry.py | 4 +- uv.lock | 24 ++++++++++++ 6 files changed, 121 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bf7a19a..8fe7d73 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,9 +42,12 @@ jobs: run: uv run ruff format --check src/ type-check: - name: Type Check + name: Type Check (${{ matrix.type-checker }}) runs-on: ubuntu-latest timeout-minutes: 10 + strategy: + matrix: + type-checker: [mypy, pyright] steps: - uses: actions/checkout@v6 @@ -59,8 +62,13 @@ jobs: run: uv sync --extra dev - name: Type check with mypy + if: matrix.type-checker == 'mypy' run: uv run mypy src/promptfoo/ + - name: Type check with pyright + if: matrix.type-checker == 'pyright' + run: uv run pyright src/promptfoo/ + test: name: Test (py${{ matrix.python-version }}, ${{ matrix.os }}) runs-on: ${{ matrix.os }} diff --git a/AGENTS.md b/AGENTS.md index 84e5a2d..483a6fc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -134,7 +134,9 @@ Runs on every PR and push to main: - **Lint**: Ruff linting (`uv run ruff check src/`) - **Format Check**: Ruff formatting (`uv run ruff format --check src/`) -- **Type Check**: mypy static analysis (`uv run mypy src/promptfoo/`) +- **Type Check**: Both mypy and pyright in strict mode (run in parallel via matrix) + - `uv run mypy src/promptfoo/` - Standard Python type checker + - `uv run pyright src/promptfoo/` - Microsoft's type checker for additional coverage - **Tests**: pytest on multiple Python versions (3.9, 3.13) and OSes (Ubuntu, Windows) - **Build**: Package build validation @@ -177,7 +179,9 @@ We use **OpenID Connect (OIDC)** for secure, credential-free PyPI publishing: - **Linter**: Ruff with extended rule sets (isort, pycodestyle, flake8-bugbear, etc.) - **Formatter**: Ruff (replaces Black) -- **Type Checker**: mypy with strict settings +- **Type Checkers**: Both **mypy** and **pyright** in strict mode for comprehensive coverage + - **mypy**: The standard Python type checker with strict mode and additional error codes + - **pyright**: Microsoft's fast type checker that catches different issues than mypy - **Package Manager**: uv (Astral's fast Python package manager) ### Running Checks Locally @@ -195,9 +199,15 @@ uv run ruff check src/ --fix # Format code uv run ruff format src/ -# Type check +# Type check with mypy (strict mode) uv run mypy src/promptfoo/ +# Type check with pyright (strict mode) +uv run pyright src/promptfoo/ + +# Run both type checkers (recommended before PR) +uv run mypy src/promptfoo/ && uv run pyright src/promptfoo/ + # Run tests uv run pytest ``` @@ -276,6 +286,7 @@ git checkout -b feat/my-feature-name uv run ruff check src/ --fix uv run ruff format src/ uv run mypy src/promptfoo/ +uv run pyright src/promptfoo/ uv run pytest # 4. Commit with conventional commit message @@ -303,6 +314,7 @@ git checkout -b fix/bug-description uv run ruff check src/ --fix uv run ruff format src/ uv run mypy src/promptfoo/ +uv run pyright src/promptfoo/ uv run pytest # 4. Commit with conventional commit message @@ -443,5 +455,5 @@ git push --force --- -**Last Updated**: 2026-01-05 +**Last Updated**: 2026-01-11 **Maintained By**: @promptfoo/engineering diff --git a/pyproject.toml b/pyproject.toml index cf1e18a..814cf18 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ dependencies = [ dev = [ "pytest>=8.4.0", "mypy>=1.16.0", + "pyright>=1.1.400", "ruff>=0.12.0", "types-pyyaml>=6.0.0", ] @@ -92,13 +93,74 @@ quote-style = "double" [tool.mypy] python_version = "3.9" -warn_return_any = true -warn_unused_configs = true -warn_redundant_casts = true +# Enable strict mode for comprehensive type checking +strict = true +# Additional strictness beyond --strict warn_unreachable = true -no_implicit_optional = true -strict_equality = true +enable_error_code = [ + "ignore-without-code", + "redundant-cast", + "truthy-bool", + "truthy-iterable", + "unused-awaitable", +] +# Output formatting show_error_codes = true +show_column_numbers = true pretty = true -check_untyped_defs = true -disallow_incomplete_defs = true +# Error handling +warn_unused_ignores = true + +[[tool.mypy.overrides]] +module = "posthog.*" +ignore_missing_imports = true + +[tool.pyright] +pythonVersion = "3.9" +pythonPlatform = "All" +typeCheckingMode = "strict" +include = ["src/promptfoo"] +exclude = ["tests", "**/__pycache__"] +# Report all errors in strict mode +reportMissingImports = true +reportMissingTypeStubs = false # Disable for posthog which lacks full stubs +reportUnusedImport = true +reportUnusedClass = true +reportUnusedFunction = true +reportUnusedVariable = true +reportDuplicateImport = true +reportPrivateUsage = true +reportConstantRedefinition = true +reportIncompatibleMethodOverride = true +reportIncompatibleVariableOverride = true +reportInconsistentConstructor = true +reportOverlappingOverload = true +reportUninitializedInstanceVariable = true +reportCallInDefaultInitializer = true +reportUnnecessaryIsInstance = true +reportUnnecessaryCast = true +reportUnnecessaryComparison = true +reportUnnecessaryContains = true +reportImplicitStringConcatenation = false +reportUnusedCallResult = false +reportUnusedExpression = true +reportUnnecessaryTypeIgnoreComment = false # Allow type: ignore for mypy compatibility +reportMatchNotExhaustive = true +# Relax some strict checks for third-party libraries without full stubs +reportUnknownMemberType = false +reportUnknownArgumentType = false +reportUnknownVariableType = false +reportUnknownParameterType = false + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = ["test_*.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] +addopts = [ + "-v", + "--strict-markers", +] +markers = [ + "smoke: smoke tests that run the full CLI (slow, requires Node.js)", +] diff --git a/src/promptfoo/cli.py b/src/promptfoo/cli.py index e62be52..7bb0a67 100644 --- a/src/promptfoo/cli.py +++ b/src/promptfoo/cli.py @@ -165,7 +165,7 @@ def _requires_shell(executable: str) -> bool: return ext.lower() in _WINDOWS_SHELL_EXTENSIONS -def _run_command(cmd: list[str], env: Optional[dict[str, str]] = None) -> subprocess.CompletedProcess: +def _run_command(cmd: list[str], env: Optional[dict[str, str]] = None) -> subprocess.CompletedProcess[bytes]: """Execute a command, handling shell requirements on Windows.""" if _requires_shell(cmd[0]): return subprocess.run(subprocess.list2cmdline(cmd), shell=True, env=env) diff --git a/src/promptfoo/telemetry.py b/src/promptfoo/telemetry.py index f77f09c..f2fdb6f 100644 --- a/src/promptfoo/telemetry.py +++ b/src/promptfoo/telemetry.py @@ -172,8 +172,8 @@ def shutdown(self) -> None: """Shutdown the telemetry client and flush any pending events.""" if self._client: try: - self._client.flush() - self._client.shutdown() + self._client.flush() # type: ignore[no-untyped-call] + self._client.shutdown() # type: ignore[no-untyped-call] except Exception: pass # Silently fail finally: diff --git a/uv.lock b/uv.lock index 860104b..401a91e 100644 --- a/uv.lock +++ b/uv.lock @@ -336,6 +336,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, ] +[[package]] +name = "nodeenv" +version = "1.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/24/bf/d1bda4f6168e0b2e9e5958945e01910052158313224ada5ce1fb2e1113b8/nodeenv-1.10.0.tar.gz", hash = "sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb", size = 55611, upload-time = "2025-12-20T14:08:54.006Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" }, +] + [[package]] name = "packaging" version = "25.0" @@ -416,6 +425,7 @@ dependencies = [ [package.optional-dependencies] dev = [ { name = "mypy" }, + { name = "pyright" }, { name = "pytest", version = "8.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "pytest", version = "9.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, { name = "ruff" }, @@ -426,6 +436,7 @@ dev = [ requires-dist = [ { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.16.0" }, { name = "posthog", specifier = ">=3.0.0" }, + { name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.400" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.4.0" }, { name = "pyyaml", specifier = ">=6.0.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.12.0" }, @@ -442,6 +453,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, ] +[[package]] +name = "pyright" +version = "1.1.408" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nodeenv" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/74/b2/5db700e52554b8f025faa9c3c624c59f1f6c8841ba81ab97641b54322f16/pyright-1.1.408.tar.gz", hash = "sha256:f28f2321f96852fa50b5829ea492f6adb0e6954568d1caa3f3af3a5f555eb684", size = 4400578, upload-time = "2026-01-08T08:07:38.795Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/82/a2c93e32800940d9573fb28c346772a14778b84ba7524e691b324620ab89/pyright-1.1.408-py3-none-any.whl", hash = "sha256:090b32865f4fdb1e0e6cd82bf5618480d48eecd2eb2e70f960982a3d9a4c17c1", size = 6399144, upload-time = "2026-01-08T08:07:37.082Z" }, +] + [[package]] name = "pytest" version = "8.4.2"