Integration tests use pytest (#38)

* Integration tests use pytest

* Use pytest -s

* Integration test entry point

* Switch to pytest-gak

* Fix linting

* Compile deps for CI

Author: jfhbrook

.github/workflows/qa.yaml

@@ -31,4 +31,4 @@ jobs:
           npx pyright@latest
       - name: Run tests
         run: |
-          pytest ./tests
+          pytest ./tests --ignore=tests/test_integration.py

justfile

@@ -41,7 +41,7 @@ _upgrade:
 compile:
   uv pip compile -o requirements.txt pyproject.toml
   cp requirements.txt requirements_dev.txt
-  python3 -c 'import toml; print("\n".join(toml.load(open("pyproject.toml"))["dependency-groups"]["dev"]))' >> requirements_dev.txt
+  python3 -c 'import tomllib; print("\n".join(tomllib.load(open("pyproject.toml", "rb"))["dependency-groups"]["dev"]))' >> requirements_dev.txt
 
 clean-compile:
   rm -f requirements.txt
@@ -80,17 +80,17 @@ check:
 
 # Run tests with pytest
 test:
-  uv run pytest -vvv ./tests
+  uv run pytest -vvv ./tests --ignore=./tests/test_integration.py
   @just clean-test
 
 # Update snapshots
 snap:
   uv run pytest --snapshot-update ./tests
   @just clean-test
 
-# Run integration tests (for what they are)
+# Run integration tests
 integration:
-  uv run python ./tests/integration.py
+  uv run gaktest ./tests/test_integration.py
 
 clean-test:
   rm -f pytest_runner-*.egg

plusdeck/test.py

@@ -1,110 +1,55 @@
-import asyncio
-from collections.abc import Awaitable
-from inspect import getmembers, isfunction
 import sys
-from typing import Callable, cast, Protocol, Set, Union
+from typing import List
 
-from rich.prompt import Prompt
+from pytest import console_main
 
-"""Tools for manual testing with real hardware."""
 
+def parse_args(raw_args: List[str] = sys.argv[1:]) -> List[str]:
+    """
+    Given arguments for pytest, ensure that -s or --capture=no is set.
 
-class AbortError(Exception):
-    """A manual testing step has been aborted."""
+    This function is not entirely robust, as it doesn't implement full arguments
+    parsing as in pytest. If an option is passed "-s" or "--capture=no" as a value,
+    this will fail to add the appropriate flag. But those cases should be very
+    rare, and a full options parse is difficult.
+    """
 
-    pass
+    args: List[str] = list()
 
+    has_no_capture_flag = False
+    for i, arg in enumerate(raw_args):
+        if arg == "--capture=no" or arg == "-s":
+            has_no_capture_flag = True
+            args += raw_args[i:]
+            break
 
-def confirm(text: str) -> None:
-    """Manually confirm an expected state."""
-
-    res = Prompt.ask(text, choices=["confirm", "abort"])
-
-    if res == "abort":
-        raise AbortError("Aborted.")
-
-
-def take_action(text: str) -> None:
-    """Take a manual action before continuing."""
-
-    res = Prompt.ask(text, choices=["continue", "abort"])
-
-    if res == "abort":
-        raise AbortError("Aborted.")
-
-
-def check(text: str, expected: str) -> None:
-    """Manually check whether or not an expected state is so."""
-
-    res = Prompt.ask(text, choices=["yes", "no", "abort"])
-
-    if res == "abort":
-        raise AbortError("Aborted.")
-
-    assert res == "yes", expected
-
-
-class MarkedTest(Protocol):
-    marks: Set[str]
-
-    def __call__(self) -> Awaitable[None]: ...
-
-
-UnmarkedTest = Callable[[], Awaitable[None]]
-
-Test = Union[UnmarkedTest, MarkedTest]
-
-
-def mark(tag: str) -> Callable[[Test], MarkedTest]:
-    def decorator(test: Test) -> MarkedTest:
-        marked = cast(MarkedTest, test)
-
-        if not hasattr(test, "marks"):
-            marked.marks = set()
-
-        marked.marks.add(tag)
-
-        return marked
-
-    return decorator
-
-
-def skip(test: Test) -> MarkedTest:
-    """Skip a test."""
-
-    return mark("skip")(test)
-
+        if arg.startswith("--capture="):
+            # Drop the flag, since we're going to override it later
+            continue
 
-def marked_with(tag: str, test: Test) -> bool:
-    """Check if a test has a mark."""
+        args.append(arg)
 
-    if not hasattr(test, "marks"):
-        return False
+    if not has_no_capture_flag:
+        args.insert(0, "--capture=no")
 
-    return tag in cast(MarkedTest, test).marks
+    return args
 
 
-async def _run_tests(__name__: str) -> None:
-    for name, test in getmembers(sys.modules[__name__], isfunction):
-        if not name.startswith("test_"):
-            continue
+def main() -> int:
+    """
+    A command line entry point that calls pytest with --capture=no set.
+    """
 
-        if marked_with("skip", test):
-            print(f"=== {name} SKIPPED ===")
-            continue
+    args = parse_args()
+    # sys.argv[0] is typically the command that was run
+    args.insert(0, "pytest")
 
-        print(f"=== {name} ===")
-        try:
-            await test()
-        except Exception as exc:
-            print(f"{name} FAILED")
-            print(exc)
-        else:
-            print(f"=== {name} PASSED ===")
+    # Patch sys.argv so the pytest entry point picks it up
+    sys.argv = args
 
+    # Call the standard pytest entry point with modified args
+    return console_main()
 
-def run_tests(__name__: str) -> None:
-    """Run integration tests in module."""
 
-    loop = asyncio.get_event_loop()
-    loop.run_until_complete(_run_tests(__name__))
+if __name__ == "__main__":
+    sys.exit(main())

pyproject.toml

@@ -58,13 +58,13 @@ dev = [
   "flake8-black",
   "pytest",
   "pytest-asyncio",
+  "pytest-gak",
   "black",
   "isort",
   "jupyterlab",
   "mkdocs",
   "mkdocs-include-markdown-plugin",
   "mkdocstrings[python]",
-  "rich",
   "syrupy",
   "tox",
   "validate-pyproject[all]",

requirements_dev.txt

@@ -66,14 +66,13 @@ flake8
 flake8-black
 pytest
 pytest-asyncio
+pytest-gak
 black
 isort
 jupyterlab
 mkdocs
 mkdocs-include-markdown-plugin
 mkdocstrings[python]
-rich
 syrupy
 tox
-twine
 validate-pyproject[all]

tests/integration.py tests/test_integration.pytests/integration.py renamed to tests/test_integration.py

@@ -1,13 +1,16 @@
+import pytest
+
 from plusdeck.client import Command, create_connection, State
 from plusdeck.config import Config
-from plusdeck.test import check, confirm, run_tests, skip, take_action
 
 CONFIG = Config.from_environment()
 
 
-@skip
-async def test_manual_no_events():
-    """Plus Deck plays tapes manually without state subscription."""
+@pytest.mark.skip
+async def test_manual_no_events(check, confirm, take_action) -> None:
+    """
+    Plus Deck plays tapes manually without state subscription.
+    """
 
     confirm("There is NO tape in the deck")
 
@@ -33,8 +36,11 @@ def unexpected_state(state: State):
     client.close()
 
 
-async def test_commands_and_events():
-    """Plus Deck plays tapes with commands when subscribed."""
+@pytest.mark.asyncio
+async def test_commands_and_events(check, confirm, take_action) -> None:
+    """
+    Plus Deck plays tapes with commands when subscribed.
+    """
 
     confirm("There is NO tape in the deck")
 
@@ -107,7 +113,3 @@ def log_state(state: State) -> None:
     client.events.remove_listener("state", log_state)
 
     client.close()
-
-
-if __name__ == "__main__":
-    run_tests(__name__)