Skip to content

Commit 9a35ba2

Browse files
committed
test(runner): drive results acceptance tests through CliRunner
Replaces the subprocess-based `robotcode_cli` fixture with an in-process `click.testing.CliRunner` invocation of the `robotcode` Click group. Per-test cost was previously dominated by a fresh Python interpreter plus the Click/robotcode import graph (~0.3–0.9 s per call). With CliRunner those imports are loaded once at collection time. Robot Framework suite generation stays on subprocess (only ~11 calls per session, dwarfed by the test runtime that we just removed). Wall-clock for the full results suite: ~53 s → ~3 s (factor ~17).
1 parent 06a2c8f commit 9a35ba2

1 file changed

Lines changed: 49 additions & 31 deletions

File tree

tests/robotcode/runner/cli/results/conftest.py

Lines changed: 49 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,22 @@
11
"""Acceptance-test fixtures for `robotcode results`.
22
3-
These tests spawn a real `robotcode` subprocess (so the bundled-script /
4-
quoting layer is exercised too) and feed it `output.xml` files that are
5-
generated *once per pytest session* by running the matching Robot Framework
6-
suite under `tests/.../suites/` with the currently installed RF version.
7-
8-
Three architectural choices, set by the plan:
9-
10-
1. **subprocess** for CLI invocation — real process, real exit codes.
11-
2. **Live fixture generation** — RF runs the suites once per session, so the
12-
test matrix automatically covers every supported RF version.
13-
3. **JSON-structural + selective text snapshots** — parse `--format json`
14-
output and assert on fields; only specific text-render cases are pinned
15-
with regtest2.
3+
The tests feed the `robotcode` Click group via `click.testing.CliRunner`
4+
(in-process — no Python-startup overhead per call) with `output.xml`
5+
files that are generated *once per pytest session* by running the
6+
matching Robot Framework suite under `tests/.../suites/` with the
7+
currently installed RF version.
8+
9+
Three architectural choices:
10+
11+
1. **CliRunner** for CLI invocation — in-process invocation of the
12+
actual `robotcode` Click group. Cuts test runtime from ~50 s to a
13+
few seconds; trade-off: doesn't exercise the bundled-script
14+
shell wrapper.
15+
2. **Live fixture generation** — `python -m robot` runs the fixture
16+
suites once per session via subprocess (only ~11 calls total).
17+
3. **JSON-structural + selective text snapshots** — parse `--format
18+
json` output and assert on fields; only specific text-render cases
19+
are pinned with regtest2.
1620
"""
1721

1822
import json
@@ -25,7 +29,9 @@
2529
from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence
2630

2731
import pytest
32+
from click.testing import CliRunner as _ClickCliRunner
2833

34+
from robotcode.cli import robotcode as _robotcode_group
2935
from robotcode.core.utils.version import Version
3036
from robotcode.robot.utils import RF_VERSION
3137

@@ -118,31 +124,45 @@ def _run(
118124

119125
@pytest.fixture
120126
def robotcode_cli() -> CliRunner:
121-
"""Callable: `(args, *, cwd=None, expect_ok=True) -> CliResult`.
127+
"""Callable: `(args, *, expect_ok=True) -> CliResult`.
122128
123-
Runs `python -m robotcode.cli` so the test never depends on the
124-
`robotcode` script being on PATH. The global flags `--no-color` and
125-
`--no-pager` are pre-injected.
129+
Invokes the `robotcode` Click group in-process via
130+
`click.testing.CliRunner` — no Python startup per call, so the suite
131+
runs in seconds instead of minutes. The global flags `--no-color`
132+
and `--no-pager` are pre-injected so output is deterministic.
133+
134+
Unexpected exceptions (i.e. bugs inside the CLI, not regular
135+
UsageErrors) propagate when `expect_ok=True` so they fail loudly
136+
instead of being masked as a non-zero exit.
126137
"""
138+
runner = _ClickCliRunner()
127139

128140
def run(
129141
args: Sequence[str],
130142
*,
131-
cwd: Optional[Path] = None,
132143
expect_ok: bool = True,
133-
timeout: float = 60.0,
134144
) -> CliResult:
135-
full = ["-m", "robotcode.cli", "--no-color", "--no-pager", *args]
136-
proc = _run(full, cwd=cwd, timeout=timeout)
137-
result = CliResult(
145+
full = ["--no-color", "--no-pager", *args]
146+
result = runner.invoke(_robotcode_group, full)
147+
cli_result = CliResult(
138148
args=full,
139-
returncode=proc.returncode,
140-
stdout=proc.stdout,
141-
stderr=proc.stderr,
149+
returncode=result.exit_code,
150+
stdout=result.stdout,
151+
stderr=result.stderr or "",
142152
)
143153
if expect_ok:
144-
result.expect_ok()
145-
return result
154+
# Surface unexpected exceptions (anything other than SystemExit
155+
# raised inside the command) so they don't disguise themselves
156+
# as a "just" non-zero exit.
157+
if result.exception is not None and not isinstance(result.exception, SystemExit):
158+
raise AssertionError(
159+
f"robotcode raised {type(result.exception).__name__}: {result.exception}\n"
160+
f"args: {full}\n"
161+
f"stdout: {cli_result.stdout!r}\n"
162+
f"stderr: {cli_result.stderr!r}"
163+
) from result.exception
164+
cli_result.expect_ok()
165+
return cli_result
146166

147167
return run
148168

@@ -159,14 +179,13 @@ def run(
159179
subcommand: str,
160180
*extra: str,
161181
output_path: Optional[Path] = None,
162-
cwd: Optional[Path] = None,
163182
expect_ok: bool = True,
164183
) -> Any:
165184
args: List[str] = ["--format", "json", "results", subcommand]
166185
args.extend(extra)
167186
if output_path is not None:
168187
args.extend(["--output", str(output_path)])
169-
result = robotcode_cli(args, cwd=cwd, expect_ok=expect_ok)
188+
result = robotcode_cli(args, expect_ok=expect_ok)
170189
if not expect_ok or not result.stdout.strip():
171190
return result
172191
try:
@@ -189,13 +208,12 @@ def run(
189208
subcommand: str,
190209
*extra: str,
191210
output_path: Optional[Path] = None,
192-
cwd: Optional[Path] = None,
193211
expect_ok: bool = True,
194212
) -> CliResult:
195213
args: List[str] = ["results", subcommand, *extra]
196214
if output_path is not None:
197215
args.extend(["--output", str(output_path)])
198-
return robotcode_cli(args, cwd=cwd, expect_ok=expect_ok)
216+
return robotcode_cli(args, expect_ok=expect_ok)
199217

200218
return run
201219

0 commit comments

Comments
 (0)