Fix walltime mode

Author: toby-coleman

src/pytest_codspeed/instruments/walltime.py

@@ -292,8 +292,8 @@ def __codspeed_root_frame__(*args, **kwargs) -> T:
         if self.instrument_hooks:
             self.instrument_hooks.start_benchmark()
         for _ in range(pedantic_options.rounds):
-            start = perf_counter_ns()
             args, kwargs = pedantic_options.setup_and_get_args_kwargs()
+            start = perf_counter_ns()
             for _ in iter_range:
                 __codspeed_root_frame__(*args, **kwargs)
             end = perf_counter_ns()

tests/test_pytest_plugin_walltime.py

@@ -1,6 +1,8 @@
 import pytest
 from conftest import run_pytest_codspeed_with_mode
 
+from pytest_codspeed.config import BenchmarkMarkerOptions, CodSpeedConfig, PedanticOptions
+from pytest_codspeed.instruments.walltime import WallTimeInstrument
 from pytest_codspeed.instruments import MeasurementMode
 
 
@@ -86,3 +88,45 @@ def target(a, b, c):
     result = run_pytest_codspeed_with_mode(pytester, MeasurementMode.WallTime)
     assert result.ret == 0, "the run should have succeeded"
     result.assert_outcomes(passed=1)
+
+
+def test_benchmark_pedantic_walltime_setup_not_timed(monkeypatch: pytest.MonkeyPatch):
+    """Verify that the setup time is not included in the measurement when using pedantic mode with walltime."""
+    current_time_ns = 0
+
+    def fake_perf_counter_ns() -> int:
+        return current_time_ns
+
+    monkeypatch.setattr(
+        "pytest_codspeed.instruments.walltime.perf_counter_ns", fake_perf_counter_ns
+    )
+
+    def setup() -> tuple[tuple[int], dict[str, int]]:
+        nonlocal current_time_ns
+        current_time_ns += 200
+        return (1,), {"c": 2}
+
+    def target(a: int, c: int) -> int:
+        nonlocal current_time_ns
+        current_time_ns += 400
+        return a + c
+
+    instrument = WallTimeInstrument(CodSpeedConfig(), MeasurementMode.WallTime)
+    result = instrument.measure_pedantic(
+        BenchmarkMarkerOptions(),
+        PedanticOptions(
+            target=target,
+            setup=setup,
+            teardown=None,
+            rounds=2,
+            warmup_rounds=0,
+            iterations=1,
+        ),
+        name="test_pedantic_setup_not_timed",
+        uri="tests/test_benchmark.py::test_pedantic_setup_not_timed",
+    )
+
+    assert result == 3
+    assert len(instrument.benchmarks) == 1
+    # Two rounds should each measure target-only time (400ns), excluding setup (200ns).
+    assert instrument.benchmarks[0].stats.min_ns == 400