python(fix): teardown failure on the last test, and combined parametrize axis nesting

python/lib/sift_client/_internal/pytest_plugin/report.py

@@ -474,6 +474,11 @@ def report_context_impl(
         replay_log_file=not (disabled or offline),
         metadata=report_metadata,
         audit_log=audit_log,
+        # pytest tears this session-scoped fixture down during the LAST item's
+        # teardown phase but reports that phase's outcome afterwards, so a
+        # teardown failure on the final test is unknown here. The plugin's
+        # ``pytest_sessionfinish`` calls ``finalize`` once it can't be.
+        defer_finalize=True,
     ) as context:
         report = context.report
         meta_kv = ",".join(f"{k}={v}" for k, v in (report.metadata or {}).items()) or "-"

python/lib/sift_client/_tests/pytest_plugin/_step_status_capture.py

@@ -11,7 +11,7 @@
 
 import json
 from dataclasses import dataclass, field
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, List, Tuple
 
 from sift_client._internal.low_level_wrappers._test_results_log import parse_log_data_lines
 from sift_client.sift_types.test_report import TestStatus
@@ -170,6 +170,11 @@ def final_error_message(name: str) -> str | None:
     return step.error_messages[-1] if step and step.error_messages else None
 
 
+# Ordered ``(line index, status)`` writes, for asserting write order across
+# entry kinds. See ``status_writes``.
+StatusWrites = List[Tuple[int, TestStatus]]
+
+
 def log_events(log_path: Path) -> list[tuple[str, str, TestStatus]]:
     """Ordered ``(request_type, step_name, status)`` tuples as they appear in the log.
 
@@ -196,6 +201,38 @@ def log_events(log_path: Path) -> list[tuple[str, str, TestStatus]]:
     return events
 
 
+def status_writes(log_path: Path, step_name: str) -> tuple[StatusWrites, StatusWrites]:
+    """``(report writes, FAILED writes on ``step_name``)`` as ``(line index, status)``.
+
+    Line indices come from the same pass, so they are comparable across the two
+    lists and a test can assert the ORDER of a report write against a step write.
+    ``log_events`` can't express that, since it drops report entries.
+    """
+    if not log_path.exists():
+        return [], []
+    report_writes: StatusWrites = []
+    step_writes: StatusWrites = []
+    step_ids: set[str] = set()
+    for idx, (request_type, response_id, json_str) in enumerate(_log_data_lines(log_path)):
+        # Skip the JSON decode for measurement entries, which dominate most logs.
+        if not request_type.endswith(("TestReport", "TestStep")):
+            continue
+        payload = json.loads(json_str)
+        if request_type.endswith("TestReport"):
+            raw = payload.get("testReport", {}).get("status")
+            if raw:
+                report_writes.append((idx, _status(raw)))
+            continue
+        test_step = payload.get("testStep", {})
+        if request_type == "CreateTestStep" and test_step.get("name") == step_name and response_id:
+            step_ids.add(response_id)
+        elif request_type == "UpdateTestStep" and test_step.get("testStepId") in step_ids:
+            status = _status(test_step.get("status"))
+            if status == TestStatus.FAILED:
+                step_writes.append((idx, status))
+    return report_writes, step_writes
+
+
 def load_steps(log_path: Path) -> list[dict]:
     """Load the offline log as a list of step records keyed by hierarchy fields.
 

python/lib/sift_client/_tests/pytest_plugin/test_hierarchy.py

@@ -1561,6 +1561,38 @@ def test_lid(v):
     assert by_name["two"][0]["parent_step_id"] == parent_id
 
 
+def test_explicit_list_ids_survive_a_stacked_outer_axis(
+    pytester: pytest.Pytester, out_dir: Path
+) -> None:
+    """``ids=`` must resolve per axis, not by the item's position in the whole matrix.
+
+    An axis stacked under another one is where ``callspec.indices`` stops meaning
+    "position within this axis" (it tracks the expanded callspec on pytest 8.4),
+    so an ID list indexed by it runs off the end or picks a neighbour's entry.
+    Every ``inner`` universe must carry the same two IDs, in every ``outer`` one.
+    """
+    pytester.makepyfile(
+        test_slid=dedent(
+            """
+            import pytest
+
+            @pytest.mark.parametrize("outer", ["o1", "o2", "o3"])
+            @pytest.mark.parametrize("inner", [1, 2], ids=["one", "two"])
+            def test_slid(outer, inner):
+                pass
+            """
+        )
+    )
+    result = pytester.runpytest_inprocess("-v")
+    result.assert_outcomes(passed=6)
+    by_name = _by_name(capture.load_steps(capture.run_jsonl(out_dir)))
+    # One leaf per (outer, inner) pair, every one labelled by its own axis's ID.
+    assert len(by_name.get("one", [])) == 3
+    assert len(by_name.get("two", [])) == 3
+    assert "inner=1" not in by_name
+    assert "inner=2" not in by_name
+
+
 def test_callable_id_factory_on_inner_parametrize(pytester: pytest.Pytester, out_dir: Path) -> None:
     """A callable ``ids=`` factory is invoked per value, just as pytest does."""
     pytester.makepyfile(
@@ -1644,30 +1676,160 @@ def test_auto(v):
     assert "v=2" in by_name
 
 
-def test_combined_axis_ids_fall_back_to_name_value(
+def test_combined_axis_renders_one_frame_with_its_id(
     pytester: pytest.Pytester, out_dir: Path
 ) -> None:
-    """A combined ``"a,b"`` axis can't attribute its shared ID, so each frame uses
-    ``name=value`` rather than mislabelling both with the same combined ID.
+    """A combined ``"a,b"`` axis is ONE axis in pytest, so it is one step labelled
+    with the shared ID, not one step per argname.
     """
     pytester.makepyfile(
-        test_comb=dedent(
+        test_comb_ids=dedent(
             """
             import pytest
 
             @pytest.mark.parametrize("a,b", [(1, 2)], ids=["combined"])
-            def test_comb(a, b):
+            def test_comb_ids(a, b):
+                pass
+            """
+        )
+    )
+    result = pytester.runpytest_inprocess("-v")
+    result.assert_outcomes(passed=1)
+    by_name = _by_name(capture.load_steps(capture.run_jsonl(out_dir)))
+    assert "combined" in by_name
+    # No per-argname frames beneath it.
+    assert "a=1" not in by_name
+    assert "b=2" not in by_name
+
+
+def test_combined_axis_without_ids_joins_name_value_pairs(
+    pytester: pytest.Pytester, out_dir: Path
+) -> None:
+    """With no ``ids=``, a combined axis still renders as ONE frame, labelled with
+    the tuple's ``name=value`` pairs rather than pytest's noisier auto-ID.
+    """
+    pytester.makepyfile(
+        test_comb_noids=dedent(
+            """
+            import pytest
+
+            @pytest.mark.parametrize("a,b", [(1, 2)])
+            def test_comb_noids(a, b):
+                pass
+            """
+        )
+    )
+    result = pytester.runpytest_inprocess("-v")
+    result.assert_outcomes(passed=1)
+    by_name = _by_name(capture.load_steps(capture.run_jsonl(out_dir)))
+    assert "a=1, b=2" in by_name
+    assert "a=1" not in by_name
+
+
+def test_callable_id_factory_on_combined_axis(pytester: pytest.Pytester, out_dir: Path) -> None:
+    """A callable ``ids=`` factory runs per value and the results join with ``-``,
+    matching how pytest builds a combined axis's node-ID segment.
+    """
+    pytester.makepyfile(
+        test_comb_factory=dedent(
+            """
+            import pytest
+
+            def label(value):
+                return f"v{value}"
+
+            @pytest.mark.parametrize("a,b", [(1, 2)], ids=label)
+            def test_comb_factory(a, b):
                 pass
             """
         )
     )
     result = pytester.runpytest_inprocess("-v")
     result.assert_outcomes(passed=1)
     by_name = _by_name(capture.load_steps(capture.run_jsonl(out_dir)))
-    # The shared "combined" ID is not adopted for either per-arg frame.
-    assert "combined" not in by_name
-    assert "a=1" in by_name
-    assert "b=2" in by_name
+    assert "v1-v2" in by_name
+
+
+def test_combined_axis_stacked_with_single_axis(pytester: pytest.Pytester, out_dir: Path) -> None:
+    """A combined axis stacked under a single-name axis yields two frames, top
+    decorator outermost, each labelled by its own ID.
+    """
+    pytester.makepyfile(
+        test_comb_stacked=dedent(
+            """
+            import pytest
+
+            @pytest.mark.parametrize("v", ["hi", "lo"])
+            @pytest.mark.parametrize("a,b", [(1, 2)], ids=["combined"])
+            def test_comb_stacked(v, a, b):
+                pass
+            """
+        )
+    )
+    result = pytester.runpytest_inprocess("-v")
+    result.assert_outcomes(passed=2)
+    steps = capture.load_steps(capture.run_jsonl(out_dir))
+    by_name = _by_name(steps)
+    assert "v='hi'" in by_name
+    assert "combined" in by_name
+    assert "a=1" not in by_name
+    # Top decorator outermost: every combined-axis frame nests under a ``v=`` frame.
+    assert len(by_name["combined"]) == 2
+    for frame in by_name["combined"]:
+        ancestors = _ancestor_names(steps, frame)
+        assert "v='hi'" in ancestors or "v='lo'" in ancestors
+
+
+def test_combined_axis_under_scoped_fixture_param(pytester: pytest.Pytester, out_dir: Path) -> None:
+    """A module-scoped fixture axis above a three-argname combined axis whose
+    tuples carry a function object. One frame per axis, each labelled by its own
+    ID, and no frame carrying an object repr.
+    """
+    pytester.makepyfile(
+        test_axes=dedent(
+            """
+            import pytest
+
+            def handler_a(): pass
+            def handler_b(): pass
+
+            CASES = [
+                (handler_a, "signal.alpha", "ALPHA"),
+                (handler_b, "signal.beta", "BETA"),
+            ]
+            CASE_IDS = ["ALPHA", "BETA"]
+
+            @pytest.fixture(scope="module", params=["mode one"], ids=["mode one"])
+            def mode(request):
+                return request.param
+
+            @pytest.mark.parametrize("handler,signal_name,case_name", CASES, ids=CASE_IDS)
+            def test_signal(handler, signal_name, case_name, mode, step):
+                with step.substep("record baseline"):
+                    pass
+            """
+        )
+    )
+    result = pytester.runpytest_inprocess("-v")
+    result.assert_outcomes(passed=2)
+    steps = capture.load_steps(capture.run_jsonl(out_dir))
+    by_name = _by_name(steps)
+    assert "mode one" in by_name
+    assert "ALPHA" in by_name
+    assert "BETA" in by_name
+    # The other two argnames of the combined axis open no frames of their own.
+    assert not [n for n in by_name if n.startswith(("handler=", "signal_name="))]
+    # No label carries a function repr (heap address, unstable across runs).
+    assert not [n for n in by_name if "<function" in n]
+    # Expected chain: module → mode → test → ID → substep. Asserted over every
+    # leaf rather than a fixed index, since the inner run order is not pinned.
+    seen_ids = set()
+    for leaf in by_name["record baseline"]:
+        ancestors = _ancestor_names(steps, leaf)
+        assert ancestors[0] == "record baseline"
+        assert ancestors[2:] == ["test_signal", "mode one", "test_axes.py"]
+        seen_ids.add(ancestors[1])
+    assert seen_ids == {"ALPHA", "BETA"}
 
 
 # ---------------------------------------------------------------------------
@@ -1998,12 +2160,53 @@ def _raise(*_args: object, **_kwargs: object) -> None:
     raise RuntimeError("simulated pytest internals change")
 
 
+def test_no_ids_declared_resolves_quietly() -> None:
+    """The ordinary "author supplied no ``ids=``" path must stay silent."""
+    steps.reset_introspection_state()
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", SiftPytestPluginWarning)
+        assert steps._id_from_spec(None, [1, 2], (1,), "v") is None
+    assert steps._warned_once == set()
+
+
+def test_unmatchable_value_warns_once_per_axis() -> None:
+    """A declared ``ids=`` we cannot attribute warns once for that axis.
+
+    Once per AXIS, not once per session: when a pytest version breaks the
+    resolution every axis is affected, and naming each is what shows the scope.
+    Repeats of the same axis stay quiet so the matrix cannot flood the output.
+    """
+    steps.reset_introspection_state()
+    with pytest.warns(SiftPytestPluginWarning, match="could not match the ids="):
+        assert steps._id_from_spec(["one", "two"], [1, 2], (99,), "v") is None
+    assert steps._warned_once == {"id_unresolved:v"}
+
+    # Same axis again: latched, no second warning.
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", SiftPytestPluginWarning)
+        assert steps._id_from_spec(["one", "two"], [1, 2], (98,), "v") is None
+
+    # A different axis is its own signal and must still be reported.
+    with pytest.warns(SiftPytestPluginWarning, match="argnames 'w'"):
+        assert steps._id_from_spec(["only"], [1], (99,), "w") is None
+    assert steps._warned_once == {"id_unresolved:v", "id_unresolved:w"}
+
+
+def test_id_factory_returning_none_does_not_warn() -> None:
+    """An ID factory returning None is the author's choice, not a fault."""
+    steps.reset_introspection_state()
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", SiftPytestPluginWarning)
+        assert steps._id_from_spec(lambda v: None, [1], (1,), "v") is None
+    assert steps._warned_once == set()
+
+
 def test_fixturedefs_degrades_when_fixture_manager_missing() -> None:
     steps.reset_introspection_state()
     item = _fake_item_without_fixture_manager()
     with pytest.warns(SiftPytestPluginWarning, match="scope-aware parametrize"):
         assert steps._fixturedefs(item, "x") is None
-    assert steps._introspection_degraded is True
+    assert "introspection_degraded" in steps._warned_once
 
 
 def test_fixturedefs_degrades_when_getfixturedefs_raises() -> None:
@@ -2183,4 +2386,4 @@ def getfixturedefs(self, name: str, node_or_nodeid: object) -> object:
         warnings.simplefilter("error")  # a degradation warning would raise here
         result = steps._fixturedefs(item, "x")
     assert result == (sentinel,)
-    assert steps._introspection_degraded is False
+    assert "introspection_degraded" not in steps._warned_once

python/lib/sift_client/_tests/pytest_plugin/test_pass_fail.py

@@ -506,6 +506,50 @@ def test_x(bad_teardown):
     assert outer.statuses[-1] == TestStatus.FAILED
 
 
+def test_teardown_failure_on_last_item_reaches_the_report(inner):
+    """A teardown failure on the FINAL collected item must reach the report.
+
+    pytest tears down the session-scoped ``report_context`` fixture during the
+    last item's teardown phase and only *then* reports that phase's outcome. The
+    report status and the import-worker drain therefore have to be deferred to
+    ``pytest_sessionfinish``, or the failure is computed too late to matter and
+    its log entry is written after the worker has already stopped reading.
+    """
+    _run(
+        inner,
+        """
+        import pytest
+
+        @pytest.fixture
+        def bad_teardown():
+            yield
+            raise RuntimeError("teardown boom")
+
+        def test_x(bad_teardown):
+            assert True
+        """,
+    )
+    log_path = capture._active_log
+    assert log_path is not None
+
+    outer = capture.test_step("test_x")
+    assert outer is not None
+    assert outer.statuses[-1] == TestStatus.FAILED
+
+    report_writes, step_writes = capture.status_writes(log_path, "test_x")
+
+    # The report's LAST status write must be FAILED. Before the fix the report
+    # was finalized as PASSED inside the fixture teardown, ahead of the failure.
+    assert report_writes, "report never wrote a status"
+    assert report_writes[-1][1] == TestStatus.FAILED
+
+    # Ordering is the real guarantee: the report's terminal status has to be
+    # written AFTER the step's late FAILED update. If it precedes it, the drain
+    # preceded it too and the failure would never have been uploaded.
+    assert step_writes, "step never wrote a FAILED status"
+    assert report_writes[-1][0] > step_writes[-1][0]
+
+
 # ---------------------------------------------------------------------------
 # Collection-phase failures
 # ---------------------------------------------------------------------------