From 20c2d1b5fb87d83428489e2e8441f0a23100819a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 10 Jul 2026 08:32:57 +0000 Subject: [PATCH 01/17] Add private Windows ETW thread spawn tracer prototype Introduce threadpoolctl._thread_tracer as an experimental private package with a ctypes-based WindowsThreadSpawnTracer that counts kernel Thread Start events for a target PID via the NT Kernel Logger. Includes pure-Python event payload parsing, mocked ETW lifecycle tests that run on all platforms, and keeps the tracer out of the public threadpoolctl API. Co-authored-by: Olivier Grisel --- threadpoolctl/_thread_tracer/__init__.py | 19 + threadpoolctl/_thread_tracer/_parsing.py | 68 +++ threadpoolctl/_thread_tracer/_types.py | 32 ++ threadpoolctl/_thread_tracer/_windows_etw.py | 412 ++++++++++++++++++ .../tests/test_windows_thread_tracer.py | 158 +++++++ 5 files changed, 689 insertions(+) create mode 100644 threadpoolctl/_thread_tracer/__init__.py create mode 100644 threadpoolctl/_thread_tracer/_parsing.py create mode 100644 threadpoolctl/_thread_tracer/_types.py create mode 100644 threadpoolctl/_thread_tracer/_windows_etw.py create mode 100644 threadpoolctl/tests/test_windows_thread_tracer.py diff --git a/threadpoolctl/_thread_tracer/__init__.py b/threadpoolctl/_thread_tracer/__init__.py new file mode 100644 index 00000000..c000e4d7 --- /dev/null +++ b/threadpoolctl/_thread_tracer/__init__.py @@ -0,0 +1,19 @@ +"""Private experimental thread spawn tracer (not part of the public API).""" + +import sys + +from threadpoolctl._thread_tracer._types import ThreadSpawnStats, ThreadTracerError +from threadpoolctl._thread_tracer._windows_etw import WindowsThreadSpawnTracer + + +def windows_tracer_available(): + """Return whether the Windows ETW tracer can run on this platform.""" + return sys.platform == "win32" + + +__all__ = [ + "ThreadSpawnStats", + "ThreadTracerError", + "WindowsThreadSpawnTracer", + "windows_tracer_available", +] diff --git a/threadpoolctl/_thread_tracer/_parsing.py b/threadpoolctl/_thread_tracer/_parsing.py new file mode 100644 index 00000000..14812ee3 --- /dev/null +++ b/threadpoolctl/_thread_tracer/_parsing.py @@ -0,0 +1,68 @@ +"""Pure-Python helpers to classify kernel thread ETW events.""" + +import ctypes as ct +import struct + +# Kernel thread event opcodes (EVENT_TRACE_TYPE_* for the Thread task). +EVENT_TRACE_TYPE_START = 1 +EVENT_TRACE_TYPE_END = 2 +EVENT_TRACE_TYPE_DCSTART = 3 +EVENT_TRACE_TYPE_DCEND = 4 + +_MIN_USER_DATA_LENGTH = 8 + + +def parse_kernel_thread_payload(user_data, user_data_length): + """Extract ``(process_id, thread_id)`` from a kernel Thread event payload. + + The first two fields are stable across Thread event schema versions. Pointer + fields that follow are ignored because only the identifiers are needed. + """ + if user_data is None or user_data_length < _MIN_USER_DATA_LENGTH: + return None + + if isinstance(user_data, (bytes, bytearray)): + raw = user_data[:user_data_length] + else: + address = user_data + if not isinstance(address, int): + address = ct.cast(user_data, ct.c_void_p).value or 0 + if not address: + return None + raw = ct.string_at(address, user_data_length) + + if len(raw) < _MIN_USER_DATA_LENGTH: + return None + process_id, thread_id = struct.unpack_from(" Date: Fri, 10 Jul 2026 09:34:36 +0000 Subject: [PATCH 02/17] Add Windows ETW integration tests and drop mock-heavy unit tests Replace mocked ETW lifecycle tests with subprocess integration checks that spawn real Python, OpenMP, and BLAS threads while the kernel tracer is attached. Keep lightweight payload parsing unit tests for all platforms. Add windows_etw_admin_available() and a short flush delay on tracer stop to improve reliability on CI runners. Co-authored-by: Olivier Grisel --- threadpoolctl/_thread_tracer/__init__.py | 13 ++ threadpoolctl/_thread_tracer/_windows_etw.py | 2 + .../tests/test_windows_thread_tracer.py | 112 +------------- .../test_windows_thread_tracer_integration.py | 144 ++++++++++++++++++ 4 files changed, 160 insertions(+), 111 deletions(-) create mode 100644 threadpoolctl/tests/test_windows_thread_tracer_integration.py diff --git a/threadpoolctl/_thread_tracer/__init__.py b/threadpoolctl/_thread_tracer/__init__.py index c000e4d7..d1a4d4eb 100644 --- a/threadpoolctl/_thread_tracer/__init__.py +++ b/threadpoolctl/_thread_tracer/__init__.py @@ -11,9 +11,22 @@ def windows_tracer_available(): return sys.platform == "win32" +def windows_etw_admin_available(): + """Return whether the current process can start kernel ETW sessions.""" + if not windows_tracer_available(): + return False + try: + import ctypes + + return bool(ctypes.windll.shell32.IsUserAnAdmin()) + except (AttributeError, OSError): + return False + + __all__ = [ "ThreadSpawnStats", "ThreadTracerError", "WindowsThreadSpawnTracer", + "windows_etw_admin_available", "windows_tracer_available", ] diff --git a/threadpoolctl/_thread_tracer/_windows_etw.py b/threadpoolctl/_thread_tracer/_windows_etw.py index 10d7336d..f8a8f115 100644 --- a/threadpoolctl/_thread_tracer/_windows_etw.py +++ b/threadpoolctl/_thread_tracer/_windows_etw.py @@ -11,6 +11,7 @@ import ctypes.wintypes as wt import sys import threading +import time import uuid from threadpoolctl._thread_tracer._parsing import ( @@ -338,6 +339,7 @@ def stop(self): return self._stats.snapshot() self._stop_event.set() + time.sleep(0.2) if self._trace_handle.value: CloseTrace(self._trace_handle) if self._consumer_thread is not None: diff --git a/threadpoolctl/tests/test_windows_thread_tracer.py b/threadpoolctl/tests/test_windows_thread_tracer.py index b4d595a7..384789c1 100644 --- a/threadpoolctl/tests/test_windows_thread_tracer.py +++ b/threadpoolctl/tests/test_windows_thread_tracer.py @@ -1,15 +1,8 @@ -import ctypes as ct import struct import sys import pytest -from threadpoolctl._thread_tracer import ( - ThreadSpawnStats, - ThreadTracerError, - WindowsThreadSpawnTracer, - windows_tracer_available, -) from threadpoolctl._thread_tracer._parsing import ( EVENT_TRACE_TYPE_DCSTART, EVENT_TRACE_TYPE_END, @@ -17,6 +10,7 @@ classify_kernel_thread_event, parse_kernel_thread_payload, ) +from threadpoolctl._thread_tracer import windows_tracer_available def _pack_thread_payload(process_id, thread_id): @@ -52,107 +46,3 @@ def test_parse_kernel_thread_payload_rejects_short_buffers(): def test_windows_tracer_available_matches_platform(): assert windows_tracer_available() == (sys.platform == "win32") - - -def test_windows_tracer_start_requires_windows(monkeypatch): - if sys.platform == "win32": - pytest.skip("Platform-specific negative test") - - monkeypatch.setattr( - "threadpoolctl._thread_tracer._windows_etw.sys.platform", - "linux", - ) - monkeypatch.setattr( - "threadpoolctl._thread_tracer._windows_etw._advapi32", - None, - ) - monkeypatch.setattr( - "threadpoolctl._thread_tracer._windows_etw.StartTraceW", - None, - ) - - tracer = WindowsThreadSpawnTracer(1234) - with pytest.raises(ThreadTracerError, match="only supported on Windows"): - tracer.start() - - -def test_windows_tracer_lifecycle_with_mocked_etw(monkeypatch): - from threadpoolctl._thread_tracer import _windows_etw as etw - - monkeypatch.setattr(etw, "_advapi32", object()) - monkeypatch.setattr(etw.sys, "platform", "win32") - - spawn_payload = _pack_thread_payload(9001, 12) - existing_payload = _pack_thread_payload(9001, 34) - - def _make_record(opcode, payload): - record = etw.EVENT_RECORD() - record.EventHeader.EventDescriptor.Opcode = opcode - record.UserDataLength = len(payload) - buffer = ct.create_string_buffer(payload) - record._payload_buffer = buffer - record.UserData = ct.cast(buffer, ct.c_void_p) - return ct.pointer(record) - - events = [ - _make_record(etw.EVENT_TRACE_TYPE_DCSTART, existing_payload), - _make_record(etw.EVENT_TRACE_TYPE_START, spawn_payload), - _make_record(etw.EVENT_TRACE_TYPE_START, spawn_payload), - ] - event_state = {"index": 0, "callback": None, "stop": False} - - def fake_start_trace(session_handle, logger_name, props): - session_handle._obj.value = 1 - return etw.ERROR_SUCCESS - - def fake_open_trace(trace_logfile): - event_state["callback"] = trace_logfile._obj.EventRecordCallback - return etw.TRACEHANDLE(2) - - def fake_process_trace(trace_handle, count, start_time, end_time): - if event_state["stop"]: - return 1 - callback = event_state["callback"] - while event_state["index"] < len(events): - callback(events[event_state["index"]]) - event_state["index"] += 1 - return 1 - - def fake_close_trace(trace_handle): - event_state["stop"] = True - return etw.ERROR_SUCCESS - - def fake_control_trace(session_handle, logger_name, props, control_code): - return etw.ERROR_SUCCESS - - monkeypatch.setattr(etw, "StartTraceW", fake_start_trace) - monkeypatch.setattr(etw, "OpenTraceW", fake_open_trace) - monkeypatch.setattr(etw, "ProcessTrace", fake_process_trace) - monkeypatch.setattr(etw, "CloseTrace", fake_close_trace) - monkeypatch.setattr(etw, "ControlTraceW", fake_control_trace) - - tracer = WindowsThreadSpawnTracer(9001, include_existing_threads=True) - tracer.start() - stats = tracer.stop() - - assert stats == ThreadSpawnStats( - spawn_count=2, - existing_thread_count=1, - thread_ids=frozenset({12, 34}), - ) - - -def test_windows_tracer_reports_existing_kernel_logger_session(monkeypatch): - from threadpoolctl._thread_tracer import _windows_etw as etw - - monkeypatch.setattr(etw, "_advapi32", object()) - monkeypatch.setattr(etw.sys, "platform", "win32") - - def fake_start_trace(session_handle, logger_name, props): - return etw.ERROR_ALREADY_EXISTS - - monkeypatch.setattr(etw, "StartTraceW", fake_start_trace) - - tracer = WindowsThreadSpawnTracer(1234) - with pytest.raises(ThreadTracerError, match="already in use"): - tracer.start() diff --git a/threadpoolctl/tests/test_windows_thread_tracer_integration.py b/threadpoolctl/tests/test_windows_thread_tracer_integration.py new file mode 100644 index 00000000..15e902a4 --- /dev/null +++ b/threadpoolctl/tests/test_windows_thread_tracer_integration.py @@ -0,0 +1,144 @@ +import os +import subprocess +import sys +import textwrap +from pathlib import Path + +import pytest + +from threadpoolctl._thread_tracer import ( + ThreadTracerError, + WindowsThreadSpawnTracer, + windows_etw_admin_available, + windows_tracer_available, +) + +pytestmark = [ + pytest.mark.skipif( + not windows_tracer_available(), + reason="Windows ETW integration tests require Windows", + ), + pytest.mark.skipif( + not windows_etw_admin_available(), + reason="Windows ETW kernel tracing requires an elevated process", + ), +] + +REPO_ROOT = Path(__file__).resolve().parents[2] +TRACER_ATTACH_DELAY_SECONDS = 0.5 +TRACER_FLUSH_DELAY_SECONDS = 0.3 +SUBPROCESS_TIMEOUT_SECONDS = 60 + + +def _child_script(body): + return textwrap.dedent( + """ + import time + + time.sleep({attach_delay}) + {body} + time.sleep({flush_delay}) + """ + ).format( + attach_delay=TRACER_ATTACH_DELAY_SECONDS, + body=textwrap.indent(textwrap.dedent(body).strip(), " "), + flush_delay=TRACER_FLUSH_DELAY_SECONDS, + ) + + +def _run_traced_child(body, minimum_spawn_count): + env = os.environ.copy() + pythonpath = env.get("PYTHONPATH", "") + env["PYTHONPATH"] = ( + str(REPO_ROOT) + if not pythonpath + else str(REPO_ROOT) + os.pathsep + pythonpath + ) + + proc = subprocess.Popen( + [sys.executable, "-c", _child_script(body)], + cwd=str(REPO_ROOT), + env=env, + ) + tracer = WindowsThreadSpawnTracer(proc.pid) + try: + tracer.start() + except ThreadTracerError as exc: + proc.kill() + proc.wait(timeout=SUBPROCESS_TIMEOUT_SECONDS) + pytest.fail("failed to start Windows ETW tracer: {0}".format(exc)) + + try: + return_code = proc.wait(timeout=SUBPROCESS_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=SUBPROCESS_TIMEOUT_SECONDS) + tracer.stop() + pytest.fail("timed out waiting for traced child process") + + stats = tracer.stop() + assert return_code == 0, "traced child exited with code {0}".format(return_code) + assert stats.spawn_count >= minimum_spawn_count, ( + "expected at least {expected} thread spawn events, got {actual} ({stats})" + .format( + expected=minimum_spawn_count, + actual=stats.spawn_count, + stats=stats, + ) + ) + return stats + + +def test_tracer_counts_python_thread_spawns(): + body = """ + import threading + + num_threads = 3 + started = threading.Barrier(num_threads) + + def work(): + started.wait(timeout=5) + + threads = [threading.Thread(target=work) for _ in range(num_threads)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=5) + """ + _run_traced_child(body, minimum_spawn_count=3) + + +def test_tracer_counts_openmp_thread_spawns(): + try: + from threadpoolctl.tests._openmp_test_helper import check_openmp_n_threads # noqa: F401 + except ImportError: + pytest.skip("OpenMP test helper is not built") + + body = """ + import os + + os.environ["OMP_NUM_THREADS"] = "4" + from threadpoolctl.tests._openmp_test_helper import check_openmp_n_threads + + used = check_openmp_n_threads(1000) + assert used >= 1 + """ + _run_traced_child(body, minimum_spawn_count=4) + + +def test_tracer_counts_blas_thread_spawns(): + pytest.importorskip("numpy") + body = """ + import os + + os.environ["OMP_NUM_THREADS"] = "1" + os.environ["OPENBLAS_NUM_THREADS"] = "4" + os.environ["MKL_NUM_THREADS"] = "4" + + import numpy as np + + rng = np.random.RandomState(0) + a = rng.rand(2000, 2000) + np.dot(a, a) + """ + _run_traced_child(body, minimum_spawn_count=2) From 38a12029785f46550734d2aff38e515b41ec7ff1 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 10 Jul 2026 09:51:53 +0000 Subject: [PATCH 03/17] Derive native pool spawn expectations from threadpool_info Warm up BLAS/OpenMP pools in the parent process, then use threadpool_info (or get_threadpool_limits on older releases) to set the minimum expected ETW thread spawn count instead of hard-coded guesses. Co-authored-by: Olivier Grisel --- .../test_windows_thread_tracer_integration.py | 57 ++++++++++++++++++- 1 file changed, 54 insertions(+), 3 deletions(-) diff --git a/threadpoolctl/tests/test_windows_thread_tracer_integration.py b/threadpoolctl/tests/test_windows_thread_tracer_integration.py index 15e902a4..1e8f6fe4 100644 --- a/threadpoolctl/tests/test_windows_thread_tracer_integration.py +++ b/threadpoolctl/tests/test_windows_thread_tracer_integration.py @@ -28,6 +28,44 @@ TRACER_ATTACH_DELAY_SECONDS = 0.5 TRACER_FLUSH_DELAY_SECONDS = 0.3 SUBPROCESS_TIMEOUT_SECONDS = 60 +BLAS_THREAD_ENV_VARS = { + "OMP_NUM_THREADS": "1", + "OPENBLAS_NUM_THREADS": "4", + "MKL_NUM_THREADS": "4", +} + + +def _threadpool_info(): + try: + from threadpoolctl import threadpool_info + + return threadpool_info() + except ImportError: + from threadpoolctl import get_threadpool_limits + + return get_threadpool_limits() + + +def _module_num_threads(module): + if "num_threads" in module: + return module["num_threads"] + return module.get("n_thread") + + +def _expected_pool_thread_count(user_api): + thread_counts = [ + _module_num_threads(module) + for module in _threadpool_info() + if module.get("user_api") == user_api and _module_num_threads(module) + ] + if not thread_counts: + pytest.skip("No {0} thread pool detected".format(user_api)) + return max(thread_counts) + + +def _configure_blas_thread_env(): + for name, value in BLAS_THREAD_ENV_VARS.items(): + os.environ[name] = value def _child_script(body): @@ -110,10 +148,14 @@ def work(): def test_tracer_counts_openmp_thread_spawns(): try: - from threadpoolctl.tests._openmp_test_helper import check_openmp_n_threads # noqa: F401 + from threadpoolctl.tests._openmp_test_helper import check_openmp_n_threads except ImportError: pytest.skip("OpenMP test helper is not built") + os.environ["OMP_NUM_THREADS"] = "4" + check_openmp_n_threads(10) + expected_spawn_count = _expected_pool_thread_count("openmp") + body = """ import os @@ -123,11 +165,20 @@ def test_tracer_counts_openmp_thread_spawns(): used = check_openmp_n_threads(1000) assert used >= 1 """ - _run_traced_child(body, minimum_spawn_count=4) + _run_traced_child(body, minimum_spawn_count=expected_spawn_count) def test_tracer_counts_blas_thread_spawns(): pytest.importorskip("numpy") + _configure_blas_thread_env() + + import numpy as np + + rng = np.random.RandomState(0) + warmup = rng.rand(100, 100) + np.dot(warmup, warmup) + expected_spawn_count = _expected_pool_thread_count("blas") + body = """ import os @@ -141,4 +192,4 @@ def test_tracer_counts_blas_thread_spawns(): a = rng.rand(2000, 2000) np.dot(a, a) """ - _run_traced_child(body, minimum_spawn_count=2) + _run_traced_child(body, minimum_spawn_count=expected_spawn_count) From b0aa8d71dfa89618d328e7818b238ad8ce679a51 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 10 Jul 2026 09:54:24 +0000 Subject: [PATCH 04/17] Fix black formatting and address review feedback Remove trivial availability helpers from the private API and inline the Windows/admin checks in integration test skipif predicates. Co-authored-by: Olivier Grisel --- threadpoolctl/_thread_tracer/__init__.py | 22 ------------ threadpoolctl/_thread_tracer/_windows_etw.py | 10 +++--- .../tests/test_windows_thread_tracer.py | 6 ---- .../test_windows_thread_tracer_integration.py | 35 +++++++------------ 4 files changed, 18 insertions(+), 55 deletions(-) diff --git a/threadpoolctl/_thread_tracer/__init__.py b/threadpoolctl/_thread_tracer/__init__.py index d1a4d4eb..d54a6e04 100644 --- a/threadpoolctl/_thread_tracer/__init__.py +++ b/threadpoolctl/_thread_tracer/__init__.py @@ -1,32 +1,10 @@ """Private experimental thread spawn tracer (not part of the public API).""" -import sys - from threadpoolctl._thread_tracer._types import ThreadSpawnStats, ThreadTracerError from threadpoolctl._thread_tracer._windows_etw import WindowsThreadSpawnTracer - -def windows_tracer_available(): - """Return whether the Windows ETW tracer can run on this platform.""" - return sys.platform == "win32" - - -def windows_etw_admin_available(): - """Return whether the current process can start kernel ETW sessions.""" - if not windows_tracer_available(): - return False - try: - import ctypes - - return bool(ctypes.windll.shell32.IsUserAnAdmin()) - except (AttributeError, OSError): - return False - - __all__ = [ "ThreadSpawnStats", "ThreadTracerError", "WindowsThreadSpawnTracer", - "windows_etw_admin_available", - "windows_tracer_available", ] diff --git a/threadpoolctl/_thread_tracer/_windows_etw.py b/threadpoolctl/_thread_tracer/_windows_etw.py index f8a8f115..dd8cb18e 100644 --- a/threadpoolctl/_thread_tracer/_windows_etw.py +++ b/threadpoolctl/_thread_tracer/_windows_etw.py @@ -59,7 +59,7 @@ def from_string(cls, guid_string): guid.Data3 = int(parts[2], 16) hex_data4 = parts[3] + parts[4] for index in range(8): - guid.Data4[index] = int(hex_data4[index * 2:index * 2 + 2], 16) + guid.Data4[index] = int(hex_data4[index * 2 : index * 2 + 2], 16) return guid @@ -188,9 +188,7 @@ def _ensure_advapi32(): return _advapi32 if sys.platform != "win32": - raise ThreadTracerError( - "Windows ETW tracer is only supported on Windows" - ) + raise ThreadTracerError("Windows ETW tracer is only supported on Windows") _advapi32 = ct.windll.advapi32 @@ -232,7 +230,9 @@ def _ensure_advapi32(): def _make_trace_properties(enable_flags): max_str_len = 1024 - buf_size = ct.sizeof(EVENT_TRACE_PROPERTIES) + 2 * ct.sizeof(ct.c_wchar) * max_str_len + buf_size = ( + ct.sizeof(EVENT_TRACE_PROPERTIES) + 2 * ct.sizeof(ct.c_wchar) * max_str_len + ) buf = (ct.c_char * buf_size)() props = ct.cast(ct.pointer(buf), ct.POINTER(EVENT_TRACE_PROPERTIES)) diff --git a/threadpoolctl/tests/test_windows_thread_tracer.py b/threadpoolctl/tests/test_windows_thread_tracer.py index 384789c1..b4645069 100644 --- a/threadpoolctl/tests/test_windows_thread_tracer.py +++ b/threadpoolctl/tests/test_windows_thread_tracer.py @@ -1,5 +1,4 @@ import struct -import sys import pytest @@ -10,7 +9,6 @@ classify_kernel_thread_event, parse_kernel_thread_payload, ) -from threadpoolctl._thread_tracer import windows_tracer_available def _pack_thread_payload(process_id, thread_id): @@ -42,7 +40,3 @@ def test_classify_kernel_thread_event_ignores_other_process(): def test_parse_kernel_thread_payload_rejects_short_buffers(): assert parse_kernel_thread_payload(b"\x01\x02\x03", 3) is None - - -def test_windows_tracer_available_matches_platform(): - assert windows_tracer_available() == (sys.platform == "win32") diff --git a/threadpoolctl/tests/test_windows_thread_tracer_integration.py b/threadpoolctl/tests/test_windows_thread_tracer_integration.py index 1e8f6fe4..94477645 100644 --- a/threadpoolctl/tests/test_windows_thread_tracer_integration.py +++ b/threadpoolctl/tests/test_windows_thread_tracer_integration.py @@ -1,3 +1,4 @@ +import ctypes import os import subprocess import sys @@ -6,20 +7,15 @@ import pytest -from threadpoolctl._thread_tracer import ( - ThreadTracerError, - WindowsThreadSpawnTracer, - windows_etw_admin_available, - windows_tracer_available, -) +from threadpoolctl._thread_tracer import ThreadTracerError, WindowsThreadSpawnTracer pytestmark = [ pytest.mark.skipif( - not windows_tracer_available(), + sys.platform != "win32", reason="Windows ETW integration tests require Windows", ), pytest.mark.skipif( - not windows_etw_admin_available(), + sys.platform == "win32" and not ctypes.windll.shell32.IsUserAnAdmin(), reason="Windows ETW kernel tracing requires an elevated process", ), ] @@ -69,15 +65,13 @@ def _configure_blas_thread_env(): def _child_script(body): - return textwrap.dedent( - """ + return textwrap.dedent(""" import time time.sleep({attach_delay}) {body} time.sleep({flush_delay}) - """ - ).format( + """).format( attach_delay=TRACER_ATTACH_DELAY_SECONDS, body=textwrap.indent(textwrap.dedent(body).strip(), " "), flush_delay=TRACER_FLUSH_DELAY_SECONDS, @@ -88,9 +82,7 @@ def _run_traced_child(body, minimum_spawn_count): env = os.environ.copy() pythonpath = env.get("PYTHONPATH", "") env["PYTHONPATH"] = ( - str(REPO_ROOT) - if not pythonpath - else str(REPO_ROOT) + os.pathsep + pythonpath + str(REPO_ROOT) if not pythonpath else str(REPO_ROOT) + os.pathsep + pythonpath ) proc = subprocess.Popen( @@ -116,13 +108,12 @@ def _run_traced_child(body, minimum_spawn_count): stats = tracer.stop() assert return_code == 0, "traced child exited with code {0}".format(return_code) - assert stats.spawn_count >= minimum_spawn_count, ( - "expected at least {expected} thread spawn events, got {actual} ({stats})" - .format( - expected=minimum_spawn_count, - actual=stats.spawn_count, - stats=stats, - ) + assert ( + stats.spawn_count >= minimum_spawn_count + ), "expected at least {expected} thread spawn events, got {actual} ({stats})".format( + expected=minimum_spawn_count, + actual=stats.spawn_count, + stats=stats, ) return stats From 3a8b0edbb85d1c1938048aa113683f8bddbaa7d1 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 10 Jul 2026 09:55:00 +0000 Subject: [PATCH 05/17] Format integration tests with black 25.1.0 (CI pin) Co-authored-by: Olivier Grisel --- .../test_windows_thread_tracer_integration.py | 30 +++++-------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/threadpoolctl/tests/test_windows_thread_tracer_integration.py b/threadpoolctl/tests/test_windows_thread_tracer_integration.py index 94477645..b8af2444 100644 --- a/threadpoolctl/tests/test_windows_thread_tracer_integration.py +++ b/threadpoolctl/tests/test_windows_thread_tracer_integration.py @@ -7,6 +7,7 @@ import pytest +from threadpoolctl import threadpool_info from threadpoolctl._thread_tracer import ThreadTracerError, WindowsThreadSpawnTracer pytestmark = [ @@ -31,28 +32,11 @@ } -def _threadpool_info(): - try: - from threadpoolctl import threadpool_info - - return threadpool_info() - except ImportError: - from threadpoolctl import get_threadpool_limits - - return get_threadpool_limits() - - -def _module_num_threads(module): - if "num_threads" in module: - return module["num_threads"] - return module.get("n_thread") - - def _expected_pool_thread_count(user_api): thread_counts = [ - _module_num_threads(module) - for module in _threadpool_info() - if module.get("user_api") == user_api and _module_num_threads(module) + module["num_threads"] + for module in threadpool_info() + if module.get("user_api") == user_api and module.get("num_threads") ] if not thread_counts: pytest.skip("No {0} thread pool detected".format(user_api)) @@ -65,13 +49,15 @@ def _configure_blas_thread_env(): def _child_script(body): - return textwrap.dedent(""" + return textwrap.dedent( + """ import time time.sleep({attach_delay}) {body} time.sleep({flush_delay}) - """).format( + """ + ).format( attach_delay=TRACER_ATTACH_DELAY_SECONDS, body=textwrap.indent(textwrap.dedent(body).strip(), " "), flush_delay=TRACER_FLUSH_DELAY_SECONDS, From 6201324be665d8964fce30992516c298c4f7f235 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 10 Jul 2026 10:15:44 +0000 Subject: [PATCH 06/17] Fix package layout conflict with flit single-module structure Move the private _thread_tracer prototype and its tests out of the threadpoolctl/ directory, which conflicted with threadpoolctl.py during flit install. Update imports for the modern tests layout and OpenMP helper module names. Co-authored-by: Olivier Grisel --- .../__init__.py | 4 ++-- .../_parsing.py | 0 .../_types.py | 0 .../_windows_etw.py | 4 ++-- .../test_windows_thread_tracer.py | 2 +- .../test_windows_thread_tracer_integration.py | 21 +++++++++++-------- 6 files changed, 17 insertions(+), 14 deletions(-) rename {threadpoolctl/_thread_tracer => _thread_tracer}/__init__.py (51%) rename {threadpoolctl/_thread_tracer => _thread_tracer}/_parsing.py (100%) rename {threadpoolctl/_thread_tracer => _thread_tracer}/_types.py (100%) rename {threadpoolctl/_thread_tracer => _thread_tracer}/_windows_etw.py (98%) rename {threadpoolctl/tests => tests}/test_windows_thread_tracer.py (95%) rename {threadpoolctl/tests => tests}/test_windows_thread_tracer_integration.py (88%) diff --git a/threadpoolctl/_thread_tracer/__init__.py b/_thread_tracer/__init__.py similarity index 51% rename from threadpoolctl/_thread_tracer/__init__.py rename to _thread_tracer/__init__.py index d54a6e04..61fb6cda 100644 --- a/threadpoolctl/_thread_tracer/__init__.py +++ b/_thread_tracer/__init__.py @@ -1,7 +1,7 @@ """Private experimental thread spawn tracer (not part of the public API).""" -from threadpoolctl._thread_tracer._types import ThreadSpawnStats, ThreadTracerError -from threadpoolctl._thread_tracer._windows_etw import WindowsThreadSpawnTracer +from _thread_tracer._types import ThreadSpawnStats, ThreadTracerError +from _thread_tracer._windows_etw import WindowsThreadSpawnTracer __all__ = [ "ThreadSpawnStats", diff --git a/threadpoolctl/_thread_tracer/_parsing.py b/_thread_tracer/_parsing.py similarity index 100% rename from threadpoolctl/_thread_tracer/_parsing.py rename to _thread_tracer/_parsing.py diff --git a/threadpoolctl/_thread_tracer/_types.py b/_thread_tracer/_types.py similarity index 100% rename from threadpoolctl/_thread_tracer/_types.py rename to _thread_tracer/_types.py diff --git a/threadpoolctl/_thread_tracer/_windows_etw.py b/_thread_tracer/_windows_etw.py similarity index 98% rename from threadpoolctl/_thread_tracer/_windows_etw.py rename to _thread_tracer/_windows_etw.py index dd8cb18e..e3488b2d 100644 --- a/threadpoolctl/_thread_tracer/_windows_etw.py +++ b/_thread_tracer/_windows_etw.py @@ -14,13 +14,13 @@ import time import uuid -from threadpoolctl._thread_tracer._parsing import ( +from _thread_tracer._parsing import ( EVENT_TRACE_TYPE_DCSTART, EVENT_TRACE_TYPE_START, classify_kernel_thread_event, parse_kernel_thread_payload, ) -from threadpoolctl._thread_tracer._types import ThreadSpawnStats, ThreadTracerError +from _thread_tracer._types import ThreadSpawnStats, ThreadTracerError ERROR_SUCCESS = 0 ERROR_ALREADY_EXISTS = 183 diff --git a/threadpoolctl/tests/test_windows_thread_tracer.py b/tests/test_windows_thread_tracer.py similarity index 95% rename from threadpoolctl/tests/test_windows_thread_tracer.py rename to tests/test_windows_thread_tracer.py index b4645069..c5f6904c 100644 --- a/threadpoolctl/tests/test_windows_thread_tracer.py +++ b/tests/test_windows_thread_tracer.py @@ -2,7 +2,7 @@ import pytest -from threadpoolctl._thread_tracer._parsing import ( +from _thread_tracer._parsing import ( EVENT_TRACE_TYPE_DCSTART, EVENT_TRACE_TYPE_END, EVENT_TRACE_TYPE_START, diff --git a/threadpoolctl/tests/test_windows_thread_tracer_integration.py b/tests/test_windows_thread_tracer_integration.py similarity index 88% rename from threadpoolctl/tests/test_windows_thread_tracer_integration.py rename to tests/test_windows_thread_tracer_integration.py index b8af2444..4a39f375 100644 --- a/threadpoolctl/tests/test_windows_thread_tracer_integration.py +++ b/tests/test_windows_thread_tracer_integration.py @@ -8,7 +8,9 @@ import pytest from threadpoolctl import threadpool_info -from threadpoolctl._thread_tracer import ThreadTracerError, WindowsThreadSpawnTracer + +from _thread_tracer import ThreadTracerError, WindowsThreadSpawnTracer +from tests.utils import cython_extensions_compiled pytestmark = [ pytest.mark.skipif( @@ -21,7 +23,7 @@ ), ] -REPO_ROOT = Path(__file__).resolve().parents[2] +REPO_ROOT = Path(__file__).resolve().parents[1] TRACER_ATTACH_DELAY_SECONDS = 0.5 TRACER_FLUSH_DELAY_SECONDS = 0.3 SUBPROCESS_TIMEOUT_SECONDS = 60 @@ -123,23 +125,24 @@ def work(): _run_traced_child(body, minimum_spawn_count=3) +@pytest.mark.skipif( + not cython_extensions_compiled, + reason="OpenMP test helper is not built", +) def test_tracer_counts_openmp_thread_spawns(): - try: - from threadpoolctl.tests._openmp_test_helper import check_openmp_n_threads - except ImportError: - pytest.skip("OpenMP test helper is not built") + from tests._openmp_test_helper.openmp_helpers_inner import check_openmp_num_threads os.environ["OMP_NUM_THREADS"] = "4" - check_openmp_n_threads(10) + check_openmp_num_threads(10) expected_spawn_count = _expected_pool_thread_count("openmp") body = """ import os os.environ["OMP_NUM_THREADS"] = "4" - from threadpoolctl.tests._openmp_test_helper import check_openmp_n_threads + from tests._openmp_test_helper.openmp_helpers_inner import check_openmp_num_threads - used = check_openmp_n_threads(1000) + used = check_openmp_num_threads(1000) assert used >= 1 """ _run_traced_child(body, minimum_spawn_count=expected_spawn_count) From c31174a6aea4ee9ae04f4ceea59aca947cfad21d Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 10 Jul 2026 10:29:15 +0000 Subject: [PATCH 07/17] Fix Windows ETW consumer segfault in integration tests Use correctly-sized EVENT_TRACE_LOGFILE nested structures so OpenTraceW does not corrupt adjacent memory. Route ETW callbacks through a module- level function keyed by UserContext, and pass ProcessTrace a TRACEHANDLE array with stable logger-name references. Co-authored-by: Olivier Grisel --- _thread_tracer/_windows_etw.py | 149 +++++++++++++++++++++++++++++---- 1 file changed, 133 insertions(+), 16 deletions(-) diff --git a/_thread_tracer/_windows_etw.py b/_thread_tracer/_windows_etw.py index e3488b2d..242ae3a7 100644 --- a/_thread_tracer/_windows_etw.py +++ b/_thread_tracer/_windows_etw.py @@ -12,7 +12,6 @@ import sys import threading import time -import uuid from _thread_tracer._parsing import ( EVENT_TRACE_TYPE_DCSTART, @@ -124,6 +123,95 @@ class EVENT_RECORD(ct.Structure): ] +class EVENT_TRACE_HEADER_CLASS(ct.Structure): + _fields_ = [ + ("Type", ct.c_ubyte), + ("Level", ct.c_ubyte), + ("Version", ct.c_uint16), + ] + + +class EVENT_TRACE_HEADER(ct.Structure): + _fields_ = [ + ("Size", ct.c_ushort), + ("HeaderType", ct.c_ubyte), + ("MarkerFlags", ct.c_ubyte), + ("Class", EVENT_TRACE_HEADER_CLASS), + ("ThreadId", ct.c_ulong), + ("ProcessId", ct.c_ulong), + ("TimeStamp", wt.LARGE_INTEGER), + ("Guid", GUID), + ("ClientContext", ct.c_ulong), + ("Flags", ct.c_ulong), + ] + + +class EVENT_TRACE(ct.Structure): + _fields_ = [ + ("Header", EVENT_TRACE_HEADER), + ("InstanceId", ct.c_ulong), + ("ParentInstanceId", ct.c_ulong), + ("ParentGuid", GUID), + ("MofData", ct.c_void_p), + ("MofLength", ct.c_ulong), + ("ClientContext", ct.c_ulong), + ] + + +class SYSTEMTIME(ct.Structure): + _fields_ = [ + ("wYear", wt.WORD), + ("wMonth", wt.WORD), + ("wDayOfWeek", wt.WORD), + ("wDay", wt.WORD), + ("wHour", wt.WORD), + ("wMinute", wt.WORD), + ("wSecond", wt.WORD), + ("wMilliseconds", wt.WORD), + ] + + +class TIME_ZONE_INFORMATION(ct.Structure): + _fields_ = [ + ("Bias", ct.c_long), + ("StandardName", ct.c_wchar * 32), + ("StandardDate", SYSTEMTIME), + ("StandardBias", ct.c_long), + ("DaylightName", ct.c_wchar * 32), + ("DaylightDate", SYSTEMTIME), + ("DaylightBias", ct.c_long), + ] + + +class TRACE_LOGFILE_HEADER(ct.Structure): + _fields_ = [ + ("BufferSize", ct.c_ulong), + ("MajorVersion", ct.c_byte), + ("MinorVersion", ct.c_byte), + ("SubVersion", ct.c_byte), + ("SubMinorVersion", ct.c_byte), + ("ProviderVersion", ct.c_ulong), + ("NumberOfProcessors", ct.c_ulong), + ("EndTime", wt.LARGE_INTEGER), + ("TimerResolution", ct.c_ulong), + ("MaximumFileSize", ct.c_ulong), + ("LogFileMode", ct.c_ulong), + ("BuffersWritten", ct.c_ulong), + ("StartBuffers", ct.c_ulong), + ("PointerSize", ct.c_ulong), + ("EventsLost", ct.c_ulong), + ("CpuSpeedInMHz", ct.c_ulong), + ("LoggerName", ct.c_wchar_p), + ("LogFileName", ct.c_wchar_p), + ("TimeZone", TIME_ZONE_INFORMATION), + ("BootTime", wt.LARGE_INTEGER), + ("PerfFreq", wt.LARGE_INTEGER), + ("StartTime", wt.LARGE_INTEGER), + ("ReservedFlags", ct.c_ulong), + ("BuffersLost", ct.c_ulong), + ] + + class EVENT_TRACE_PROPERTIES(ct.Structure): _fields_ = [ ("Wnode", WNODE_HEADER), @@ -162,8 +250,8 @@ class EVENT_TRACE_LOGFILE(ct.Structure): ("CurrentTime", ct.c_longlong), ("BuffersRead", ct.c_ulong), ("ProcessTraceMode", ct.c_ulong), - ("CurrentEvent", ct.c_void_p), - ("LogfileHeader", ct.c_void_p), + ("CurrentEvent", EVENT_TRACE), + ("LogfileHeader", TRACE_LOGFILE_HEADER), ("BufferCallback", ct.c_void_p), ("BufferSize", ct.c_ulong), ("Filled", ct.c_ulong), @@ -173,6 +261,22 @@ class EVENT_TRACE_LOGFILE(ct.Structure): ("Context", ct.c_void_p), ] +_TRACERS_BY_CONTEXT = {} +_TRACERS_LOCK = threading.Lock() + + +@EVENT_RECORD_CALLBACK +def _event_record_callback(record_pointer): + record = record_pointer.contents + context = record.UserContext + if not context: + return + with _TRACERS_LOCK: + tracer = _TRACERS_BY_CONTEXT.get(context) + if tracer is None: + return + tracer._handle_event_record(record) + _advapi32 = None StartTraceW = None @@ -295,7 +399,8 @@ def __init__(self, pid, include_existing_threads=False): self._trace_properties_buf = None self._trace_properties = None self._trace_logfile = None - self._callback = None + self._logger_name = None + self._trace_context = None self._consumer_thread = None self._stop_event = threading.Event() self._started = False @@ -331,6 +436,7 @@ def start(self): raise ThreadTracerError(status, ct.FormatError(status)) self._session_started = True + time.sleep(0.1) self._install_consumer() self._started = True @@ -342,9 +448,14 @@ def stop(self): time.sleep(0.2) if self._trace_handle.value: CloseTrace(self._trace_handle) + self._trace_handle = TRACEHANDLE(0) if self._consumer_thread is not None: self._consumer_thread.join(timeout=10) self._consumer_thread = None + if self._trace_context is not None: + with _TRACERS_LOCK: + _TRACERS_BY_CONTEXT.pop(self._trace_context.value, None) + self._trace_context = None if self._session_started: status = ControlTraceW( @@ -362,36 +473,42 @@ def stop(self): def _install_consumer(self): self._trace_logfile = EVENT_TRACE_LOGFILE() - self._trace_logfile.LoggerName = KERNEL_LOGGER_NAME + self._logger_name = KERNEL_LOGGER_NAME + self._trace_logfile.LoggerName = self._logger_name self._trace_logfile.ProcessTraceMode = ( PROCESS_TRACE_MODE_REAL_TIME | PROCESS_TRACE_MODE_EVENT_RECORD ) - self._trace_logfile.Context = ct.c_void_p(id(self)) - self._callback = EVENT_RECORD_CALLBACK(self._on_event_record) - self._trace_logfile.EventRecordCallback = self._callback - - self._trace_handle = OpenTraceW(ct.byref(self._trace_logfile)) - if self._trace_handle == INVALID_PROCESSTRACE_HANDLE: + self._trace_context = ct.c_void_p(id(self)) + self._trace_logfile.Context = self._trace_context + with _TRACERS_LOCK: + _TRACERS_BY_CONTEXT[self._trace_context.value] = self + self._trace_logfile.EventRecordCallback = _event_record_callback + + trace_handle = OpenTraceW(ct.byref(self._trace_logfile)) + if trace_handle in (0, INVALID_PROCESSTRACE_HANDLE): + with _TRACERS_LOCK: + _TRACERS_BY_CONTEXT.pop(self._trace_context.value, None) + self._trace_context = None raise ThreadTracerError(ct.get_last_error(), "OpenTraceW failed") + self._trace_handle = TRACEHANDLE(trace_handle) self._consumer_thread = threading.Thread( target=self._consume_trace, - name="threadpoolctl-etw-consumer-{0}".format(uuid.uuid4().hex[:8]), + name="threadpoolctl-etw-consumer", ) self._consumer_thread.daemon = True self._consumer_thread.start() def _consume_trace(self): - trace_handle = self._trace_handle + handles = (TRACEHANDLE * 1)(self._trace_handle) while not self._stop_event.is_set(): - status = ProcessTrace(ct.byref(trace_handle), 1, None, None) + status = ProcessTrace(handles, 1, None, None) if status != ERROR_SUCCESS: break if self._stop_event.is_set(): break - def _on_event_record(self, record_pointer): - record = record_pointer.contents + def _handle_event_record(self, record): opcode = record.EventHeader.EventDescriptor.Opcode if opcode not in (EVENT_TRACE_TYPE_START, EVENT_TRACE_TYPE_DCSTART): return From a30a5c1fb822f07445a5d24609daf910bbb17b3c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 10 Jul 2026 10:42:15 +0000 Subject: [PATCH 08/17] Stabilize Windows ETW integration tests on CI Drop the threading.Barrier child synchronization that could break under load, derive native pool spawn expectations as team_size minus one, and capture child stderr to simplify future CI debugging. Co-authored-by: Olivier Grisel --- .../test_windows_thread_tracer_integration.py | 32 ++++++++++++------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/tests/test_windows_thread_tracer_integration.py b/tests/test_windows_thread_tracer_integration.py index 4a39f375..c68fcd97 100644 --- a/tests/test_windows_thread_tracer_integration.py +++ b/tests/test_windows_thread_tracer_integration.py @@ -42,7 +42,9 @@ def _expected_pool_thread_count(user_api): ] if not thread_counts: pytest.skip("No {0} thread pool detected".format(user_api)) - return max(thread_counts) + # Native pools report team size including the calling thread, while ETW + # Start events only observe newly created OS threads. + return max(1, max(thread_counts) - 1) def _configure_blas_thread_env(): @@ -77,6 +79,9 @@ def _run_traced_child(body, minimum_spawn_count): [sys.executable, "-c", _child_script(body)], cwd=str(REPO_ROOT), env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, ) tracer = WindowsThreadSpawnTracer(proc.pid) try: @@ -87,15 +92,23 @@ def _run_traced_child(body, minimum_spawn_count): pytest.fail("failed to start Windows ETW tracer: {0}".format(exc)) try: - return_code = proc.wait(timeout=SUBPROCESS_TIMEOUT_SECONDS) + stdout, stderr = proc.communicate(timeout=SUBPROCESS_TIMEOUT_SECONDS) + return_code = proc.returncode except subprocess.TimeoutExpired: proc.kill() - proc.wait(timeout=SUBPROCESS_TIMEOUT_SECONDS) + stdout, stderr = proc.communicate(timeout=SUBPROCESS_TIMEOUT_SECONDS) tracer.stop() pytest.fail("timed out waiting for traced child process") stats = tracer.stop() - assert return_code == 0, "traced child exited with code {0}".format(return_code) + if return_code != 0: + pytest.fail( + "traced child exited with code {0}\nstdout:\n{1}\nstderr:\n{2}".format( + return_code, + stdout, + stderr, + ) + ) assert ( stats.spawn_count >= minimum_spawn_count ), "expected at least {expected} thread spawn events, got {actual} ({stats})".format( @@ -110,17 +123,14 @@ def test_tracer_counts_python_thread_spawns(): body = """ import threading - num_threads = 3 - started = threading.Barrier(num_threads) - def work(): - started.wait(timeout=5) + pass - threads = [threading.Thread(target=work) for _ in range(num_threads)] + threads = [threading.Thread(target=work) for _ in range(3)] for thread in threads: thread.start() for thread in threads: - thread.join(timeout=5) + thread.join() """ _run_traced_child(body, minimum_spawn_count=3) @@ -142,7 +152,7 @@ def test_tracer_counts_openmp_thread_spawns(): os.environ["OMP_NUM_THREADS"] = "4" from tests._openmp_test_helper.openmp_helpers_inner import check_openmp_num_threads - used = check_openmp_num_threads(1000) + used = check_openmp_num_threads(100) assert used >= 1 """ _run_traced_child(body, minimum_spawn_count=expected_spawn_count) From c5fb7f4537e3fb3c2bcc7d70857a7b746fd5781d Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 10 Jul 2026 10:54:50 +0000 Subject: [PATCH 09/17] Fix indentation of traced subprocess scripts Co-authored-by: Olivier Grisel --- tests/test_windows_thread_tracer_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_windows_thread_tracer_integration.py b/tests/test_windows_thread_tracer_integration.py index c68fcd97..034253f6 100644 --- a/tests/test_windows_thread_tracer_integration.py +++ b/tests/test_windows_thread_tracer_integration.py @@ -63,7 +63,7 @@ def _child_script(body): """ ).format( attach_delay=TRACER_ATTACH_DELAY_SECONDS, - body=textwrap.indent(textwrap.dedent(body).strip(), " "), + body=textwrap.dedent(body).strip(), flush_delay=TRACER_FLUSH_DELAY_SECONDS, ) From 1aa8ae7c8684e1573dacdee96b4acafbdbb9c723 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 10 Jul 2026 13:20:47 +0000 Subject: [PATCH 10/17] Temporarily disable flexiblas CI matrix jobs for faster iteration Co-authored-by: Olivier Grisel --- .github/workflows/test.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 39fc19e2..84559de7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -84,12 +84,12 @@ jobs: PACKAGER: "conda-forge" BLAS: "openblas" INSTALL_LIBOMP: "conda-forge" - # MacOS env with FlexiBLAS - - name: pylatest_flexiblas - os: macos-latest - PYTHON_VERSION: "*" - INSTALL_BLAS: "flexiblas" - PLATFORM_SPECIFIC_PACKAGES: "llvm-openmp" + # MacOS env with FlexiBLAS — temporarily disabled on this PR for faster CI iteration. + # - name: pylatest_flexiblas + # os: macos-latest + # PYTHON_VERSION: "*" + # INSTALL_BLAS: "flexiblas" + # PLATFORM_SPECIFIC_PACKAGES: "llvm-openmp" # Linux environments to test that packages that comes with Ubuntu 22.04 # are correctly handled. @@ -197,14 +197,14 @@ jobs: BLIS_CC: "gcc-12" BLIS_ENABLE_THREADING: "no" - # Linux env with FlexiBLAS - - name: pylatest_flexiblas - os: ubuntu-latest - PYTHON_VERSION: "*" - INSTALL_BLAS: "flexiblas" - PLATFORM_SPECIFIC_PACKAGES: "mkl" - CC_OUTER_LOOP: "gcc" - CC_INNER_LOOP: "gcc" + # Linux env with FlexiBLAS — temporarily disabled on this PR for faster CI iteration. + # - name: pylatest_flexiblas + # os: ubuntu-latest + # PYTHON_VERSION: "*" + # INSTALL_BLAS: "flexiblas" + # PLATFORM_SPECIFIC_PACKAGES: "mkl" + # CC_OUTER_LOOP: "gcc" + # CC_INNER_LOOP: "gcc" env: ${{ matrix }} From 8d91f9a97beb05044128806c18b4fedaa782f021 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 10 Jul 2026 13:25:28 +0000 Subject: [PATCH 11/17] Return stats from traced child helper; assert exact spawn counts in tests _run_traced_child now only runs the subprocess and tracer, returning ThreadSpawnStats. Each integration test asserts the exact expected spawn count explicitly instead of checking a minimum threshold. Co-authored-by: Olivier Grisel --- .../test_windows_thread_tracer_integration.py | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/tests/test_windows_thread_tracer_integration.py b/tests/test_windows_thread_tracer_integration.py index 034253f6..cc23d0d9 100644 --- a/tests/test_windows_thread_tracer_integration.py +++ b/tests/test_windows_thread_tracer_integration.py @@ -34,7 +34,7 @@ } -def _expected_pool_thread_count(user_api): +def _expected_pool_spawn_count(user_api): thread_counts = [ module["num_threads"] for module in threadpool_info() @@ -68,7 +68,7 @@ def _child_script(body): ) -def _run_traced_child(body, minimum_spawn_count): +def _run_traced_child(body): env = os.environ.copy() pythonpath = env.get("PYTHONPATH", "") env["PYTHONPATH"] = ( @@ -109,13 +109,6 @@ def _run_traced_child(body, minimum_spawn_count): stderr, ) ) - assert ( - stats.spawn_count >= minimum_spawn_count - ), "expected at least {expected} thread spawn events, got {actual} ({stats})".format( - expected=minimum_spawn_count, - actual=stats.spawn_count, - stats=stats, - ) return stats @@ -132,7 +125,9 @@ def work(): for thread in threads: thread.join() """ - _run_traced_child(body, minimum_spawn_count=3) + stats = _run_traced_child(body) + assert stats.spawn_count == 3 + assert stats.existing_thread_count == 0 @pytest.mark.skipif( @@ -144,7 +139,7 @@ def test_tracer_counts_openmp_thread_spawns(): os.environ["OMP_NUM_THREADS"] = "4" check_openmp_num_threads(10) - expected_spawn_count = _expected_pool_thread_count("openmp") + expected_spawn_count = _expected_pool_spawn_count("openmp") body = """ import os @@ -155,7 +150,9 @@ def test_tracer_counts_openmp_thread_spawns(): used = check_openmp_num_threads(100) assert used >= 1 """ - _run_traced_child(body, minimum_spawn_count=expected_spawn_count) + stats = _run_traced_child(body) + assert stats.spawn_count == expected_spawn_count + assert stats.existing_thread_count == 0 def test_tracer_counts_blas_thread_spawns(): @@ -167,7 +164,7 @@ def test_tracer_counts_blas_thread_spawns(): rng = np.random.RandomState(0) warmup = rng.rand(100, 100) np.dot(warmup, warmup) - expected_spawn_count = _expected_pool_thread_count("blas") + expected_spawn_count = _expected_pool_spawn_count("blas") body = """ import os @@ -182,4 +179,6 @@ def test_tracer_counts_blas_thread_spawns(): a = rng.rand(2000, 2000) np.dot(a, a) """ - _run_traced_child(body, minimum_spawn_count=expected_spawn_count) + stats = _run_traced_child(body) + assert stats.spawn_count == expected_spawn_count + assert stats.existing_thread_count == 0 From c67ca9e8ebb3f876fe976665df07534ab09d9d03 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 10 Jul 2026 13:40:37 +0000 Subject: [PATCH 12/17] Account for child startup threads in strict spawn assertions CI shows Windows traced children emit a stable startup overhead on top of work-specific spawns. Measure that baseline once per module and assert total counts as startup + expected work spawns instead of bare totals. Co-authored-by: Olivier Grisel --- .../test_windows_thread_tracer_integration.py | 56 ++++++++++++++++--- 1 file changed, 48 insertions(+), 8 deletions(-) diff --git a/tests/test_windows_thread_tracer_integration.py b/tests/test_windows_thread_tracer_integration.py index cc23d0d9..3ee9feff 100644 --- a/tests/test_windows_thread_tracer_integration.py +++ b/tests/test_windows_thread_tracer_integration.py @@ -33,6 +33,19 @@ "MKL_NUM_THREADS": "4", } +# Fresh ``python -c`` children routinely spawn a few interpreter threads +# during the tracer attach window. CI shows a stable startup overhead of three +# spawns across Windows matrix jobs; work-specific spawns sit on top of that. +PYTHON_THREAD_WORK_SPAWN_COUNT = 3 +BASELINE_CHILD_BODY = "pass" + + +@pytest.fixture(scope="module") +def child_startup_spawn_count(): + stats = _run_traced_child(BASELINE_CHILD_BODY) + assert stats.existing_thread_count == 0 + return stats.spawn_count + def _expected_pool_spawn_count(user_api): thread_counts = [ @@ -47,6 +60,21 @@ def _expected_pool_spawn_count(user_api): return max(1, max(thread_counts) - 1) +def _assert_total_spawn_count(stats, child_startup_spawn_count, work_spawn_count): + expected_total = child_startup_spawn_count + work_spawn_count + assert stats.spawn_count == expected_total, ( + "expected {expected_total} spawns " + "({startup} child startup + {work} work), " + "got {actual} ({stats})".format( + expected_total=expected_total, + startup=child_startup_spawn_count, + work=work_spawn_count, + actual=stats.spawn_count, + stats=stats, + ) + ) + + def _configure_blas_thread_env(): for name, value in BLAS_THREAD_ENV_VARS.items(): os.environ[name] = value @@ -112,7 +140,7 @@ def _run_traced_child(body): return stats -def test_tracer_counts_python_thread_spawns(): +def test_tracer_counts_python_thread_spawns(child_startup_spawn_count): body = """ import threading @@ -126,7 +154,11 @@ def work(): thread.join() """ stats = _run_traced_child(body) - assert stats.spawn_count == 3 + _assert_total_spawn_count( + stats, + child_startup_spawn_count, + PYTHON_THREAD_WORK_SPAWN_COUNT, + ) assert stats.existing_thread_count == 0 @@ -134,12 +166,12 @@ def work(): not cython_extensions_compiled, reason="OpenMP test helper is not built", ) -def test_tracer_counts_openmp_thread_spawns(): +def test_tracer_counts_openmp_thread_spawns(child_startup_spawn_count): from tests._openmp_test_helper.openmp_helpers_inner import check_openmp_num_threads os.environ["OMP_NUM_THREADS"] = "4" check_openmp_num_threads(10) - expected_spawn_count = _expected_pool_spawn_count("openmp") + expected_work_spawn_count = _expected_pool_spawn_count("openmp") body = """ import os @@ -151,11 +183,15 @@ def test_tracer_counts_openmp_thread_spawns(): assert used >= 1 """ stats = _run_traced_child(body) - assert stats.spawn_count == expected_spawn_count + _assert_total_spawn_count( + stats, + child_startup_spawn_count, + expected_work_spawn_count, + ) assert stats.existing_thread_count == 0 -def test_tracer_counts_blas_thread_spawns(): +def test_tracer_counts_blas_thread_spawns(child_startup_spawn_count): pytest.importorskip("numpy") _configure_blas_thread_env() @@ -164,7 +200,7 @@ def test_tracer_counts_blas_thread_spawns(): rng = np.random.RandomState(0) warmup = rng.rand(100, 100) np.dot(warmup, warmup) - expected_spawn_count = _expected_pool_spawn_count("blas") + expected_work_spawn_count = _expected_pool_spawn_count("blas") body = """ import os @@ -180,5 +216,9 @@ def test_tracer_counts_blas_thread_spawns(): np.dot(a, a) """ stats = _run_traced_child(body) - assert stats.spawn_count == expected_spawn_count + _assert_total_spawn_count( + stats, + child_startup_spawn_count, + expected_work_spawn_count, + ) assert stats.existing_thread_count == 0 From 077951eba7fda0f3b144bfc5f4477d40f33ff391 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 10 Jul 2026 13:53:42 +0000 Subject: [PATCH 13/17] Derive Python thread expectations from measured noop baseline Remove the hardcoded PYTHON_THREAD_WORK_SPAWN_COUNT constant. Each test asserts spawn counts relative to a traced noop child subprocess, with extra spawns defined by the test itself (thread count, OpenMP, or BLAS). Co-authored-by: Olivier Grisel --- .../test_windows_thread_tracer_integration.py | 58 ++++++++++--------- 1 file changed, 30 insertions(+), 28 deletions(-) diff --git a/tests/test_windows_thread_tracer_integration.py b/tests/test_windows_thread_tracer_integration.py index 3ee9feff..0f1661cb 100644 --- a/tests/test_windows_thread_tracer_integration.py +++ b/tests/test_windows_thread_tracer_integration.py @@ -33,16 +33,13 @@ "MKL_NUM_THREADS": "4", } -# Fresh ``python -c`` children routinely spawn a few interpreter threads -# during the tracer attach window. CI shows a stable startup overhead of three -# spawns across Windows matrix jobs; work-specific spawns sit on top of that. -PYTHON_THREAD_WORK_SPAWN_COUNT = 3 -BASELINE_CHILD_BODY = "pass" +NOOP_CHILD_BODY = "pass" @pytest.fixture(scope="module") -def child_startup_spawn_count(): - stats = _run_traced_child(BASELINE_CHILD_BODY) +def noop_child_spawn_count(): + """Spawn count for a traced child that only sleeps (no extra work).""" + stats = _run_traced_child(NOOP_CHILD_BODY) assert stats.existing_thread_count == 0 return stats.spawn_count @@ -60,15 +57,17 @@ def _expected_pool_spawn_count(user_api): return max(1, max(thread_counts) - 1) -def _assert_total_spawn_count(stats, child_startup_spawn_count, work_spawn_count): - expected_total = child_startup_spawn_count + work_spawn_count +def _assert_spawn_count_relative_to_baseline( + stats, baseline_spawn_count, extra_spawn_count +): + expected_total = baseline_spawn_count + extra_spawn_count assert stats.spawn_count == expected_total, ( "expected {expected_total} spawns " - "({startup} child startup + {work} work), " + "({baseline} noop baseline + {extra} extra), " "got {actual} ({stats})".format( expected_total=expected_total, - startup=child_startup_spawn_count, - work=work_spawn_count, + baseline=baseline_spawn_count, + extra=extra_spawn_count, actual=stats.spawn_count, stats=stats, ) @@ -140,24 +139,27 @@ def _run_traced_child(body): return stats -def test_tracer_counts_python_thread_spawns(child_startup_spawn_count): +def test_tracer_counts_python_thread_spawns(noop_child_spawn_count): + num_threads = 3 body = """ import threading def work(): pass - threads = [threading.Thread(target=work) for _ in range(3)] + threads = [threading.Thread(target=work) for _ in range({num_threads})] for thread in threads: thread.start() for thread in threads: thread.join() - """ + """.format( + num_threads=num_threads + ) stats = _run_traced_child(body) - _assert_total_spawn_count( + _assert_spawn_count_relative_to_baseline( stats, - child_startup_spawn_count, - PYTHON_THREAD_WORK_SPAWN_COUNT, + noop_child_spawn_count, + num_threads, ) assert stats.existing_thread_count == 0 @@ -166,12 +168,12 @@ def work(): not cython_extensions_compiled, reason="OpenMP test helper is not built", ) -def test_tracer_counts_openmp_thread_spawns(child_startup_spawn_count): +def test_tracer_counts_openmp_thread_spawns(noop_child_spawn_count): from tests._openmp_test_helper.openmp_helpers_inner import check_openmp_num_threads os.environ["OMP_NUM_THREADS"] = "4" check_openmp_num_threads(10) - expected_work_spawn_count = _expected_pool_spawn_count("openmp") + expected_extra_spawn_count = _expected_pool_spawn_count("openmp") body = """ import os @@ -183,15 +185,15 @@ def test_tracer_counts_openmp_thread_spawns(child_startup_spawn_count): assert used >= 1 """ stats = _run_traced_child(body) - _assert_total_spawn_count( + _assert_spawn_count_relative_to_baseline( stats, - child_startup_spawn_count, - expected_work_spawn_count, + noop_child_spawn_count, + expected_extra_spawn_count, ) assert stats.existing_thread_count == 0 -def test_tracer_counts_blas_thread_spawns(child_startup_spawn_count): +def test_tracer_counts_blas_thread_spawns(noop_child_spawn_count): pytest.importorskip("numpy") _configure_blas_thread_env() @@ -200,7 +202,7 @@ def test_tracer_counts_blas_thread_spawns(child_startup_spawn_count): rng = np.random.RandomState(0) warmup = rng.rand(100, 100) np.dot(warmup, warmup) - expected_work_spawn_count = _expected_pool_spawn_count("blas") + expected_extra_spawn_count = _expected_pool_spawn_count("blas") body = """ import os @@ -216,9 +218,9 @@ def test_tracer_counts_blas_thread_spawns(child_startup_spawn_count): np.dot(a, a) """ stats = _run_traced_child(body) - _assert_total_spawn_count( + _assert_spawn_count_relative_to_baseline( stats, - child_startup_spawn_count, - expected_work_spawn_count, + noop_child_spawn_count, + expected_extra_spawn_count, ) assert stats.existing_thread_count == 0 From bb262fe7cdad2af32d6a30f47840e9dfbacda5f9 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 10 Jul 2026 14:15:08 +0000 Subject: [PATCH 14/17] Add integration test for existing-thread DCStart rundown events Delay tracer attach until worker threads are already alive and enable include_existing_threads so ETW rundown is observable. Measure a noop baseline with the same attach timing and assert extra existing counts relative to that baseline. Co-authored-by: Olivier Grisel --- .../test_windows_thread_tracer_integration.py | 84 +++++++++++++++++-- 1 file changed, 78 insertions(+), 6 deletions(-) diff --git a/tests/test_windows_thread_tracer_integration.py b/tests/test_windows_thread_tracer_integration.py index 0f1661cb..5cb39388 100644 --- a/tests/test_windows_thread_tracer_integration.py +++ b/tests/test_windows_thread_tracer_integration.py @@ -3,6 +3,7 @@ import subprocess import sys import textwrap +import time from pathlib import Path import pytest @@ -26,6 +27,9 @@ REPO_ROOT = Path(__file__).resolve().parents[1] TRACER_ATTACH_DELAY_SECONDS = 0.5 TRACER_FLUSH_DELAY_SECONDS = 0.3 +# Attach after the child body has created and blocked its worker threads so ETW +# rundown can observe them as pre-existing (DCStart) rather than new spawns. +EXISTING_THREAD_TRACER_DELAY_SECONDS = TRACER_ATTACH_DELAY_SECONDS + 0.5 SUBPROCESS_TIMEOUT_SECONDS = 60 BLAS_THREAD_ENV_VARS = { "OMP_NUM_THREADS": "1", @@ -40,10 +44,20 @@ def noop_child_spawn_count(): """Spawn count for a traced child that only sleeps (no extra work).""" stats = _run_traced_child(NOOP_CHILD_BODY) - assert stats.existing_thread_count == 0 return stats.spawn_count +@pytest.fixture(scope="module") +def noop_child_existing_thread_count(): + """Existing-thread rundown count for a noop child traced after a delay.""" + stats = _run_traced_child( + NOOP_CHILD_BODY, + include_existing_threads=True, + tracer_start_delay=EXISTING_THREAD_TRACER_DELAY_SECONDS, + ) + return stats.existing_thread_count + + def _expected_pool_spawn_count(user_api): thread_counts = [ module["num_threads"] @@ -74,6 +88,23 @@ def _assert_spawn_count_relative_to_baseline( ) +def _assert_existing_count_relative_to_baseline( + stats, baseline_existing_count, extra_existing_count +): + expected_total = baseline_existing_count + extra_existing_count + assert stats.existing_thread_count == expected_total, ( + "expected {expected_total} existing threads " + "({baseline} noop baseline + {extra} extra), " + "got {actual} ({stats})".format( + expected_total=expected_total, + baseline=baseline_existing_count, + extra=extra_existing_count, + actual=stats.existing_thread_count, + stats=stats, + ) + ) + + def _configure_blas_thread_env(): for name, value in BLAS_THREAD_ENV_VARS.items(): os.environ[name] = value @@ -95,7 +126,11 @@ def _child_script(body): ) -def _run_traced_child(body): +def _run_traced_child( + body, + include_existing_threads=False, + tracer_start_delay=0, +): env = os.environ.copy() pythonpath = env.get("PYTHONPATH", "") env["PYTHONPATH"] = ( @@ -110,7 +145,12 @@ def _run_traced_child(body): stderr=subprocess.PIPE, text=True, ) - tracer = WindowsThreadSpawnTracer(proc.pid) + if tracer_start_delay: + time.sleep(tracer_start_delay) + tracer = WindowsThreadSpawnTracer( + proc.pid, + include_existing_threads=include_existing_threads, + ) try: tracer.start() except ThreadTracerError as exc: @@ -161,7 +201,41 @@ def work(): noop_child_spawn_count, num_threads, ) - assert stats.existing_thread_count == 0 + + +def test_tracer_counts_existing_python_threads( + noop_child_existing_thread_count, +): + num_threads = 3 + body = """ + import threading + import time + + hold = threading.Event() + + def work(): + hold.wait(timeout=10) + + threads = [threading.Thread(target=work) for _ in range({num_threads})] + for thread in threads: + thread.start() + time.sleep(2) + hold.set() + for thread in threads: + thread.join() + """.format( + num_threads=num_threads + ) + stats = _run_traced_child( + body, + include_existing_threads=True, + tracer_start_delay=EXISTING_THREAD_TRACER_DELAY_SECONDS, + ) + _assert_existing_count_relative_to_baseline( + stats, + noop_child_existing_thread_count, + num_threads, + ) @pytest.mark.skipif( @@ -190,7 +264,6 @@ def test_tracer_counts_openmp_thread_spawns(noop_child_spawn_count): noop_child_spawn_count, expected_extra_spawn_count, ) - assert stats.existing_thread_count == 0 def test_tracer_counts_blas_thread_spawns(noop_child_spawn_count): @@ -223,4 +296,3 @@ def test_tracer_counts_blas_thread_spawns(noop_child_spawn_count): noop_child_spawn_count, expected_extra_spawn_count, ) - assert stats.existing_thread_count == 0 From b3418c10f97da2c56e922ca7b05cf41f52f83dba Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 10 Jul 2026 14:21:40 +0000 Subject: [PATCH 15/17] Keep noop child alive for existing-thread baseline measurement The delayed tracer attach must find the child still running; a bare pass body finishes before attach and yielded a zero existing-thread baseline. Co-authored-by: Olivier Grisel --- tests/test_windows_thread_tracer_integration.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_windows_thread_tracer_integration.py b/tests/test_windows_thread_tracer_integration.py index 5cb39388..f925d0fd 100644 --- a/tests/test_windows_thread_tracer_integration.py +++ b/tests/test_windows_thread_tracer_integration.py @@ -38,6 +38,12 @@ } NOOP_CHILD_BODY = "pass" +# Keep the child alive long enough for a delayed tracer attach when measuring +# DCStart rundown events (see EXISTING_THREAD_TRACER_DELAY_SECONDS). +NOOP_EXISTING_CHILD_BODY = """ +import time +time.sleep(2) +""" @pytest.fixture(scope="module") @@ -51,7 +57,7 @@ def noop_child_spawn_count(): def noop_child_existing_thread_count(): """Existing-thread rundown count for a noop child traced after a delay.""" stats = _run_traced_child( - NOOP_CHILD_BODY, + NOOP_EXISTING_CHILD_BODY, include_existing_threads=True, tracer_start_delay=EXISTING_THREAD_TRACER_DELAY_SECONDS, ) From 9d952b02053bb8d01f8c7fee7dc3be4723af603b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 10 Jul 2026 14:50:01 +0000 Subject: [PATCH 16/17] Address review feedback and re-enable flexiblas CI jobs Use WINFUNCTYPE unconditionally in the Windows-only ETW module instead of a cross-platform callback fallback. Restore the macOS and Linux flexiblas matrix entries in the CI workflow. Co-authored-by: Olivier Grisel --- .github/workflows/test.yml | 28 ++++++++++++++-------------- _thread_tracer/_windows_etw.py | 5 +---- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 84559de7..39fc19e2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -84,12 +84,12 @@ jobs: PACKAGER: "conda-forge" BLAS: "openblas" INSTALL_LIBOMP: "conda-forge" - # MacOS env with FlexiBLAS — temporarily disabled on this PR for faster CI iteration. - # - name: pylatest_flexiblas - # os: macos-latest - # PYTHON_VERSION: "*" - # INSTALL_BLAS: "flexiblas" - # PLATFORM_SPECIFIC_PACKAGES: "llvm-openmp" + # MacOS env with FlexiBLAS + - name: pylatest_flexiblas + os: macos-latest + PYTHON_VERSION: "*" + INSTALL_BLAS: "flexiblas" + PLATFORM_SPECIFIC_PACKAGES: "llvm-openmp" # Linux environments to test that packages that comes with Ubuntu 22.04 # are correctly handled. @@ -197,14 +197,14 @@ jobs: BLIS_CC: "gcc-12" BLIS_ENABLE_THREADING: "no" - # Linux env with FlexiBLAS — temporarily disabled on this PR for faster CI iteration. - # - name: pylatest_flexiblas - # os: ubuntu-latest - # PYTHON_VERSION: "*" - # INSTALL_BLAS: "flexiblas" - # PLATFORM_SPECIFIC_PACKAGES: "mkl" - # CC_OUTER_LOOP: "gcc" - # CC_INNER_LOOP: "gcc" + # Linux env with FlexiBLAS + - name: pylatest_flexiblas + os: ubuntu-latest + PYTHON_VERSION: "*" + INSTALL_BLAS: "flexiblas" + PLATFORM_SPECIFIC_PACKAGES: "mkl" + CC_OUTER_LOOP: "gcc" + CC_INNER_LOOP: "gcc" env: ${{ matrix }} diff --git a/_thread_tracer/_windows_etw.py b/_thread_tracer/_windows_etw.py index 242ae3a7..b2715e81 100644 --- a/_thread_tracer/_windows_etw.py +++ b/_thread_tracer/_windows_etw.py @@ -239,10 +239,7 @@ class EVENT_TRACE_LOGFILE(ct.Structure): pass -if sys.platform == "win32": - EVENT_RECORD_CALLBACK = ct.WINFUNCTYPE(None, ct.POINTER(EVENT_RECORD)) -else: - EVENT_RECORD_CALLBACK = ct.CFUNCTYPE(None, ct.POINTER(EVENT_RECORD)) +EVENT_RECORD_CALLBACK = ct.WINFUNCTYPE(None, ct.POINTER(EVENT_RECORD)) EVENT_TRACE_LOGFILE._fields_ = [ ("LogFileName", ct.c_wchar_p), From a0a0f618fc6df4641e30f34536c5682480b4542e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 10 Jul 2026 15:11:43 +0000 Subject: [PATCH 17/17] Allow importing the Windows tracer module on non-Windows platforms Defer WINFUNCTYPE and the ETW callback registration to win32 only so importing _thread_tracer succeeds on Linux and macOS CI jobs. Usage still raises ThreadTracerError when starting a trace off Windows. Co-authored-by: Olivier Grisel --- _thread_tracer/_windows_etw.py | 35 ++++++++++++++++++----------- tests/test_windows_thread_tracer.py | 14 ++++++++++++ 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/_thread_tracer/_windows_etw.py b/_thread_tracer/_windows_etw.py index b2715e81..a7176511 100644 --- a/_thread_tracer/_windows_etw.py +++ b/_thread_tracer/_windows_etw.py @@ -239,7 +239,12 @@ class EVENT_TRACE_LOGFILE(ct.Structure): pass -EVENT_RECORD_CALLBACK = ct.WINFUNCTYPE(None, ct.POINTER(EVENT_RECORD)) +if sys.platform == "win32": + EVENT_RECORD_CALLBACK = ct.WINFUNCTYPE(None, ct.POINTER(EVENT_RECORD)) + _event_record_callback_type = EVENT_RECORD_CALLBACK +else: + EVENT_RECORD_CALLBACK = None + _event_record_callback_type = ct.c_void_p EVENT_TRACE_LOGFILE._fields_ = [ ("LogFileName", ct.c_wchar_p), @@ -253,7 +258,7 @@ class EVENT_TRACE_LOGFILE(ct.Structure): ("BufferSize", ct.c_ulong), ("Filled", ct.c_ulong), ("EventsLost", ct.c_ulong), - ("EventRecordCallback", EVENT_RECORD_CALLBACK), + ("EventRecordCallback", _event_record_callback_type), ("IsKernelTrace", ct.c_ulong), ("Context", ct.c_void_p), ] @@ -261,18 +266,22 @@ class EVENT_TRACE_LOGFILE(ct.Structure): _TRACERS_BY_CONTEXT = {} _TRACERS_LOCK = threading.Lock() +if sys.platform == "win32": -@EVENT_RECORD_CALLBACK -def _event_record_callback(record_pointer): - record = record_pointer.contents - context = record.UserContext - if not context: - return - with _TRACERS_LOCK: - tracer = _TRACERS_BY_CONTEXT.get(context) - if tracer is None: - return - tracer._handle_event_record(record) + @EVENT_RECORD_CALLBACK + def _event_record_callback(record_pointer): + record = record_pointer.contents + context = record.UserContext + if not context: + return + with _TRACERS_LOCK: + tracer = _TRACERS_BY_CONTEXT.get(context) + if tracer is None: + return + tracer._handle_event_record(record) + +else: + _event_record_callback = None _advapi32 = None diff --git a/tests/test_windows_thread_tracer.py b/tests/test_windows_thread_tracer.py index c5f6904c..44582812 100644 --- a/tests/test_windows_thread_tracer.py +++ b/tests/test_windows_thread_tracer.py @@ -1,4 +1,5 @@ import struct +import sys import pytest @@ -11,6 +12,19 @@ ) +@pytest.mark.skipif(sys.platform == "win32", reason="non-Windows import check") +def test_windows_tracer_imports_without_error_on_non_windows(): + from _thread_tracer import ( + ThreadSpawnStats, + ThreadTracerError, + WindowsThreadSpawnTracer, + ) + + tracer = WindowsThreadSpawnTracer(1) + with pytest.raises(ThreadTracerError, match="only supported on Windows"): + tracer.start() + + def _pack_thread_payload(process_id, thread_id): return struct.pack("