FIX intermittent Windows module enumeration failure (#217)

.github/workflows/test.yml

@@ -51,22 +51,26 @@ jobs:
             PYTHON_VERSION: "*"
             PACKAGER: "conda-forge"
             BLAS: "mkl"
+            INSTALL_OPENCV: "true"
           # Windows env with numpy, scipy, OpenBLAS installed through conda-forge
           - name: py311_conda_forge_openblas
             os: windows-latest
             PYTHON_VERSION: "3.11"
             PACKAGER: "conda-forge"
             BLAS: "openblas"
+            INSTALL_OPENCV: "true"
           # Windows env with numpy, scipy installed through conda
           - name: py310_conda
             os: windows-latest
             PYTHON_VERSION: "3.10"
             PACKAGER: "conda"
+            INSTALL_OPENCV: "true"
           # Windows env with numpy, scipy installed through pip
           - name: py39_pip
             os: windows-latest
             PYTHON_VERSION: "3.9"
             PACKAGER: "pip"
+            INSTALL_OPENCV: "true"
 
           # MacOS env with OpenMP installed through homebrew
           - name: py39_conda_homebrew_libomp
@@ -157,6 +161,7 @@ jobs:
             PYTHON_VERSION: "*"
             BLAS: "openblas"
             OPENBLAS_THREADING_LAYER: "openmp"
+            INSTALL_OPENCV: "true"
             CC_OUTER_LOOP: "gcc"
             CC_INNER_LOOP: "gcc"
           # Linux environment with no numpy and heterogeneous OpenMP runtimes.

CHANGES.md

@@ -1,6 +1,12 @@
 3.7.0 (TBD)
 ===========
 
+- Fixed an intermittent `OSError` on Windows when DLLs are loaded or unloaded
+  concurrently during library discovery (for example when importing conda-forge
+  OpenCV). Windows module enumeration now uses a snapshot-first approach with
+  graceful per-module fallbacks.
+  https://github.com/joblib/threadpoolctl/pull/219
+
 - Only warn about simultaneous `libomp` and `libiomp` usage on Linux, where the
   incompatibility is known to cause crashes.
 

continuous_integration/install.sh

@@ -65,6 +65,9 @@ if [[ "$PACKAGER" == "conda" ]]; then
         if [[ -n "$BLAS" ]]; then
             TO_INSTALL="$TO_INSTALL blas=*=$BLAS"
         fi
+    fi
+    if [[ "$INSTALL_OPENCV" == "true" ]]; then
+        TO_INSTALL="$TO_INSTALL opencv"
     fi
 	make_conda "defaults" "$TO_INSTALL"
 
@@ -73,6 +76,9 @@ elif [[ "$PACKAGER" == "conda-forge" ]]; then
     if [[ "$BLAS" == "openblas" && "$OPENBLAS_THREADING_LAYER" == "openmp" ]]; then
         TO_INSTALL="$TO_INSTALL libopenblas=*=*openmp*"
     fi
+    if [[ "$INSTALL_OPENCV" == "true" ]]; then
+        TO_INSTALL="$TO_INSTALL opencv"
+    fi
     make_conda "conda-forge" "$TO_INSTALL"
 
 elif [[ "$PACKAGER" == "pip" ]]; then
@@ -82,6 +88,9 @@ elif [[ "$PACKAGER" == "pip" ]]; then
     if [[ "$NO_NUMPY" != "true" ]]; then
         pip install numpy scipy
     fi
+    if [[ "$INSTALL_OPENCV" == "true" ]]; then
+        pip install opencv-python
+    fi
 
 elif [[ "$PACKAGER" == "pip-dev" ]]; then
     # Use conda to build an empty python env and then use pip to install

tests/test_threadpoolctl.py

@@ -4,14 +4,19 @@
 import re
 import subprocess
 import sys
+import ctypes
+import shutil
 
+import threadpoolctl
 from threadpoolctl import threadpool_limits, threadpool_info
 from threadpoolctl import ThreadpoolController
 from threadpoolctl import _ALL_PREFIXES, _ALL_USER_APIS
 
 from .utils import cython_extensions_compiled
 from .utils import check_nested_prange_blas
 from .utils import libopenblas_paths
+from .utils import get_openblas_dll_path
+from .utils import make_long_windows_path
 from .utils import scipy
 from .utils import threadpool_info_from_subprocess
 from .utils import select
@@ -793,3 +798,108 @@ def test_custom_controller():
         assert mylib_controller.num_threads == 1
 
     assert ThreadpoolController().info() == original_info
+
+
+def test_threadpool_controller_repeated_init():
+    """Stress-test repeated library discovery.
+
+    Non-regression test for a Windows-specific problem where DLLs loaded or
+    unloaded concurrently during discovery could raise an OSError. The failure
+    was originally reproduced after importing OpenCV on Windows conda-forge;
+    see https://github.com/joblib/threadpoolctl/issues/217
+    """
+    pytest.importorskip("cv2")
+
+    for _ in range(100):
+        ThreadpoolController()
+
+
+@pytest.mark.skipif(sys.platform != "win32", reason="Windows-only test")
+def test_windows_library_path_longer_than_max_path(tmp_path):
+    """OpenBLAS loaded from a path longer than MAX_PATH is discovered.
+
+    Regression test inspired by the local repro in
+    https://github.com/joblib/threadpoolctl/pull/189#issuecomment-2714235916
+    """
+    src_dll = get_openblas_dll_path()
+    if src_dll is None:
+        pytest.skip("Requires OpenBLAS on Windows")
+
+    long_path = make_long_windows_path(
+        tmp_path, "libopenblas_long_path_test.dll", min_length=261
+    )
+    extended_path = os.path.abspath(str(long_path))
+    if not extended_path.startswith("\\\\?\\"):
+        if extended_path.startswith("\\\\"):
+            extended_path = "\\\\?\\UNC\\" + extended_path[2:]
+        else:
+            extended_path = "\\\\?\\" + extended_path
+    shutil.copy2(src_dll, extended_path)
+    ctypes.CDLL(extended_path)
+
+    expected_path = os.path.abspath(str(long_path))
+    if expected_path.startswith("\\\\?\\"):
+        expected_path = expected_path[4:]
+        if expected_path.startswith("UNC\\"):
+            expected_path = "\\\\" + expected_path[4:]
+    expected_path = os.path.normcase(os.path.normpath(expected_path))
+    openblas_info = ThreadpoolController().select(internal_api="openblas").info()
+
+    long_path_entries = [info for info in openblas_info if len(info["filepath"]) > 260]
+    assert len(long_path_entries) >= 1
+    normalized_filepaths = []
+    for info in long_path_entries:
+        filepath = info["filepath"]
+        if filepath.startswith("\\\\?\\"):
+            filepath = filepath[4:]
+            if filepath.startswith("UNC\\"):
+                filepath = "\\\\" + filepath[4:]
+        normalized_filepaths.append(os.path.normcase(os.path.normpath(filepath)))
+    assert expected_path in normalized_filepaths
+
+
+@pytest.mark.skipif(sys.platform != "win32", reason="Windows-only test")
+def test_windows_library_path_exceeds_internal_limit(tmp_path, monkeypatch):
+    """Libraries with a path longer than the internal limit are ignored.
+
+    Regression test inspired by the local repro in
+    https://github.com/joblib/threadpoolctl/pull/189#issuecomment-2714235916
+    """
+    src_dll = get_openblas_dll_path()
+    if src_dll is None:
+        pytest.skip("Requires OpenBLAS on Windows")
+
+    monkeypatch.setattr(threadpoolctl, "_WINDOWS_MAX_LIBRARY_PATH_LENGTH", 300)
+
+    long_path = make_long_windows_path(
+        tmp_path, "libopenblas_path_too_long.dll", min_length=400
+    )
+    extended_path = os.path.abspath(str(long_path))
+    if not extended_path.startswith("\\\\?\\"):
+        if extended_path.startswith("\\\\"):
+            extended_path = "\\\\?\\UNC\\" + extended_path[2:]
+        else:
+            extended_path = "\\\\?\\" + extended_path
+    shutil.copy2(src_dll, extended_path)
+    ctypes.CDLL(extended_path)
+
+    expected_path = os.path.abspath(str(long_path))
+    if expected_path.startswith("\\\\?\\"):
+        expected_path = expected_path[4:]
+        if expected_path.startswith("UNC\\"):
+            expected_path = "\\\\" + expected_path[4:]
+    expected_path = os.path.normcase(os.path.normpath(expected_path))
+    with pytest.warns(RuntimeWarning, match="path too long"):
+        info = ThreadpoolController().info()
+
+    filepaths = set()
+    for entry in info:
+        if "filepath" not in entry:
+            continue
+        filepath = entry["filepath"]
+        if filepath.startswith("\\\\?\\"):
+            filepath = filepath[4:]
+            if filepath.startswith("UNC\\"):
+                filepath = "\\\\" + filepath[4:]
+        filepaths.add(os.path.normcase(os.path.normpath(filepath)))
+    assert expected_path not in filepaths

tests/utils.py

@@ -1,9 +1,10 @@
-import os
 import json
+import os
 import sys
 import threadpoolctl
 from glob import glob
 from os.path import dirname, normpath
+from pathlib import Path
 from subprocess import check_output
 
 # Path to shipped openblas for libraries such as numpy or scipy
@@ -17,9 +18,21 @@
     np.dot(np.ones(1000), np.ones(1000))
 
     libopenblas_patterns.append(os.path.join(np.__path__[0], ".libs", "libopenblas*"))
+    numpy_site_packages = os.path.dirname(np.__path__[0])
+    libopenblas_patterns.append(
+        os.path.join(numpy_site_packages, "numpy.libs", "libscipy_openblas*.dll")
+    )
 except ImportError:
     pass
 
+if sys.platform == "win32":
+    libopenblas_patterns.append(
+        os.path.join(sys.prefix, "Library", "bin", "openblas*.dll")
+    )
+    libopenblas_patterns.append(
+        os.path.join(sys.prefix, "Library", "bin", "libopenblas*.dll")
+    )
+
 
 try:
     import scipy
@@ -30,6 +43,10 @@
     libopenblas_patterns.append(
         os.path.join(scipy.__path__[0], ".libs", "libopenblas*")
     )
+    scipy_site_packages = os.path.dirname(scipy.__path__[0])
+    libopenblas_patterns.append(
+        os.path.join(scipy_site_packages, "scipy.libs", "libscipy_openblas*.dll")
+    )
 except ImportError:
     scipy = None
 
@@ -86,3 +103,45 @@ def select(info, **kwargs):
     ]
 
     return selected_info
+
+
+def get_openblas_dll_path():
+    """Return a path to an OpenBLAS DLL that can be copied for Windows tests."""
+    if libopenblas_paths:
+        return sorted(
+            libopenblas_paths,
+            key=lambda path: (
+                0
+                if "libscipy_openblas" in os.path.basename(path).lower()
+                else 1 if "libopenblas" in os.path.basename(path).lower() else 2
+            ),
+        )[0]
+
+    controllers = (
+        threadpoolctl.ThreadpoolController()
+        .select(internal_api="openblas")
+        .lib_controllers
+    )
+    if not controllers:
+        return None
+
+    filepath = controllers[0].filepath
+    if os.path.isfile(filepath):
+        return filepath
+    return None
+
+
+def make_long_windows_path(base_dir, filename, min_length=261):
+    """Nest padded directories under base_dir until base/.../filename >= min_length."""
+    padding_segments = ["a" * 100, "b" * 100, "c" * 100, "d" * 100]
+    current = Path(base_dir)
+    segment_index = 0
+    target = current / filename
+
+    while len(str(target)) < min_length:
+        current = current / padding_segments[segment_index % len(padding_segments)]
+        segment_index += 1
+        target = current / filename
+
+    target.parent.mkdir(parents=True, exist_ok=True)
+    return target