From 5925e2156227f2eb5ac43533048404192c4233b0 Mon Sep 17 00:00:00 2001 From: tomMoral Date: Wed, 27 Mar 2019 15:05:19 +0100 Subject: [PATCH 001/187] CI enable coverage on unix repo --- continuous_integration/posix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/continuous_integration/posix.yml b/continuous_integration/posix.yml index 8e486799..a31b33c8 100644 --- a/continuous_integration/posix.yml +++ b/continuous_integration/posix.yml @@ -8,9 +8,9 @@ jobs: pool: vmImage: ${{ parameters.vmImage }} variables: - TEST_DIR: '$(Agent.WorkFolder)/tmp_folder' VIRTUALENV: 'testvenv' JUNITXML: 'test-data.xml' + CODECOV_TOKEN: 'cee0e505-c12e-4139-aa43-621fb16a2347' strategy: matrix: ${{ insert }}: ${{ parameters.matrix }} From 981393e9e3fa6fdcf7aaece119dc75c13be70781 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Wed, 27 Mar 2019 15:41:44 +0100 Subject: [PATCH 002/187] add gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..b909b665 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +*.pyc + +threadpoolctl.egg-info/ \ No newline at end of file From 01158ae87b8558a98860ccc0f863c84a16137e65 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Wed, 27 Mar 2019 15:42:27 +0100 Subject: [PATCH 003/187] new line --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b909b665..325558d2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ *.pyc -threadpoolctl.egg-info/ \ No newline at end of file +threadpoolctl.egg-info/ From ef3a4b43b045ada14a7f825349833da13e0e8005 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 27 Mar 2019 16:54:37 +0100 Subject: [PATCH 004/187] ENH improve threadpool_limits ctx manager overhead (#2) Skip redundant call in load operations. This should improve significantly the performance issue reported in #1. --- benchmarks/bench_context_manager_overhead.py | 28 ++++ threadpoolctl/_threadpool_limiters.py | 134 ++++++++++-------- threadpoolctl/tests/test_threadpool_limits.py | 32 ++--- 3 files changed, 115 insertions(+), 79 deletions(-) create mode 100644 benchmarks/bench_context_manager_overhead.py diff --git a/benchmarks/bench_context_manager_overhead.py b/benchmarks/bench_context_manager_overhead.py new file mode 100644 index 00000000..e1a9114b --- /dev/null +++ b/benchmarks/bench_context_manager_overhead.py @@ -0,0 +1,28 @@ +import time +from argparse import ArgumentParser +from pprint import pprint +from statistics import mean, stdev +from threadpoolctl import get_threadpool_limits, threadpool_limits + +parser = ArgumentParser(description='Measure threadpool_limits call overhead.') +parser.add_argument('--import', dest="packages", nargs='+', + help='Python packages to import to load threadpool enabled' + ' libraries.') +parser.add_argument("--n-calls", type=int, default=100, + help="Number of iterations") + +args = parser.parse_args() +for package_name in args.packages: + __import__(package_name) + +pprint(get_threadpool_limits()) + +timings = [] +for _ in range(args.n_calls): + t = time.time() + with threadpool_limits(limits=1): + pass + timings.append(time.time() - t) + +print(f"Overhead per call: {mean(timings) * 1000:.3f} " + f"+/-{stdev(timings) * 1000:.3f} ms") diff --git a/threadpoolctl/_threadpool_limiters.py b/threadpoolctl/_threadpool_limiters.py index 68a0bea3..8791fe67 100644 --- a/threadpoolctl/_threadpool_limiters.py +++ b/threadpoolctl/_threadpool_limiters.py @@ -14,7 +14,6 @@ if sys.platform == "darwin": - # On OSX, we can get a runtime error due to multiple OpenMP libraries # loaded simultaneously. This can happen for instance when calling BLAS # inside a prange. Setting the following environment variable allows @@ -92,27 +91,29 @@ def _get_limit(prefix, user_api, limits): @_format_docstring(ALL_PREFIXES=ALL_PREFIXES, INTERNAL_APIS=ALL_INTERNAL_APIS) -def _set_threadpool_limits(limits=None, user_api=None): - """Limit the maximal number of threads for threadpools in supported C-lib +def _set_threadpool_limits(limits=None, user_api=None, + return_original_limits=False): + """Limit the maximal number of threads for threadpools in supported libs Set the maximal number of threads that can be used in thread pools used in - the supported C-libraries to `limit`. This function works for libraries - that are already loaded in the interpreter and can be changed dynamically. + the supported native libraries to `limit`. This function works for + libraries that are already loaded in the interpreter and can be changed + dynamically. The `limits` parameter can be either an integer or a dict to specify the maximal number of thread that can be used in thread pools. If it is an - integer, sets the maximum number of thread to `limits` for each C-lib - selected by `user_api`. If it is a dictionary `{{key: max_threads}}`, - this function sets a custom maximum number of thread for each `key` which - can be either a `user_api` or a `prefix` for a specific library. - If None, this function does not do anything. - - The `user_api` parameter selects particular APIs of C-libs to limit. Used - only if `limits` is an int. If it is None, this function will apply to all - supported C-libs. If it is "blas", it will limit only BLAS supported C-libs - and if it is "openmp", only OpenMP supported C-libs will be limited. Note - that the latter can affect the number of threads used by the BLAS C-libs if - they rely on OpenMP. + integer, sets the maximum number of thread to `limits` for each library + selected by `user_api`. If it is a dictionary `{{key: max_threads}}`, this + function sets a custom maximum number of thread for each `key` which can be + either a `user_api` or a `prefix` for a specific library. If None, this + function does not do anything. + + The `user_api` parameter selects particular APIs of libraries to limit. + Used only if `limits` is an int. If it is None, this function will apply to + all supported libraries. If it is "blas", it will limit only BLAS supported + libraries and if it is "openmp", only OpenMP supported libraries will be + limited. Note that the latter can affect the number of threads used by the + BLAS libraries if they rely on OpenMP. Return a list with all the supported modules that have been found. Each module is represented by a dict with the following information: @@ -122,7 +123,12 @@ def _set_threadpool_limits(limits=None, user_api=None): - 'internal_api': internal API.s Possible values are {INTERNAL_APIS}. - 'module_path': path to the loaded module. - 'version': version of the library implemented (if available). - - 'n_thread': current thread limit. + - 'n_thread': current thread limit if return_original_limits is False or + the original limit if return_original_limits is True. + - 'set_num_threads': callable to set the maximum number of threads + - 'get_num_threads': callable to get the current number of threads + - 'dynlib': the instance of ctypes.CDLL use to access the dynamic + library. """ if isinstance(limits, int) or limits is None: if user_api is None: @@ -154,14 +160,15 @@ def _set_threadpool_limits(limits=None, user_api=None): modules = _load_modules(prefixes=prefixes, user_api=user_api) for module in modules: n_thread = _get_limit(module['prefix'], module['user_api'], limits) + if return_original_limits: + module['n_thread'] = module['get_num_threads']() + if n_thread is not None: set_func = module['set_num_threads'] set_func(n_thread) - # Store the module and remove un-necessary info - module['n_thread'] = module['get_num_threads']() - del module['set_num_threads'], module['get_num_threads'] - del module['clib'] + if not return_original_limits: + module['n_thread'] = module['get_num_threads']() report_threadpool_size.append(module) return report_threadpool_size @@ -187,30 +194,29 @@ def get_threadpool_limits(): module['n_thread'] = module['get_num_threads']() # Remove the wrapper for the module and its function del module['set_num_threads'], module['get_num_threads'] - del module['clib'] + del module['dynlib'] report_threadpool_size.append(module) return report_threadpool_size -def get_version(clib, internal_api): +def get_version(dynlib, internal_api): if internal_api == "mkl": - return _get_mkl_version(clib) + return _get_mkl_version(dynlib) elif internal_api == "openmp": # There is no way to get the version number programmatically in # OpenMP. return None elif internal_api == "openblas": - return _get_openblas_version(clib) + return _get_openblas_version(dynlib) else: raise NotImplementedError("Unsupported API {}".format(internal_api)) -def _get_mkl_version(mkl_clib): - """Return the MKL version - """ +def _get_mkl_version(mkl_dynlib): + """Return the MKL version""" res = ctypes.create_string_buffer(200) - mkl_clib.mkl_get_version_string(res, 200) + mkl_dynlib.mkl_get_version_string(res, 200) version = res.value.decode('utf-8') group = re.search(r"Version ([^ ]+) ", version) @@ -219,13 +225,13 @@ def _get_mkl_version(mkl_clib): return version.strip() -def _get_openblas_version(openblas_clib): +def _get_openblas_version(openblas_dynlib): """Return the OpenBLAS version None means OpenBLAS is not loaded or version < 0.3.4, since OpenBLAS did not expose its version before that. """ - get_config = getattr(openblas_clib, "openblas_get_config") + get_config = getattr(openblas_dynlib, "openblas_get_config") get_config.restype = ctypes.c_char_p config = get_config().split() if config[0] == b"OpenBLAS": @@ -243,13 +249,13 @@ def _load_modules(prefixes=None, user_api=None): if user_api is None: user_api = [] if sys.platform == "darwin": - return _find_modules_with_clibs_dyld( + return _find_modules_with_dyld( prefixes=prefixes, user_api=user_api) elif sys.platform == "win32": return _find_modules_with_enum_process_module_ex( prefixes=prefixes, user_api=user_api) else: - return _find_modules_with_clibs_dl_iterate_phdr( + return _find_modules_with_dl_iterate_phdr( prefixes=prefixes, user_api=user_api) @@ -271,16 +277,18 @@ def _match_module(module_info, prefix, prefixes, user_api): def _make_module_info(module_path, module_info, prefix): """Make a dict with the information from the module.""" module_path = os.path.normpath(module_path) - clib = ctypes.CDLL(module_path) + dynlib = ctypes.CDLL(module_path) internal_api = module_info['internal_api'] - set_func = getattr(clib, MAP_API_TO_FUNC[internal_api]['set_num_threads'], + set_func = getattr(dynlib, + MAP_API_TO_FUNC[internal_api]['set_num_threads'], lambda n_thread: None) - get_func = getattr(clib, MAP_API_TO_FUNC[internal_api]['get_num_threads'], + get_func = getattr(dynlib, + MAP_API_TO_FUNC[internal_api]['get_num_threads'], lambda: None) module_info = module_info.copy() - module_info.update(clib=clib, module_path=module_path, prefix=prefix, + module_info.update(dynlib=dynlib, module_path=module_path, prefix=prefix, set_num_threads=set_func, get_num_threads=get_func, - version=get_version(clib, internal_api)) + version=get_version(dynlib, internal_api)) return module_info @@ -292,7 +300,7 @@ def _get_module_info_from_path(module_path, prefixes, user_api, modules): modules.append(_make_module_info(module_path, info, prefix)) -def _find_modules_with_clibs_dl_iterate_phdr(prefixes, user_api): +def _find_modules_with_dl_iterate_phdr(prefixes, user_api): """Loop through loaded libraries and return binders on supported ones This function is expected to work on POSIX system only. @@ -302,7 +310,6 @@ def _find_modules_with_clibs_dl_iterate_phdr(prefixes, user_api): Copyright (c) 2017, Intel Corporation published under the BSD 3-Clause license """ - libc = _get_libc() if not hasattr(libc, "dl_iterate_phdr"): # pragma: no cover return [] @@ -312,7 +319,6 @@ def _find_modules_with_clibs_dl_iterate_phdr(prefixes, user_api): # Callback function for `dl_iterate_phdr` which is called for every # module loaded in the current process until it returns 1. def match_module_callback(info, size, data): - # Get the path of the current module module_path = info.contents.dlpi_name if module_path: @@ -335,7 +341,7 @@ def match_module_callback(info, size, data): return _modules -def _find_modules_with_clibs_dyld(prefixes, user_api): +def _find_modules_with_dyld(prefixes, user_api): """Loop through loaded libraries and return binders on supported ones This function is expected to work on OSX system only @@ -447,29 +453,30 @@ class threadpool_limits: block. Set the maximal number of threads that can be used in thread pools used in - the supported C-libraries to `limit`. This function works for libraries - that are already loaded in the interpreter and can be changed dynamically. + the supported libraries to `limit`. This function works for libraries that + are already loaded in the interpreter and can be changed dynamically. The `limits` parameter can be either an integer or a dict to specify the maximal number of thread that can be used in thread pools. If it is an - integer, sets the maximum number of thread to `limits` for each C-lib - selected by `user_api`. If it is a dictionary `{{key: max_threads}}`, - this function sets a custom maximum number of thread for each `key` which - can be either a `user_api` or a `prefix` for a specific library. - If None, this function does not do anything. - - The `user_api` parameter selects particular APIs of C-libs to limit. Used - only if `limits` is an int. If it is None, this function will apply to all - supported C-libs. If it is "blas", it will limit only BLAS supported C-libs - and if it is "openmp", only OpenMP supported C-libs will be limited. Note - that the latter can affect the number of threads used by the BLAS C-libs if - they rely on OpenMP. + integer, sets the maximum number of thread to `limits` for each library + selected by `user_api`. If it is a dictionary `{{key: max_threads}}`, this + function sets a custom maximum number of thread for each `key` which can be + either a `user_api` or a `prefix` for a specific library. If None, this + function does not do anything. + + The `user_api` parameter selects particular APIs of libraries to limit. + Used only if `limits` is an int. If it is None, this function will apply to + all supported libraries. If it is "blas", it will limit only BLAS supported + libraries and if it is "openmp", only OpenMP supported libraries will be + limited. Note that the latter can affect the number of threads used by the + BLAS libraries if they rely on OpenMP. """ def __init__(self, limits=None, user_api=None): - self._enabled = limits is not None - if self._enabled: - self.old_limits = get_threadpool_limits() - _set_threadpool_limits(limits=limits, user_api=user_api) + if limits is not None: + self.original_limits = _set_threadpool_limits( + limits=limits, user_api=user_api, return_original_limits=True) + else: + self.original_limits = None def __enter__(self): pass @@ -478,5 +485,6 @@ def __exit__(self, type, value, traceback): self.unregister() def unregister(self): - if self._enabled: - _set_threadpool_limits(limits=self.old_limits) + if self.original_limits is not None: + for module in self.original_limits: + module['set_num_threads'](module['n_thread']) diff --git a/threadpoolctl/tests/test_threadpool_limits.py b/threadpoolctl/tests/test_threadpool_limits.py index b951ee93..949e6aa2 100644 --- a/threadpoolctl/tests/test_threadpool_limits.py +++ b/threadpoolctl/tests/test_threadpool_limits.py @@ -23,7 +23,7 @@ def test_threadpool_limits(openblas_present, mkl_present, prefix): prefix_found = len([1 for module in old_limits if prefix == module['prefix']]) - old_limits = {clib['prefix']: clib['n_thread'] for clib in old_limits} + old_limits = {dynlib['prefix']: dynlib['n_thread'] for dynlib in old_limits} if not prefix_found: have_mkl = len({'mkl_rt', 'libmkl_rt'}.intersection(old_limits)) > 0 @@ -36,34 +36,37 @@ def test_threadpool_limits(openblas_present, mkl_present, prefix): try: new_limits = _set_threadpool_limits(limits={prefix: 1}) - new_limits = {clib['prefix']: clib['n_thread'] for clib in new_limits} + new_limits = {dynlib['prefix']: dynlib['n_thread'] + for dynlib in new_limits} assert new_limits[prefix] == 1 threadpool_limits(limits={prefix: 3}) new_limits = get_threadpool_limits() - new_limits = {clib['prefix']: clib['n_thread'] for clib in new_limits} + new_limits = {dynlib['prefix']: dynlib['n_thread'] + for dynlib in new_limits} assert new_limits[prefix] in (3, old_limits[prefix]) finally: # Avoid having side effects in case of failures threadpool_limits(limits=old_limits) new_limits = get_threadpool_limits() - new_limits = {clib['prefix']: clib['n_thread'] for clib in new_limits} + new_limits = {dynlib['prefix']: dynlib['n_thread'] + for dynlib in new_limits} assert new_limits[prefix] == old_limits[prefix] @pytest.mark.parametrize("user_api", (None, "blas", "openmp")) def test_set_threadpool_limits_apis(user_api): - # Check that the number of threads used by the multithreaded C-libs can be - # modified dynamically. - + # Check that the number of threads used by the multithreaded libraries can + # be modified dynamically. if user_api is None: api_modules = ('blas', 'openmp') else: api_modules = (user_api,) old_limits = get_threadpool_limits() - old_limits = {clib['prefix']: clib['n_thread'] for clib in old_limits} + old_limits = {dynlib['prefix']: dynlib['n_thread'] + for dynlib in old_limits} try: new_limits = _set_threadpool_limits(limits=1, user_api=user_api) @@ -91,7 +94,6 @@ def test_set_threadpool_limits_apis(user_api): def test_set_threadpool_limits_bad_input(): # Check that appropriate errors are raised for invalid arguments - match = re.escape("user_api must be either in {} or None." .format(ALL_USER_APIS)) with pytest.raises(ValueError, match=match): @@ -105,18 +107,19 @@ def test_set_threadpool_limits_bad_input(): @pytest.mark.parametrize("user_api", (None, "blas", "openmp")) def test_thread_limit_context(user_api): # Tests the thread limits context manager - if user_api is None: apis = ('blas', 'openmp') else: apis = (user_api,) old_limits = get_threadpool_limits() - old_limits = {clib['prefix']: clib['n_thread'] for clib in old_limits} + old_limits = {dynlib['prefix']: dynlib['n_thread'] + for dynlib in old_limits} with threadpool_limits(limits=None, user_api=user_api): limits = get_threadpool_limits() - limits = {clib['prefix']: clib['n_thread'] for clib in limits} + limits = {dynlib['prefix']: dynlib['n_thread'] + for dynlib in limits} assert limits == old_limits with threadpool_limits(limits=1, user_api=user_api): @@ -131,7 +134,7 @@ def test_thread_limit_context(user_api): assert module['n_thread'] == old_limits[module['prefix']] limits = get_threadpool_limits() - limits = {clib['prefix']: clib['n_thread'] for clib in limits} + limits = {dynlib['prefix']: dynlib['n_thread'] for dynlib in limits} assert limits == old_limits @@ -140,7 +143,6 @@ def test_thread_limit_context(user_api): def test_openmp_limit_num_threads(n_threads): # checks that OpenMP effectively uses the number of threads requested by # the context manager - from ._openmp_test_helper import check_openmp_n_threads old_num_threads = check_openmp_n_threads(100) @@ -151,7 +153,6 @@ def test_openmp_limit_num_threads(n_threads): def test_shipped_openblas(): - libopenblas = [ctypes.CDLL(path) for path in libopenblas_paths] old_limits = [blas.openblas_get_num_threads() for blas in libopenblas] @@ -167,7 +168,6 @@ def test_shipped_openblas(): @pytest.mark.skipif(len(libopenblas_paths) < 2, reason="need at least 2 shipped openblas library") def test_multiple_shipped_openblas(): - libopenblas = [ctypes.CDLL(path) for path in libopenblas_paths] old_limits = [blas.openblas_get_num_threads() for blas in libopenblas] From e173b3106c222909e54e8a5a627471d45409afab Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 27 Mar 2019 17:35:19 +0100 Subject: [PATCH 005/187] Fix in benchmark script --- benchmarks/bench_context_manager_overhead.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/bench_context_manager_overhead.py b/benchmarks/bench_context_manager_overhead.py index e1a9114b..ea550d6c 100644 --- a/benchmarks/bench_context_manager_overhead.py +++ b/benchmarks/bench_context_manager_overhead.py @@ -5,7 +5,7 @@ from threadpoolctl import get_threadpool_limits, threadpool_limits parser = ArgumentParser(description='Measure threadpool_limits call overhead.') -parser.add_argument('--import', dest="packages", nargs='+', +parser.add_argument('--import', dest="packages", default=[], nargs='+', help='Python packages to import to load threadpool enabled' ' libraries.') parser.add_argument("--n-calls", type=int, default=100, From f8c38699315eb9acdfb0e8e918ca1fd1a1267ff9 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 27 Mar 2019 18:07:09 +0100 Subject: [PATCH 006/187] Cache system libraries lookups --- benchmarks/bench_context_manager_overhead.py | 4 ++-- threadpoolctl/_threadpool_limiters.py | 21 +++++++++++++++----- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/benchmarks/bench_context_manager_overhead.py b/benchmarks/bench_context_manager_overhead.py index ea550d6c..2e096ca2 100644 --- a/benchmarks/bench_context_manager_overhead.py +++ b/benchmarks/bench_context_manager_overhead.py @@ -24,5 +24,5 @@ pass timings.append(time.time() - t) -print(f"Overhead per call: {mean(timings) * 1000:.3f} " - f"+/-{stdev(timings) * 1000:.3f} ms") +print(f"Overhead per call: {mean(timings) * 1e3:.3f} " + f"+/-{stdev(timings) * 1e3:.3f} ms") diff --git a/threadpoolctl/_threadpool_limiters.py b/threadpoolctl/_threadpool_limiters.py index 8791fe67..b20ecae8 100644 --- a/threadpoolctl/_threadpool_limiters.py +++ b/threadpoolctl/_threadpool_limiters.py @@ -12,6 +12,9 @@ from .utils import _format_docstring +# Cache for libc under POSIX and a few system libraries under Windows +_system_libraries = {} + if sys.platform == "darwin": # On OSX, we can get a runtime error due to multiple OpenMP libraries @@ -434,15 +437,23 @@ def _find_modules_with_enum_process_module_ex(prefixes, user_api): def _get_libc(): """Load the lib-C for unix systems.""" - libc_name = find_library("c") - if libc_name is None: # pragma: no cover - return None - return ctypes.CDLL(libc_name) + libc = _system_libraries.get("libc") + if libc is None: + libc_name = find_library("c") + if libc_name is None: # pragma: no cover + return None + libc = ctypes.CDLL(libc_name) + _system_libraries["libc"] = libc + return libc def _get_windll(dll_name): """Load a windows DLL""" - return ctypes.WinDLL("{}.dll".format(dll_name)) + dll = _system_libraries.get(dll_name) + if dll is None: + dll = ctypes.WinDLL("{}.dll".format(dll_name)) + _system_libraries[dll_name] = dll + return dll class threadpool_limits: From 3504afae753aa552d28f5f3678bca10d7060866c Mon Sep 17 00:00:00 2001 From: Thomas Moreau Date: Thu, 28 Mar 2019 08:58:03 +0100 Subject: [PATCH 007/187] CI fix coverage reports for conda based builds (#5) The report were not correctly uploaded to codecov. Also remove some unsused code. --- .azure_pipeline.yml | 13 +++++++++++++ .codecov.yml | 2 ++ .coveragerc | 3 +++ .gitignore | 9 ++++++++- continuous_integration/install.sh | 6 ++++-- continuous_integration/posix.yml | 4 ---- continuous_integration/test_script.cmd | 4 +--- continuous_integration/test_script.sh | 2 +- continuous_integration/upload_codecov.sh | 12 ++++++++++-- continuous_integration/windows.yml | 6 +----- .../tests/_openmp_test_helper/.gitignore | 3 +++ threadpoolctl/tests/utils.py | 15 ++------------- 12 files changed, 48 insertions(+), 31 deletions(-) create mode 100644 .codecov.yml create mode 100644 .coveragerc create mode 100644 threadpoolctl/tests/_openmp_test_helper/.gitignore diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml index 15daa590..313c4ac4 100644 --- a/.azure_pipeline.yml +++ b/.azure_pipeline.yml @@ -1,4 +1,12 @@ # Adapted from https://github.com/pandas-dev/pandas/blob/master/azure-pipelines.yml + +# Global variables for all jobs +variables: + VIRTUALENV: 'testvenv' + JUNITXML: 'test-data.xml' + CODECOV_TOKEN: 'cee0e505-c12e-4139-aa43-621fb16a2347' + + jobs: - template: continuous_integration/windows.yml @@ -35,6 +43,11 @@ jobs: pylatest_conda: PACKAGER: 'conda' VERSION_PYTHON: '*' + # Linux environment with no numpy. + pylatest_conda_nonumpy: + PACKAGER: 'conda' + NO_NUMPY: 'true' + VERSION_PYTHON: '*' - template: continuous_integration/posix.yml parameters: diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 00000000..50979600 --- /dev/null +++ b/.codecov.yml @@ -0,0 +1,2 @@ +comment: off + diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 00000000..f25962cc --- /dev/null +++ b/.coveragerc @@ -0,0 +1,3 @@ +[run] +source=threadpoolctl + diff --git a/.gitignore b/.gitignore index 325558d2..e1b4f986 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,10 @@ +# Python cache files *.pyc +__pycache__ -threadpoolctl.egg-info/ +# Python install files +*.egg-info/ + +# Coverage data +.coverage +/htmlcov diff --git a/continuous_integration/install.sh b/continuous_integration/install.sh index 4f552be0..21980b5e 100755 --- a/continuous_integration/install.sh +++ b/continuous_integration/install.sh @@ -25,8 +25,10 @@ make_conda() { } if [[ "$PACKAGER" == "conda" ]]; then - TO_INSTALL="python=$VERSION_PYTHON pip pytest pytest-cov \ - numpy cython" + TO_INSTALL="python=$VERSION_PYTHON pip pytest pytest-cov cython" + if [[ "$NO_NUMPY" != "true" ]]; then + TO_INSTALL="$TO_INSTALL numpy" + fi make_conda $TO_INSTALL diff --git a/continuous_integration/posix.yml b/continuous_integration/posix.yml index a31b33c8..a80816ca 100644 --- a/continuous_integration/posix.yml +++ b/continuous_integration/posix.yml @@ -7,10 +7,6 @@ jobs: - job: ${{ parameters.name }} pool: vmImage: ${{ parameters.vmImage }} - variables: - VIRTUALENV: 'testvenv' - JUNITXML: 'test-data.xml' - CODECOV_TOKEN: 'cee0e505-c12e-4139-aa43-621fb16a2347' strategy: matrix: ${{ insert }}: ${{ parameters.matrix }} diff --git a/continuous_integration/test_script.cmd b/continuous_integration/test_script.cmd index 29d718c0..bb9bafc7 100644 --- a/continuous_integration/test_script.cmd +++ b/continuous_integration/test_script.cmd @@ -1,5 +1,3 @@ -set DEFAULT_PYTEST_ARGS=-vlx --cov=threadpoolctl - call activate %VIRTUALENV% -pytest --junitxml=%JUNITXML% %DEFAULT_PYTEST_ARGS% +pytest -vlx --junitxml=%JUNITXML% --cov=threadpoolctl diff --git a/continuous_integration/test_script.sh b/continuous_integration/test_script.sh index 6372b09b..605c0f17 100755 --- a/continuous_integration/test_script.sh +++ b/continuous_integration/test_script.sh @@ -9,5 +9,5 @@ elif [[ "$PACKAGER" == "ubuntu" ]]; then fi set -x -pytest -vl --junitxml=$JUNITXML --cov=threadpoolctl +pytest -vlx --junitxml=$JUNITXML --cov=threadpoolctl set +x diff --git a/continuous_integration/upload_codecov.sh b/continuous_integration/upload_codecov.sh index f188ba15..1fb312c0 100755 --- a/continuous_integration/upload_codecov.sh +++ b/continuous_integration/upload_codecov.sh @@ -2,6 +2,14 @@ set -e -python -m pip install --user codecov +if [[ "$PACKAGER" == "conda" ]]; then + source activate $VIRTUALENV +elif [[ "$PACKAGER" == "pip" ]]; then + source activate $VIRTUALENV +elif [[ "$PACKAGER" == "ubuntu" ]]; then + source $VIRTUALENV/bin/activate +fi -python -m codecov || echo "codecov upload failed" +pip install codecov + +codecov || echo "codecov upload failed" diff --git a/continuous_integration/windows.yml b/continuous_integration/windows.yml index d05ba3dc..efbe510e 100644 --- a/continuous_integration/windows.yml +++ b/continuous_integration/windows.yml @@ -8,10 +8,7 @@ jobs: - job: ${{ parameters.name }} pool: vmImage: ${{ parameters.vmImage }} - variables: - VIRTUALENV: 'testvenv' - JUNITXML: 'test-data.xml' - CODECOV_TOKEN: 'cee0e505-c12e-4139-aa43-621fb16a2347' + strategy: matrix: ${{ insert }}: ${{ parameters.matrix }} @@ -19,7 +16,6 @@ jobs: steps: - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" displayName: Add conda to PATH - condition: eq(variables['PACKAGER'], 'conda') - script: | continuous_integration\\install.cmd displayName: 'Install' diff --git a/threadpoolctl/tests/_openmp_test_helper/.gitignore b/threadpoolctl/tests/_openmp_test_helper/.gitignore new file mode 100644 index 00000000..2e5170f4 --- /dev/null +++ b/threadpoolctl/tests/_openmp_test_helper/.gitignore @@ -0,0 +1,3 @@ +*.c +*.so +/build diff --git a/threadpoolctl/tests/utils.py b/threadpoolctl/tests/utils.py index 6ee2ed4b..aa6754b6 100644 --- a/threadpoolctl/tests/utils.py +++ b/threadpoolctl/tests/utils.py @@ -19,17 +19,11 @@ def test_func(*args, **kwargs): import numpy as np np.dot(np.ones(1000), np.ones(1000)) - def with_numpy(func): - """A decorator to skip tests requiring numpy.""" - return func - libopenblas_patterns.append(os.path.join(np.__path__[0], ".libs", "libopenblas*")) except ImportError: - def with_numpy(func): - """A decorator to skip tests requiring numpy.""" - return skip_func('Test require numpy') + pass try: @@ -47,20 +41,15 @@ def with_numpy(func): # A decorator to run tests only when check_openmp_n_threads is available try: - from ._openmp_test_helper import check_openmp_n_threads + from ._openmp_test_helper import check_openmp_n_threads # noqa: F401 def with_check_openmp_n_threads(func): """A decorator to skip tests if check_openmp_n_threads is not compiled. """ return func - def _run_check_openmp_n_threads(*args): - return check_openmp_n_threads(*args) - except ImportError: def with_check_openmp_n_threads(func): """A decorator to skip tests if check_openmp_n_threads is not compiled. """ return skip_func('Test requires check_openmp_n_threads to be compiled') - - _run_check_openmp_n_threads = None From 3d02c31b60e3ce3e62ce4551c9c5783766a23e7d Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Fri, 29 Mar 2019 17:08:16 +0100 Subject: [PATCH 008/187] CLN single module project (#9) * Use a single file for this project * Add a LICENSE * Improve .gitignore --- .gitignore | 12 +- LICENSE | 24 ++++ MANIFEST.in | 4 + continuous_integration/build_test_ext.sh | 4 +- setup.py | 7 +- {threadpoolctl/tests => tests}/__init__.py | 0 .../_openmp_test_helper/__init__.py | 0 .../_openmp_test_helper/openmp_helpers.pyx | 9 +- .../_openmp_test_helper/setup.py | 0 .../test_threadpoolctl.py | 54 +++++---- {threadpoolctl/tests => tests}/utils.py | 6 +- ...threadpool_limiters.py => threadpoolctl.py | 112 +++++++++++------- threadpoolctl/__init__.py | 7 -- threadpoolctl/utils.py | 8 -- 14 files changed, 147 insertions(+), 100 deletions(-) create mode 100644 LICENSE create mode 100644 MANIFEST.in rename {threadpoolctl/tests => tests}/__init__.py (100%) rename {threadpoolctl/tests => tests}/_openmp_test_helper/__init__.py (100%) rename {threadpoolctl/tests => tests}/_openmp_test_helper/openmp_helpers.pyx (64%) rename {threadpoolctl/tests => tests}/_openmp_test_helper/setup.py (100%) rename threadpoolctl/tests/test_threadpool_limits.py => tests/test_threadpoolctl.py (76%) rename {threadpoolctl/tests => tests}/utils.py (88%) rename threadpoolctl/_threadpool_limiters.py => threadpoolctl.py (84%) delete mode 100644 threadpoolctl/__init__.py delete mode 100644 threadpoolctl/utils.py diff --git a/.gitignore b/.gitignore index e1b4f986..5ce98657 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,18 @@ -# Python cache files +# Python and Cython generated files *.pyc __pycache__ +*.so +*.dylib +*.c -# Python install files +# Python install files, build and release artifacts *.egg-info/ +build +dist # Coverage data .coverage /htmlcov + +# Developer tools +.vscode diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..f2927f5f --- /dev/null +++ b/LICENSE @@ -0,0 +1,24 @@ +Copyright (c) 2019, threadpoolctl contributors + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..86cfc8af --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,4 @@ +include *.md *.py *.cfg *.ini +include LICENSE +recursive-include tests *.py *.pyx +global-exclude *~ *.swp diff --git a/continuous_integration/build_test_ext.sh b/continuous_integration/build_test_ext.sh index 0160a9ad..a5572e51 100644 --- a/continuous_integration/build_test_ext.sh +++ b/continuous_integration/build_test_ext.sh @@ -1,4 +1,4 @@ -cd threadpoolctl/tests/_openmp_test_helper +cd tests/_openmp_test_helper python setup.py build_ext -i || echo 'No openmp' -cd ../.. \ No newline at end of file +cd ../.. diff --git a/setup.py b/setup.py index b89146ad..b6c4f5ca 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ # Function to parse __version__ in `threadpoolctl` def find_version(): here = os.path.abspath(os.path.dirname(__file__)) - with open(os.path.join(here, 'threadpoolctl', '__init__.py'), 'r') as fp: + with open(os.path.join(here, 'threadpoolctl.py'), 'r') as fp: version_file = fp.read() version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M) @@ -54,14 +54,15 @@ def run(self): setup( name='threadpoolctl', version=find_version(), - description=("Python helpers to limit the number of thread used in " - "thread-pool backed parallelism for C-libraries"), + description=("Python helpers to limit the number of threads used in " + "native libraries with thread-pools."), long_description=open('README.md', 'rb').read().decode('utf-8'), long_description_content_type='text/markdown', url='https://github.com/joblib/threadpoolctl/', author='Thomas Moreau', author_email='thomas.moreau.2010@gmail.com', packages=packages, + py_modules=["threadpoolctl"], zip_safe=False, license='BSD', classifiers=[ diff --git a/threadpoolctl/tests/__init__.py b/tests/__init__.py similarity index 100% rename from threadpoolctl/tests/__init__.py rename to tests/__init__.py diff --git a/threadpoolctl/tests/_openmp_test_helper/__init__.py b/tests/_openmp_test_helper/__init__.py similarity index 100% rename from threadpoolctl/tests/_openmp_test_helper/__init__.py rename to tests/_openmp_test_helper/__init__.py diff --git a/threadpoolctl/tests/_openmp_test_helper/openmp_helpers.pyx b/tests/_openmp_test_helper/openmp_helpers.pyx similarity index 64% rename from threadpoolctl/tests/_openmp_test_helper/openmp_helpers.pyx rename to tests/_openmp_test_helper/openmp_helpers.pyx index 02b6da02..5a6cbc3a 100644 --- a/threadpoolctl/tests/_openmp_test_helper/openmp_helpers.pyx +++ b/tests/_openmp_test_helper/openmp_helpers.pyx @@ -4,9 +4,12 @@ from cython.parallel import prange from libc.stdlib cimport malloc, free -def check_openmp_n_threads(int n): - """Run a short parallel section, and return the number of threads that - where effectively used by openmp.""" +def check_openmp_num_threads(int n): + """Run a short parallel section with OpenMP + + Return the number of threads that where effectively used by the + OpenMP runtime. + """ cdef long n_sum = 0 cdef int i, num_threads diff --git a/threadpoolctl/tests/_openmp_test_helper/setup.py b/tests/_openmp_test_helper/setup.py similarity index 100% rename from threadpoolctl/tests/_openmp_test_helper/setup.py rename to tests/_openmp_test_helper/setup.py diff --git a/threadpoolctl/tests/test_threadpool_limits.py b/tests/test_threadpoolctl.py similarity index 76% rename from threadpoolctl/tests/test_threadpool_limits.py rename to tests/test_threadpoolctl.py index 949e6aa2..c87350a0 100644 --- a/threadpoolctl/tests/test_threadpool_limits.py +++ b/tests/test_threadpoolctl.py @@ -5,10 +5,10 @@ from threadpoolctl import threadpool_limits from threadpoolctl import get_threadpool_limits -from threadpoolctl._threadpool_limiters import _set_threadpool_limits -from threadpoolctl._threadpool_limiters import ALL_PREFIXES, ALL_USER_APIS +from threadpoolctl import _set_threadpool_limits +from threadpoolctl import _ALL_PREFIXES, _ALL_USER_APIS -from .utils import with_check_openmp_n_threads, libopenblas_paths +from .utils import with_check_openmp_num_threads, libopenblas_paths def should_skip_module(module): @@ -17,13 +17,14 @@ def should_skip_module(module): return module['internal_api'] == "openblas" and module['version'] is None -@pytest.mark.parametrize("prefix", ALL_PREFIXES) +@pytest.mark.parametrize("prefix", _ALL_PREFIXES) def test_threadpool_limits(openblas_present, mkl_present, prefix): old_limits = get_threadpool_limits() prefix_found = len([1 for module in old_limits if prefix == module['prefix']]) - old_limits = {dynlib['prefix']: dynlib['n_thread'] for dynlib in old_limits} + old_limits = {dynlib['prefix']: dynlib['num_threads'] + for dynlib in old_limits} if not prefix_found: have_mkl = len({'mkl_rt', 'libmkl_rt'}.intersection(old_limits)) > 0 @@ -36,13 +37,13 @@ def test_threadpool_limits(openblas_present, mkl_present, prefix): try: new_limits = _set_threadpool_limits(limits={prefix: 1}) - new_limits = {dynlib['prefix']: dynlib['n_thread'] + new_limits = {dynlib['prefix']: dynlib['num_threads'] for dynlib in new_limits} assert new_limits[prefix] == 1 threadpool_limits(limits={prefix: 3}) new_limits = get_threadpool_limits() - new_limits = {dynlib['prefix']: dynlib['n_thread'] + new_limits = {dynlib['prefix']: dynlib['num_threads'] for dynlib in new_limits} assert new_limits[prefix] in (3, old_limits[prefix]) finally: @@ -50,7 +51,7 @@ def test_threadpool_limits(openblas_present, mkl_present, prefix): threadpool_limits(limits=old_limits) new_limits = get_threadpool_limits() - new_limits = {dynlib['prefix']: dynlib['n_thread'] + new_limits = {dynlib['prefix']: dynlib['num_threads'] for dynlib in new_limits} assert new_limits[prefix] == old_limits[prefix] @@ -65,7 +66,7 @@ def test_set_threadpool_limits_apis(user_api): api_modules = (user_api,) old_limits = get_threadpool_limits() - old_limits = {dynlib['prefix']: dynlib['n_thread'] + old_limits = {dynlib['prefix']: dynlib['num_threads'] for dynlib in old_limits} try: @@ -74,7 +75,7 @@ def test_set_threadpool_limits_apis(user_api): if should_skip_module(module): continue if module['user_api'] in api_modules: - assert module['n_thread'] == 1 + assert module['num_threads'] == 1 threadpool_limits(limits=3, user_api=user_api) new_limits = get_threadpool_limits() @@ -82,20 +83,21 @@ def test_set_threadpool_limits_apis(user_api): if should_skip_module(module): continue if module['user_api'] in api_modules: - assert module['n_thread'] in (3, old_limits[module['prefix']]) + assert module['num_threads'] in ( + 3, old_limits[module['prefix']]) finally: # Avoid having side effects on other tests in case of failure threadpool_limits(limits=old_limits) new_limits = get_threadpool_limits() for module in new_limits: - assert module['n_thread'] == old_limits[module['prefix']] + assert module['num_threads'] == old_limits[module['prefix']] def test_set_threadpool_limits_bad_input(): # Check that appropriate errors are raised for invalid arguments match = re.escape("user_api must be either in {} or None." - .format(ALL_USER_APIS)) + .format(_ALL_USER_APIS)) with pytest.raises(ValueError, match=match): threadpool_limits(limits=1, user_api="wrong") @@ -113,12 +115,12 @@ def test_thread_limit_context(user_api): apis = (user_api,) old_limits = get_threadpool_limits() - old_limits = {dynlib['prefix']: dynlib['n_thread'] + old_limits = {dynlib['prefix']: dynlib['num_threads'] for dynlib in old_limits} with threadpool_limits(limits=None, user_api=user_api): limits = get_threadpool_limits() - limits = {dynlib['prefix']: dynlib['n_thread'] + limits = {dynlib['prefix']: dynlib['num_threads'] for dynlib in limits} assert limits == old_limits @@ -129,27 +131,27 @@ def test_thread_limit_context(user_api): if should_skip_module(module): continue elif module['user_api'] in apis: - assert module['n_thread'] == 1 + assert module['num_threads'] == 1 else: - assert module['n_thread'] == old_limits[module['prefix']] + assert module['num_threads'] == old_limits[module['prefix']] limits = get_threadpool_limits() - limits = {dynlib['prefix']: dynlib['n_thread'] for dynlib in limits} + limits = {dynlib['prefix']: dynlib['num_threads'] for dynlib in limits} assert limits == old_limits -@with_check_openmp_n_threads -@pytest.mark.parametrize('n_threads', [1, 2, 4]) -def test_openmp_limit_num_threads(n_threads): +@with_check_openmp_num_threads +@pytest.mark.parametrize('num_threads', [1, 2, 4]) +def test_openmp_limit_num_threads(num_threads): # checks that OpenMP effectively uses the number of threads requested by # the context manager - from ._openmp_test_helper import check_openmp_n_threads + from ._openmp_test_helper import check_openmp_num_threads - old_num_threads = check_openmp_n_threads(100) + old_num_threads = check_openmp_num_threads(100) - with threadpool_limits(limits=n_threads): - assert check_openmp_n_threads(100) in (n_threads, old_num_threads) - assert check_openmp_n_threads(100) == old_num_threads + with threadpool_limits(limits=num_threads): + assert check_openmp_num_threads(100) in (num_threads, old_num_threads) + assert check_openmp_num_threads(100) == old_num_threads def test_shipped_openblas(): diff --git a/threadpoolctl/tests/utils.py b/tests/utils.py similarity index 88% rename from threadpoolctl/tests/utils.py rename to tests/utils.py index aa6754b6..f5f8a758 100644 --- a/threadpoolctl/tests/utils.py +++ b/tests/utils.py @@ -41,15 +41,15 @@ def test_func(*args, **kwargs): # A decorator to run tests only when check_openmp_n_threads is available try: - from ._openmp_test_helper import check_openmp_n_threads # noqa: F401 + from ._openmp_test_helper import check_openmp_num_threads # noqa: F401 - def with_check_openmp_n_threads(func): + def with_check_openmp_num_threads(func): """A decorator to skip tests if check_openmp_n_threads is not compiled. """ return func except ImportError: - def with_check_openmp_n_threads(func): + def with_check_openmp_num_threads(func): """A decorator to skip tests if check_openmp_n_threads is not compiled. """ return skip_func('Test requires check_openmp_n_threads to be compiled') diff --git a/threadpoolctl/_threadpool_limiters.py b/threadpoolctl.py similarity index 84% rename from threadpoolctl/_threadpool_limiters.py rename to threadpoolctl.py index b20ecae8..d3417c2a 100644 --- a/threadpoolctl/_threadpool_limiters.py +++ b/threadpoolctl.py @@ -1,16 +1,23 @@ - -############################################################################# -# The following provides utilities to load C-libraries that relies on thread -# pools and limit the maximal number of thread that can be used. -# -# +"""threadpoolctl + +This module provides utilities to introspect native libraries that relies on +thread pools (notably BLAS and OpenMP implementations) and dynamically set the +maximal number of threads they can use. +""" +# License: BSD 3-Clause + +# The code to introspect dynamically loaded libraries on POSIX systems is +# adapted from code by Intel developper @anton-malakhov available at +# https://github.com/IntelPython/smp (Copyright (c) 2017, Intel Corporation) +# and also published under the BSD 3-Clause license import os import re import sys import ctypes from ctypes.util import find_library -from .utils import _format_docstring +__version__ = '1.0.0.dev0' +__all__ = ["threadpool_limits", "get_threadpool_limits"] # Cache for libc under POSIX and a few system libraries under Windows _system_libraries = {} @@ -31,23 +38,25 @@ # Structure to cast the info on dynamically loaded library. See # https://linux.die.net/man/3/dl_iterate_phdr for more details. -UINT_SYSTEM = ctypes.c_uint64 if sys.maxsize > 2**32 else ctypes.c_uint32 -UINT_HALF_SYSTEM = ctypes.c_uint32 if sys.maxsize > 2**32 else ctypes.c_uint16 + +_SYSTEM_UINT = ctypes.c_uint64 if sys.maxsize > 2**32 else ctypes.c_uint32 +_SYSTEM_UINT_HALF = ctypes.c_uint32 if sys.maxsize > 2**32 else ctypes.c_uint16 -class dl_phdr_info(ctypes.Structure): +class _dl_phdr_info(ctypes.Structure): _fields_ = [ - ("dlpi_addr", UINT_SYSTEM), # Base address of object + ("dlpi_addr", _SYSTEM_UINT), # Base address of object ("dlpi_name", ctypes.c_char_p), # path to the library ("dlpi_phdr", ctypes.c_void_p), # pointer on dlpi_headers - ("dlpi_phnum", UINT_HALF_SYSTEM) # number of element in dlpi_phdr + ("dlpi_phnum", _SYSTEM_UINT_HALF) # number of element in dlpi_phdr ] # List of the supported implementations. The items hold the prefix of loaded # shared objects, the name of the internal_api to call, matching the # MAP_API_TO_FUNC keys and the name of the user_api, in {"blas", "openmp"}. -SUPPORTED_IMPLEMENTATIONS = [ + +_SUPPORTED_IMPLEMENTATIONS = [ { "user_api": "openmp", "internal_api": "openmp", @@ -66,7 +75,8 @@ class dl_phdr_info(ctypes.Structure): ] # map a internal_api (openmp, openblas, mkl) to set and get functions -MAP_API_TO_FUNC = { + +_MAP_API_TO_FUNC = { "openmp": { "set_num_threads": "omp_set_num_threads", "get_num_threads": "omp_get_max_threads"}, @@ -79,10 +89,19 @@ class dl_phdr_info(ctypes.Structure): } # Helpers for the doc and test names -ALL_USER_APIS = set(impl['user_api'] for impl in SUPPORTED_IMPLEMENTATIONS) -ALL_PREFIXES = [prefix for impl in SUPPORTED_IMPLEMENTATIONS - for prefix in impl['filename_prefixes']] -ALL_INTERNAL_APIS = list(MAP_API_TO_FUNC.keys()) + +_ALL_USER_APIS = set(impl['user_api'] for impl in _SUPPORTED_IMPLEMENTATIONS) +_ALL_PREFIXES = [prefix for impl in _SUPPORTED_IMPLEMENTATIONS + for prefix in impl['filename_prefixes']] +_ALL_INTERNAL_APIS = list(_MAP_API_TO_FUNC.keys()) + + +def _format_docstring(*args, **kwargs): + def decorator(o): + o.__doc__ = o.__doc__.format(*args, **kwargs) + return o + + return decorator def _get_limit(prefix, user_api, limits): @@ -93,7 +112,8 @@ def _get_limit(prefix, user_api, limits): return None -@_format_docstring(ALL_PREFIXES=ALL_PREFIXES, INTERNAL_APIS=ALL_INTERNAL_APIS) +@_format_docstring(ALL_PREFIXES=_ALL_PREFIXES, + INTERNAL_APIS=_ALL_INTERNAL_APIS) def _set_threadpool_limits(limits=None, user_api=None, return_original_limits=False): """Limit the maximal number of threads for threadpools in supported libs @@ -126,8 +146,8 @@ def _set_threadpool_limits(limits=None, user_api=None, - 'internal_api': internal API.s Possible values are {INTERNAL_APIS}. - 'module_path': path to the loaded module. - 'version': version of the library implemented (if available). - - 'n_thread': current thread limit if return_original_limits is False or - the original limit if return_original_limits is True. + - 'num_threads': current thread limit if return_original_limits is False + or the original limit if return_original_limits is True. - 'set_num_threads': callable to set the maximum number of threads - 'get_num_threads': callable to get the current number of threads - 'dynlib': the instance of ctypes.CDLL use to access the dynamic @@ -135,19 +155,19 @@ def _set_threadpool_limits(limits=None, user_api=None, """ if isinstance(limits, int) or limits is None: if user_api is None: - user_api = ALL_USER_APIS - elif user_api in ALL_USER_APIS: + user_api = _ALL_USER_APIS + elif user_api in _ALL_USER_APIS: user_api = (user_api,) else: raise ValueError("user_api must be either in {} or None. Got {} " - "instead.".format(ALL_USER_APIS, user_api)) + "instead.".format(_ALL_USER_APIS, user_api)) limits = {api: limits for api in user_api} prefixes = [] else: if isinstance(limits, list): # This should be a list of module, for compatibility with # the result from get_threadpool_limits. - limits = {module['prefix']: module['n_thread'] + limits = {module['prefix']: module['num_threads'] for module in limits} if not isinstance(limits, dict): @@ -156,28 +176,29 @@ def _set_threadpool_limits(limits=None, user_api=None, # With a dictionary, can set both specific limit for given modules # and global limit for user_api. Fetch each separately. - prefixes = [module for module in limits if module in ALL_PREFIXES] - user_api = [module for module in limits if module in ALL_USER_APIS] + prefixes = [module for module in limits if module in _ALL_PREFIXES] + user_api = [module for module in limits if module in _ALL_USER_APIS] report_threadpool_size = [] modules = _load_modules(prefixes=prefixes, user_api=user_api) for module in modules: - n_thread = _get_limit(module['prefix'], module['user_api'], limits) + num_threads = _get_limit(module['prefix'], module['user_api'], limits) if return_original_limits: - module['n_thread'] = module['get_num_threads']() + module['num_threads'] = module['get_num_threads']() - if n_thread is not None: + if num_threads is not None: set_func = module['set_num_threads'] - set_func(n_thread) + set_func(num_threads) if not return_original_limits: - module['n_thread'] = module['get_num_threads']() + module['num_threads'] = module['get_num_threads']() report_threadpool_size.append(module) return report_threadpool_size -@_format_docstring(ALL_PREFIXES=ALL_PREFIXES, INTERNAL_APIS=ALL_INTERNAL_APIS) +@_format_docstring(ALL_PREFIXES=_ALL_PREFIXES, + INTERNAL_APIS=_ALL_INTERNAL_APIS) def get_threadpool_limits(): """Return the maximal number of threads for threadpools in supported C-lib. @@ -189,12 +210,12 @@ def get_threadpool_limits(): - 'internal_api': internal API. Possible values are {INTERNAL_APIS}. - 'module_path': path to the loaded module. - 'version': version of the library implemented (if available). - - 'n_thread': current thread limit. + - 'num_threads': current thread limit. """ report_threadpool_size = [] - modules = _load_modules(user_api=ALL_USER_APIS) + modules = _load_modules(user_api=_ALL_USER_APIS) for module in modules: - module['n_thread'] = module['get_num_threads']() + module['num_threads'] = module['get_num_threads']() # Remove the wrapper for the module and its function del module['set_num_threads'], module['get_num_threads'] del module['dynlib'] @@ -203,7 +224,7 @@ def get_threadpool_limits(): return report_threadpool_size -def get_version(dynlib, internal_api): +def _get_version(dynlib, internal_api): if internal_api == "mkl": return _get_mkl_version(dynlib) elif internal_api == "openmp": @@ -242,7 +263,6 @@ def _get_openblas_version(openblas_dynlib): return None -################################################################# # Loading utilities for dynamically linked shared objects def _load_modules(prefixes=None, user_api=None): @@ -283,21 +303,21 @@ def _make_module_info(module_path, module_info, prefix): dynlib = ctypes.CDLL(module_path) internal_api = module_info['internal_api'] set_func = getattr(dynlib, - MAP_API_TO_FUNC[internal_api]['set_num_threads'], - lambda n_thread: None) + _MAP_API_TO_FUNC[internal_api]['set_num_threads'], + lambda num_threads: None) get_func = getattr(dynlib, - MAP_API_TO_FUNC[internal_api]['get_num_threads'], + _MAP_API_TO_FUNC[internal_api]['get_num_threads'], lambda: None) module_info = module_info.copy() module_info.update(dynlib=dynlib, module_path=module_path, prefix=prefix, set_num_threads=set_func, get_num_threads=get_func, - version=get_version(dynlib, internal_api)) + version=_get_version(dynlib, internal_api)) return module_info def _get_module_info_from_path(module_path, prefixes, user_api, modules): module_name = os.path.basename(module_path).lower() - for info in SUPPORTED_IMPLEMENTATIONS: + for info in _SUPPORTED_IMPLEMENTATIONS: prefix = _check_prefix(module_name, info['filename_prefixes']) if _match_module(info, prefix, prefixes, user_api): modules.append(_make_module_info(module_path, info, prefix)) @@ -335,10 +355,10 @@ def match_module_callback(info, size, data): c_func_signature = ctypes.CFUNCTYPE( ctypes.c_int, # Return type - ctypes.POINTER(dl_phdr_info), ctypes.c_size_t, ctypes.c_char_p) + ctypes.POINTER(_dl_phdr_info), ctypes.c_size_t, ctypes.c_char_p) c_match_module_callback = c_func_signature(match_module_callback) - data = ctypes.c_char_p(''.encode('utf-8')) + data = ctypes.c_char_p(b'') libc.dl_iterate_phdr(c_match_module_callback, data) return _modules @@ -498,4 +518,4 @@ def __exit__(self, type, value, traceback): def unregister(self): if self.original_limits is not None: for module in self.original_limits: - module['set_num_threads'](module['n_thread']) + module['set_num_threads'](module['num_threads']) diff --git a/threadpoolctl/__init__.py b/threadpoolctl/__init__.py deleted file mode 100644 index cae49eb1..00000000 --- a/threadpoolctl/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from ._threadpool_limiters import threadpool_limits -from ._threadpool_limiters import get_threadpool_limits - -__version__ = '0.1.dev0' - - -__all__ = ["threadpool_limits", "get_threadpool_limits"] diff --git a/threadpoolctl/utils.py b/threadpoolctl/utils.py deleted file mode 100644 index 6f85db32..00000000 --- a/threadpoolctl/utils.py +++ /dev/null @@ -1,8 +0,0 @@ - - -def _format_docstring(*args, **kwargs): - def decorator(o): - o.__doc__ = o.__doc__.format(*args, **kwargs) - return o - - return decorator From 949af6b3d0d4921258cca9c26fa9b4d18010bb84 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Fri, 29 Mar 2019 17:37:29 +0100 Subject: [PATCH 009/187] Use flit to build the project and generate the wheel --- MANIFEST.in | 4 - README.md | 27 ++++++ continuous_integration/install.sh | 5 +- dev-requirements.txt | 4 + pyproject.toml | 9 ++ setup.cfg | 5 ++ setup.py | 89 ------------------- .../tests/_openmp_test_helper/.gitignore | 3 - 8 files changed, 47 insertions(+), 99 deletions(-) delete mode 100644 MANIFEST.in create mode 100644 dev-requirements.txt create mode 100644 pyproject.toml create mode 100644 setup.cfg delete mode 100644 setup.py delete mode 100644 threadpoolctl/tests/_openmp_test_helper/.gitignore diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 86cfc8af..00000000 --- a/MANIFEST.in +++ /dev/null @@ -1,4 +0,0 @@ -include *.md *.py *.cfg *.ini -include LICENSE -recursive-include tests *.py *.pyx -global-exclude *~ *.swp diff --git a/README.md b/README.md index e54e3755..8869dd42 100644 --- a/README.md +++ b/README.md @@ -7,10 +7,19 @@ Python helpers to limit the number of threads used in threadpool-backed parallel # Installation +To install the last published version from PyPI: + ``` pip install threadpoolctl ``` +or to install from the source repository in developer mode: + +``` +pip install flit +flit install --symlink +``` + # Usage ```python @@ -23,3 +32,21 @@ with threadpool_limits(limits=1, user_api='blas'): # with thread-parallelism or openmp calls. ... ``` + +# Maintainers + +To make a release: + +``` +pip install flit +flit build +``` + +Check the contents of `dist/`. + +If everything is fine, make a commit for the release, tag it, push the +tag to github and then: + +``` +flit publish +``` diff --git a/continuous_integration/install.sh b/continuous_integration/install.sh index 21980b5e..747a97f0 100755 --- a/continuous_integration/install.sh +++ b/continuous_integration/install.sh @@ -40,12 +40,11 @@ elif [[ "$PACKAGER" == "ubuntu" ]]; then fi -python -m pip install coverage - bash ./continuous_integration/build_test_ext.sh python --version python -c "import numpy; print('numpy %s' % numpy.__version__)" || echo "no numpy" python -c "import scipy; print('scipy %s' % scipy.__version__)" || echo "no scipy" -pip install -e . +python -m pip install -r dev-requirements.txt +python -m flit install --symlink diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 00000000..0218af3f --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1,4 @@ +flit +coverage +pytest +cython diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..f5ca0b86 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,9 @@ +[build-system] +requires = ["flit"] +build-backend = "flit.buildapi" + +[tool.flit.metadata] +module = "threadpoolctl" +author = "Thomas Moreau" +author-email = "thomas.moreau.2010@gmail.com" +home-page = "https://github.com/joblib/threadpoolctl" \ No newline at end of file diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..055ac83d --- /dev/null +++ b/setup.cfg @@ -0,0 +1,5 @@ +[metadata] +license_file = LICENSE + +[wheel] +universal=1 diff --git a/setup.py b/setup.py deleted file mode 100644 index b6c4f5ca..00000000 --- a/setup.py +++ /dev/null @@ -1,89 +0,0 @@ -import os -import re -import shutil -from setuptools import setup, find_packages -from distutils.command.clean import clean as Clean - -packages = find_packages( - exclude=['threadpoolctl.tests', 'threadpoolctl.tests._openmp_test_helper']) - - -# Function to parse __version__ in `threadpoolctl` -def find_version(): - here = os.path.abspath(os.path.dirname(__file__)) - with open(os.path.join(here, 'threadpoolctl.py'), 'r') as fp: - version_file = fp.read() - version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", - version_file, re.M) - if version_match: - return version_match.group(1) - raise RuntimeError("Unable to find version string.") - - -# Custom clean command to remove build artifacts -class CleanCommand(Clean): - description = "Remove build artifacts from the source tree" - - def run(self): - Clean.run(self) - # Remove c files if we are not within a sdist package - cwd = os.path.abspath(os.path.dirname(__file__)) - remove_c_files = not os.path.exists(os.path.join(cwd, 'PKG-INFO')) - if remove_c_files: - print('Will remove generated .c files') - if os.path.exists('build'): - shutil.rmtree('build') - for dirpath, dirnames, filenames in os.walk('.'): - for filename in filenames: - if any(filename.endswith(suffix) for suffix in - (".so", ".pyd", ".dll", ".pyc")): - os.unlink(os.path.join(dirpath, filename)) - continue - extension = os.path.splitext(filename)[1] - if remove_c_files and extension in ['.c', '.cpp']: - pyx_file = str.replace(filename, extension, '.pyx') - if os.path.exists(os.path.join(dirpath, pyx_file)): - os.unlink(os.path.join(dirpath, filename)) - for dirname in dirnames: - if dirname == '__pycache__': - shutil.rmtree(os.path.join(dirpath, dirname)) - - -cmdclass = {'clean': CleanCommand} - -setup( - name='threadpoolctl', - version=find_version(), - description=("Python helpers to limit the number of threads used in " - "native libraries with thread-pools."), - long_description=open('README.md', 'rb').read().decode('utf-8'), - long_description_content_type='text/markdown', - url='https://github.com/joblib/threadpoolctl/', - author='Thomas Moreau', - author_email='thomas.moreau.2010@gmail.com', - packages=packages, - py_modules=["threadpoolctl"], - zip_safe=False, - license='BSD', - classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Environment :: Console', - 'Intended Audience :: Developers', - 'Intended Audience :: Science/Research', - 'License :: OSI Approved :: BSD License', - 'Operating System :: OS Independent', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Topic :: Scientific/Engineering', - 'Topic :: Utilities', - 'Topic :: Software Development :: Libraries', - ], - cmdclass=cmdclass, - platforms='any', - tests_require=['pytest'], -) diff --git a/threadpoolctl/tests/_openmp_test_helper/.gitignore b/threadpoolctl/tests/_openmp_test_helper/.gitignore deleted file mode 100644 index 2e5170f4..00000000 --- a/threadpoolctl/tests/_openmp_test_helper/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -*.c -*.so -/build From 23b998f8baf0db0a6bd5297d5d716d4e36606891 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Fri, 29 Mar 2019 18:16:30 +0100 Subject: [PATCH 010/187] Fix and improve README.md --- README.md | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 8869dd42..c03cced6 100644 --- a/README.md +++ b/README.md @@ -1,39 +1,51 @@ # Thread-pool Controls [![Build Status](https://dev.azure.com/karementmoi/loky/_apis/build/status/joblib.threadpoolctl?branchName=master)](https://dev.azure.com/karementmoi/loky/_build/latest?definitionId=2&branchName=master) [![codecov](https://codecov.io/gh/joblib/threadpoolctl/branch/master/graph/badge.svg)](https://codecov.io/gh/joblib/threadpoolctl) +Python helpers to limit the number of threads used in the +threadpool-backed of common native libraries used for scientific +computing and data science (e.g. BLAS and OpenMP). +Fine control of the underlying thread-pool size can be useful in +workloads that involve nested parallelism so as to mitigate +oversubscription issues. -Python helpers to limit the number of threads used in threadpool-backed parallelism for C-libraries. +## Installation - -# Installation - -To install the last published version from PyPI: +- For users, you can install the last published version from PyPI: ``` pip install threadpoolctl ``` -or to install from the source repository in developer mode: +- For contributors, you can install from the source repository in + developer mode: ``` -pip install flit +pip install -r dev-requirements.txt flit install --symlink ``` -# Usage +then you can run the tests with pytest: + +``` +pytest +``` + +## Usage ```python from threadpoolctl import threadpool_limits +import numpy as np with threadpool_limits(limits=1, user_api='blas'): # In this block, calls to blas implementation (like openblas or MKL) # will be limited to use only one thread. They can thus be used jointly - # with thread-parallelism or openmp calls. - ... + # with thread-parallelism. + a = np.random.randn(1000, 1000) + a_squared = a @ a ``` -# Maintainers +## Maintainers To make a release: From 4308b51080e884211704bb9dc875c384b0f4e325 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Fri, 29 Mar 2019 18:17:34 +0100 Subject: [PATCH 011/187] Remove useless file --- setup.cfg | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 055ac83d..00000000 --- a/setup.cfg +++ /dev/null @@ -1,5 +0,0 @@ -[metadata] -license_file = LICENSE - -[wheel] -universal=1 From 26810c2bfe256c77920be2bd06824cbb6d2c3ffc Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Fri, 29 Mar 2019 18:18:51 +0100 Subject: [PATCH 012/187] cosmit --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f5ca0b86..1a9ee5ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,4 +6,4 @@ build-backend = "flit.buildapi" module = "threadpoolctl" author = "Thomas Moreau" author-email = "thomas.moreau.2010@gmail.com" -home-page = "https://github.com/joblib/threadpoolctl" \ No newline at end of file +home-page = "https://github.com/joblib/threadpoolctl" From fecd893c1bceb3658e5d7ecf96d7310bc6df7d72 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Sat, 30 Mar 2019 18:56:15 +0100 Subject: [PATCH 013/187] Refactoring to introduce threadpool_info --- tests/_openmp_test_helper/__init__.py | 4 +- tests/test_threadpoolctl.py | 168 ++++++++++---------------- threadpoolctl.py | 77 ++++++------ 3 files changed, 100 insertions(+), 149 deletions(-) diff --git a/tests/_openmp_test_helper/__init__.py b/tests/_openmp_test_helper/__init__.py index 4a436e9e..247bed99 100644 --- a/tests/_openmp_test_helper/__init__.py +++ b/tests/_openmp_test_helper/__init__.py @@ -1,4 +1,4 @@ -from .openmp_helpers import check_openmp_n_threads +from .openmp_helpers import check_openmp_num_threads -__all__ = ["check_openmp_n_threads"] +__all__ = ["check_openmp_num_threads"] diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index c87350a0..b4e7cb71 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -3,9 +3,7 @@ import pytest -from threadpoolctl import threadpool_limits -from threadpoolctl import get_threadpool_limits -from threadpoolctl import _set_threadpool_limits +from threadpoolctl import threadpool_limits, threadpool_info from threadpoolctl import _ALL_PREFIXES, _ALL_USER_APIS from .utils import with_check_openmp_num_threads, libopenblas_paths @@ -18,83 +16,83 @@ def should_skip_module(module): @pytest.mark.parametrize("prefix", _ALL_PREFIXES) -def test_threadpool_limits(openblas_present, mkl_present, prefix): - old_limits = get_threadpool_limits() - - prefix_found = len([1 for module in old_limits - if prefix == module['prefix']]) - old_limits = {dynlib['prefix']: dynlib['num_threads'] - for dynlib in old_limits} - +def test_threadpool_limits_by_prefix(openblas_present, mkl_present, prefix): + original_infos = threadpool_info() + original_num_threads = {info["filepath"]: info['num_threads'] + for info in original_infos} + found_mkl = any([True for info in original_infos + if info["prefix"] in ('mkl_rt', 'libmkl_rt')]) + prefix_found = len([info["prefix"] for info in original_infos]) if not prefix_found: - have_mkl = len({'mkl_rt', 'libmkl_rt'}.intersection(old_limits)) > 0 - if "mkl_rt" in prefix and mkl_present and not have_mkl: + if "mkl_rt" in prefix and mkl_present and not found_mkl: raise RuntimeError("Could not load the MKL prefix") elif prefix == "libopenblas" and openblas_present: raise RuntimeError("Could not load the OpenBLAS prefix") else: pytest.skip("Need {} support".format(prefix)) - try: - new_limits = _set_threadpool_limits(limits={prefix: 1}) - new_limits = {dynlib['prefix']: dynlib['num_threads'] - for dynlib in new_limits} - assert new_limits[prefix] == 1 + with threadpool_limits(limits={prefix: 1}): + for module in threadpool_info(): + if should_skip_module(module): + continue + num_threads, filepath = module["num_threads"], module["filepath"] + if module["prefix"] == prefix: + assert num_threads == 1 + else: + assert num_threads == original_num_threads[filepath] - threadpool_limits(limits={prefix: 3}) - new_limits = get_threadpool_limits() - new_limits = {dynlib['prefix']: dynlib['num_threads'] - for dynlib in new_limits} - assert new_limits[prefix] in (3, old_limits[prefix]) - finally: - # Avoid having side effects in case of failures - threadpool_limits(limits=old_limits) + with threadpool_limits(limits={prefix: 3}): + for module in threadpool_info(): + if should_skip_module(module): + continue + num_threads, filepath = module["num_threads"], module["filepath"] + if module["prefix"] == prefix: + expected_num_threads = min(3, original_num_threads[filepath]) + assert num_threads == expected_num_threads + else: + assert num_threads == original_num_threads[filepath] - new_limits = get_threadpool_limits() - new_limits = {dynlib['prefix']: dynlib['num_threads'] - for dynlib in new_limits} - assert new_limits[prefix] == old_limits[prefix] + assert threadpool_info() == original_infos @pytest.mark.parametrize("user_api", (None, "blas", "openmp")) -def test_set_threadpool_limits_apis(user_api): +def test_set_threadpool_limits_by_api(user_api): # Check that the number of threads used by the multithreaded libraries can # be modified dynamically. if user_api is None: - api_modules = ('blas', 'openmp') + user_apis = ("blas", "openmp") else: - api_modules = (user_api,) + user_apis = (user_api,) - old_limits = get_threadpool_limits() - old_limits = {dynlib['prefix']: dynlib['num_threads'] - for dynlib in old_limits} + original_infos = threadpool_info() + original_num_threads = {info["filepath"]: info['num_threads'] + for info in original_infos} - try: - new_limits = _set_threadpool_limits(limits=1, user_api=user_api) - for module in new_limits: + with threadpool_limits(limits=1, user_api=user_api): + for module in threadpool_info(): if should_skip_module(module): continue - if module['user_api'] in api_modules: - assert module['num_threads'] == 1 + num_threads, filepath = module["num_threads"], module["filepath"] + if module["user_api"] in user_apis: + assert num_threads == 1 + else: + assert num_threads == original_num_threads[filepath] - threadpool_limits(limits=3, user_api=user_api) - new_limits = get_threadpool_limits() - for module in new_limits: + with threadpool_limits(limits=3, user_api=user_api): + for module in threadpool_info(): if should_skip_module(module): continue - if module['user_api'] in api_modules: - assert module['num_threads'] in ( - 3, old_limits[module['prefix']]) - finally: - # Avoid having side effects on other tests in case of failure - threadpool_limits(limits=old_limits) + num_threads, filepath = module["num_threads"], module["filepath"] + if module["user_api"] in user_apis: + expected_num_threads = min(3, original_num_threads[filepath]) + assert num_threads == expected_num_threads + else: + assert num_threads == original_num_threads[filepath] - new_limits = get_threadpool_limits() - for module in new_limits: - assert module['num_threads'] == old_limits[module['prefix']] + assert threadpool_info() == original_infos -def test_set_threadpool_limits_bad_input(): +def test_threadpool_limits_bad_input(): # Check that appropriate errors are raised for invalid arguments match = re.escape("user_api must be either in {} or None." .format(_ALL_USER_APIS)) @@ -106,40 +104,6 @@ def test_set_threadpool_limits_bad_input(): threadpool_limits(limits=(1, 2, 3)) -@pytest.mark.parametrize("user_api", (None, "blas", "openmp")) -def test_thread_limit_context(user_api): - # Tests the thread limits context manager - if user_api is None: - apis = ('blas', 'openmp') - else: - apis = (user_api,) - - old_limits = get_threadpool_limits() - old_limits = {dynlib['prefix']: dynlib['num_threads'] - for dynlib in old_limits} - - with threadpool_limits(limits=None, user_api=user_api): - limits = get_threadpool_limits() - limits = {dynlib['prefix']: dynlib['num_threads'] - for dynlib in limits} - assert limits == old_limits - - with threadpool_limits(limits=1, user_api=user_api): - limits = get_threadpool_limits() - - for module in limits: - if should_skip_module(module): - continue - elif module['user_api'] in apis: - assert module['num_threads'] == 1 - else: - assert module['num_threads'] == old_limits[module['prefix']] - - limits = get_threadpool_limits() - limits = {dynlib['prefix']: dynlib['num_threads'] for dynlib in limits} - assert limits == old_limits - - @with_check_openmp_num_threads @pytest.mark.parametrize('num_threads', [1, 2, 4]) def test_openmp_limit_num_threads(num_threads): @@ -155,28 +119,22 @@ def test_openmp_limit_num_threads(num_threads): def test_shipped_openblas(): - libopenblas = [ctypes.CDLL(path) for path in libopenblas_paths] - - old_limits = [blas.openblas_get_num_threads() for blas in libopenblas] + all_openblases = [ctypes.CDLL(path) for path in libopenblas_paths] + original_num_threads = [blas.openblas_get_num_threads() + for blas in all_openblases] with threadpool_limits(1): - assert all([blas.openblas_get_num_threads() == 1 - for blas in libopenblas]) + for openblas in all_openblases: + assert openblas.openblas_get_num_threads() == 1 - assert all([blas.openblas_get_num_threads() == l - for blas, l in zip(libopenblas, old_limits)]) + assert original_num_threads == [openblas.openblas_get_num_threads() + for openblas in all_openblases] @pytest.mark.skipif(len(libopenblas_paths) < 2, reason="need at least 2 shipped openblas library") def test_multiple_shipped_openblas(): - libopenblas = [ctypes.CDLL(path) for path in libopenblas_paths] - - old_limits = [blas.openblas_get_num_threads() for blas in libopenblas] - - with threadpool_limits(1): - assert all([blas.openblas_get_num_threads() == 1 - for blas in libopenblas]) - - assert all([blas.openblas_get_num_threads() == l - for blas, l in zip(libopenblas, old_limits)]) + # This redundant test is meant to make it easier to see if the system + # has 2 or more active openblas runtimes available just be reading the + # pytest report (whether or not this test has been skipped). + test_shipped_openblas() diff --git a/threadpoolctl.py b/threadpoolctl.py index d3417c2a..332e187f 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -17,7 +17,7 @@ from ctypes.util import find_library __version__ = '1.0.0.dev0' -__all__ = ["threadpool_limits", "get_threadpool_limits"] +__all__ = ["threadpool_limits", "threadpool_info"] # Cache for libc under POSIX and a few system libraries under Windows _system_libraries = {} @@ -91,7 +91,8 @@ class _dl_phdr_info(ctypes.Structure): # Helpers for the doc and test names _ALL_USER_APIS = set(impl['user_api'] for impl in _SUPPORTED_IMPLEMENTATIONS) -_ALL_PREFIXES = [prefix for impl in _SUPPORTED_IMPLEMENTATIONS +_ALL_PREFIXES = [prefix + for impl in _SUPPORTED_IMPLEMENTATIONS for prefix in impl['filename_prefixes']] _ALL_INTERNAL_APIS = list(_MAP_API_TO_FUNC.keys()) @@ -144,7 +145,7 @@ def _set_threadpool_limits(limits=None, user_api=None, Possible values are {ALL_PREFIXES}. - 'prefix' : prefix of the specific implementation of this module. - 'internal_api': internal API.s Possible values are {INTERNAL_APIS}. - - 'module_path': path to the loaded module. + - 'filepath': path to the loaded module. - 'version': version of the library implemented (if available). - 'num_threads': current thread limit if return_original_limits is False or the original limit if return_original_limits is True. @@ -179,7 +180,6 @@ def _set_threadpool_limits(limits=None, user_api=None, prefixes = [module for module in limits if module in _ALL_PREFIXES] user_api = [module for module in limits if module in _ALL_USER_APIS] - report_threadpool_size = [] modules = _load_modules(prefixes=prefixes, user_api=user_api) for module in modules: num_threads = _get_limit(module['prefix'], module['user_api'], limits) @@ -192,36 +192,31 @@ def _set_threadpool_limits(limits=None, user_api=None, if not return_original_limits: module['num_threads'] = module['get_num_threads']() - report_threadpool_size.append(module) + return modules - return report_threadpool_size - -@_format_docstring(ALL_PREFIXES=_ALL_PREFIXES, - INTERNAL_APIS=_ALL_INTERNAL_APIS) -def get_threadpool_limits(): - """Return the maximal number of threads for threadpools in supported C-lib. +@_format_docstring(INTERNAL_APIS=_ALL_INTERNAL_APIS) +def threadpool_info(): + """Return the maximal number of threads for each detected library. Return a list with all the supported modules that have been found. Each module is represented by a dict with the following information: - - 'filename-prefixes' : possible prefixes for the given internal_api. - Possible values are {ALL_PREFIXES}. - - 'prefix' : prefix of the specific implementation of this module. + - 'prefix' : filename prefix of the specific implementation. + - 'filepath': path to the loaded module. - 'internal_api': internal API. Possible values are {INTERNAL_APIS}. - - 'module_path': path to the loaded module. - 'version': version of the library implemented (if available). - - 'num_threads': current thread limit. + - 'num_threads': the current thread limit. """ - report_threadpool_size = [] + infos = [] modules = _load_modules(user_api=_ALL_USER_APIS) for module in modules: module['num_threads'] = module['get_num_threads']() # Remove the wrapper for the module and its function del module['set_num_threads'], module['get_num_threads'] del module['dynlib'] - report_threadpool_size.append(module) - - return report_threadpool_size + del module['filename_prefixes'] + infos.append(module) + return infos def _get_version(dynlib, internal_api): @@ -272,8 +267,7 @@ def _load_modules(prefixes=None, user_api=None): if user_api is None: user_api = [] if sys.platform == "darwin": - return _find_modules_with_dyld( - prefixes=prefixes, user_api=user_api) + return _find_modules_with_dyld(prefixes=prefixes, user_api=user_api) elif sys.platform == "win32": return _find_modules_with_enum_process_module_ex( prefixes=prefixes, user_api=user_api) @@ -297,10 +291,10 @@ def _match_module(module_info, prefix, prefixes, user_api): module_info['user_api'] in user_api) -def _make_module_info(module_path, module_info, prefix): +def _make_module_info(filepath, module_info, prefix): """Make a dict with the information from the module.""" - module_path = os.path.normpath(module_path) - dynlib = ctypes.CDLL(module_path) + filepath = os.path.normpath(filepath) + dynlib = ctypes.CDLL(filepath) internal_api = module_info['internal_api'] set_func = getattr(dynlib, _MAP_API_TO_FUNC[internal_api]['set_num_threads'], @@ -309,18 +303,18 @@ def _make_module_info(module_path, module_info, prefix): _MAP_API_TO_FUNC[internal_api]['get_num_threads'], lambda: None) module_info = module_info.copy() - module_info.update(dynlib=dynlib, module_path=module_path, prefix=prefix, + module_info.update(dynlib=dynlib, filepath=filepath, prefix=prefix, set_num_threads=set_func, get_num_threads=get_func, version=_get_version(dynlib, internal_api)) return module_info -def _get_module_info_from_path(module_path, prefixes, user_api, modules): - module_name = os.path.basename(module_path).lower() +def _get_module_info_from_path(filepath, prefixes, user_api, modules): + filename = os.path.basename(filepath) for info in _SUPPORTED_IMPLEMENTATIONS: - prefix = _check_prefix(module_name, info['filename_prefixes']) + prefix = _check_prefix(filename, info['filename_prefixes']) if _match_module(info, prefix, prefixes, user_api): - modules.append(_make_module_info(module_path, info, prefix)) + modules.append(_make_module_info(filepath, info, prefix)) def _find_modules_with_dl_iterate_phdr(prefixes, user_api): @@ -343,13 +337,13 @@ def _find_modules_with_dl_iterate_phdr(prefixes, user_api): # module loaded in the current process until it returns 1. def match_module_callback(info, size, data): # Get the path of the current module - module_path = info.contents.dlpi_name - if module_path: - module_path = module_path.decode("utf-8") + filepath = info.contents.dlpi_name + if filepath: + filepath = filepath.decode("utf-8") # Store the module in cls_thread_locals._module if it is # supported and selected - _get_module_info_from_path(module_path, prefixes, user_api, + _get_module_info_from_path(filepath, prefixes, user_api, _modules) return 0 @@ -379,13 +373,12 @@ def _find_modules_with_dyld(prefixes, user_api): libc._dyld_get_image_name.restype = ctypes.c_char_p for i in range(n_dyld): - module_path = ctypes.string_at(libc._dyld_get_image_name(i)) - module_path = module_path.decode("utf-8") + filepath = ctypes.string_at(libc._dyld_get_image_name(i)) + filepath = filepath.decode("utf-8") - # Store the module in cls_thread_locals._module if it is - # supported and selected - _get_module_info_from_path(module_path, prefixes, user_api, - _modules) + # Store the module in cls_thread_locals._module if it is supported and + # selected + _get_module_info_from_path(filepath, prefixes, user_api, _modules) return _modules @@ -443,11 +436,11 @@ def _find_modules_with_enum_process_module_ex(prefixes, user_api): h_process, h_module, ctypes.byref(buf), ctypes.byref(n_size)): raise OSError('GetModuleFileNameEx failed') - module_path = buf.value + filepath = buf.value # Store the module in cls_thread_locals._module if it is # supported and selected - _get_module_info_from_path(module_path, prefixes, user_api, + _get_module_info_from_path(filepath, prefixes, user_api, _modules) finally: kernel_32.CloseHandle(h_process) From a82b414b2948711c4bb89a6d8d5d0bcd4678f0ff Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Sat, 30 Mar 2019 19:01:01 +0100 Subject: [PATCH 014/187] Fix handling of num_threads > 1 in tests --- tests/test_threadpoolctl.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index b4e7cb71..c9c80377 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -47,8 +47,8 @@ def test_threadpool_limits_by_prefix(openblas_present, mkl_present, prefix): continue num_threads, filepath = module["num_threads"], module["filepath"] if module["prefix"] == prefix: - expected_num_threads = min(3, original_num_threads[filepath]) - assert num_threads == expected_num_threads + expected_num_threads = (3, original_num_threads[filepath]) + assert num_threads in expected_num_threads else: assert num_threads == original_num_threads[filepath] @@ -84,8 +84,8 @@ def test_set_threadpool_limits_by_api(user_api): continue num_threads, filepath = module["num_threads"], module["filepath"] if module["user_api"] in user_apis: - expected_num_threads = min(3, original_num_threads[filepath]) - assert num_threads == expected_num_threads + expected_num_threads = (3, original_num_threads[filepath]) + assert num_threads in expected_num_threads else: assert num_threads == original_num_threads[filepath] From 20046d58641666b002928588133a30a8993fc92e Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Sun, 31 Mar 2019 03:54:05 +0200 Subject: [PATCH 015/187] Special case interaction between libiomp and MKL --- tests/test_threadpoolctl.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index c9c80377..b42d5465 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -20,11 +20,11 @@ def test_threadpool_limits_by_prefix(openblas_present, mkl_present, prefix): original_infos = threadpool_info() original_num_threads = {info["filepath"]: info['num_threads'] for info in original_infos} - found_mkl = any([True for info in original_infos + mkl_found = any([True for info in original_infos if info["prefix"] in ('mkl_rt', 'libmkl_rt')]) prefix_found = len([info["prefix"] for info in original_infos]) if not prefix_found: - if "mkl_rt" in prefix and mkl_present and not found_mkl: + if "mkl_rt" in prefix and mkl_present and not mkl_found: raise RuntimeError("Could not load the MKL prefix") elif prefix == "libopenblas" and openblas_present: raise RuntimeError("Could not load the OpenBLAS prefix") @@ -38,6 +38,10 @@ def test_threadpool_limits_by_prefix(openblas_present, mkl_present, prefix): num_threads, filepath = module["num_threads"], module["filepath"] if module["prefix"] == prefix: assert num_threads == 1 + elif "mkl_rt" in module["prefix"] and prefix == "libiomp": + # MKL is impacted by a change in the thread pool of Intel + # implementation of the OpenMP runtime: + assert num_threads == 1 else: assert num_threads == original_num_threads[filepath] @@ -46,8 +50,12 @@ def test_threadpool_limits_by_prefix(openblas_present, mkl_present, prefix): if should_skip_module(module): continue num_threads, filepath = module["num_threads"], module["filepath"] + expected_num_threads = (3, original_num_threads[filepath]) if module["prefix"] == prefix: - expected_num_threads = (3, original_num_threads[filepath]) + assert num_threads in expected_num_threads + elif "mkl_rt" in module["prefix"] and prefix == "libiomp": + # MKL is impacted by a change in the thread pool of Intel + # implementation of the OpenMP runtime: assert num_threads in expected_num_threads else: assert num_threads == original_num_threads[filepath] @@ -83,8 +91,8 @@ def test_set_threadpool_limits_by_api(user_api): if should_skip_module(module): continue num_threads, filepath = module["num_threads"], module["filepath"] + expected_num_threads = (3, original_num_threads[filepath]) if module["user_api"] in user_apis: - expected_num_threads = (3, original_num_threads[filepath]) assert num_threads in expected_num_threads else: assert num_threads == original_num_threads[filepath] From ccf7e4cb2d7d4fe59bdc841506fee452da757479 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Sun, 31 Mar 2019 13:46:56 +0200 Subject: [PATCH 016/187] Fix prefix filtering logic in test --- tests/test_threadpoolctl.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index b42d5465..127890b6 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -22,7 +22,8 @@ def test_threadpool_limits_by_prefix(openblas_present, mkl_present, prefix): for info in original_infos} mkl_found = any([True for info in original_infos if info["prefix"] in ('mkl_rt', 'libmkl_rt')]) - prefix_found = len([info["prefix"] for info in original_infos]) + prefix_found = len([info["prefix"] for info in original_infos + if info["prefix"] == prefix]) if not prefix_found: if "mkl_rt" in prefix and mkl_present and not mkl_found: raise RuntimeError("Could not load the MKL prefix") From d56f64f645d049fd0af591b46bd34e1faf355b37 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Sun, 31 Mar 2019 13:54:28 +0200 Subject: [PATCH 017/187] More informative readme --- README.md | 49 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c03cced6..74570aea 100644 --- a/README.md +++ b/README.md @@ -12,26 +12,69 @@ oversubscription issues. - For users, you can install the last published version from PyPI: -``` +```bash pip install threadpoolctl ``` - For contributors, you can install from the source repository in developer mode: -``` +```bash pip install -r dev-requirements.txt flit install --symlink ``` then you can run the tests with pytest: -``` +```bash pytest ``` ## Usage +- Introspect the current state of the threadpool-enabled runtime + libraries that are loaded when importing Python packages: + +```python +>>> from threadpoolctl import threadpool_info +>>> from pprint import pprint +>>> pprint(threadpool_info()) +[] + +>>> import numpy +>>> pprint(threadpool_info()) +[{'filepath': '/opt/venvs/py37/lib/python3.7/site-packages/numpy/.libs/libopenblasp-r0-382c8f3a.3.5.dev.so', + 'internal_api': 'openblas', + 'num_threads': 4, + 'prefix': 'libopenblas', + 'user_api': 'blas', + 'version': '0.3.5.dev'}] + +>>> import xgboost +>>> pprint(threadpool_info()) +[{'filepath': '/opt/venvs/py37/lib/python3.7/site-packages/numpy/.libs/libopenblasp-r0-382c8f3a.3.5.dev.so', + 'internal_api': 'openblas', + 'num_threads': 4, + 'prefix': 'libopenblas', + 'user_api': 'blas', + 'version': '0.3.5.dev'}, + {'filepath': '/opt/venvs/py37/lib/python3.7/site-packages/scipy/.libs/libopenblasp-r0-8dca6697.3.0.dev.so', + 'internal_api': 'openblas', + 'num_threads': 4, + 'prefix': 'libopenblas', + 'user_api': 'blas', + 'version': None}, + {'filepath': '/usr/lib/x86_64-linux-gnu/libgomp.so.1', + 'internal_api': 'openmp', + 'num_threads': 4, + 'prefix': 'libgomp', + 'user_api': 'openmp', + 'version': None}] +``` + +- Control the number of threads used by the underlying runtime libraries + in specific sections of your Python program: + ```python from threadpoolctl import threadpool_limits import numpy as np From 5ec2a7166d7e30ef25dcefb861ce5ce20301db66 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Sun, 31 Mar 2019 20:37:37 +0200 Subject: [PATCH 018/187] Add test for threadpool_limits used as a function with side effects --- tests/test_threadpoolctl.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index 127890b6..50f79aee 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -101,6 +101,26 @@ def test_set_threadpool_limits_by_api(user_api): assert threadpool_info() == original_infos +def test_threadpool_limits_function_with_side_effect(): + # Check that threadpool_limits can be used as a function with + # side effects instead of a context manager. + original_infos = threadpool_info() + + threadpool_limits(limits=1) + try: + for module in threadpool_info(): + if should_skip_module(module): + continue + assert module["num_threads"] == 1 + finally: + # Restore the original limits so that this test does not have any + # side-effect. + threadpool_limits(limits={info["prefix"]: info["num_threads"] + for info in original_infos}) + + assert threadpool_info() == original_infos + + def test_threadpool_limits_bad_input(): # Check that appropriate errors are raised for invalid arguments match = re.escape("user_api must be either in {} or None." From e05ae8b1414c93ea53a50a3a28c8900ac5eba2ed Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Sun, 31 Mar 2019 21:25:05 +0200 Subject: [PATCH 019/187] Various small private API improvements to simplify code and increase coverage --- tests/test_threadpoolctl.py | 5 ++--- threadpoolctl.py | 21 ++++++++++----------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index 50f79aee..f577eaee 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -115,8 +115,7 @@ def test_threadpool_limits_function_with_side_effect(): finally: # Restore the original limits so that this test does not have any # side-effect. - threadpool_limits(limits={info["prefix"]: info["num_threads"] - for info in original_infos}) + threadpool_limits(limits=original_infos) assert threadpool_info() == original_infos @@ -129,7 +128,7 @@ def test_threadpool_limits_bad_input(): threadpool_limits(limits=1, user_api="wrong") with pytest.raises(TypeError, - match="limits must either be an int, a dict or None"): + match="limits must either be an int, a list or a dict"): threadpool_limits(limits=(1, 2, 3)) diff --git a/threadpoolctl.py b/threadpoolctl.py index 332e187f..02c776f3 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -115,7 +115,7 @@ def _get_limit(prefix, user_api, limits): @_format_docstring(ALL_PREFIXES=_ALL_PREFIXES, INTERNAL_APIS=_ALL_INTERNAL_APIS) -def _set_threadpool_limits(limits=None, user_api=None, +def _set_threadpool_limits(limits, user_api=None, return_original_limits=False): """Limit the maximal number of threads for threadpools in supported libs @@ -129,8 +129,7 @@ def _set_threadpool_limits(limits=None, user_api=None, integer, sets the maximum number of thread to `limits` for each library selected by `user_api`. If it is a dictionary `{{key: max_threads}}`, this function sets a custom maximum number of thread for each `key` which can be - either a `user_api` or a `prefix` for a specific library. If None, this - function does not do anything. + either a `user_api` or a `prefix` for a specific library. The `user_api` parameter selects particular APIs of libraries to limit. Used only if `limits` is an int. If it is None, this function will apply to @@ -141,7 +140,7 @@ def _set_threadpool_limits(limits=None, user_api=None, Return a list with all the supported modules that have been found. Each module is represented by a dict with the following information: - - 'filename-prefixes' : possible prefixes for the given internal_api. + - 'filename_prefixes' : possible prefixes for the given internal_api. Possible values are {ALL_PREFIXES}. - 'prefix' : prefix of the specific implementation of this module. - 'internal_api': internal API.s Possible values are {INTERNAL_APIS}. @@ -154,7 +153,7 @@ def _set_threadpool_limits(limits=None, user_api=None, - 'dynlib': the instance of ctypes.CDLL use to access the dynamic library. """ - if isinstance(limits, int) or limits is None: + if isinstance(limits, int): if user_api is None: user_api = _ALL_USER_APIS elif user_api in _ALL_USER_APIS: @@ -167,12 +166,12 @@ def _set_threadpool_limits(limits=None, user_api=None, else: if isinstance(limits, list): # This should be a list of module, for compatibility with - # the result from get_threadpool_limits. + # the result from threadpool_info. limits = {module['prefix']: module['num_threads'] for module in limits} if not isinstance(limits, dict): - raise TypeError("limits must either be an int, a dict or None." + raise TypeError("limits must either be an int, a list or a dict." " Got {} instead".format(type(limits))) # With a dictionary, can set both specific limit for given modules @@ -497,10 +496,10 @@ class threadpool_limits: """ def __init__(self, limits=None, user_api=None): if limits is not None: - self.original_limits = _set_threadpool_limits( + self._original_limits = _set_threadpool_limits( limits=limits, user_api=user_api, return_original_limits=True) else: - self.original_limits = None + self._original_limits = None def __enter__(self): pass @@ -509,6 +508,6 @@ def __exit__(self, type, value, traceback): self.unregister() def unregister(self): - if self.original_limits is not None: - for module in self.original_limits: + if self._original_limits is not None: + for module in self._original_limits: module['set_num_threads'](module['num_threads']) From a6acd869cbf697b16c3e9f8114b8509e4a66c0fa Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Sun, 31 Mar 2019 21:31:39 +0200 Subject: [PATCH 020/187] Add test for manual unregister --- tests/test_threadpoolctl.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index f577eaee..72d47007 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -120,6 +120,26 @@ def test_threadpool_limits_function_with_side_effect(): assert threadpool_info() == original_infos +def test_threadpool_limits_manual_unregister(): + # Check that threadpool_limits can be used as an object with that hold + # the original state of the threadpools that can be restored thanks to the + # dedicated unregister method + original_infos = threadpool_info() + + limits = threadpool_limits(limits=1) + try: + for module in threadpool_info(): + if should_skip_module(module): + continue + assert module["num_threads"] == 1 + finally: + # Restore the original limits so that this test does not have any + # side-effect. + limits.unregister() + + assert threadpool_info() == original_infos + + def test_threadpool_limits_bad_input(): # Check that appropriate errors are raised for invalid arguments match = re.escape("user_api must be either in {} or None." From 4b36e6422159600da832128ddcefb783a0cc39a9 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Sun, 31 Mar 2019 21:37:35 +0200 Subject: [PATCH 021/187] Avoid warnings in Windows CI logs --- continuous_integration/install.cmd | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/continuous_integration/install.cmd b/continuous_integration/install.cmd index 38e896f1..b58a6036 100644 --- a/continuous_integration/install.cmd +++ b/continuous_integration/install.cmd @@ -21,11 +21,12 @@ pip --version if "%PACKAGER%" == "conda" (%CONDA_INSTALL% numpy=1.15 pytest cython) if "%PACKAGER%" == "pip" (%PIP_INSTALL% numpy scipy pytest cython) -@rem Install extra dependency +@rem Install extra developer dependencies +pip install -r dev-requirements.txt pip install -q coverage pytest-cov @rem Install package -pip install -e . +flit install --symlink @rem Build the cython test helper for openmp bash ./continuous_integration/build_test_ext.sh From 27247964b0bf90f3f27668c1d89d61a9e1a01535 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Sun, 31 Mar 2019 21:41:07 +0200 Subject: [PATCH 022/187] Add test to cover the limits=None case --- tests/test_threadpoolctl.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index 72d47007..bc457353 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -120,6 +120,15 @@ def test_threadpool_limits_function_with_side_effect(): assert threadpool_info() == original_infos +def test_set_threadpool_limits_no_limit(): + # Check that limits=None does nothing. + original_infos = threadpool_info() + with threadpool_limits(limits=None): + assert threadpool_info() == original_infos + + assert threadpool_info() == original_infos + + def test_threadpool_limits_manual_unregister(): # Check that threadpool_limits can be used as an object with that hold # the original state of the threadpools that can be restored thanks to the From 70c994614ab40eefed0b6587e86f823c92c76a22 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Sun, 31 Mar 2019 22:10:07 +0200 Subject: [PATCH 023/187] Simplify private API of _set_threadpool_limits --- threadpoolctl.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/threadpoolctl.py b/threadpoolctl.py index 02c776f3..96723ada 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -115,8 +115,7 @@ def _get_limit(prefix, user_api, limits): @_format_docstring(ALL_PREFIXES=_ALL_PREFIXES, INTERNAL_APIS=_ALL_INTERNAL_APIS) -def _set_threadpool_limits(limits, user_api=None, - return_original_limits=False): +def _set_threadpool_limits(limits, user_api=None): """Limit the maximal number of threads for threadpools in supported libs Set the maximal number of threads that can be used in thread pools used in @@ -146,8 +145,7 @@ def _set_threadpool_limits(limits, user_api=None, - 'internal_api': internal API.s Possible values are {INTERNAL_APIS}. - 'filepath': path to the loaded module. - 'version': version of the library implemented (if available). - - 'num_threads': current thread limit if return_original_limits is False - or the original limit if return_original_limits is True. + - 'num_threads': the theadpool size limit before changing it. - 'set_num_threads': callable to set the maximum number of threads - 'get_num_threads': callable to get the current number of threads - 'dynlib': the instance of ctypes.CDLL use to access the dynamic @@ -182,15 +180,12 @@ def _set_threadpool_limits(limits, user_api=None, modules = _load_modules(prefixes=prefixes, user_api=user_api) for module in modules: num_threads = _get_limit(module['prefix'], module['user_api'], limits) - if return_original_limits: - module['num_threads'] = module['get_num_threads']() + module['num_threads'] = module['get_num_threads']() if num_threads is not None: set_func = module['set_num_threads'] set_func(num_threads) - if not return_original_limits: - module['num_threads'] = module['get_num_threads']() return modules @@ -497,7 +492,7 @@ class threadpool_limits: def __init__(self, limits=None, user_api=None): if limits is not None: self._original_limits = _set_threadpool_limits( - limits=limits, user_api=user_api, return_original_limits=True) + limits=limits, user_api=user_api) else: self._original_limits = None From 327ce2698b31977e4bdca62023f5f7ec78618409 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Sun, 31 Mar 2019 22:22:58 +0200 Subject: [PATCH 024/187] Update API in benchmark --- benchmarks/bench_context_manager_overhead.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/bench_context_manager_overhead.py b/benchmarks/bench_context_manager_overhead.py index 2e096ca2..7977c8a8 100644 --- a/benchmarks/bench_context_manager_overhead.py +++ b/benchmarks/bench_context_manager_overhead.py @@ -2,7 +2,7 @@ from argparse import ArgumentParser from pprint import pprint from statistics import mean, stdev -from threadpoolctl import get_threadpool_limits, threadpool_limits +from threadpoolctl import threadpool_info, threadpool_limits parser = ArgumentParser(description='Measure threadpool_limits call overhead.') parser.add_argument('--import', dest="packages", default=[], nargs='+', @@ -15,7 +15,7 @@ for package_name in args.packages: __import__(package_name) -pprint(get_threadpool_limits()) +pprint(threadpool_info()) timings = [] for _ in range(args.n_calls): From 3aea85014080d01412cafad290f6a7a8d916de4e Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Sun, 31 Mar 2019 22:27:25 +0200 Subject: [PATCH 025/187] Fix formatting of README.md --- README.md | 44 ++++++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 74570aea..22206179 100644 --- a/README.md +++ b/README.md @@ -10,30 +10,32 @@ oversubscription issues. ## Installation -- For users, you can install the last published version from PyPI: +- For users, install the last published version from PyPI: -```bash -pip install threadpoolctl -``` + ```bash + pip install threadpoolctl + ``` -- For contributors, you can install from the source repository in - developer mode: +- For contributors, install from the source repository in developer + mode: -```bash -pip install -r dev-requirements.txt -flit install --symlink -``` + ```bash + pip install -r dev-requirements.txt + flit install --symlink + ``` -then you can run the tests with pytest: + then you run the tests with pytest: -```bash -pytest -``` + ```bash + pytest + ``` ## Usage -- Introspect the current state of the threadpool-enabled runtime - libraries that are loaded when importing Python packages: +### Runtime Introspection + +Introspect the current state of the threadpool-enabled runtime libraries +that are loaded when importing Python packages: ```python >>> from threadpoolctl import threadpool_info @@ -72,8 +74,10 @@ pytest 'version': None}] ``` -- Control the number of threads used by the underlying runtime libraries - in specific sections of your Python program: +### Set the maximum size of thread-pools + +Control the number of threads used by the underlying runtime libraries +in specific sections of your Python program: ```python from threadpoolctl import threadpool_limits @@ -92,7 +96,7 @@ with threadpool_limits(limits=1, user_api='blas'): To make a release: -``` +```bash pip install flit flit build ``` @@ -102,6 +106,6 @@ Check the contents of `dist/`. If everything is fine, make a commit for the release, tag it, push the tag to github and then: -``` +```bash flit publish ``` From 47e3e3f76c6d694c239d74ddc403c5623be0a2e0 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Sun, 31 Mar 2019 22:37:09 +0200 Subject: [PATCH 026/187] Improve CI configuration --- continuous_integration/install.cmd | 3 +-- continuous_integration/install.sh | 3 +-- dev-requirements.txt | 1 + 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/continuous_integration/install.cmd b/continuous_integration/install.cmd index b58a6036..4f9e3330 100644 --- a/continuous_integration/install.cmd +++ b/continuous_integration/install.cmd @@ -22,8 +22,7 @@ if "%PACKAGER%" == "conda" (%CONDA_INSTALL% numpy=1.15 pytest cython) if "%PACKAGER%" == "pip" (%PIP_INSTALL% numpy scipy pytest cython) @rem Install extra developer dependencies -pip install -r dev-requirements.txt -pip install -q coverage pytest-cov +pip install -q -r dev-requirements.txt @rem Install package flit install --symlink diff --git a/continuous_integration/install.sh b/continuous_integration/install.sh index 747a97f0..cf1bca7f 100755 --- a/continuous_integration/install.sh +++ b/continuous_integration/install.sh @@ -36,15 +36,14 @@ elif [[ "$PACKAGER" == "ubuntu" ]]; then sudo apt-get install python3-scipy libatlas3-base libatlas-base-dev libatlas-dev libopenblas-base python3-virtualenv python3 -m virtualenv --system-site-packages --python=python3 $VIRTUALENV source $VIRTUALENV/bin/activate - python -m pip install pytest pytest-cov cython fi +python -m pip install -q -r dev-requirements.txt bash ./continuous_integration/build_test_ext.sh python --version python -c "import numpy; print('numpy %s' % numpy.__version__)" || echo "no numpy" python -c "import scipy; print('scipy %s' % scipy.__version__)" || echo "no scipy" -python -m pip install -r dev-requirements.txt python -m flit install --symlink diff --git a/dev-requirements.txt b/dev-requirements.txt index 0218af3f..d118f1e7 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,4 +1,5 @@ flit coverage pytest +pytest-cov cython From 34d27d2a8771c7b8cb5c01b05873b637d1f15981 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Sun, 31 Mar 2019 22:40:52 +0200 Subject: [PATCH 027/187] Improve project metadata --- pyproject.toml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 1a9ee5ee..481dc9a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,3 +7,15 @@ module = "threadpoolctl" author = "Thomas Moreau" author-email = "thomas.moreau.2010@gmail.com" home-page = "https://github.com/joblib/threadpoolctl" +description-file = "README.md" +classifiers = [ + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Topic :: Software Development :: Libraries :: Python Modules", +] \ No newline at end of file From df55e83df9699719e757f9e15863c83b08ec1c88 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Thu, 25 Apr 2019 11:40:22 +0200 Subject: [PATCH 028/187] Use the Azure pipeline from the joblib org --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 22206179..bad2cc61 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Thread-pool Controls [![Build Status](https://dev.azure.com/karementmoi/loky/_apis/build/status/joblib.threadpoolctl?branchName=master)](https://dev.azure.com/karementmoi/loky/_build/latest?definitionId=2&branchName=master) [![codecov](https://codecov.io/gh/joblib/threadpoolctl/branch/master/graph/badge.svg)](https://codecov.io/gh/joblib/threadpoolctl) +# Thread-pool Controls [![Build Status](https://dev.azure.com/joblib/threadpoolctl/_apis/build/status/joblib.threadpoolctl?branchName=master)](https://dev.azure.com/joblib/threadpoolctl/_build/latest?definitionId=1&branchName=master) [![codecov](https://codecov.io/gh/joblib/threadpoolctl/branch/master/graph/badge.svg)](https://codecov.io/gh/joblib/threadpoolctl) Python helpers to limit the number of threads used in the threadpool-backed of common native libraries used for scientific From 57979da09ed34b2e6848ea972a0c87409da809cb Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Mon, 3 Jun 2019 15:34:03 +0200 Subject: [PATCH 029/187] [MRG] make it possible to protect against oversubsription for various threadpool nesting cases (#16) --- .azure_pipeline.yml | 41 ++++- README.md | 9 + continuous_integration/build_test_ext.sh | 16 +- continuous_integration/display_versions.py | 23 +++ continuous_integration/install.cmd | 2 +- continuous_integration/install.sh | 32 +++- continuous_integration/posix.yml | 2 +- continuous_integration/test_script.cmd | 2 +- continuous_integration/test_script.sh | 8 +- tests/_openmp_test_helper/__init__.py | 12 +- tests/_openmp_test_helper/build_utils.py | 23 +++ .../nested_prange_blas.pyx | 42 +++++ tests/_openmp_test_helper/openmp_helpers.pyx | 22 --- .../openmp_helpers_inner.pxd | 1 + .../openmp_helpers_inner.pyx | 42 +++++ .../openmp_helpers_outer.pyx | 26 +++ tests/_openmp_test_helper/setup.py | 33 ---- tests/_openmp_test_helper/setup_inner.py | 37 +++++ .../setup_nested_prange_blas.py | 34 ++++ tests/_openmp_test_helper/setup_outer.py | 37 +++++ tests/test_threadpoolctl.py | 154 +++++++++++++----- tests/utils.py | 3 +- threadpoolctl.py | 53 ++++-- 23 files changed, 522 insertions(+), 132 deletions(-) mode change 100644 => 100755 continuous_integration/build_test_ext.sh create mode 100644 continuous_integration/display_versions.py create mode 100644 tests/_openmp_test_helper/build_utils.py create mode 100644 tests/_openmp_test_helper/nested_prange_blas.pyx delete mode 100644 tests/_openmp_test_helper/openmp_helpers.pyx create mode 100644 tests/_openmp_test_helper/openmp_helpers_inner.pxd create mode 100644 tests/_openmp_test_helper/openmp_helpers_inner.pyx create mode 100644 tests/_openmp_test_helper/openmp_helpers_outer.pyx delete mode 100644 tests/_openmp_test_helper/setup.py create mode 100644 tests/_openmp_test_helper/setup_inner.py create mode 100644 tests/_openmp_test_helper/setup_nested_prange_blas.py create mode 100644 tests/_openmp_test_helper/setup_outer.py diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml index 313c4ac4..d5f43231 100644 --- a/.azure_pipeline.yml +++ b/.azure_pipeline.yml @@ -32,22 +32,47 @@ jobs: matrix: # Linux environment to test that packages that comes with Ubuntu Xenial # 16.04 are correctly handled. - py35_np_atlas: + py35_ubuntu_atlas_gcc_gcc: PACKAGER: 'ubuntu' + APT_BLAS: 'libatlas3-base libatlas-base-dev libatlas-dev' VERSION_PYTHON: '3.5' - # Linux + Python 3.6 - py36_conda_openblas: + CC_OUTER_LOOP: 'gcc' + CC_INNER_LOOP: 'gcc' + py35_ubuntu_openblas_gcc_gcc: + PACKAGER: 'ubuntu' + APT_BLAS: 'libopenblas-base libopenblas-dev' + VERSION_PYTHON: '3.5' + CC_OUTER_LOOP: 'gcc' + CC_INNER_LOOP: 'gcc' + # Linux + Python 3.6 and homogeneous runtime nesting. + py36_conda_openblas_clang_clang: PACKAGER: 'conda' VERSION_PYTHON: '3.6' + NO_MKL: 'true' + CC_OUTER_LOOP: 'clang-8' + CC_INNER_LOOP: 'clang-8' # Linux environment to test the latest available dependencies and MKL. - pylatest_conda: + # and heterogeneous OpenMP runtime nesting. + pylatest_conda_mkl_clang_gcc: PACKAGER: 'conda' VERSION_PYTHON: '*' - # Linux environment with no numpy. - pylatest_conda_nonumpy: + CC_OUTER_LOOP: 'clang-8' + CC_INNER_LOOP: 'gcc' + # Same but with numpy / scipy installed with pip from PyPI and switched + # compilers. + pylatest_pip_openblas_gcc_clang: + PACKAGER: 'pip' + VERSION_PYTHON: '*' + CC_OUTER_LOOP: 'gcc' + CC_INNER_LOOP: 'clang-8' + # Linux environment with no numpy and heterogeneous OpenMP runtime + # nesting. + pylatest_conda_nonumpy_gcc_clang: PACKAGER: 'conda' NO_NUMPY: 'true' VERSION_PYTHON: '*' + CC_OUTER_LOOP: 'gcc' + CC_INNER_LOOP: 'clang-8' - template: continuous_integration/posix.yml parameters: @@ -56,4 +81,6 @@ jobs: matrix: pylatest_conda: PACKAGER: 'conda' - VERSION_PYTHON: '*' \ No newline at end of file + VERSION_PYTHON: '*' + CC_OUTER_LOOP: 'clang' + CC_INNER_LOOP: 'clang' diff --git a/README.md b/README.md index bad2cc61..082e6874 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,15 @@ with threadpool_limits(limits=1, user_api='blas'): a_squared = a @ a ``` +### Known limitation + +`threadpool_limits` does not act as expected in nested parallel loops +managed by distinct OpenMP runtime implementations (for instance libgomp +from GCC and libomp from clang/llvm or libiomp from ICC). + +See the `test_openmp_nesting()` function in `tests/test_threadpoolctl.py` +for an example. + ## Maintainers To make a release: diff --git a/continuous_integration/build_test_ext.sh b/continuous_integration/build_test_ext.sh old mode 100644 new mode 100755 index a5572e51..7b91c7e1 --- a/continuous_integration/build_test_ext.sh +++ b/continuous_integration/build_test_ext.sh @@ -1,4 +1,14 @@ +#!/bin/bash -cd tests/_openmp_test_helper -python setup.py build_ext -i || echo 'No openmp' -cd ../.. +set -e + +pushd tests/_openmp_test_helper +rm -rf *.c *.so *.dylib build/ +python setup_inner.py build_ext -i +python setup_outer.py build_ext -i + +# skip scipy required extension if no numpy +if [[ "$NO_NUMPY" != "true" ]]; then + python setup_nested_prange_blas.py build_ext -i +fi +popd diff --git a/continuous_integration/display_versions.py b/continuous_integration/display_versions.py new file mode 100644 index 00000000..aae3ebb7 --- /dev/null +++ b/continuous_integration/display_versions.py @@ -0,0 +1,23 @@ +from threadpoolctl import threadpool_info +from pprint import pprint + +try: + import numpy as np + print("numpy", np.__version__) +except ImportError: + pass + +try: + import scipy + import scipy.linalg + print("scipy", scipy.__version__) +except ImportError: + pass + +try: + from tests._openmp_test_helper import * # noqa +except ImportError: + pass + + +pprint(threadpool_info()) diff --git a/continuous_integration/install.cmd b/continuous_integration/install.cmd index 4f9e3330..263fdabb 100644 --- a/continuous_integration/install.cmd +++ b/continuous_integration/install.cmd @@ -18,7 +18,7 @@ python --version pip --version @rem Install dependencies with either conda or pip. -if "%PACKAGER%" == "conda" (%CONDA_INSTALL% numpy=1.15 pytest cython) +if "%PACKAGER%" == "conda" (%CONDA_INSTALL% numpy=1.15 scipy pytest cython) if "%PACKAGER%" == "pip" (%PIP_INSTALL% numpy scipy pytest cython) @rem Install extra developer dependencies diff --git a/continuous_integration/install.sh b/continuous_integration/install.sh index cf1bca7f..34af9408 100755 --- a/continuous_integration/install.sh +++ b/continuous_integration/install.sh @@ -5,7 +5,7 @@ set -e UNAMESTR=`uname` if [[ "$UNAMESTR" == "Darwin" ]]; then - # install OpenMP not present by default on osx + # Install a compiler with a working openmp HOMEBREW_NO_AUTO_UPDATE=1 brew install libomp # enable OpenMP support for Apple-clang @@ -16,6 +16,14 @@ if [[ "$UNAMESTR" == "Darwin" ]]; then export CXXFLAGS="$CXXFLAGS -I/usr/local/opt/libomp/include" export LDFLAGS="$LDFLAGS -L/usr/local/opt/libomp/lib -lomp" export DYLD_LIBRARY_PATH=/usr/local/opt/libomp/lib + +elif [[ "$CC_OUTER_LOOP" == "clang-8" || "$CC_INNER_LOOP" == "clang-8" ]]; then + # Assume Ubuntu: install a recent version of clang and libomp + echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main" | sudo tee -a /etc/apt/sources.list.d/llvm.list + echo "deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main" | sudo tee -a /etc/apt/sources.list.d/llvm.list + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - + sudo apt update + sudo apt install clang-8 libomp-8-dev fi make_conda() { @@ -25,15 +33,29 @@ make_conda() { } if [[ "$PACKAGER" == "conda" ]]; then - TO_INSTALL="python=$VERSION_PYTHON pip pytest pytest-cov cython" + TO_INSTALL="python=$VERSION_PYTHON pip" if [[ "$NO_NUMPY" != "true" ]]; then - TO_INSTALL="$TO_INSTALL numpy" + TO_INSTALL="$TO_INSTALL numpy scipy" + if [[ "$NO_MKL" == "true" ]]; then + TO_INSTALL="$TO_INSTALL nomkl" + fi fi - make_conda $TO_INSTALL +elif [[ "$PACKAGER" == "pip" ]]; then + # Use conda to build an empty python env and then use pip to install + # numpy and scipy + TO_INSTALL="python=$VERSION_PYTHON pip" + make_conda $TO_INSTALL + if [[ "$NO_NUMPY" != "true" ]]; then + pip install numpy scipy + fi + elif [[ "$PACKAGER" == "ubuntu" ]]; then - sudo apt-get install python3-scipy libatlas3-base libatlas-base-dev libatlas-dev libopenblas-base python3-virtualenv + # Remove the ubuntu toolchain PPA that seems to be invalid: + # https://github.com/scikit-learn/scikit-learn/pull/13934 + sudo add-apt-repository --remove ppa:ubuntu-toolchain-r/test + sudo apt-get install python3-scipy python3-virtualenv $APT_BLAS python3 -m virtualenv --system-site-packages --python=python3 $VIRTUALENV source $VIRTUALENV/bin/activate fi diff --git a/continuous_integration/posix.yml b/continuous_integration/posix.yml index a80816ca..816f526a 100644 --- a/continuous_integration/posix.yml +++ b/continuous_integration/posix.yml @@ -14,7 +14,7 @@ jobs: steps: - bash: echo "##vso[task.prependpath]$CONDA/bin" displayName: Add conda to PATH - condition: eq(variables['PACKAGER'], 'conda') + condition: or(eq(variables['PACKAGER'], 'conda'), eq(variables['PACKAGER'], 'pip')) - script: | continuous_integration/install.sh displayName: 'Install' diff --git a/continuous_integration/test_script.cmd b/continuous_integration/test_script.cmd index bb9bafc7..680f377c 100644 --- a/continuous_integration/test_script.cmd +++ b/continuous_integration/test_script.cmd @@ -1,3 +1,3 @@ call activate %VIRTUALENV% -pytest -vlx --junitxml=%JUNITXML% --cov=threadpoolctl +pytest -vlrxXs --junitxml=%JUNITXML% --cov=threadpoolctl diff --git a/continuous_integration/test_script.sh b/continuous_integration/test_script.sh index 605c0f17..f2c2b78a 100755 --- a/continuous_integration/test_script.sh +++ b/continuous_integration/test_script.sh @@ -4,10 +4,14 @@ set -e if [[ "$PACKAGER" == "conda" ]]; then source activate $VIRTUALENV +elif [[ "$PACKAGER" == "pip" ]]; then + # we actually use conda to install the base environment: + source activate $VIRTUALENV elif [[ "$PACKAGER" == "ubuntu" ]]; then source $VIRTUALENV/bin/activate fi set -x -pytest -vlx --junitxml=$JUNITXML --cov=threadpoolctl -set +x +PYTHONPATH="." python continuous_integration/display_versions.py + +pytest -vlrxXs --junitxml=$JUNITXML --cov=threadpoolctl diff --git a/tests/_openmp_test_helper/__init__.py b/tests/_openmp_test_helper/__init__.py index 247bed99..192c8e21 100644 --- a/tests/_openmp_test_helper/__init__.py +++ b/tests/_openmp_test_helper/__init__.py @@ -1,4 +1,12 @@ -from .openmp_helpers import check_openmp_num_threads +from .openmp_helpers_inner import check_openmp_num_threads +from .openmp_helpers_inner import get_compiler as get_inner_compiler +from .openmp_helpers_outer import check_nested_openmp_loops +from .openmp_helpers_outer import get_compiler as get_outer_compiler +from .nested_prange_blas import check_nested_prange_blas -__all__ = ["check_openmp_num_threads"] +__all__ = ["check_openmp_num_threads", + "check_nested_openmp_loops", + "check_nested_prange_blas", + "get_inner_compiler", + "get_outer_compiler"] diff --git a/tests/_openmp_test_helper/build_utils.py b/tests/_openmp_test_helper/build_utils.py new file mode 100644 index 00000000..06cc52bb --- /dev/null +++ b/tests/_openmp_test_helper/build_utils.py @@ -0,0 +1,23 @@ +import os +import sys + + +def set_cc_variables(var_name="CC"): + cc_var = os.environ.get(var_name) + if cc_var is not None: + os.environ["CC"] = cc_var + if sys.platform == "darwin": + os.environ["LDSHARED"] = ( + cc_var + " -bundle -undefined dynamic_lookup") + else: + os.environ["LDSHARED"] = cc_var + " -shared" + + return cc_var + + +def get_openmp_flag(): + if sys.platform == "win32": + return ["/openmp"] + elif sys.platform == "darwin" and 'openmp' in os.getenv('CPPFLAGS', ''): + return [] + return ["-fopenmp"] diff --git a/tests/_openmp_test_helper/nested_prange_blas.pyx b/tests/_openmp_test_helper/nested_prange_blas.pyx new file mode 100644 index 00000000..79a19f12 --- /dev/null +++ b/tests/_openmp_test_helper/nested_prange_blas.pyx @@ -0,0 +1,42 @@ +cimport openmp +from cython.parallel import prange + +import numpy as np +from scipy.linalg.cython_blas cimport dgemm + +from threadpoolctl import threadpool_info + + +def check_nested_prange_blas(double[:, ::1] A, double[:, ::1] B, int nthreads): + """Run multithreaded BLAS calls within OpenMP parallel loop""" + cdef: + int m = A.shape[0] + int n = B.shape[0] + int k = A.shape[1] + + double[:, ::1] C = np.empty((m, n)) + int n_chunks = 100 + int chunk_size = A.shape[0] // n_chunks + + char* trans = 't' + char* no_trans = 'n' + double alpha = 1.0 + double beta = 0.0 + + int i + int prange_num_threads + + threadpool_infos = None + + for i in prange(n_chunks, num_threads=nthreads, nogil=True): + dgemm(trans, no_trans, &n, &chunk_size, &k, + &alpha, &B[0, 0], &k, &A[i * chunk_size, 0], &k, + &beta, &C[i * chunk_size, 0], &n) + + prange_num_threads = openmp.omp_get_num_threads() + + if i == 0: + with gil: + threadpool_infos = threadpool_info() + + return np.asarray(C), prange_num_threads, threadpool_infos diff --git a/tests/_openmp_test_helper/openmp_helpers.pyx b/tests/_openmp_test_helper/openmp_helpers.pyx deleted file mode 100644 index 5a6cbc3a..00000000 --- a/tests/_openmp_test_helper/openmp_helpers.pyx +++ /dev/null @@ -1,22 +0,0 @@ -import cython -cimport openmp -from cython.parallel import prange -from libc.stdlib cimport malloc, free - - -def check_openmp_num_threads(int n): - """Run a short parallel section with OpenMP - - Return the number of threads that where effectively used by the - OpenMP runtime. - """ - cdef long n_sum = 0 - cdef int i, num_threads - - for i in prange(n, nogil=True): - num_threads = openmp.omp_get_num_threads() - n_sum += i - - assert n_sum == (n - 1) * n / 2 - - return num_threads diff --git a/tests/_openmp_test_helper/openmp_helpers_inner.pxd b/tests/_openmp_test_helper/openmp_helpers_inner.pxd new file mode 100644 index 00000000..50005710 --- /dev/null +++ b/tests/_openmp_test_helper/openmp_helpers_inner.pxd @@ -0,0 +1 @@ +cdef int inner_openmp_loop(int) nogil \ No newline at end of file diff --git a/tests/_openmp_test_helper/openmp_helpers_inner.pyx b/tests/_openmp_test_helper/openmp_helpers_inner.pyx new file mode 100644 index 00000000..9fdffca0 --- /dev/null +++ b/tests/_openmp_test_helper/openmp_helpers_inner.pyx @@ -0,0 +1,42 @@ +cimport openmp +from cython.parallel import prange + + +def check_openmp_num_threads(int n): + """Run a short parallel section with OpenMP + + Return the number of threads that where effectively used by the + OpenMP runtime. + """ + cdef int num_threads = -1 + + with nogil: + num_threads = inner_openmp_loop(n) + return num_threads + + +cdef int inner_openmp_loop(int n) nogil: + """Run a short parallel section with OpenMP + + Return the number of threads that where effectively used by the + OpenMP runtime. + + This function is expected to run without the GIL and can be called + by an outer OpenMP / prange loop written in Cython in anoter file. + """ + cdef long n_sum = 0 + cdef int i, num_threads + + for i in prange(n): + num_threads = openmp.omp_get_num_threads() + n_sum += i + + if n_sum != (n - 1) * n / 2: + # error + return -1 + + return num_threads + + +def get_compiler(): + return CC_INNER_LOOP diff --git a/tests/_openmp_test_helper/openmp_helpers_outer.pyx b/tests/_openmp_test_helper/openmp_helpers_outer.pyx new file mode 100644 index 00000000..ffed13ef --- /dev/null +++ b/tests/_openmp_test_helper/openmp_helpers_outer.pyx @@ -0,0 +1,26 @@ +cimport openmp +from cython.parallel import prange +from openmp_helpers_inner cimport inner_openmp_loop + + +def check_nested_openmp_loops(int n, nthreads=None): + """Run a short parallel section with OpenMP with nested calls + + The inner OpenMP loop has not necessarily been built/linked with the + same runtime OpenMP runtime. + """ + cdef: + int outer_num_threads = -1 + int inner_num_threads = -1 + int num_threads = nthreads or openmp.omp_get_max_threads() + int i + + for i in prange(n, num_threads=num_threads, nogil=True): + inner_num_threads = inner_openmp_loop(n) + outer_num_threads = openmp.omp_get_num_threads() + + return outer_num_threads, inner_num_threads + + +def get_compiler(): + return CC_OUTER_LOOP diff --git a/tests/_openmp_test_helper/setup.py b/tests/_openmp_test_helper/setup.py deleted file mode 100644 index 8d23276b..00000000 --- a/tests/_openmp_test_helper/setup.py +++ /dev/null @@ -1,33 +0,0 @@ -import os -import sys -from distutils.core import setup -from Cython.Build import cythonize -from distutils.extension import Extension - - -if sys.platform == "darwin": - os.environ["CC"] = "gcc-4.9" - os.environ["CXX"] = "g++-4.9" - -if sys.platform == "win32": - extra_compile_args = ["/openmp"] - extra_link_args = None -else: - extra_compile_args = ["-fopenmp"] - extra_link_args = ['-fopenmp'] - -ext_modules = [ - Extension( - "openmp_helpers", - ["openmp_helpers.pyx"], - extra_compile_args=extra_compile_args, - extra_link_args=extra_link_args - ) -] - -setup( - name='_openmp_test_helper', - ext_modules=cythonize( - ext_modules, - language_level=3), -) diff --git a/tests/_openmp_test_helper/setup_inner.py b/tests/_openmp_test_helper/setup_inner.py new file mode 100644 index 00000000..af3e40df --- /dev/null +++ b/tests/_openmp_test_helper/setup_inner.py @@ -0,0 +1,37 @@ +import os +from distutils.core import setup +from Cython.Build import cythonize +from distutils.extension import Extension + +from build_utils import set_cc_variables +from build_utils import get_openmp_flag + + +original_environ = os.environ.copy() +try: + # Make it possible to compile the 2 OpenMP enabled Cython extensions + # with different compilers and therefore different OpenMP runtimes. + inner_loop_cc_var = set_cc_variables("CC_INNER_LOOP") + openmp_flag = get_openmp_flag() + + ext_modules = [ + Extension( + "openmp_helpers_inner", + ["openmp_helpers_inner.pyx"], + extra_compile_args=openmp_flag, + extra_link_args=openmp_flag + ) + ] + + setup( + name='_openmp_test_helper_inner', + ext_modules=cythonize( + ext_modules, + compiler_directives={'language_level': 3, + 'boundscheck': False, + 'wraparound': False}, + compile_time_env={"CC_INNER_LOOP": inner_loop_cc_var or "unknown"}) + ) + +finally: + os.environ.update(original_environ) diff --git a/tests/_openmp_test_helper/setup_nested_prange_blas.py b/tests/_openmp_test_helper/setup_nested_prange_blas.py new file mode 100644 index 00000000..746e573d --- /dev/null +++ b/tests/_openmp_test_helper/setup_nested_prange_blas.py @@ -0,0 +1,34 @@ +import os +from distutils.core import setup +from Cython.Build import cythonize +from distutils.extension import Extension + +from build_utils import set_cc_variables +from build_utils import get_openmp_flag + + +original_environ = os.environ.copy() +try: + set_cc_variables("CC_OUTER_LOOP") + openmp_flag = get_openmp_flag() + + ext_modules = [ + Extension( + "nested_prange_blas", + ["nested_prange_blas.pyx"], + extra_compile_args=openmp_flag, + extra_link_args=openmp_flag + ) + ] + + setup( + name='_openmp_test_helper_nested_prange_blas', + ext_modules=cythonize( + ext_modules, + compiler_directives={'language_level': 3, + 'boundscheck': False, + 'wraparound': False}) + ) + +finally: + os.environ.update(original_environ) diff --git a/tests/_openmp_test_helper/setup_outer.py b/tests/_openmp_test_helper/setup_outer.py new file mode 100644 index 00000000..56c9bd2a --- /dev/null +++ b/tests/_openmp_test_helper/setup_outer.py @@ -0,0 +1,37 @@ +import os +from distutils.core import setup +from Cython.Build import cythonize +from distutils.extension import Extension + +from build_utils import set_cc_variables +from build_utils import get_openmp_flag + + +original_environ = os.environ.copy() +try: + # Make it possible to compile the 2 OpenMP enabled Cython extensions + # with different compilers and therefore different OpenMP runtimes. + outer_loop_cc_var = set_cc_variables("CC_OUTER_LOOP") + openmp_flag = get_openmp_flag() + + ext_modules = [ + Extension( + "openmp_helpers_outer", + ["openmp_helpers_outer.pyx"], + extra_compile_args=openmp_flag, + extra_link_args=openmp_flag + ) + ] + + setup( + name="_openmp_test_helper_outer", + ext_modules=cythonize( + ext_modules, + compiler_directives={'language_level': 3, + 'boundscheck': False, + 'wraparound': False}, + compile_time_env={"CC_OUTER_LOOP": outer_loop_cc_var or "unknown"}) + ) + +finally: + os.environ.update(original_environ) diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index bc457353..5713be76 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -6,20 +6,26 @@ from threadpoolctl import threadpool_limits, threadpool_info from threadpoolctl import _ALL_PREFIXES, _ALL_USER_APIS -from .utils import with_check_openmp_num_threads, libopenblas_paths +from .utils import with_check_openmp_num_threads +from .utils import libopenblas_paths +from .utils import scipy -def should_skip_module(module): +def is_old_openblas(module): # Possible bug in getting maximum number of threads with OpenBLAS < 0.2.16 # and OpenBLAS does not expose its version before 0.3.4. return module['internal_api'] == "openblas" and module['version'] is None +def effective_num_threads(nthreads, max_threads): + if nthreads is None or nthreads > max_threads: + return max_threads + return nthreads + + @pytest.mark.parametrize("prefix", _ALL_PREFIXES) def test_threadpool_limits_by_prefix(openblas_present, mkl_present, prefix): original_infos = threadpool_info() - original_num_threads = {info["filepath"]: info['num_threads'] - for info in original_infos} mkl_found = any([True for info in original_infos if info["prefix"] in ('mkl_rt', 'libmkl_rt')]) prefix_found = len([info["prefix"] for info in original_infos @@ -30,36 +36,21 @@ def test_threadpool_limits_by_prefix(openblas_present, mkl_present, prefix): elif prefix == "libopenblas" and openblas_present: raise RuntimeError("Could not load the OpenBLAS prefix") else: - pytest.skip("Need {} support".format(prefix)) + pytest.skip("{} runtime missing".format(prefix)) with threadpool_limits(limits={prefix: 1}): for module in threadpool_info(): - if should_skip_module(module): + if is_old_openblas(module): continue - num_threads, filepath = module["num_threads"], module["filepath"] if module["prefix"] == prefix: - assert num_threads == 1 - elif "mkl_rt" in module["prefix"] and prefix == "libiomp": - # MKL is impacted by a change in the thread pool of Intel - # implementation of the OpenMP runtime: - assert num_threads == 1 - else: - assert num_threads == original_num_threads[filepath] + assert module["num_threads"] == 1 with threadpool_limits(limits={prefix: 3}): for module in threadpool_info(): - if should_skip_module(module): + if is_old_openblas(module): continue - num_threads, filepath = module["num_threads"], module["filepath"] - expected_num_threads = (3, original_num_threads[filepath]) if module["prefix"] == prefix: - assert num_threads in expected_num_threads - elif "mkl_rt" in module["prefix"] and prefix == "libiomp": - # MKL is impacted by a change in the thread pool of Intel - # implementation of the OpenMP runtime: - assert num_threads in expected_num_threads - else: - assert num_threads == original_num_threads[filepath] + assert module["num_threads"] <= 3 assert threadpool_info() == original_infos @@ -74,29 +65,20 @@ def test_set_threadpool_limits_by_api(user_api): user_apis = (user_api,) original_infos = threadpool_info() - original_num_threads = {info["filepath"]: info['num_threads'] - for info in original_infos} with threadpool_limits(limits=1, user_api=user_api): for module in threadpool_info(): - if should_skip_module(module): + if is_old_openblas(module): continue - num_threads, filepath = module["num_threads"], module["filepath"] if module["user_api"] in user_apis: - assert num_threads == 1 - else: - assert num_threads == original_num_threads[filepath] + assert module["num_threads"] == 1 with threadpool_limits(limits=3, user_api=user_api): for module in threadpool_info(): - if should_skip_module(module): + if is_old_openblas(module): continue - num_threads, filepath = module["num_threads"], module["filepath"] - expected_num_threads = (3, original_num_threads[filepath]) if module["user_api"] in user_apis: - assert num_threads in expected_num_threads - else: - assert num_threads == original_num_threads[filepath] + assert module["num_threads"] <= 3 assert threadpool_info() == original_infos @@ -109,7 +91,7 @@ def test_threadpool_limits_function_with_side_effect(): threadpool_limits(limits=1) try: for module in threadpool_info(): - if should_skip_module(module): + if is_old_openblas(module): continue assert module["num_threads"] == 1 finally: @@ -138,7 +120,7 @@ def test_threadpool_limits_manual_unregister(): limits = threadpool_limits(limits=1) try: for module in threadpool_info(): - if should_skip_module(module): + if is_old_openblas(module): continue assert module["num_threads"] == 1 finally: @@ -175,6 +157,65 @@ def test_openmp_limit_num_threads(num_threads): assert check_openmp_num_threads(100) == old_num_threads +@with_check_openmp_num_threads +@pytest.mark.parametrize('nthreads_outer', [None, 1, 2, 4]) +def test_openmp_nesting(nthreads_outer): + # checks that OpenMP effectively uses the number of threads requested by + # the context manager + from ._openmp_test_helper import check_nested_openmp_loops + from ._openmp_test_helper import get_inner_compiler + from ._openmp_test_helper import get_outer_compiler + + inner_cc = get_inner_compiler() + outer_cc = get_outer_compiler() + + outer_num_threads, inner_num_threads = check_nested_openmp_loops(10) + + original_infos = threadpool_info() + openmp_infos = [info for info in original_infos + if info["user_api"] == "openmp"] + + if "gcc" in (inner_cc, outer_cc): + assert "libgomp" in [info["prefix"] for info in openmp_infos] + + if "clang" in (inner_cc, outer_cc): + assert "libomp" in [info["prefix"] for info in openmp_infos] + + if inner_cc == outer_cc: + # The openmp runtime should be shared by default, meaning that + # the inner loop should automatically be run serially by the OpenMP + # runtime. + assert inner_num_threads == 1 + else: + # There should be at least 2 OpenMP runtime detected. + assert len(openmp_infos) >= 2 + + with threadpool_limits(limits=1) as threadpoolctx: + max_threads = threadpoolctx.get_original_num_threads('openmp') + nthreads = effective_num_threads(nthreads_outer, max_threads) + + outer_num_threads, inner_num_threads = \ + check_nested_openmp_loops(10, nthreads) + + # The state of the original state of all threadpools should have been + # restored. + assert threadpool_info() == original_infos + + # The number of threads available in the outer loop should not have been + # decreased: + assert outer_num_threads == nthreads + + # The number of threads available in the inner loop should have been + # set to 1 so avoid oversubscription and preserve performance: + if inner_cc != outer_cc: + if inner_num_threads != 1: + # XXX: this does not always work when nesting independent openmp + # implementations. See: https://github.com/jeremiedbb/Nested_OpenMP + pytest.xfail("Inner OpenMP num threads was %d instead of 1" + % inner_num_threads) + assert inner_num_threads == 1 + + def test_shipped_openblas(): all_openblases = [ctypes.CDLL(path) for path in libopenblas_paths] original_num_threads = [blas.openblas_get_num_threads() @@ -195,3 +236,38 @@ def test_multiple_shipped_openblas(): # has 2 or more active openblas runtimes available just be reading the # pytest report (whether or not this test has been skipped). test_shipped_openblas() + + +@pytest.mark.skipif(scipy is None, reason="requires scipy") +@pytest.mark.parametrize('nthreads_outer', [None, 1, 2, 4]) +def test_nested_prange_blas(nthreads_outer): + import numpy as np + + blas_info = [module for module in threadpool_info() + if module["user_api"] == "blas"] + for module in threadpool_info(): + if is_old_openblas(module): + # OpenBLAS 0.3.3 and older are known to cause an unrecoverable + # deadlock at process shutdown time (after pytest has exited). + pytest.skip("Old OpenBLAS: skipping test to avoid deadlock") + + from ._openmp_test_helper import check_nested_prange_blas + A = np.ones((1000, 10)) + B = np.ones((100, 10)) + + with threadpool_limits(limits=1) as threadpoolctx: + max_threads = threadpoolctx.get_original_num_threads('openmp') + nthreads = effective_num_threads(nthreads_outer, max_threads) + + result = check_nested_prange_blas(A, B, nthreads) + C, prange_num_threads, threadpool_infos = result + + assert np.allclose(C, np.dot(A, B.T)) + assert prange_num_threads == nthreads + + nested_blas_info = [module for module in threadpool_infos + if module["user_api"] == "blas"] + + assert len(nested_blas_info) == len(blas_info) + for module in nested_blas_info: + assert module['num_threads'] == 1 diff --git a/tests/utils.py b/tests/utils.py index f5f8a758..71aa903f 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -29,11 +29,12 @@ def test_func(*args, **kwargs): try: import scipy import scipy.linalg # noqa: F401 + scipy.linalg.svd([[1, 2], [3, 4]]) libopenblas_patterns.append(os.path.join(scipy.__path__[0], ".libs", "libopenblas*")) except ImportError: - pass + scipy = None libopenblas_paths = set(path for pattern in libopenblas_patterns for path in glob(pattern)) diff --git a/threadpoolctl.py b/threadpoolctl.py index 96723ada..a8dea083 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -14,6 +14,7 @@ import re import sys import ctypes +import warnings from ctypes.util import find_library __version__ = '1.0.0.dev0' @@ -23,17 +24,18 @@ _system_libraries = {} -if sys.platform == "darwin": - # On OSX, we can get a runtime error due to multiple OpenMP libraries - # loaded simultaneously. This can happen for instance when calling BLAS - # inside a prange. Setting the following environment variable allows - # multiple OpenMP libraries to be loaded. It should not degrade - # performances since we manually take care of potential over-subscription - # performance issues, in sections of the code where nested OpenMP loops can - # happen, by dynamically reconfiguring the inner OpenMP runtime to - # temporarily disable it while under the scope of the outer OpenMP parallel - # section. - os.environ.setdefault("KMP_DUPLICATE_LIB_OK", "True") +# One can get runtime errors or even segfaults due to multiple OpenMP libraries +# loaded simultaneously which can happen easily in Python when importing and +# using compiled extensions built with different compilers and therefore +# different OpenMP runtimes in the same program. In particular libiomp (used by +# Intel ICC) and libomp used by clang/llvm tend to crash. This can happen for +# instance when calling BLAS inside a prange. Setting the following environment +# variable allows multiple OpenMP libraries to be loaded. It should not degrade +# performances since we manually take care of potential over-subscription +# performance issues, in sections of the code where nested OpenMP loops can +# happen, by dynamically reconfiguring the inner OpenMP runtime to temporarily +# disable it while under the scope of the outer OpenMP parallel section. +os.environ.setdefault("KMP_DUPLICATE_LIB_OK", "True") # Structure to cast the info on dynamically loaded library. See @@ -179,9 +181,12 @@ def _set_threadpool_limits(limits, user_api=None): modules = _load_modules(prefixes=prefixes, user_api=user_api) for module in modules: - num_threads = _get_limit(module['prefix'], module['user_api'], limits) - module['num_threads'] = module['get_num_threads']() + # Workaround clang bug (TODO: report it) + module['get_num_threads']() + for module in modules: + module['num_threads'] = module['get_num_threads']() + num_threads = _get_limit(module['prefix'], module['user_api'], limits) if num_threads is not None: set_func = module['set_num_threads'] set_func(num_threads) @@ -304,7 +309,9 @@ def _make_module_info(filepath, module_info, prefix): def _get_module_info_from_path(filepath, prefixes, user_api, modules): - filename = os.path.basename(filepath) + # `lower` required to take account of OpenMP dll case on Windows + # (vcomp, VCOMP, Vcomp, ...) + filename = os.path.basename(filepath).lower() for info in _SUPPORTED_IMPLEMENTATIONS: prefix = _check_prefix(filename, info['filename_prefixes']) if _match_module(info, prefix, prefixes, user_api): @@ -497,7 +504,7 @@ def __init__(self, limits=None, user_api=None): self._original_limits = None def __enter__(self): - pass + return self def __exit__(self, type, value, traceback): self.unregister() @@ -506,3 +513,19 @@ def unregister(self): if self._original_limits is not None: for module in self._original_limits: module['set_num_threads'](module['num_threads']) + + def get_original_num_threads(self, user_api): + limits = [module['num_threads'] for module in self._original_limits + if module['user_api'] == user_api] + + limits = set(limits) + n_limits = len(limits) + + if n_limits == 1: + return limits.pop() + elif n_limits == 0: + return None + else: + warnings.warn("Multiple value possible for user_api='{}'. " + "Returning the minimum.".format(user_api)) + return min(limits) From 862d06983f728a3f1bb01eb51cee76ac91437bc9 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Mon, 3 Jun 2019 15:42:21 +0200 Subject: [PATCH 030/187] Python 2 is not tested hence not supported --- pyproject.toml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 481dc9a0..4e76e43d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,14 +8,13 @@ author = "Thomas Moreau" author-email = "thomas.moreau.2010@gmail.com" home-page = "https://github.com/joblib/threadpoolctl" description-file = "README.md" +requires-python = ">=3.5" classifiers = [ "Intended Audience :: Developers", "License :: OSI Approved :: BSD License", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Topic :: Software Development :: Libraries :: Python Modules", -] \ No newline at end of file +] From f1b5e47928d865275b71e73c1897115565155247 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Mon, 3 Jun 2019 15:40:15 +0200 Subject: [PATCH 031/187] Release 1.0.0 --- CHANGES.md | 4 ++++ threadpoolctl.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 CHANGES.md diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 00000000..bc8315c4 --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,4 @@ +1.0.0 (2019-06-03) +================== + +Initial release. diff --git a/threadpoolctl.py b/threadpoolctl.py index a8dea083..54bd616d 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -17,7 +17,7 @@ import warnings from ctypes.util import find_library -__version__ = '1.0.0.dev0' +__version__ = '1.0.0' __all__ = ["threadpool_limits", "threadpool_info"] # Cache for libc under POSIX and a few system libraries under Windows From 7b2cd9c957cb164c920b4dd6e23b0760db1a648f Mon Sep 17 00:00:00 2001 From: jeremiedbb <34657725+jeremiedbb@users.noreply.github.com> Date: Thu, 18 Jul 2019 11:17:34 +0200 Subject: [PATCH 032/187] add pytest_cache to gitignore (#24) --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 5ce98657..0b6ad0a9 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,6 @@ dist # Developer tools .vscode + +# pytest +.pytest_cache From c855827a52e33e8e7c03dc00db1d13611f70a147 Mon Sep 17 00:00:00 2001 From: jeremiedbb <34657725+jeremiedbb@users.noreply.github.com> Date: Mon, 9 Sep 2019 18:14:20 +0200 Subject: [PATCH 033/187] CI: Try to use conda-forge compilers on macOS (#29) --- .azure_pipeline.yml | 11 ++++++++++- continuous_integration/install.sh | 33 ++++++++++++++++++------------- continuous_integration/posix.yml | 5 +++++ 3 files changed, 34 insertions(+), 15 deletions(-) diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml index d5f43231..7f51aa13 100644 --- a/.azure_pipeline.yml +++ b/.azure_pipeline.yml @@ -79,8 +79,17 @@ jobs: name: macOS vmImage: xcode9-macos10.13 matrix: - pylatest_conda: + # MacOS environment with OpenMP installed through homebrew + pylatest_conda_homebrew_libomp: PACKAGER: 'conda' VERSION_PYTHON: '*' CC_OUTER_LOOP: 'clang' CC_INNER_LOOP: 'clang' + INSTALL_LIBOMP: 'homebrew' + # MacOS environment with OpenMP installed through conda-forge compilers + pylatest_conda_conda_forge_clang: + PACKAGER: 'conda' + VERSION_PYTHON: '*' + CC_OUTER_LOOP: 'clang' + CC_INNER_LOOP: 'clang' + INSTALL_LIBOMP: 'conda-forge' diff --git a/continuous_integration/install.sh b/continuous_integration/install.sh index 34af9408..6d112bf9 100755 --- a/continuous_integration/install.sh +++ b/continuous_integration/install.sh @@ -4,20 +4,7 @@ set -e UNAMESTR=`uname` -if [[ "$UNAMESTR" == "Darwin" ]]; then - # Install a compiler with a working openmp - HOMEBREW_NO_AUTO_UPDATE=1 brew install libomp - - # enable OpenMP support for Apple-clang - export CC=/usr/bin/clang - export CXX=/usr/bin/clang++ - export CPPFLAGS="$CPPFLAGS -Xpreprocessor -fopenmp" - export CFLAGS="$CFLAGS -I/usr/local/opt/libomp/include" - export CXXFLAGS="$CXXFLAGS -I/usr/local/opt/libomp/include" - export LDFLAGS="$LDFLAGS -L/usr/local/opt/libomp/lib -lomp" - export DYLD_LIBRARY_PATH=/usr/local/opt/libomp/lib - -elif [[ "$CC_OUTER_LOOP" == "clang-8" || "$CC_INNER_LOOP" == "clang-8" ]]; then +if [[ "$CC_OUTER_LOOP" == "clang-8" || "$CC_INNER_LOOP" == "clang-8" ]]; then # Assume Ubuntu: install a recent version of clang and libomp echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main" | sudo tee -a /etc/apt/sources.list.d/llvm.list echo "deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main" | sudo tee -a /etc/apt/sources.list.d/llvm.list @@ -28,6 +15,24 @@ fi make_conda() { TO_INSTALL="$@" + if [[ "$UNAMESTR" == "Darwin" ]]; then + if [[ "$INSTALL_LIBOMP" == "conda-forge" ]]; then + # Install an OpenMP-enabled clang/llvm from conda-forge + TO_INSTALL="$TO_INSTALL conda-forge::compilers conda-forge::llvm-openmp" + export LDFLAGS="$LDFLAGS -Wl,-rpath,$CONDA/envs/$VIRTUALENV/lib -L$CONDA/envs/$VIRTUALENV/lib" + export CFLAGS="$CFLAGS -I$CONDA/envs/$VIRTUALENV/include" + + elif [[ "$INSTALL_LIBOMP" == "homebrew" ]]; then + # Install a compiler with a working openmp + HOMEBREW_NO_AUTO_UPDATE=1 brew install libomp + + # enable OpenMP support for Apple-clang + export CC=/usr/bin/clang + export CPPFLAGS="$CPPFLAGS -Xpreprocessor -fopenmp" + export CFLAGS="$CFLAGS -I/usr/local/opt/libomp/include" + export LDFLAGS="$LDFLAGS -Wl,-rpath,/usr/local/opt/libomp/lib -L/usr/local/opt/libomp/lib -lomp" + fi + fi conda create -n $VIRTUALENV -q --yes $TO_INSTALL source activate $VIRTUALENV } diff --git a/continuous_integration/posix.yml b/continuous_integration/posix.yml index 816f526a..54edf669 100644 --- a/continuous_integration/posix.yml +++ b/continuous_integration/posix.yml @@ -15,6 +15,11 @@ jobs: - bash: echo "##vso[task.prependpath]$CONDA/bin" displayName: Add conda to PATH condition: or(eq(variables['PACKAGER'], 'conda'), eq(variables['PACKAGER'], 'pip')) + - bash: sudo chown -R $USER $CONDA + # On Hosted macOS, the agent user doesn't have ownership of Miniconda's installation directory/ + # We need to take ownership if we want to update conda or install packages globally + displayName: Take ownership of conda installation + condition: eq('${{ parameters.name }}', 'macOS') - script: | continuous_integration/install.sh displayName: 'Install' From 93c8b9c543f290c4cc06c17d595ff71be191b4b2 Mon Sep 17 00:00:00 2001 From: jeremiedbb <34657725+jeremiedbb@users.noreply.github.com> Date: Mon, 9 Sep 2019 18:28:15 +0200 Subject: [PATCH 034/187] [MRG] ENH Add BLIS support (#23) --- .azure_pipeline.yml | 25 +++++++++ continuous_integration/install_with_blis.sh | 53 +++++++++++++++++++ continuous_integration/posix.yml | 7 ++- .../nested_prange_blas.pyx | 47 +++++++++++----- .../setup_nested_prange_blas.py | 7 ++- tests/test_threadpoolctl.py | 16 ++++-- threadpoolctl.py | 24 ++++++++- 7 files changed, 158 insertions(+), 21 deletions(-) create mode 100755 continuous_integration/install_with_blis.sh diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml index 7f51aa13..6f3e26ba 100644 --- a/.azure_pipeline.yml +++ b/.azure_pipeline.yml @@ -73,6 +73,31 @@ jobs: VERSION_PYTHON: '*' CC_OUTER_LOOP: 'gcc' CC_INNER_LOOP: 'clang-8' + # Linux environment with numpy linked to BLIS + pylatest_blis_gcc_clang: + PACKAGER: 'conda' + VERSION_PYTHON: '*' + INSTALL_BLIS: 'true' + BLIS_NUM_THREADS: '4' + CC_OUTER_LOOP: 'gcc' + CC_INNER_LOOP: 'gcc' + BLIS_CC: 'clang-8' + pylatest_blis_clang_gcc: + PACKAGER: 'conda' + VERSION_PYTHON: '*' + INSTALL_BLIS: 'true' + BLIS_NUM_THREADS: '4' + CC_OUTER_LOOP: 'clang-8' + CC_INNER_LOOP: 'clang-8' + BLIS_CC: 'gcc' + pylatest_blis_sinlge_threaded: + PACKAGER: 'conda' + VERSION_PYTHON: '*' + INSTALL_BLIS: 'true' + BLIS_NUM_THREADS: '1' + CC_OUTER_LOOP: 'gcc' + CC_INNER_LOOP: 'gcc' + BLIS_CC: 'gcc' - template: continuous_integration/posix.yml parameters: diff --git a/continuous_integration/install_with_blis.sh b/continuous_integration/install_with_blis.sh new file mode 100755 index 00000000..010c3b64 --- /dev/null +++ b/continuous_integration/install_with_blis.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +set -e + +pushd .. +ABS_PATH=$(pwd) +popd + +# Assume Ubuntu: install a recent version of clang and libomp +echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main" | sudo tee -a /etc/apt/sources.list.d/llvm.list +echo "deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main" | sudo tee -a /etc/apt/sources.list.d/llvm.list +wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - +sudo apt update +sudo apt install clang-8 libomp-8-dev + +# create conda env +conda create -n $VIRTUALENV -q --yes python=$VERSION_PYTHON pip cython +source activate $VIRTUALENV + +pushd .. + +# build & install blis +mkdir BLIS_install +git clone https://github.com/flame/blis.git +pushd blis +./configure --prefix=$ABS_PATH/BLIS_install --enable-cblas --enable-threading=openmp CC=$BLIS_CC auto +make -j4 +make install +popd + +# build & install numpy +git clone https://github.com/numpy/numpy.git +pushd numpy +echo "[blis] +libraries = blis +library_dirs = $ABS_PATH/BLIS_install/lib +include_dirs = $ABS_PATH/BLIS_install/include/blis +runtime_library_dirs = $ABS_PATH/BLIS_install/lib" > site.cfg +python setup.py build_ext -i +pip install -e . +popd + +popd + +python -m pip install -q -r dev-requirements.txt +CFLAGS=-I$ABS_PATH/BLIS_install/include/blis LDFLAGS=-L$ABS_PATH/BLIS_install/lib \ + bash ./continuous_integration/build_test_ext.sh + +python --version +python -c "import numpy; print('numpy %s' % numpy.__version__)" || echo "no numpy" +python -c "import scipy; print('scipy %s' % scipy.__version__)" || echo "no scipy" + +python -m flit install --symlink diff --git a/continuous_integration/posix.yml b/continuous_integration/posix.yml index 54edf669..fff39a3d 100644 --- a/continuous_integration/posix.yml +++ b/continuous_integration/posix.yml @@ -22,7 +22,12 @@ jobs: condition: eq('${{ parameters.name }}', 'macOS') - script: | continuous_integration/install.sh - displayName: 'Install' + displayName: 'Install without BLIS' + condition: ne(variables['INSTALL_BLIS'], 'true') + - script: | + continuous_integration/install_with_blis.sh + displayName: 'Install with BLIS' + condition: eq(variables['INSTALL_BLIS'], 'true') - script: | continuous_integration/test_script.sh displayName: 'Test Library' diff --git a/tests/_openmp_test_helper/nested_prange_blas.pyx b/tests/_openmp_test_helper/nested_prange_blas.pyx index 79a19f12..a6615fe5 100644 --- a/tests/_openmp_test_helper/nested_prange_blas.pyx +++ b/tests/_openmp_test_helper/nested_prange_blas.pyx @@ -1,8 +1,24 @@ cimport openmp -from cython.parallel import prange +from cython.parallel import parallel, prange import numpy as np -from scipy.linalg.cython_blas cimport dgemm + +IF USE_BLIS: + cdef extern from 'cblas.h' nogil: + ctypedef enum CBLAS_ORDER: + CblasRowMajor=101 + CblasColMajor=102 + ctypedef enum CBLAS_TRANSPOSE: + CblasNoTrans=111 + CblasTrans=112 + CblasConjTrans=113 + void dgemm 'cblas_dgemm' ( + CBLAS_ORDER Order, CBLAS_TRANSPOSE TransA, + CBLAS_TRANSPOSE TransB, int M, int N, + int K, double alpha, double *A, int lda, + double *B, int ldb, double beta, double *C, int ldc) +ELSE: + from scipy.linalg.cython_blas cimport dgemm from threadpoolctl import threadpool_info @@ -25,18 +41,25 @@ def check_nested_prange_blas(double[:, ::1] A, double[:, ::1] B, int nthreads): int i int prange_num_threads + int *prange_num_threads_ptr = &prange_num_threads - threadpool_infos = None + threadpool_infos = [None] - for i in prange(n_chunks, num_threads=nthreads, nogil=True): - dgemm(trans, no_trans, &n, &chunk_size, &k, - &alpha, &B[0, 0], &k, &A[i * chunk_size, 0], &k, - &beta, &C[i * chunk_size, 0], &n) + with nogil, parallel(num_threads=nthreads): + if openmp.omp_get_thread_num() == 0: + with gil: + threadpool_infos[0] = threadpool_info() - prange_num_threads = openmp.omp_get_num_threads() + prange_num_threads_ptr[0] = openmp.omp_get_num_threads() - if i == 0: - with gil: - threadpool_infos = threadpool_info() + for i in prange(n_chunks): + IF USE_BLIS: + dgemm(CblasRowMajor, CblasNoTrans, CblasTrans, + chunk_size, n, k, alpha, &A[i * chunk_size, 0], k, + &B[0, 0], k, beta, &C[i * chunk_size, 0], n) + ELSE: + dgemm(trans, no_trans, &n, &chunk_size, &k, + &alpha, &B[0, 0], &k, &A[i * chunk_size, 0], &k, + &beta, &C[i * chunk_size, 0], &n) - return np.asarray(C), prange_num_threads, threadpool_infos + return np.asarray(C), prange_num_threads, threadpool_infos[0] diff --git a/tests/_openmp_test_helper/setup_nested_prange_blas.py b/tests/_openmp_test_helper/setup_nested_prange_blas.py index 746e573d..54da058f 100644 --- a/tests/_openmp_test_helper/setup_nested_prange_blas.py +++ b/tests/_openmp_test_helper/setup_nested_prange_blas.py @@ -12,12 +12,16 @@ set_cc_variables("CC_OUTER_LOOP") openmp_flag = get_openmp_flag() + use_blis = os.getenv('INSTALL_BLIS', False) + libraries = ['blis'] if use_blis else [] + ext_modules = [ Extension( "nested_prange_blas", ["nested_prange_blas.pyx"], extra_compile_args=openmp_flag, - extra_link_args=openmp_flag + extra_link_args=openmp_flag, + libraries=libraries ) ] @@ -25,6 +29,7 @@ name='_openmp_test_helper_nested_prange_blas', ext_modules=cythonize( ext_modules, + compile_time_env={'USE_BLIS': use_blis}, compiler_directives={'language_level': 3, 'boundscheck': False, 'wraparound': False}) diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index 5713be76..636e9acd 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -245,11 +245,17 @@ def test_nested_prange_blas(nthreads_outer): blas_info = [module for module in threadpool_info() if module["user_api"] == "blas"] - for module in threadpool_info(): - if is_old_openblas(module): - # OpenBLAS 0.3.3 and older are known to cause an unrecoverable - # deadlock at process shutdown time (after pytest has exited). - pytest.skip("Old OpenBLAS: skipping test to avoid deadlock") + + blis_linked = any([module['internal_api'] == 'blis' + for module in threadpool_info()]) + if not blis_linked: + # numpy can be linked to BLIS for CBLAS and OpenBLAS for LAPACK. In that + # case this test will run BLIS gemm so no need to skip. + for module in threadpool_info(): + if is_old_openblas(module): + # OpenBLAS 0.3.3 and older are known to cause an unrecoverable + # deadlock at process shutdown time (after pytest has exited). + pytest.skip("Old OpenBLAS: skipping test to avoid deadlock") from ._openmp_test_helper import check_nested_prange_blas A = np.ones((1000, 10)) diff --git a/threadpoolctl.py b/threadpoolctl.py index 54bd616d..347c92df 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -74,6 +74,11 @@ class _dl_phdr_info(ctypes.Structure): "internal_api": "mkl", "filename_prefixes": ("libmkl_rt", "mkl_rt",), }, + { + "user_api": "blas", + "internal_api": "blis", + "filename_prefixes": ("libblis",), + }, ] # map a internal_api (openmp, openblas, mkl) to set and get functions @@ -88,6 +93,9 @@ class _dl_phdr_info(ctypes.Structure): "mkl": { "set_num_threads": "MKL_Set_Num_Threads", "get_num_threads": "MKL_Get_Max_Threads"}, + "blis": { + "set_num_threads": "bli_thread_set_num_threads", + "get_num_threads": "bli_thread_get_num_threads"} } # Helpers for the doc and test names @@ -110,9 +118,8 @@ def decorator(o): def _get_limit(prefix, user_api, limits): if prefix in limits: return limits[prefix] - if user_api in limits: + else: return limits[user_api] - return None @_format_docstring(ALL_PREFIXES=_ALL_PREFIXES, @@ -210,6 +217,10 @@ def threadpool_info(): modules = _load_modules(user_api=_ALL_USER_APIS) for module in modules: module['num_threads'] = module['get_num_threads']() + # by default BLIS is single-threaded and get_num_threads returns -1. + # we map it to 1 for consistency with other libraries. + if module['num_threads'] == -1 and module['internal_api'] == 'blis': + module['num_threads'] = 1 # Remove the wrapper for the module and its function del module['set_num_threads'], module['get_num_threads'] del module['dynlib'] @@ -227,6 +238,8 @@ def _get_version(dynlib, internal_api): return None elif internal_api == "openblas": return _get_openblas_version(dynlib) + elif internal_api == "blis": + return _get_blis_version(dynlib) else: raise NotImplementedError("Unsupported API {}".format(internal_api)) @@ -257,6 +270,13 @@ def _get_openblas_version(openblas_dynlib): return None +def _get_blis_version(blis_dynlib): + """Return the BLIS version""" + get_version = getattr(blis_dynlib, "bli_info_get_version_str") + get_version.restype = ctypes.c_char_p + return get_version().decode('utf-8') + + # Loading utilities for dynamically linked shared objects def _load_modules(prefixes=None, user_api=None): From 683d0e207e014584fcf86fc8977edc2d7550dc5a Mon Sep 17 00:00:00 2001 From: jeremiedbb <34657725+jeremiedbb@users.noreply.github.com> Date: Wed, 11 Sep 2019 11:07:37 +0200 Subject: [PATCH 035/187] Fully disable threading of BLIS to improve coverage (#30) --- .azure_pipeline.yml | 2 +- continuous_integration/install_with_blis.sh | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml index 6f3e26ba..440d3746 100644 --- a/.azure_pipeline.yml +++ b/.azure_pipeline.yml @@ -90,7 +90,7 @@ jobs: CC_OUTER_LOOP: 'clang-8' CC_INNER_LOOP: 'clang-8' BLIS_CC: 'gcc' - pylatest_blis_sinlge_threaded: + pylatest_blis_no_threading: PACKAGER: 'conda' VERSION_PYTHON: '*' INSTALL_BLIS: 'true' diff --git a/continuous_integration/install_with_blis.sh b/continuous_integration/install_with_blis.sh index 010c3b64..d6316f20 100755 --- a/continuous_integration/install_with_blis.sh +++ b/continuous_integration/install_with_blis.sh @@ -23,7 +23,12 @@ pushd .. mkdir BLIS_install git clone https://github.com/flame/blis.git pushd blis -./configure --prefix=$ABS_PATH/BLIS_install --enable-cblas --enable-threading=openmp CC=$BLIS_CC auto +if [[ "$BLIS_NUM_THREADS" == "1" ]]; then + THREADING='no' +else + THREADING='openmp' +fi +./configure --prefix=$ABS_PATH/BLIS_install --enable-cblas --enable-threading=$THREADING CC=$BLIS_CC auto make -j4 make install popd From 96fa9b5f78211ad7e6ca1ba3405448e2fb5ae805 Mon Sep 17 00:00:00 2001 From: jeremiedbb <34657725+jeremiedbb@users.noreply.github.com> Date: Thu, 12 Sep 2019 14:37:50 +0200 Subject: [PATCH 036/187] remove unnecessary llvm-openmp (#31) --- continuous_integration/install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/continuous_integration/install.sh b/continuous_integration/install.sh index 6d112bf9..f94d8791 100755 --- a/continuous_integration/install.sh +++ b/continuous_integration/install.sh @@ -18,9 +18,9 @@ make_conda() { if [[ "$UNAMESTR" == "Darwin" ]]; then if [[ "$INSTALL_LIBOMP" == "conda-forge" ]]; then # Install an OpenMP-enabled clang/llvm from conda-forge - TO_INSTALL="$TO_INSTALL conda-forge::compilers conda-forge::llvm-openmp" - export LDFLAGS="$LDFLAGS -Wl,-rpath,$CONDA/envs/$VIRTUALENV/lib -L$CONDA/envs/$VIRTUALENV/lib" + TO_INSTALL="$TO_INSTALL conda-forge::compilers" export CFLAGS="$CFLAGS -I$CONDA/envs/$VIRTUALENV/include" + export LDFLAGS="$LDFLAGS -Wl,-rpath,$CONDA/envs/$VIRTUALENV/lib -L$CONDA/envs/$VIRTUALENV/lib" elif [[ "$INSTALL_LIBOMP" == "homebrew" ]]; then # Install a compiler with a working openmp From fa3263a4ddf1f3acf0c594b814a25757957a968e Mon Sep 17 00:00:00 2001 From: jeremiedbb <34657725+jeremiedbb@users.noreply.github.com> Date: Thu, 12 Sep 2019 15:02:21 +0200 Subject: [PATCH 037/187] MAINT: Fix + Refactor get_original_num_threads & add a test (#32) --- tests/test_threadpoolctl.py | 28 ++++++++++++++++++++++++-- threadpoolctl.py | 40 +++++++++++++++++++++++++------------ 2 files changed, 53 insertions(+), 15 deletions(-) diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index 636e9acd..d7698456 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -191,7 +191,7 @@ def test_openmp_nesting(nthreads_outer): assert len(openmp_infos) >= 2 with threadpool_limits(limits=1) as threadpoolctx: - max_threads = threadpoolctx.get_original_num_threads('openmp') + max_threads = threadpoolctx.get_original_num_threads()['openmp'] nthreads = effective_num_threads(nthreads_outer, max_threads) outer_num_threads, inner_num_threads = \ @@ -262,7 +262,7 @@ def test_nested_prange_blas(nthreads_outer): B = np.ones((100, 10)) with threadpool_limits(limits=1) as threadpoolctx: - max_threads = threadpoolctx.get_original_num_threads('openmp') + max_threads = threadpoolctx.get_original_num_threads()['openmp'] nthreads = effective_num_threads(nthreads_outer, max_threads) result = check_nested_prange_blas(A, B, nthreads) @@ -277,3 +277,27 @@ def test_nested_prange_blas(nthreads_outer): assert len(nested_blas_info) == len(blas_info) for module in nested_blas_info: assert module['num_threads'] == 1 + + +@pytest.mark.parametrize("limit", [1, None]) +def test_get_original_num_threads(limit): + with threadpool_limits(limits=2, user_api='blas') as ctl: + # set different blas num threads to start with (when multiple openblas) + if ctl._original_limits: + ctl._original_limits[0]['set_num_threads'](1) + + original_infos = threadpool_info() + with threadpool_limits(limits=limit, user_api='blas') as threadpoolctx: + original_num_threads = threadpoolctx.get_original_num_threads() + + assert 'openmp' not in original_num_threads + + if 'blas' in [module['user_api'] for module in original_infos]: + assert original_num_threads['blas'] >= 1 + else: + assert original_num_threads['blas'] is None + + if len(libopenblas_paths) >= 2: + with pytest.warns(None, match='Multiple value possible'): + expected = min([module['num_threads'] for module in original_infos]) + assert original_num_threads['blas'] == expected diff --git a/threadpoolctl.py b/threadpoolctl.py index 347c92df..0287a7e2 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -517,6 +517,8 @@ class threadpool_limits: BLAS libraries if they rely on OpenMP. """ def __init__(self, limits=None, user_api=None): + self._user_api = _ALL_USER_APIS if user_api is None else [user_api] + if limits is not None: self._original_limits = _set_threadpool_limits( limits=limits, user_api=user_api) @@ -534,18 +536,30 @@ def unregister(self): for module in self._original_limits: module['set_num_threads'](module['num_threads']) - def get_original_num_threads(self, user_api): - limits = [module['num_threads'] for module in self._original_limits - if module['user_api'] == user_api] + def get_original_num_threads(self): + original_limits = self._original_limits or threadpool_info() - limits = set(limits) - n_limits = len(limits) + num_threads = {} + warning_apis = [] - if n_limits == 1: - return limits.pop() - elif n_limits == 0: - return None - else: - warnings.warn("Multiple value possible for user_api='{}'. " - "Returning the minimum.".format(user_api)) - return min(limits) + for user_api in self._user_api: + limits = [module['num_threads'] for module in original_limits + if module['user_api'] == user_api] + limits = set(limits) + n_limits = len(limits) + + if n_limits == 1: + limit = limits.pop() + elif n_limits == 0: + limit = None + else: + limit = min(limits) + warning_apis.append(user_api) + + num_threads[user_api] = limit + + if warning_apis: + warnings.warn("Multiple value possible for following user apis: " + + ', '.join(warning_apis) + ". Returning the minimum.") + + return num_threads From b7b0be3be163b6e153d609bcd8bc8e3ae1cc89f3 Mon Sep 17 00:00:00 2001 From: jeremiedbb <34657725+jeremiedbb@users.noreply.github.com> Date: Thu, 12 Sep 2019 16:29:11 +0200 Subject: [PATCH 038/187] [MRG] Support libraries linked to symlinks (#34) --- .azure_pipeline.yml | 6 ++++++ continuous_integration/install.sh | 6 ++++++ continuous_integration/posix.yml | 2 +- continuous_integration/test_script.sh | 2 +- threadpoolctl.py | 17 +++++++++++++++++ 5 files changed, 31 insertions(+), 2 deletions(-) diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml index 440d3746..4feee5ac 100644 --- a/.azure_pipeline.yml +++ b/.azure_pipeline.yml @@ -65,6 +65,12 @@ jobs: VERSION_PYTHON: '*' CC_OUTER_LOOP: 'gcc' CC_INNER_LOOP: 'clang-8' + # Linux environment with numpy from conda-forge channel + pylatest_conda_forge: + PACKAGER: 'conda-forge' + VERSION_PYTHON: '*' + CC_OUTER_LOOP: 'gcc' + CC_INNER_LOOP: 'gcc' # Linux environment with no numpy and heterogeneous OpenMP runtime # nesting. pylatest_conda_nonumpy_gcc_clang: diff --git a/continuous_integration/install.sh b/continuous_integration/install.sh index f94d8791..fe9a224d 100755 --- a/continuous_integration/install.sh +++ b/continuous_integration/install.sh @@ -47,6 +47,12 @@ if [[ "$PACKAGER" == "conda" ]]; then fi make_conda $TO_INSTALL +elif [[ "$PACKAGER" == "conda-forge" ]]; then + conda config --prepend channels conda-forge + conda config --set channel_priority strict + TO_INSTALL="python=$VERSION_PYTHON numpy scipy" + make_conda $TO_INSTALL + elif [[ "$PACKAGER" == "pip" ]]; then # Use conda to build an empty python env and then use pip to install # numpy and scipy diff --git a/continuous_integration/posix.yml b/continuous_integration/posix.yml index fff39a3d..5e0ee94a 100644 --- a/continuous_integration/posix.yml +++ b/continuous_integration/posix.yml @@ -14,7 +14,7 @@ jobs: steps: - bash: echo "##vso[task.prependpath]$CONDA/bin" displayName: Add conda to PATH - condition: or(eq(variables['PACKAGER'], 'conda'), eq(variables['PACKAGER'], 'pip')) + condition: or(startsWith(variables['PACKAGER'], 'conda'), eq(variables['PACKAGER'], 'pip')) - bash: sudo chown -R $USER $CONDA # On Hosted macOS, the agent user doesn't have ownership of Miniconda's installation directory/ # We need to take ownership if we want to update conda or install packages globally diff --git a/continuous_integration/test_script.sh b/continuous_integration/test_script.sh index f2c2b78a..2d48fff4 100755 --- a/continuous_integration/test_script.sh +++ b/continuous_integration/test_script.sh @@ -2,7 +2,7 @@ set -e -if [[ "$PACKAGER" == "conda" ]]; then +if [[ "$PACKAGER" == conda* ]]; then source activate $VIRTUALENV elif [[ "$PACKAGER" == "pip" ]]; then # we actually use conda to install the base environment: diff --git a/threadpoolctl.py b/threadpoolctl.py index 0287a7e2..372377ce 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -23,6 +23,9 @@ # Cache for libc under POSIX and a few system libraries under Windows _system_libraries = {} +# Cache for calls to os.path.realpath on system libraries to reduce the +# impact of slow system calls (e.g. stat) on slow filesystem +_realpaths = dict() # One can get runtime errors or even segfaults due to multiple OpenMP libraries # loaded simultaneously which can happen easily in Python when importing and @@ -107,6 +110,18 @@ class _dl_phdr_info(ctypes.Structure): _ALL_INTERNAL_APIS = list(_MAP_API_TO_FUNC.keys()) +def _realpath(filepath, cache_limit=10000): + """Small caching wrapper around os.path.realpath to limit system calls""" + rpath = _realpaths.get(filepath) + if rpath is None: + rpath = os.path.realpath(filepath) + if len(_realpaths) < cache_limit: + # If we drop support for Python 2.7, we could use functools.lru_cache + # with maxsize=10000 instead. + _realpaths[filepath] = rpath + return rpath + + def _format_docstring(*args, **kwargs): def decorator(o): o.__doc__ = o.__doc__.format(*args, **kwargs) @@ -329,6 +344,8 @@ def _make_module_info(filepath, module_info, prefix): def _get_module_info_from_path(filepath, prefixes, user_api, modules): + # Required to resolve symlinks + filepath =_realpath(filepath) # `lower` required to take account of OpenMP dll case on Windows # (vcomp, VCOMP, Vcomp, ...) filename = os.path.basename(filepath).lower() From 88be98b2bcd387a734b7f856b7388403a6771dcd Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Thu, 12 Sep 2019 17:39:31 +0200 Subject: [PATCH 039/187] Prepare the 1.1.0 release (#35) --- .gitignore | 2 ++ CHANGES.md | 15 +++++++++++++++ README.md | 4 ++++ threadpoolctl.py | 2 +- 4 files changed, 22 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 0b6ad0a9..a0afdb17 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ # Python and Cython generated files *.pyc __pycache__ +.cache +.pytest_cache *.so *.dylib *.c diff --git a/CHANGES.md b/CHANGES.md index bc8315c4..2f62af08 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,18 @@ +1.1.0 (2019-09-12) +================== + +- Detect libraries referenced by symlinks (e.g. BLAS libraries from + conda-forge). + https://github.com/joblib/threadpoolctl/pull/34 + +- New method `get_original_num_threads` on the `threadpool_limits` + context manager to cheaply access the initial state of the runtime. + https://github.com/joblib/threadpoolctl/pull/32 + +- Add support for BLIS. + https://github.com/joblib/threadpoolctl/pull/23 + + 1.0.0 (2019-06-03) ================== diff --git a/README.md b/README.md index 082e6874..3d1bec9e 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,10 @@ for an example. To make a release: +Bump the version number (`__version__`) in `threadpoolctl.py`. + +Build the distribution archives: + ```bash pip install flit flit build diff --git a/threadpoolctl.py b/threadpoolctl.py index 372377ce..524ae02d 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -17,7 +17,7 @@ import warnings from ctypes.util import find_library -__version__ = '1.0.0' +__version__ = '1.1.0' __all__ = ["threadpool_limits", "threadpool_info"] # Cache for libc under POSIX and a few system libraries under Windows From ab7b2e664f6fe1d56e121de1a1a58cf978e5df89 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Thu, 12 Sep 2019 17:44:05 +0200 Subject: [PATCH 040/187] Better changelog --- CHANGES.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 2f62af08..4850acd5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,13 +5,18 @@ conda-forge). https://github.com/joblib/threadpoolctl/pull/34 -- New method `get_original_num_threads` on the `threadpool_limits` - context manager to cheaply access the initial state of the runtime. - https://github.com/joblib/threadpoolctl/pull/32 - - Add support for BLIS. https://github.com/joblib/threadpoolctl/pull/23 +- Breaking change: method `get_original_num_threads` on the `threadpool_limits` + context manager to cheaply access the initial state of the runtime: + - drop the `user_api` parameter; + - instead return a dict `{user_api: num_threads}`; + - fixed a bug when the limit parameter of `threadpool_limits` was set to + `None`. + + https://github.com/joblib/threadpoolctl/pull/32 + 1.0.0 (2019-06-03) ================== From f6e65387fd364e7fe93f7774bdf3660442ae81ea Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Fri, 13 Sep 2019 09:27:58 +0200 Subject: [PATCH 041/187] Back to dev mode --- threadpoolctl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/threadpoolctl.py b/threadpoolctl.py index 524ae02d..428a75fc 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -17,7 +17,7 @@ import warnings from ctypes.util import find_library -__version__ = '1.1.0' +__version__ = '1.2.0.dev0' __all__ = ["threadpool_limits", "threadpool_info"] # Cache for libc under POSIX and a few system libraries under Windows From e8c9dc6ebd5565a75284aed31beec2e439d4f80e Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Fri, 13 Sep 2019 16:17:36 +0200 Subject: [PATCH 042/187] Give credits to smp in README.md --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index 3d1bec9e..dcef5638 100644 --- a/README.md +++ b/README.md @@ -122,3 +122,14 @@ tag to github and then: ```bash flit publish ``` + +### Credits + +The initial dynamic library introspection code was written by @anton-malakhov +for the smp package available at https://github.com/IntelPython/smp . + +threadpoolctl extends this for other operationg systems. Contrary to smp, +threadpoolctl does not attempt to limit the size of Python multiprocessing +pools (threads or processes) or set operating system-level CPU affinity +constraints: threadpoolctl only interacts with native libraries via their +public runtime APIs. From 447602579f304cb43361b64a621ffb9bbed0ae01 Mon Sep 17 00:00:00 2001 From: jeremiedbb <34657725+jeremiedbb@users.noreply.github.com> Date: Wed, 25 Sep 2019 12:32:30 +0200 Subject: [PATCH 043/187] [MRG] install llvm-openmp in macOS conda-forge job again (#39) * trigger ci * add llvm-openmp back --- continuous_integration/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/continuous_integration/install.sh b/continuous_integration/install.sh index fe9a224d..98df5777 100755 --- a/continuous_integration/install.sh +++ b/continuous_integration/install.sh @@ -18,7 +18,7 @@ make_conda() { if [[ "$UNAMESTR" == "Darwin" ]]; then if [[ "$INSTALL_LIBOMP" == "conda-forge" ]]; then # Install an OpenMP-enabled clang/llvm from conda-forge - TO_INSTALL="$TO_INSTALL conda-forge::compilers" + TO_INSTALL="$TO_INSTALL conda-forge::compilers conda-forge::llvm-openmp" export CFLAGS="$CFLAGS -I$CONDA/envs/$VIRTUALENV/include" export LDFLAGS="$LDFLAGS -Wl,-rpath,$CONDA/envs/$VIRTUALENV/lib -L$CONDA/envs/$VIRTUALENV/lib" From c6deb9c324e19736e1d885b352ce1b1f46f7826e Mon Sep 17 00:00:00 2001 From: jeremiedbb <34657725+jeremiedbb@users.noreply.github.com> Date: Tue, 22 Oct 2019 10:11:37 +0200 Subject: [PATCH 044/187] [MRG] UNIX: dlopen with RTLD_NOLOAD flag (#42) --- threadpoolctl.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/threadpoolctl.py b/threadpoolctl.py index 428a75fc..fe7960c6 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -57,6 +57,13 @@ class _dl_phdr_info(ctypes.Structure): ] +# The RTLD_NOLOAD flag for loading shared libraries is not defined on Windows. +try: + _RTLD_NOLOAD = os.RTLD_NOLOAD +except AttributeError: + _RTLD_NOLOAD = ctypes.DEFAULT_MODE + + # List of the supported implementations. The items hold the prefix of loaded # shared objects, the name of the internal_api to call, matching the # MAP_API_TO_FUNC keys and the name of the user_api, in {"blas", "openmp"}. @@ -328,7 +335,7 @@ def _match_module(module_info, prefix, prefixes, user_api): def _make_module_info(filepath, module_info, prefix): """Make a dict with the information from the module.""" filepath = os.path.normpath(filepath) - dynlib = ctypes.CDLL(filepath) + dynlib = ctypes.CDLL(filepath, mode=_RTLD_NOLOAD) internal_api = module_info['internal_api'] set_func = getattr(dynlib, _MAP_API_TO_FUNC[internal_api]['set_num_threads'], @@ -493,7 +500,7 @@ def _get_libc(): libc_name = find_library("c") if libc_name is None: # pragma: no cover return None - libc = ctypes.CDLL(libc_name) + libc = ctypes.CDLL(libc_name, mode=_RTLD_NOLOAD) _system_libraries["libc"] = libc return libc From 2a56bbb19d854d1eadf79d4d1d067c1d07b3ced7 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Tue, 22 Oct 2019 10:13:14 +0200 Subject: [PATCH 045/187] Fix indentation level --- threadpoolctl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/threadpoolctl.py b/threadpoolctl.py index fe7960c6..8e4e1376 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -59,9 +59,9 @@ class _dl_phdr_info(ctypes.Structure): # The RTLD_NOLOAD flag for loading shared libraries is not defined on Windows. try: - _RTLD_NOLOAD = os.RTLD_NOLOAD + _RTLD_NOLOAD = os.RTLD_NOLOAD except AttributeError: - _RTLD_NOLOAD = ctypes.DEFAULT_MODE + _RTLD_NOLOAD = ctypes.DEFAULT_MODE # List of the supported implementations. The items hold the prefix of loaded From af7d7fd58f3a9e2ba53246038eb213ff5a1bd3cb Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 23 Oct 2019 12:28:27 +0200 Subject: [PATCH 046/187] CI use recent gcc version to build Blis under Linux (#46) --- .azure_pipeline.yml | 4 ++-- continuous_integration/install_with_blis.sh | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml index 4feee5ac..af4cd24d 100644 --- a/.azure_pipeline.yml +++ b/.azure_pipeline.yml @@ -95,7 +95,7 @@ jobs: BLIS_NUM_THREADS: '4' CC_OUTER_LOOP: 'clang-8' CC_INNER_LOOP: 'clang-8' - BLIS_CC: 'gcc' + BLIS_CC: 'gcc-8' pylatest_blis_no_threading: PACKAGER: 'conda' VERSION_PYTHON: '*' @@ -103,7 +103,7 @@ jobs: BLIS_NUM_THREADS: '1' CC_OUTER_LOOP: 'gcc' CC_INNER_LOOP: 'gcc' - BLIS_CC: 'gcc' + BLIS_CC: 'gcc-8' - template: continuous_integration/posix.yml parameters: diff --git a/continuous_integration/install_with_blis.sh b/continuous_integration/install_with_blis.sh index d6316f20..356452f1 100755 --- a/continuous_integration/install_with_blis.sh +++ b/continuous_integration/install_with_blis.sh @@ -17,6 +17,10 @@ sudo apt install clang-8 libomp-8-dev conda create -n $VIRTUALENV -q --yes python=$VERSION_PYTHON pip cython source activate $VIRTUALENV +if [[ "$BLIS_CC" == "gcc-8" ]]; then + sudo apt install gcc-8 +fi + pushd .. # build & install blis From 706b85fa96ba4ff5f64cfef92ad46c6a599bd3a4 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Thu, 24 Oct 2019 16:02:05 +0200 Subject: [PATCH 047/187] expose threading layer of mkl --- threadpoolctl.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/threadpoolctl.py b/threadpoolctl.py index 8e4e1376..23d0f4ee 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -243,6 +243,9 @@ def threadpool_info(): # we map it to 1 for consistency with other libraries. if module['num_threads'] == -1 and module['internal_api'] == 'blis': module['num_threads'] = 1 + if module['internal_api'] == 'mkl': + layer = _get_mkl_threading_layer(module['dynlib']) + module['threading_layer'] = layer # Remove the wrapper for the module and its function del module['set_num_threads'], module['get_num_threads'] del module['dynlib'] @@ -251,6 +254,17 @@ def threadpool_info(): return infos +def _get_mkl_threading_layer(mkl_dynlib): + """Return the threading layer of MKL""" + # The function mkl_set_threading_layer returns the current threading layer + # Calling it with an invalid threading layer allows us to safely get + # the threading layer + set_threading_layer = getattr(mkl_dynlib, "MKL_Set_Threading_Layer") + layer_map = {0: "intel", 1: "sequential", 2: "pgi", + 3: "gnu", 4: "tbb", -1: "not specified"} + return layer_map[set_threading_layer(-1)] + + def _get_version(dynlib, internal_api): if internal_api == "mkl": return _get_mkl_version(dynlib) From 4ebace5aba695bbbe6aaae1cc2d27a7af4c7d345 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Fri, 25 Oct 2019 11:34:52 +0200 Subject: [PATCH 048/187] add a test and explicitly set the threading layer in a ci job --- .azure_pipeline.yml | 1 + tests/test_threadpoolctl.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml index af4cd24d..41bde87a 100644 --- a/.azure_pipeline.yml +++ b/.azure_pipeline.yml @@ -58,6 +58,7 @@ jobs: VERSION_PYTHON: '*' CC_OUTER_LOOP: 'clang-8' CC_INNER_LOOP: 'gcc' + MKL_THREADING_LAYER: 'INTEL' # Same but with numpy / scipy installed with pip from PyPI and switched # compilers. pylatest_pip_openblas_gcc_clang: diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index d7698456..5083a23d 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -1,3 +1,4 @@ +import os import re import ctypes import pytest @@ -301,3 +302,17 @@ def test_get_original_num_threads(limit): with pytest.warns(None, match='Multiple value possible'): expected = min([module['num_threads'] for module in original_infos]) assert original_num_threads['blas'] == expected + + +def test_mkl_threading_layer(): + mkl_info = [module for module in threadpool_info() + if module['internal_api'] == 'mkl'] + + if not mkl_info: + pytest.skip("mkl runtime missing") + + expected_layer = os.getenv("MKL_THREADING_LAYER") + actual_layer = mkl_info[0]['threading_layer'] + + if expected_layer: + assert actual_layer == expected_layer.lower() From 9d81b07695b9f6c5e8cd595c47fa3ed3feab041b Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Fri, 25 Oct 2019 11:36:46 +0200 Subject: [PATCH 049/187] cln --- tests/test_threadpoolctl.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index 5083a23d..c0d2c130 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -305,11 +305,13 @@ def test_get_original_num_threads(limit): def test_mkl_threading_layer(): + # Check that threadpool_info correctly recovers the threading layer used + # by mkl mkl_info = [module for module in threadpool_info() if module['internal_api'] == 'mkl'] if not mkl_info: - pytest.skip("mkl runtime missing") + pytest.skip("requires MKL") expected_layer = os.getenv("MKL_THREADING_LAYER") actual_layer = mkl_info[0]['threading_layer'] From b7fa4e822e309e92ff905c9702a0026ad34ed218 Mon Sep 17 00:00:00 2001 From: jeremiedbb <34657725+jeremiedbb@users.noreply.github.com> Date: Fri, 25 Oct 2019 17:25:28 +0200 Subject: [PATCH 050/187] [MRG] CI: add linting to one of the ci jobs (#50) --- .azure_pipeline.yml | 1 + benchmarks/bench_context_manager_overhead.py | 4 ++-- continuous_integration/posix.yml | 5 +++++ tests/test_threadpoolctl.py | 7 ++++--- threadpoolctl.py | 11 ++++++----- 5 files changed, 18 insertions(+), 10 deletions(-) diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml index af4cd24d..31392826 100644 --- a/.azure_pipeline.yml +++ b/.azure_pipeline.yml @@ -38,6 +38,7 @@ jobs: VERSION_PYTHON: '3.5' CC_OUTER_LOOP: 'gcc' CC_INNER_LOOP: 'gcc' + LINT: 'true' py35_ubuntu_openblas_gcc_gcc: PACKAGER: 'ubuntu' APT_BLAS: 'libopenblas-base libopenblas-dev' diff --git a/benchmarks/bench_context_manager_overhead.py b/benchmarks/bench_context_manager_overhead.py index 7977c8a8..34c1d92e 100644 --- a/benchmarks/bench_context_manager_overhead.py +++ b/benchmarks/bench_context_manager_overhead.py @@ -24,5 +24,5 @@ pass timings.append(time.time() - t) -print(f"Overhead per call: {mean(timings) * 1e3:.3f} " - f"+/-{stdev(timings) * 1e3:.3f} ms") +print("Overhead per call: {:.3f} +/-{:.3f} ms" + .format(mean(timings) * 1e3, stdev(timings) * 1e3)) diff --git a/continuous_integration/posix.yml b/continuous_integration/posix.yml index 5e0ee94a..b6cc5b18 100644 --- a/continuous_integration/posix.yml +++ b/continuous_integration/posix.yml @@ -12,6 +12,11 @@ jobs: ${{ insert }}: ${{ parameters.matrix }} steps: + - script: | + pip3 install flake8 + python3 -m flake8 . + displayName: Lint + condition: eq(variables['LINT'], 'true') - bash: echo "##vso[task.prependpath]$CONDA/bin" displayName: Add conda to PATH condition: or(startsWith(variables['PACKAGER'], 'conda'), eq(variables['PACKAGER'], 'pip')) diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index d7698456..201c5ab4 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -249,8 +249,8 @@ def test_nested_prange_blas(nthreads_outer): blis_linked = any([module['internal_api'] == 'blis' for module in threadpool_info()]) if not blis_linked: - # numpy can be linked to BLIS for CBLAS and OpenBLAS for LAPACK. In that - # case this test will run BLIS gemm so no need to skip. + # numpy can be linked to BLIS for CBLAS and OpenBLAS for LAPACK. In + # that case this test will run BLIS gemm so no need to skip. for module in threadpool_info(): if is_old_openblas(module): # OpenBLAS 0.3.3 and older are known to cause an unrecoverable @@ -299,5 +299,6 @@ def test_get_original_num_threads(limit): if len(libopenblas_paths) >= 2: with pytest.warns(None, match='Multiple value possible'): - expected = min([module['num_threads'] for module in original_infos]) + expected = min( + [module['num_threads'] for module in original_infos]) assert original_num_threads['blas'] == expected diff --git a/threadpoolctl.py b/threadpoolctl.py index 8e4e1376..2c6dc515 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -123,8 +123,8 @@ def _realpath(filepath, cache_limit=10000): if rpath is None: rpath = os.path.realpath(filepath) if len(_realpaths) < cache_limit: - # If we drop support for Python 2.7, we could use functools.lru_cache - # with maxsize=10000 instead. + # If we drop support for Python 2.7, we could use + # functools.lru_cache with maxsize=10000 instead. _realpaths[filepath] = rpath return rpath @@ -352,7 +352,7 @@ def _make_module_info(filepath, module_info, prefix): def _get_module_info_from_path(filepath, prefixes, user_api, modules): # Required to resolve symlinks - filepath =_realpath(filepath) + filepath = _realpath(filepath) # `lower` required to take account of OpenMP dll case on Windows # (vcomp, VCOMP, Vcomp, ...) filename = os.path.basename(filepath).lower() @@ -583,7 +583,8 @@ def get_original_num_threads(self): num_threads[user_api] = limit if warning_apis: - warnings.warn("Multiple value possible for following user apis: " - + ', '.join(warning_apis) + ". Returning the minimum.") + warnings.warn( + "Multiple value possible for following user apis: " + + ', '.join(warning_apis) + ". Returning the minimum.") return num_threads From 91dfa970d4bf792119755c1706842e59d0c4eb3a Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Wed, 6 Nov 2019 13:57:47 +0100 Subject: [PATCH 051/187] add change log entry --- CHANGES.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 4850acd5..35fe4605 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,11 @@ +1.2.0 (TBD) +=========== + +- Expose MKL threading layer in informations displayed by `threadpool_info`. + This information is referenced in the `threading_layer` field. + https://github.com/joblib/threadpoolctl/pull/48 + + 1.1.0 (2019-09-12) ================== From 2e5f20fc0ee022dd9704e826edc5accca5ba90e9 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Wed, 6 Nov 2019 14:20:17 +0100 Subject: [PATCH 052/187] pin python version when installing from pip --- .azure_pipeline.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml index 31392826..df5133f3 100644 --- a/.azure_pipeline.yml +++ b/.azure_pipeline.yml @@ -59,11 +59,11 @@ jobs: VERSION_PYTHON: '*' CC_OUTER_LOOP: 'clang-8' CC_INNER_LOOP: 'gcc' - # Same but with numpy / scipy installed with pip from PyPI and switched - # compilers. - pylatest_pip_openblas_gcc_clang: + # Linux + Python 3.7 with numpy / scipy installed with pip from PyPI and + # heterogeneous openmp runtimes. + py37_pip_openblas_gcc_clang: PACKAGER: 'pip' - VERSION_PYTHON: '*' + VERSION_PYTHON: '3.7' CC_OUTER_LOOP: 'gcc' CC_INNER_LOOP: 'clang-8' # Linux environment with numpy from conda-forge channel From c61a40d24f0bb58e32c4120d7670bbb438d0ed14 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Wed, 13 Nov 2019 17:31:30 +0100 Subject: [PATCH 053/187] trigger ci From c6e5a62b83c1100c17d51db6fe8152f3d7203587 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Wed, 13 Nov 2019 17:38:02 +0100 Subject: [PATCH 054/187] apt-get update --- continuous_integration/install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/continuous_integration/install.sh b/continuous_integration/install.sh index 98df5777..920cffe2 100755 --- a/continuous_integration/install.sh +++ b/continuous_integration/install.sh @@ -66,6 +66,7 @@ elif [[ "$PACKAGER" == "ubuntu" ]]; then # Remove the ubuntu toolchain PPA that seems to be invalid: # https://github.com/scikit-learn/scikit-learn/pull/13934 sudo add-apt-repository --remove ppa:ubuntu-toolchain-r/test + sudo apt-get update sudo apt-get install python3-scipy python3-virtualenv $APT_BLAS python3 -m virtualenv --system-site-packages --python=python3 $VIRTUALENV source $VIRTUALENV/bin/activate From 712804e798ca0ccf594fe85dd50a9522953b84f9 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Thu, 14 Nov 2019 16:04:08 +0100 Subject: [PATCH 055/187] huge refactor --- tests/test_threadpoolctl.py | 186 +++---- threadpoolctl.py | 1015 +++++++++++++++++++---------------- 2 files changed, 633 insertions(+), 568 deletions(-) diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index 75d88895..21834c1c 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -1,9 +1,7 @@ import os import re -import ctypes import pytest - from threadpoolctl import threadpool_limits, threadpool_info from threadpoolctl import _ALL_PREFIXES, _ALL_USER_APIS @@ -15,7 +13,7 @@ def is_old_openblas(module): # Possible bug in getting maximum number of threads with OpenBLAS < 0.2.16 # and OpenBLAS does not expose its version before 0.3.4. - return module['internal_api'] == "openblas" and module['version'] is None + return module["internal_api"] == "openblas" and module["version"] is None def effective_num_threads(nthreads, max_threads): @@ -25,61 +23,44 @@ def effective_num_threads(nthreads, max_threads): @pytest.mark.parametrize("prefix", _ALL_PREFIXES) -def test_threadpool_limits_by_prefix(openblas_present, mkl_present, prefix): +@pytest.mark.parametrize("limit", [1, 3]) +def test_threadpool_limits_by_prefix(prefix, limit): + # Check that the maximum number of threads can be set by prefix original_infos = threadpool_info() - mkl_found = any([True for info in original_infos - if info["prefix"] in ('mkl_rt', 'libmkl_rt')]) - prefix_found = len([info["prefix"] for info in original_infos - if info["prefix"] == prefix]) - if not prefix_found: - if "mkl_rt" in prefix and mkl_present and not mkl_found: - raise RuntimeError("Could not load the MKL prefix") - elif prefix == "libopenblas" and openblas_present: - raise RuntimeError("Could not load the OpenBLAS prefix") - else: - pytest.skip("{} runtime missing".format(prefix)) - - with threadpool_limits(limits={prefix: 1}): - for module in threadpool_info(): - if is_old_openblas(module): - continue - if module["prefix"] == prefix: - assert module["num_threads"] == 1 - with threadpool_limits(limits={prefix: 3}): - for module in threadpool_info(): + modules_matching_prefix = original_infos.get_modules("prefix", prefix) + if not modules_matching_prefix: + pytest.skip("Requires {} runtime".format(prefix)) + + with threadpool_limits(limits={prefix: limit}): + for module in modules_matching_prefix: if is_old_openblas(module): continue - if module["prefix"] == prefix: - assert module["num_threads"] <= 3 + # threadpool_limits only sets an upper bound on the number of + # threads. + assert 0 < module.get_num_threads() <= limit assert threadpool_info() == original_infos @pytest.mark.parametrize("user_api", (None, "blas", "openmp")) -def test_set_threadpool_limits_by_api(user_api): - # Check that the number of threads used by the multithreaded libraries can - # be modified dynamically. - if user_api is None: - user_apis = ("blas", "openmp") - else: - user_apis = (user_api,) - +@pytest.mark.parametrize("limit", [1, 3]) +def test_set_threadpool_limits_by_api(user_api, limit): + # Check that the maximum number of threads can be set by user_api original_infos = threadpool_info() - with threadpool_limits(limits=1, user_api=user_api): - for module in threadpool_info(): - if is_old_openblas(module): - continue - if module["user_api"] in user_apis: - assert module["num_threads"] == 1 + modules_matching_api = original_infos.get_modules("user_api", user_api) + if not modules_matching_api: + user_apis = _ALL_USER_APIS if user_api is None else [user_api] + pytest.skip("Requires a library which api is in {}".format(user_apis)) - with threadpool_limits(limits=3, user_api=user_api): - for module in threadpool_info(): + with threadpool_limits(limits=limit, user_api=user_api): + for module in modules_matching_api: if is_old_openblas(module): continue - if module["user_api"] in user_apis: - assert module["num_threads"] <= 3 + # threadpool_limits only sets an upper bound on the number of + # threads. + assert 0 < module.get_num_threads() <= limit assert threadpool_info() == original_infos @@ -113,8 +94,8 @@ def test_set_threadpool_limits_no_limit(): def test_threadpool_limits_manual_unregister(): - # Check that threadpool_limits can be used as an object with that hold - # the original state of the threadpools that can be restored thanks to the + # Check that threadpool_limits can be used as an object which holds the + # original state of the threadpools and that can be restored thanks to the # dedicated unregister method original_infos = threadpool_info() @@ -145,7 +126,7 @@ def test_threadpool_limits_bad_input(): @with_check_openmp_num_threads -@pytest.mark.parametrize('num_threads', [1, 2, 4]) +@pytest.mark.parametrize("num_threads", [1, 2, 4]) def test_openmp_limit_num_threads(num_threads): # checks that OpenMP effectively uses the number of threads requested by # the context manager @@ -159,10 +140,10 @@ def test_openmp_limit_num_threads(num_threads): @with_check_openmp_num_threads -@pytest.mark.parametrize('nthreads_outer', [None, 1, 2, 4]) +@pytest.mark.parametrize("nthreads_outer", [None, 1, 2, 4]) def test_openmp_nesting(nthreads_outer): # checks that OpenMP effectively uses the number of threads requested by - # the context manager + # the context manager when nested in an outer OpenMP loop. from ._openmp_test_helper import check_nested_openmp_loops from ._openmp_test_helper import get_inner_compiler from ._openmp_test_helper import get_outer_compiler @@ -173,14 +154,13 @@ def test_openmp_nesting(nthreads_outer): outer_num_threads, inner_num_threads = check_nested_openmp_loops(10) original_infos = threadpool_info() - openmp_infos = [info for info in original_infos - if info["user_api"] == "openmp"] + openmp_infos = original_infos.get_modules("user_api", "openmp") if "gcc" in (inner_cc, outer_cc): - assert "libgomp" in [info["prefix"] for info in openmp_infos] + assert original_infos.get_modules("prefix", "libgomp") if "clang" in (inner_cc, outer_cc): - assert "libomp" in [info["prefix"] for info in openmp_infos] + assert original_infos.get_modules("prefix", "libomp") if inner_cc == outer_cc: # The openmp runtime should be shared by default, meaning that @@ -192,9 +172,11 @@ def test_openmp_nesting(nthreads_outer): assert len(openmp_infos) >= 2 with threadpool_limits(limits=1) as threadpoolctx: - max_threads = threadpoolctx.get_original_num_threads()['openmp'] + max_threads = threadpoolctx.get_original_num_threads()["openmp"] nthreads = effective_num_threads(nthreads_outer, max_threads) + # Ask outer loop to run on nthreads threads and inner loop run on 1 + # thread outer_num_threads, inner_num_threads = \ check_nested_openmp_loops(10, nthreads) @@ -218,16 +200,17 @@ def test_openmp_nesting(nthreads_outer): def test_shipped_openblas(): - all_openblases = [ctypes.CDLL(path) for path in libopenblas_paths] - original_num_threads = [blas.openblas_get_num_threads() - for blas in all_openblases] + # checks that OpenBLAS effectively uses the number of threads requested by + # the context manager + original_info = threadpool_info() + + openblas_modules = original_info.get_modules("internal_api", "openblas") with threadpool_limits(1): - for openblas in all_openblases: - assert openblas.openblas_get_num_threads() == 1 + for module in openblas_modules: + assert module.get_num_threads() == 1 - assert original_num_threads == [openblas.openblas_get_num_threads() - for openblas in all_openblases] + assert original_info == threadpool_info() @pytest.mark.skipif(len(libopenblas_paths) < 2, @@ -240,30 +223,32 @@ def test_multiple_shipped_openblas(): @pytest.mark.skipif(scipy is None, reason="requires scipy") -@pytest.mark.parametrize('nthreads_outer', [None, 1, 2, 4]) +@pytest.mark.parametrize("nthreads_outer", [None, 1, 2, 4]) def test_nested_prange_blas(nthreads_outer): + # Check that the BLAS linked to scipy effectively uses the number of + # threads requested by the context manager when nested in an outer OpenMP + # loop. import numpy as np + from ._openmp_test_helper import check_nested_prange_blas - blas_info = [module for module in threadpool_info() - if module["user_api"] == "blas"] + original_info = threadpool_info() - blis_linked = any([module['internal_api'] == 'blis' - for module in threadpool_info()]) - if not blis_linked: - # numpy can be linked to BLIS for CBLAS and OpenBLAS for LAPACK. In - # that case this test will run BLIS gemm so no need to skip. - for module in threadpool_info(): - if is_old_openblas(module): - # OpenBLAS 0.3.3 and older are known to cause an unrecoverable - # deadlock at process shutdown time (after pytest has exited). - pytest.skip("Old OpenBLAS: skipping test to avoid deadlock") + blas_info = original_info.get_modules("user_api", "blas") + blis_info = original_info.get_modules("internal_api", "blis") + + # skip if the BLAS used by numpy is an old openblas. OpenBLAS 0.3.3 and + # older are known to cause an unrecoverable deadlock at process shutdown + # time (after pytest has exited). + # numpy can be linked to BLIS for CBLAS and OpenBLAS for LAPACK. In that + # case this test will run BLIS gemm so no need to skip. + if not blis_info and any(is_old_openblas(module) for module in blas_info): + pytest.skip("Old OpenBLAS: skipping test to avoid deadlock") - from ._openmp_test_helper import check_nested_prange_blas A = np.ones((1000, 10)) B = np.ones((100, 10)) with threadpool_limits(limits=1) as threadpoolctx: - max_threads = threadpoolctx.get_original_num_threads()['openmp'] + max_threads = threadpoolctx.get_original_num_threads()["openmp"] nthreads = effective_num_threads(nthreads_outer, max_threads) result = check_nested_prange_blas(A, B, nthreads) @@ -272,50 +257,49 @@ def test_nested_prange_blas(nthreads_outer): assert np.allclose(C, np.dot(A, B.T)) assert prange_num_threads == nthreads - nested_blas_info = [module for module in threadpool_infos - if module["user_api"] == "blas"] - + nested_blas_info = threadpool_infos.get_modules("user_api", "blas") assert len(nested_blas_info) == len(blas_info) for module in nested_blas_info: - assert module['num_threads'] == 1 + assert module["num_threads"] == 1 + + assert original_info == threadpool_info() @pytest.mark.parametrize("limit", [1, None]) def test_get_original_num_threads(limit): - with threadpool_limits(limits=2, user_api='blas') as ctl: + # Tests the method get_original_num_threads of the context manager + with threadpool_limits(limits=2, user_api="blas") as ctl: # set different blas num threads to start with (when multiple openblas) - if ctl._original_limits: - ctl._original_limits[0]['set_num_threads'](1) + if ctl._original_info: + ctl._original_info[0].set_num_threads(1) original_infos = threadpool_info() - with threadpool_limits(limits=limit, user_api='blas') as threadpoolctx: + with threadpool_limits(limits=limit, user_api="blas") as threadpoolctx: original_num_threads = threadpoolctx.get_original_num_threads() - assert 'openmp' not in original_num_threads + assert "openmp" not in original_num_threads - if 'blas' in [module['user_api'] for module in original_infos]: - assert original_num_threads['blas'] >= 1 + blas_infos = original_infos.get_modules("user_api", "blas") + if blas_infos: + expected = min(module["num_threads"] for module in blas_infos) + assert original_num_threads["blas"] == expected else: - assert original_num_threads['blas'] is None + assert original_num_threads["blas"] is None if len(libopenblas_paths) >= 2: - with pytest.warns(None, match='Multiple value possible'): - expected = min( - [module['num_threads'] for module in original_infos]) - assert original_num_threads['blas'] == expected + with pytest.warns(None, match="Multiple value possible"): + threadpoolctx.get_original_num_threads() def test_mkl_threading_layer(): # Check that threadpool_info correctly recovers the threading layer used # by mkl - mkl_info = [module for module in threadpool_info() - if module['internal_api'] == 'mkl'] - - if not mkl_info: - pytest.skip("requires MKL") - + mkl_info = threadpool_info().get_modules("internal_api", "mkl") expected_layer = os.getenv("MKL_THREADING_LAYER") - actual_layer = mkl_info[0]['threading_layer'] - if expected_layer: - assert actual_layer == expected_layer.lower() + if not (mkl_info and expected_layer): + pytest.skip("requires MKL and the environment variable " + "MKL_THREADING_LAYER set") + + actual_layer = mkl_info[0]["threading_layer"] + assert actual_layer == expected_layer.lower() diff --git a/threadpoolctl.py b/threadpoolctl.py index b4004519..fde0d8f3 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -16,16 +16,12 @@ import ctypes import warnings from ctypes.util import find_library +from abc import ABC, abstractmethod +from collections import UserDict, UserList -__version__ = '1.2.0.dev0' +__version__ = "1.2.0.dev0" __all__ = ["threadpool_limits", "threadpool_info"] -# Cache for libc under POSIX and a few system libraries under Windows -_system_libraries = {} - -# Cache for calls to os.path.realpath on system libraries to reduce the -# impact of slow system calls (e.g. stat) on slow filesystem -_realpaths = dict() # One can get runtime errors or even segfaults due to multiple OpenMP libraries # loaded simultaneously which can happen easily in Python when importing and @@ -40,10 +36,8 @@ # disable it while under the scope of the outer OpenMP parallel section. os.environ.setdefault("KMP_DUPLICATE_LIB_OK", "True") - # Structure to cast the info on dynamically loaded library. See # https://linux.die.net/man/3/dl_iterate_phdr for more details. - _SYSTEM_UINT = ctypes.c_uint64 if sys.maxsize > 2**32 else ctypes.c_uint32 _SYSTEM_UINT_HALF = ctypes.c_uint32 if sys.maxsize > 2**32 else ctypes.c_uint16 @@ -51,8 +45,8 @@ class _dl_phdr_info(ctypes.Structure): _fields_ = [ ("dlpi_addr", _SYSTEM_UINT), # Base address of object - ("dlpi_name", ctypes.c_char_p), # path to the library - ("dlpi_phdr", ctypes.c_void_p), # pointer on dlpi_headers + ("dlpi_name", ctypes.c_char_p), # path to the library + ("dlpi_phdr", ctypes.c_void_p), # pointer on dlpi_headers ("dlpi_phnum", _SYSTEM_UINT_HALF) # number of element in dlpi_phdr ] @@ -64,69 +58,41 @@ class _dl_phdr_info(ctypes.Structure): _RTLD_NOLOAD = ctypes.DEFAULT_MODE -# List of the supported implementations. The items hold the prefix of loaded -# shared objects, the name of the internal_api to call, matching the -# MAP_API_TO_FUNC keys and the name of the user_api, in {"blas", "openmp"}. - -_SUPPORTED_IMPLEMENTATIONS = [ - { +# List of the supported libraries. The items hold the possible prefixes of +# loaded shared objects, the name of the internal_api to call, the name of the +# user_api and the name of the class to instanciate to create a module object +# from this library. +_SUPPORTED_MODULES = { + "openmp": { "user_api": "openmp", - "internal_api": "openmp", "filename_prefixes": ("libiomp", "libgomp", "libomp", "vcomp",), + "module_class": "_OpenMPModule" }, - { + "openblas": { "user_api": "blas", - "internal_api": "openblas", "filename_prefixes": ("libopenblas",), + "module_class": "_OpenBLASModule" }, - { + "mkl": { "user_api": "blas", - "internal_api": "mkl", "filename_prefixes": ("libmkl_rt", "mkl_rt",), + "module_class": "_MKLModule" }, - { + "blis": { "user_api": "blas", - "internal_api": "blis", "filename_prefixes": ("libblis",), - }, -] - -# map a internal_api (openmp, openblas, mkl) to set and get functions - -_MAP_API_TO_FUNC = { - "openmp": { - "set_num_threads": "omp_set_num_threads", - "get_num_threads": "omp_get_max_threads"}, - "openblas": { - "set_num_threads": "openblas_set_num_threads", - "get_num_threads": "openblas_get_num_threads"}, - "mkl": { - "set_num_threads": "MKL_Set_Num_Threads", - "get_num_threads": "MKL_Get_Max_Threads"}, - "blis": { - "set_num_threads": "bli_thread_set_num_threads", - "get_num_threads": "bli_thread_get_num_threads"} + "module_class": "_BLISModule" + } } # Helpers for the doc and test names - -_ALL_USER_APIS = set(impl['user_api'] for impl in _SUPPORTED_IMPLEMENTATIONS) -_ALL_PREFIXES = [prefix - for impl in _SUPPORTED_IMPLEMENTATIONS - for prefix in impl['filename_prefixes']] -_ALL_INTERNAL_APIS = list(_MAP_API_TO_FUNC.keys()) - - -def _realpath(filepath, cache_limit=10000): - """Small caching wrapper around os.path.realpath to limit system calls""" - rpath = _realpaths.get(filepath) - if rpath is None: - rpath = os.path.realpath(filepath) - if len(_realpaths) < cache_limit: - # If we drop support for Python 2.7, we could use - # functools.lru_cache with maxsize=10000 instead. - _realpaths[filepath] = rpath - return rpath +_ALL_USER_APIS = set(d["user_api"] for lib, d in _SUPPORTED_MODULES.items()) +_ALL_INTERNAL_APIS = list(_SUPPORTED_MODULES.keys()) +_ALL_PREFIXES = [prefix for _, d in _SUPPORTED_MODULES.items() + for prefix in d["filename_prefixes"]] +_ALL_BLAS_LIRARIES = [lib for lib, d in _SUPPORTED_MODULES.items() + if d["user_api"] == "blas"] +_ALL_OPENMP_LIBRARIES = list(_SUPPORTED_MODULES["openmp"]["filename_prefixes"]) def _format_docstring(*args, **kwargs): @@ -137,397 +103,30 @@ def decorator(o): return decorator -def _get_limit(prefix, user_api, limits): - if prefix in limits: - return limits[prefix] - else: - return limits[user_api] - - -@_format_docstring(ALL_PREFIXES=_ALL_PREFIXES, +@_format_docstring(USER_APIS=list(_ALL_USER_APIS), INTERNAL_APIS=_ALL_INTERNAL_APIS) -def _set_threadpool_limits(limits, user_api=None): - """Limit the maximal number of threads for threadpools in supported libs - - Set the maximal number of threads that can be used in thread pools used in - the supported native libraries to `limit`. This function works for - libraries that are already loaded in the interpreter and can be changed - dynamically. - - The `limits` parameter can be either an integer or a dict to specify the - maximal number of thread that can be used in thread pools. If it is an - integer, sets the maximum number of thread to `limits` for each library - selected by `user_api`. If it is a dictionary `{{key: max_threads}}`, this - function sets a custom maximum number of thread for each `key` which can be - either a `user_api` or a `prefix` for a specific library. - - The `user_api` parameter selects particular APIs of libraries to limit. - Used only if `limits` is an int. If it is None, this function will apply to - all supported libraries. If it is "blas", it will limit only BLAS supported - libraries and if it is "openmp", only OpenMP supported libraries will be - limited. Note that the latter can affect the number of threads used by the - BLAS libraries if they rely on OpenMP. - - Return a list with all the supported modules that have been found. Each - module is represented by a dict with the following information: - - 'filename_prefixes' : possible prefixes for the given internal_api. - Possible values are {ALL_PREFIXES}. - - 'prefix' : prefix of the specific implementation of this module. - - 'internal_api': internal API.s Possible values are {INTERNAL_APIS}. - - 'filepath': path to the loaded module. - - 'version': version of the library implemented (if available). - - 'num_threads': the theadpool size limit before changing it. - - 'set_num_threads': callable to set the maximum number of threads - - 'get_num_threads': callable to get the current number of threads - - 'dynlib': the instance of ctypes.CDLL use to access the dynamic - library. - """ - if isinstance(limits, int): - if user_api is None: - user_api = _ALL_USER_APIS - elif user_api in _ALL_USER_APIS: - user_api = (user_api,) - else: - raise ValueError("user_api must be either in {} or None. Got {} " - "instead.".format(_ALL_USER_APIS, user_api)) - limits = {api: limits for api in user_api} - prefixes = [] - else: - if isinstance(limits, list): - # This should be a list of module, for compatibility with - # the result from threadpool_info. - limits = {module['prefix']: module['num_threads'] - for module in limits} - - if not isinstance(limits, dict): - raise TypeError("limits must either be an int, a list or a dict." - " Got {} instead".format(type(limits))) - - # With a dictionary, can set both specific limit for given modules - # and global limit for user_api. Fetch each separately. - prefixes = [module for module in limits if module in _ALL_PREFIXES] - user_api = [module for module in limits if module in _ALL_USER_APIS] - - modules = _load_modules(prefixes=prefixes, user_api=user_api) - for module in modules: - # Workaround clang bug (TODO: report it) - module['get_num_threads']() - - for module in modules: - module['num_threads'] = module['get_num_threads']() - num_threads = _get_limit(module['prefix'], module['user_api'], limits) - if num_threads is not None: - set_func = module['set_num_threads'] - set_func(num_threads) - - return modules - - -@_format_docstring(INTERNAL_APIS=_ALL_INTERNAL_APIS) def threadpool_info(): """Return the maximal number of threads for each detected library. Return a list with all the supported modules that have been found. Each module is represented by a dict with the following information: - - 'prefix' : filename prefix of the specific implementation. - - 'filepath': path to the loaded module. - - 'internal_api': internal API. Possible values are {INTERNAL_APIS}. - - 'version': version of the library implemented (if available). - - 'num_threads': the current thread limit. - """ - infos = [] - modules = _load_modules(user_api=_ALL_USER_APIS) - for module in modules: - module['num_threads'] = module['get_num_threads']() - # by default BLIS is single-threaded and get_num_threads returns -1. - # we map it to 1 for consistency with other libraries. - if module['num_threads'] == -1 and module['internal_api'] == 'blis': - module['num_threads'] = 1 - if module['internal_api'] == 'mkl': - layer = _get_mkl_threading_layer(module['dynlib']) - module['threading_layer'] = layer - # Remove the wrapper for the module and its function - del module['set_num_threads'], module['get_num_threads'] - del module['dynlib'] - del module['filename_prefixes'] - infos.append(module) - return infos - - -def _get_mkl_threading_layer(mkl_dynlib): - """Return the threading layer of MKL""" - # The function mkl_set_threading_layer returns the current threading layer - # Calling it with an invalid threading layer allows us to safely get - # the threading layer - set_threading_layer = getattr(mkl_dynlib, "MKL_Set_Threading_Layer") - layer_map = {0: "intel", 1: "sequential", 2: "pgi", - 3: "gnu", 4: "tbb", -1: "not specified"} - return layer_map[set_threading_layer(-1)] - - -def _get_version(dynlib, internal_api): - if internal_api == "mkl": - return _get_mkl_version(dynlib) - elif internal_api == "openmp": - # There is no way to get the version number programmatically in - # OpenMP. - return None - elif internal_api == "openblas": - return _get_openblas_version(dynlib) - elif internal_api == "blis": - return _get_blis_version(dynlib) - else: - raise NotImplementedError("Unsupported API {}".format(internal_api)) - - -def _get_mkl_version(mkl_dynlib): - """Return the MKL version""" - res = ctypes.create_string_buffer(200) - mkl_dynlib.mkl_get_version_string(res, 200) - - version = res.value.decode('utf-8') - group = re.search(r"Version ([^ ]+) ", version) - if group is not None: - version = group.groups()[0] - return version.strip() - - -def _get_openblas_version(openblas_dynlib): - """Return the OpenBLAS version - - None means OpenBLAS is not loaded or version < 0.3.4, since OpenBLAS - did not expose its version before that. - """ - get_config = getattr(openblas_dynlib, "openblas_get_config") - get_config.restype = ctypes.c_char_p - config = get_config().split() - if config[0] == b"OpenBLAS": - return config[1].decode('utf-8') - return None - - -def _get_blis_version(blis_dynlib): - """Return the BLIS version""" - get_version = getattr(blis_dynlib, "bli_info_get_version_str") - get_version.restype = ctypes.c_char_p - return get_version().decode('utf-8') - - -# Loading utilities for dynamically linked shared objects - -def _load_modules(prefixes=None, user_api=None): - """Loop through loaded libraries and return supported ones.""" - if prefixes is None: - prefixes = [] - if user_api is None: - user_api = [] - if sys.platform == "darwin": - return _find_modules_with_dyld(prefixes=prefixes, user_api=user_api) - elif sys.platform == "win32": - return _find_modules_with_enum_process_module_ex( - prefixes=prefixes, user_api=user_api) - else: - return _find_modules_with_dl_iterate_phdr( - prefixes=prefixes, user_api=user_api) - - -def _check_prefix(library_basename, filename_prefixes): - """Return the prefix library_basename starts with or None if none matches - """ - for prefix in filename_prefixes: - if library_basename.startswith(prefix): - return prefix - return None - - -def _match_module(module_info, prefix, prefixes, user_api): - """Return True if this module should be selected.""" - return prefix is not None and (prefix in prefixes or - module_info['user_api'] in user_api) - - -def _make_module_info(filepath, module_info, prefix): - """Make a dict with the information from the module.""" - filepath = os.path.normpath(filepath) - dynlib = ctypes.CDLL(filepath, mode=_RTLD_NOLOAD) - internal_api = module_info['internal_api'] - set_func = getattr(dynlib, - _MAP_API_TO_FUNC[internal_api]['set_num_threads'], - lambda num_threads: None) - get_func = getattr(dynlib, - _MAP_API_TO_FUNC[internal_api]['get_num_threads'], - lambda: None) - module_info = module_info.copy() - module_info.update(dynlib=dynlib, filepath=filepath, prefix=prefix, - set_num_threads=set_func, get_num_threads=get_func, - version=_get_version(dynlib, internal_api)) - return module_info - - -def _get_module_info_from_path(filepath, prefixes, user_api, modules): - # Required to resolve symlinks - filepath = _realpath(filepath) - # `lower` required to take account of OpenMP dll case on Windows - # (vcomp, VCOMP, Vcomp, ...) - filename = os.path.basename(filepath).lower() - for info in _SUPPORTED_IMPLEMENTATIONS: - prefix = _check_prefix(filename, info['filename_prefixes']) - if _match_module(info, prefix, prefixes, user_api): - modules.append(_make_module_info(filepath, info, prefix)) - - -def _find_modules_with_dl_iterate_phdr(prefixes, user_api): - """Loop through loaded libraries and return binders on supported ones - - This function is expected to work on POSIX system only. - This code is adapted from code by Intel developper @anton-malakhov - available at https://github.com/IntelPython/smp - - Copyright (c) 2017, Intel Corporation published under the BSD 3-Clause - license - """ - libc = _get_libc() - if not hasattr(libc, "dl_iterate_phdr"): # pragma: no cover - return [] - - _modules = [] - - # Callback function for `dl_iterate_phdr` which is called for every - # module loaded in the current process until it returns 1. - def match_module_callback(info, size, data): - # Get the path of the current module - filepath = info.contents.dlpi_name - if filepath: - filepath = filepath.decode("utf-8") - - # Store the module in cls_thread_locals._module if it is - # supported and selected - _get_module_info_from_path(filepath, prefixes, user_api, - _modules) - return 0 - - c_func_signature = ctypes.CFUNCTYPE( - ctypes.c_int, # Return type - ctypes.POINTER(_dl_phdr_info), ctypes.c_size_t, ctypes.c_char_p) - c_match_module_callback = c_func_signature(match_module_callback) - - data = ctypes.c_char_p(b'') - libc.dl_iterate_phdr(c_match_module_callback, data) - - return _modules - - -def _find_modules_with_dyld(prefixes, user_api): - """Loop through loaded libraries and return binders on supported ones - - This function is expected to work on OSX system only - """ - libc = _get_libc() - if not hasattr(libc, "_dyld_image_count"): # pragma: no cover - return [] - - _modules = [] - - n_dyld = libc._dyld_image_count() - libc._dyld_get_image_name.restype = ctypes.c_char_p - for i in range(n_dyld): - filepath = ctypes.string_at(libc._dyld_get_image_name(i)) - filepath = filepath.decode("utf-8") + - "user_api" : user API. Possible values are {USER_APIS}. + - "internal_api": internal API. Possible values are {INTERNAL_APIS}. + - "prefix" : filename prefix of the specific implementation. + - "filepath": path to the loaded module. + - "version": version of the library (if available). + - "num_threads": the current thread limit. - # Store the module in cls_thread_locals._module if it is supported and - # selected - _get_module_info_from_path(filepath, prefixes, user_api, _modules) - - return _modules - - -def _find_modules_with_enum_process_module_ex(prefixes, user_api): - """Loop through loaded libraries and return binders on supported ones - - This function is expected to work on windows system only. - This code is adapted from code by Philipp Hagemeister @phihag available - at https://stackoverflow.com/questions/17474574 + In addition, each module may contain internal_api specific entries. """ - from ctypes.wintypes import DWORD, HMODULE, MAX_PATH - - PROCESS_QUERY_INFORMATION = 0x0400 - PROCESS_VM_READ = 0x0010 - - LIST_MODULES_ALL = 0x03 - - ps_api = _get_windll('Psapi') - kernel_32 = _get_windll('kernel32') - - h_process = kernel_32.OpenProcess( - PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, - False, os.getpid()) - if not h_process: # pragma: no cover - raise OSError('Could not open PID %s' % os.getpid()) - - _modules = [] - try: - buf_count = 256 - needed = DWORD() - # Grow the buffer until it becomes large enough to hold all the - # module headers - while True: - buf = (HMODULE * buf_count)() - buf_size = ctypes.sizeof(buf) - if not ps_api.EnumProcessModulesEx( - h_process, ctypes.byref(buf), buf_size, - ctypes.byref(needed), LIST_MODULES_ALL): - raise OSError('EnumProcessModulesEx failed') - if buf_size >= needed.value: - break - buf_count = needed.value // (buf_size // buf_count) - - count = needed.value // (buf_size // buf_count) - h_modules = map(HMODULE, buf[:count]) - - # Loop through all the module headers and get the module path - buf = ctypes.create_unicode_buffer(MAX_PATH) - n_size = DWORD() - for h_module in h_modules: - - # Get the path of the current module - if not ps_api.GetModuleFileNameExW( - h_process, h_module, ctypes.byref(buf), - ctypes.byref(n_size)): - raise OSError('GetModuleFileNameEx failed') - filepath = buf.value - - # Store the module in cls_thread_locals._module if it is - # supported and selected - _get_module_info_from_path(filepath, prefixes, user_api, - _modules) - finally: - kernel_32.CloseHandle(h_process) - - return _modules - - -def _get_libc(): - """Load the lib-C for unix systems.""" - libc = _system_libraries.get("libc") - if libc is None: - libc_name = find_library("c") - if libc_name is None: # pragma: no cover - return None - libc = ctypes.CDLL(libc_name, mode=_RTLD_NOLOAD) - _system_libraries["libc"] = libc - return libc - - -def _get_windll(dll_name): - """Load a windows DLL""" - dll = _system_libraries.get(dll_name) - if dll is None: - dll = ctypes.WinDLL("{}.dll".format(dll_name)) - _system_libraries[dll_name] = dll - return dll + return _ThreadpoolInfo(user_api=_ALL_USER_APIS) +@_format_docstring( + USER_APIS=", ".join('"{}"'.format(api) for api in _ALL_USER_APIS), + BLAS_LIBS=", ".join(_ALL_BLAS_LIRARIES), + OPENMP_LIBS=", ".join(_ALL_OPENMP_LIBRARIES)) class threadpool_limits: """Change the maximal number of threads that can be used in thread pools. @@ -539,29 +138,36 @@ class threadpool_limits: the supported libraries to `limit`. This function works for libraries that are already loaded in the interpreter and can be changed dynamically. - The `limits` parameter can be either an integer or a dict to specify the - maximal number of thread that can be used in thread pools. If it is an - integer, sets the maximum number of thread to `limits` for each library - selected by `user_api`. If it is a dictionary `{{key: max_threads}}`, this - function sets a custom maximum number of thread for each `key` which can be - either a `user_api` or a `prefix` for a specific library. If None, this - function does not do anything. - - The `user_api` parameter selects particular APIs of libraries to limit. - Used only if `limits` is an int. If it is None, this function will apply to - all supported libraries. If it is "blas", it will limit only BLAS supported - libraries and if it is "openmp", only OpenMP supported libraries will be - limited. Note that the latter can affect the number of threads used by the - BLAS libraries if they rely on OpenMP. + Parameters + ---------- + limits : int, dict or None (default=None) + The maximal number of thread that can be used in thread pools + + - If int, sets the maximum number of thread to `limits` for each + library selected by `user_api`. + + - If it is a dictionary `{{key: max_threads}}`, this function sets a + custom maximum number of thread for each `key` which can be either a + `user_api` or a `prefix` for a specific library. + + - If None, this function does not do anything. + + user_api : {USER_APIS} or None (default=None) + APIs of libraries to limit. Used only if `limits` is an int. + + - If "blas", it will only limit BLAS supported libraries ({BLAS_LIBS}). + + - If "openmp", it will only limit OpenMP supported libraries + ({OPENMP_LIBS}). Note that it can affect the number of threads used + by the BLAS libraries if they rely on OpenMP. + + - If None, this function will apply to all supported libraries. """ def __init__(self, limits=None, user_api=None): - self._user_api = _ALL_USER_APIS if user_api is None else [user_api] + self._limits, self._user_api, self._prefixes = \ + self._check_params(limits, user_api) - if limits is not None: - self._original_limits = _set_threadpool_limits( - limits=limits, user_api=user_api) - else: - self._original_limits = None + self._original_info = self._set_threadpool_limits() def __enter__(self): return self @@ -570,19 +176,23 @@ def __exit__(self, type, value, traceback): self.unregister() def unregister(self): - if self._original_limits is not None: - for module in self._original_limits: - module['set_num_threads'](module['num_threads']) + if self._original_info is not None: + for module in self._original_info: + module.set_num_threads(module["num_threads"]) def get_original_num_threads(self): - original_limits = self._original_limits or threadpool_info() + """Original num_threads from before calling threadpool_limits + + Return a dict `{user_api: num_threads}`. + """ + original_limits = self._original_info or threadpool_info() num_threads = {} warning_apis = [] for user_api in self._user_api: - limits = [module['num_threads'] for module in original_limits - if module['user_api'] == user_api] + limits = [module["num_threads"] for module in + original_limits.get_modules("user_api", user_api)] limits = set(limits) n_limits = len(limits) @@ -599,6 +209,477 @@ def get_original_num_threads(self): if warning_apis: warnings.warn( "Multiple value possible for following user apis: " - + ', '.join(warning_apis) + ". Returning the minimum.") + + ", ".join(warning_apis) + ". Returning the minimum.") return num_threads + + def _check_params(self, limits, user_api): + """Suitable values for the _limits, _user_api and _prefixes attributes + """ + if limits is None or isinstance(limits, int): + if user_api is None: + user_api = _ALL_USER_APIS + elif user_api in _ALL_USER_APIS: + user_api = [user_api] + else: + raise ValueError( + "user_api must be either in {} or None. Got " + "{} instead.".format(_ALL_USER_APIS, user_api)) + + if limits is not None: + limits = {api: limits for api in user_api} + prefixes = [] + else: + if isinstance(limits, _ThreadpoolInfo): + # This should be a list of modules, for compatibility with + # the result from threadpool_info. + limits = {module["prefix"]: module["num_threads"] + for module in limits} + + if not isinstance(limits, dict): + raise TypeError("limits must either be an int, a list or a " + "dict. Got {} instead".format(type(limits))) + + # With a dictionary, can set both specific limit for given modules + # and global limit for user_api. Fetch each separately. + prefixes = [prefix for prefix in limits if prefix in _ALL_PREFIXES] + user_api = [api for api in limits if api in _ALL_USER_APIS] + + return limits, user_api, prefixes + + def _set_threadpool_limits(self): + """Change the maximal number of threads in selected thread pools. + + Return a list with all the supported modules that have been found + matching `self._prefixes` and `self._user_api`. + """ + if self._limits is None: + return None + else: + modules = _ThreadpoolInfo(prefixes=self._prefixes, + user_api=self._user_api) + for module in modules: + # self._limits is a dict {key: num_threads} where key is either + # a prefix or a user_api. If a module matches both, the limit + # corresponding to the prefix is chosed. + if module["prefix"] in self._limits: + num_threads = self._limits[module["prefix"]] + else: + num_threads = self._limits[module["user_api"]] + + if num_threads is not None: + module.set_num_threads(num_threads) + return modules + + +@_format_docstring( + PREFIXES=", ".join(_ALL_PREFIXES), + USER_APIS=", ".join('"{}"'.format(api) for api in _ALL_USER_APIS), + BLAS_LIBS=", ".join(_ALL_BLAS_LIRARIES), + OPENMP_LIBS=", ".join(_ALL_OPENMP_LIBRARIES)) +class _ThreadpoolInfo(UserList): + """List of all supported modules that have been found + + Parameters + ---------- + prefixes : list of prefixes or None (default=None) + Select libraries matching the requested prefixes. Supported prefixes + are {PREFIXES}. If None, libraries are not selected by their prefix. + + user_api : {USER_APIS} or None (default=None) + Select libraries matching the requested API. + + - If "blas", BLAS supported libraries ({BLAS_LIBS}) are selected. + + - If "openmp", OpenMP supported libraries ({OPENMP_LIBS}) are selected. + + - If None, libraries are not selected by their `user_api`. + + Note + ---- + Is is possible to select libraries both by prefixes and by user_api. All + libraries matching one or the other will be selected. + """ + # Cache for libc under POSIX and a few system libraries under Windows + _system_libraries = dict() + # Cache for calls to os.path.realpath on system libraries to reduce the + # impact of slow system calls (e.g. stat) on slow filesystem + _realpaths = dict() + + def __init__(self, prefixes=None, user_api=None): + super().__init__() + self.prefixes = [] if prefixes is None else prefixes + self.user_api = [] if user_api is None else user_api + self._load_modules() + + def get_modules(self, key, values): + """Return all modules such that values contains module[key]""" + if key == "user_api" and values is None: + values = list(_ALL_USER_APIS) + if not isinstance(values, list): + values = [values] + return [module for module in self if module[key] in values] + + def _load_modules(self): + """Loop through loaded libraries and store supported ones.""" + if sys.platform == "darwin": + self._find_modules_with_dyld() + elif sys.platform == "win32": + self._find_modules_with_enum_process_module_ex() + else: + self._find_modules_with_dl_iterate_phdr() + + def _find_modules_with_dl_iterate_phdr(self): + """Loop through loaded libraries and return binders on supported ones + + This function is expected to work on POSIX system only. + This code is adapted from code by Intel developper @anton-malakhov + available at https://github.com/IntelPython/smp + + Copyright (c) 2017, Intel Corporation published under the BSD 3-Clause + license + """ + libc = self._get_libc() + if not hasattr(libc, "dl_iterate_phdr"): # pragma: no cover + return [] + + # Callback function for `dl_iterate_phdr` which is called for every + # module loaded in the current process until it returns 1. + def match_module_callback(info, size, data): + # Get the path of the current module + filepath = info.contents.dlpi_name + if filepath: + filepath = filepath.decode("utf-8") + + # Store the module if it is supported and selected + self._make_module_from_path(filepath) + return 0 + + c_func_signature = ctypes.CFUNCTYPE( + ctypes.c_int, # Return type + ctypes.POINTER(_dl_phdr_info), ctypes.c_size_t, ctypes.c_char_p) + c_match_module_callback = c_func_signature(match_module_callback) + + data = ctypes.c_char_p(b"") + libc.dl_iterate_phdr(c_match_module_callback, data) + + def _find_modules_with_dyld(self): + """Loop through loaded libraries and return binders on supported ones + + This function is expected to work on OSX system only + """ + libc = self._get_libc() + if not hasattr(libc, "_dyld_image_count"): # pragma: no cover + return [] + + n_dyld = libc._dyld_image_count() + libc._dyld_get_image_name.restype = ctypes.c_char_p + + for i in range(n_dyld): + filepath = ctypes.string_at(libc._dyld_get_image_name(i)) + filepath = filepath.decode("utf-8") + + # Store the module if it is supported and selected + self._make_module_from_path(filepath) + + def _find_modules_with_enum_process_module_ex(self): + """Loop through loaded libraries and return binders on supported ones + + This function is expected to work on windows system only. + This code is adapted from code by Philipp Hagemeister @phihag available + at https://stackoverflow.com/questions/17474574 + """ + from ctypes.wintypes import DWORD, HMODULE, MAX_PATH + + PROCESS_QUERY_INFORMATION = 0x0400 + PROCESS_VM_READ = 0x0010 + + LIST_MODULES_ALL = 0x03 + + ps_api = self._get_windll("Psapi") + kernel_32 = self._get_windll("kernel32") + + h_process = kernel_32.OpenProcess( + PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, + False, os.getpid()) + if not h_process: # pragma: no cover + raise OSError("Could not open PID %s" % os.getpid()) + + try: + buf_count = 256 + needed = DWORD() + # Grow the buffer until it becomes large enough to hold all the + # module headers + while True: + buf = (HMODULE * buf_count)() + buf_size = ctypes.sizeof(buf) + if not ps_api.EnumProcessModulesEx( + h_process, ctypes.byref(buf), buf_size, + ctypes.byref(needed), LIST_MODULES_ALL): + raise OSError("EnumProcessModulesEx failed") + if buf_size >= needed.value: + break + buf_count = needed.value // (buf_size // buf_count) + + count = needed.value // (buf_size // buf_count) + h_modules = map(HMODULE, buf[:count]) + + # Loop through all the module headers and get the module path + buf = ctypes.create_unicode_buffer(MAX_PATH) + n_size = DWORD() + for h_module in h_modules: + + # Get the path of the current module + if not ps_api.GetModuleFileNameExW( + h_process, h_module, ctypes.byref(buf), + ctypes.byref(n_size)): + raise OSError("GetModuleFileNameEx failed") + filepath = buf.value + + # Store the module if it is supported and selected + self._make_module_from_path(filepath) + finally: + kernel_32.CloseHandle(h_process) + + def _make_module_from_path(self, filepath): + """Store a module if it is supported and selected""" + # Required to resolve symlinks + filepath = self._realpath(filepath) + # `lower` required to take account of OpenMP dll case on Windows + # (vcomp, VCOMP, Vcomp, ...) + filename = os.path.basename(filepath).lower() + + # Loop through supported modules to find if this filename corresponds + # to a supported module. + for _, candidate_module in _SUPPORTED_MODULES.items(): + # check if filename matches a supported prefix + prefix = self._check_prefix(filename, + candidate_module["filename_prefixes"]) + + # filename does not match any of the prefixes of the candidate + # module. move to next module. + if prefix is None: + continue + + # filename matches a prefix. Check if it matches the request. If + # so, create and store the module. + if (prefix in self.prefixes or + candidate_module["user_api"] in self.user_api): + module_class = globals()[candidate_module["module_class"]] + module = module_class(filepath, prefix) + self.append(module) + + def _check_prefix(self, library_basename, filename_prefixes): + """Return the prefix library_basename starts with + + Return None if none matches. + """ + for prefix in filename_prefixes: + if library_basename.startswith(prefix): + return prefix + return None + + def _get_libc(self): + """Load the lib-C for unix systems.""" + libc = self._system_libraries.get("libc") + if libc is None: + libc_name = find_library("c") + if libc_name is None: # pragma: no cover + return None + libc = ctypes.CDLL(libc_name, mode=_RTLD_NOLOAD) + self._system_libraries["libc"] = libc + return libc + + def _get_windll(self, dll_name): + """Load a windows DLL""" + dll = self._system_libraries.get(dll_name) + if dll is None: + dll = ctypes.WinDLL("{}.dll".format(dll_name)) + self._system_libraries[dll_name] = dll + return dll + + def _realpath(self, filepath, cache_limit=10000): + """Small caching wrapper around os.path.realpath to limit system calls + """ + rpath = self._realpaths.get(filepath) + if rpath is None: + rpath = os.path.realpath(filepath) + if len(self._realpaths) < cache_limit: + # If we drop support for Python 2.7, we could use + # functools.lru_cache with maxsize=10000 instead. + self._realpaths[filepath] = rpath + return rpath + + +class _Module(UserDict, ABC): + """Abstract base class for the modules + + A module is represented by a dict with the following information: + - "user_api" : user API. Possible values are {USER_APIS}. + - "internal_api" : internal API. Possible values are {INTERNAL_APIS}. + - "prefix" : filename prefix of the specific implementation. + - "filepath" : path to the loaded module. + - "version" : version of the library (if available). + - "num_threads" : the current thread limit. + + In addition, each module may contain internal_api specific entries. + """ + def __init__(self, filepath=None, prefix=None): + super().__init__() + self["user_api"] = self.user_api + self["internal_api"] = self.internal_api + self["prefix"] = prefix + self["filepath"] = filepath + self.dynlib = ctypes.CDLL(filepath, mode=_RTLD_NOLOAD) + self["version"] = self.get_version() + self["num_threads"] = self.get_num_threads() + self._add_extra_info() + + @property + @abstractmethod + def user_api(self): + pass + + @property + @abstractmethod + def internal_api(self): + pass + + @abstractmethod + def get_version(self): + """Return the version of the shared library""" + pass + + @abstractmethod + def get_num_threads(self): + """Return the maximum number of threads available to use""" + pass + + @abstractmethod + def set_num_threads(self, num_threads): + """Set the maximum number of threads to use""" + pass + + @abstractmethod + def _add_extra_info(self): + """Add additional module specific information""" + pass + + +class _OpenBLASModule(_Module): + """Module class for OpenBLAS""" + user_api = "blas" + internal_api = "openblas" + + def get_version(self): + # None means OpenBLAS is not loaded or version < 0.3.4, since OpenBLAS + # did not expose its version before that. + get_config = getattr(self.dynlib, "openblas_get_config", + lambda: None) + get_config.restype = ctypes.c_char_p + config = get_config().split() + if config[0] == b"OpenBLAS": + return config[1].decode("utf-8") + return None + + def get_num_threads(self): + get_func = getattr(self.dynlib, "openblas_get_num_threads", + lambda: None) + return get_func() + + def set_num_threads(self, num_threads): + set_func = getattr(self.dynlib, "openblas_set_num_threads", + lambda num_threads: None) + return set_func(num_threads) + + def _add_extra_info(self): + pass + + +class _BLISModule(_Module): + """Module class for BLIS""" + user_api = "blas" + internal_api = "blis" + + def get_version(self): + get_version_ = getattr(self.dynlib, "bli_info_get_version_str", + lambda: None) + get_version_.restype = ctypes.c_char_p + return get_version_().decode("utf-8") + + def get_num_threads(self): + get_func = getattr(self.dynlib, "bli_thread_get_num_threads", + lambda: None) + num_threads = get_func() + # by default BLIS is single-threaded and get_num_threads + # returns -1. We map it to 1 for consistency with other libraries. + return 1 if num_threads == -1 else num_threads + + def set_num_threads(self, num_threads): + set_func = getattr(self.dynlib, "bli_thread_set_num_threads", + lambda num_threads: None) + return set_func(num_threads) + + def _add_extra_info(self): + pass + + +class _MKLModule(_Module): + """Module class for MKL""" + user_api = "blas" + internal_api = "mkl" + + def get_version(self): + res = ctypes.create_string_buffer(200) + self.dynlib.mkl_get_version_string(res, 200) + + version = res.value.decode("utf-8") + group = re.search(r"Version ([^ ]+) ", version) + if group is not None: + version = group.groups()[0] + return version.strip() + + def get_num_threads(self): + get_func = getattr(self.dynlib, "MKL_Get_Max_Threads", lambda: None) + return get_func() + + def set_num_threads(self, num_threads): + set_func = getattr(self.dynlib, "MKL_Set_Num_Threads", + lambda num_threads: None) + return set_func(num_threads) + + def _add_extra_info(self): + self["threading_layer"] = self.get_threading_layer() + + def get_threading_layer(self): + """Return the threading layer of MKL""" + # The function mkl_set_threading_layer returns the current threading + # layer. Calling it with an invalid threading layer allows us to safely + # get the threading layer + set_threading_layer = getattr(self.dynlib, "MKL_Set_Threading_Layer", + lambda layer: -1) + layer_map = {0: "intel", 1: "sequential", 2: "pgi", + 3: "gnu", 4: "tbb", -1: "not specified"} + return layer_map[set_threading_layer(-1)] + + +class _OpenMPModule(_Module): + """Module class for OpenMP""" + user_api = "openmp" + internal_api = "openmp" + + def get_version(self): + # There is no way to get the version number programmatically in OpenMP. + return None + + def get_num_threads(self): + get_func = getattr(self.dynlib, "omp_get_max_threads", lambda: None) + return get_func() + + def set_num_threads(self, num_threads): + set_func = getattr(self.dynlib, "omp_set_num_threads", + lambda num_threads: None) + return set_func(num_threads) + + def _add_extra_info(self): + pass From 8b693b63bfe12c002b05cc3ec21016a34b48943b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Thu, 14 Nov 2019 16:07:02 +0100 Subject: [PATCH 056/187] MNT TST clean unused legacy pytest fixtures (#52) --- conftest.py | 35 ----------------------------------- tests/test_threadpoolctl.py | 27 ++++++++++----------------- 2 files changed, 10 insertions(+), 52 deletions(-) delete mode 100644 conftest.py diff --git a/conftest.py b/conftest.py deleted file mode 100644 index 6f3e2e54..00000000 --- a/conftest.py +++ /dev/null @@ -1,35 +0,0 @@ -import pytest -import warnings - - -def pytest_addoption(parser): - parser.addoption("--openblas-present", action='store_true', - help="Fail test_limit_openblas_threads if BLAS is not " - "found") - parser.addoption("--mkl-present", action='store_true', - help="Fail test_limit_mkl_threads if MKL is not " - "found") - - -@pytest.fixture -def openblas_present(request): - """Fail the test if OpenBLAS is not found""" - return request.config.getoption("--openblas-present") - - -@pytest.fixture -def mkl_present(request): - """Fail the test if MKL is not found""" - return request.config.getoption("--mkl-present") - - -def pytest_configure(config): - """Verify the environment for testing threadpoolctl""" - warnings.simplefilter('always') - - # When using this option, make sure numpy is accessible - if config.getoption("--mkl-present"): - try: - import numpy as np # noqa: F401 - except ImportError: - raise ImportError("Need 'numpy' with option --mkl-present") diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index 75d88895..9c54702b 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -25,19 +25,11 @@ def effective_num_threads(nthreads, max_threads): @pytest.mark.parametrize("prefix", _ALL_PREFIXES) -def test_threadpool_limits_by_prefix(openblas_present, mkl_present, prefix): +def test_threadpool_limits_by_prefix(prefix): original_infos = threadpool_info() - mkl_found = any([True for info in original_infos - if info["prefix"] in ('mkl_rt', 'libmkl_rt')]) - prefix_found = len([info["prefix"] for info in original_infos - if info["prefix"] == prefix]) - if not prefix_found: - if "mkl_rt" in prefix and mkl_present and not mkl_found: - raise RuntimeError("Could not load the MKL prefix") - elif prefix == "libopenblas" and openblas_present: - raise RuntimeError("Could not load the OpenBLAS prefix") - else: - pytest.skip("{} runtime missing".format(prefix)) + + if prefix not in [info["prefix"] for info in original_infos]: + pytest.skip("Requires {} runtime".format(prefix)) with threadpool_limits(limits={prefix: 1}): for module in threadpool_info(): @@ -51,6 +43,8 @@ def test_threadpool_limits_by_prefix(openblas_present, mkl_present, prefix): if is_old_openblas(module): continue if module["prefix"] == prefix: + # threadpool_limits only sets an upper bound on the number of + # threads. assert module["num_threads"] <= 3 assert threadpool_info() == original_infos @@ -60,13 +54,10 @@ def test_threadpool_limits_by_prefix(openblas_present, mkl_present, prefix): def test_set_threadpool_limits_by_api(user_api): # Check that the number of threads used by the multithreaded libraries can # be modified dynamically. - if user_api is None: - user_apis = ("blas", "openmp") - else: - user_apis = (user_api,) - original_infos = threadpool_info() + user_apis = _ALL_USER_APIS if user_api is None else (user_api,) + with threadpool_limits(limits=1, user_api=user_api): for module in threadpool_info(): if is_old_openblas(module): @@ -79,6 +70,8 @@ def test_set_threadpool_limits_by_api(user_api): if is_old_openblas(module): continue if module["user_api"] in user_apis: + # threadpool_limits only sets an upper bound on the number of + # threads. assert module["num_threads"] <= 3 assert threadpool_info() == original_infos From f38ff0d52d654f1b89640ef50b65342b2bb5ce3c Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Thu, 14 Nov 2019 17:48:06 +0100 Subject: [PATCH 057/187] don't try to cover abstract methods --- threadpoolctl.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/threadpoolctl.py b/threadpoolctl.py index fde0d8f3..ea588a11 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -538,32 +538,32 @@ def __init__(self, filepath=None, prefix=None): @property @abstractmethod def user_api(self): - pass + pass # pragma: no cover @property @abstractmethod def internal_api(self): - pass + pass # pragma: no cover @abstractmethod def get_version(self): """Return the version of the shared library""" - pass + pass # pragma: no cover @abstractmethod def get_num_threads(self): """Return the maximum number of threads available to use""" - pass + pass # pragma: no cover @abstractmethod def set_num_threads(self, num_threads): """Set the maximum number of threads to use""" - pass + pass # pragma: no cover @abstractmethod def _add_extra_info(self): """Add additional module specific information""" - pass + pass # pragma: no cover class _OpenBLASModule(_Module): From 7c66f55a305453c3d545ef9468b59344501508a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Sun, 17 Nov 2019 22:41:57 +0100 Subject: [PATCH 058/187] [MRG] MNT CI Make pytest ignore _openmp_test_helper when discovering tests (#56) --- conftest.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 conftest.py diff --git a/conftest.py b/conftest.py new file mode 100644 index 00000000..bf303839 --- /dev/null +++ b/conftest.py @@ -0,0 +1 @@ +collect_ignore = ["tests/_openmp_test_helper"] From 3de325f88ef1b30abac0b440aa404e74b921acf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Wed, 20 Nov 2019 19:23:53 +0100 Subject: [PATCH 059/187] MNT CI Add a meta-test ensuring no test is always skipped (#57) --- .azure_pipeline.yml | 246 ++++++++++-------- .../check_no_test_skipped.py | 38 +++ continuous_integration/posix.yml | 1 + continuous_integration/windows.yml | 1 + tests/test_threadpoolctl.py | 8 +- tests/utils.py | 24 +- 6 files changed, 179 insertions(+), 139 deletions(-) create mode 100644 continuous_integration/check_no_test_skipped.py diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml index 955a222a..fd4f4d22 100644 --- a/.azure_pipeline.yml +++ b/.azure_pipeline.yml @@ -6,123 +6,139 @@ variables: JUNITXML: 'test-data.xml' CODECOV_TOKEN: 'cee0e505-c12e-4139-aa43-621fb16a2347' +stages: +- stage: + jobs: -jobs: + - template: continuous_integration/windows.yml + parameters: + name: Windows + vmImage: vs2017-win2016 + matrix: + py37_conda: + VERSION_PYTHON: '3.7' + PACKAGER: 'conda' + py36_conda: + VERSION_PYTHON: '3.6' + PACKAGER: 'conda' + py35_pip: + VERSION_PYTHON: '3.5' + PACKAGER: 'pip' -- template: continuous_integration/windows.yml - parameters: - name: Windows - vmImage: vs2017-win2016 - matrix: - py37_conda: - VERSION_PYTHON: '3.7' - PACKAGER: 'conda' - py36_conda: - VERSION_PYTHON: '3.6' - PACKAGER: 'conda' - py35_pip: - VERSION_PYTHON: '3.5' - PACKAGER: 'pip' + - template: continuous_integration/posix.yml + parameters: + name: Linux + vmImage: ubuntu-16.04 + matrix: + # Linux environment to test that packages that comes with Ubuntu Xenial + # 16.04 are correctly handled. + py35_ubuntu_atlas_gcc_gcc: + PACKAGER: 'ubuntu' + APT_BLAS: 'libatlas3-base libatlas-base-dev libatlas-dev' + VERSION_PYTHON: '3.5' + CC_OUTER_LOOP: 'gcc' + CC_INNER_LOOP: 'gcc' + LINT: 'true' + py35_ubuntu_openblas_gcc_gcc: + PACKAGER: 'ubuntu' + APT_BLAS: 'libopenblas-base libopenblas-dev' + VERSION_PYTHON: '3.5' + CC_OUTER_LOOP: 'gcc' + CC_INNER_LOOP: 'gcc' + # Linux + Python 3.6 and homogeneous runtime nesting. + py36_conda_openblas_clang_clang: + PACKAGER: 'conda' + VERSION_PYTHON: '3.6' + NO_MKL: 'true' + CC_OUTER_LOOP: 'clang-8' + CC_INNER_LOOP: 'clang-8' + # Linux environment to test the latest available dependencies and MKL. + # and heterogeneous OpenMP runtime nesting. + pylatest_conda_mkl_clang_gcc: + PACKAGER: 'conda' + VERSION_PYTHON: '*' + CC_OUTER_LOOP: 'clang-8' + CC_INNER_LOOP: 'gcc' + MKL_THREADING_LAYER: 'INTEL' + # Linux + Python 3.7 with numpy / scipy installed with pip from PyPI and + # heterogeneous openmp runtimes. + py37_pip_openblas_gcc_clang: + PACKAGER: 'pip' + VERSION_PYTHON: '3.7' + CC_OUTER_LOOP: 'gcc' + CC_INNER_LOOP: 'clang-8' + # Linux environment with numpy from conda-forge channel + pylatest_conda_forge: + PACKAGER: 'conda-forge' + VERSION_PYTHON: '*' + CC_OUTER_LOOP: 'gcc' + CC_INNER_LOOP: 'gcc' + # Linux environment with no numpy and heterogeneous OpenMP runtime + # nesting. + pylatest_conda_nonumpy_gcc_clang: + PACKAGER: 'conda' + NO_NUMPY: 'true' + VERSION_PYTHON: '*' + CC_OUTER_LOOP: 'gcc' + CC_INNER_LOOP: 'clang-8' + # Linux environment with numpy linked to BLIS + pylatest_blis_gcc_clang: + PACKAGER: 'conda' + VERSION_PYTHON: '*' + INSTALL_BLIS: 'true' + BLIS_NUM_THREADS: '4' + CC_OUTER_LOOP: 'gcc' + CC_INNER_LOOP: 'gcc' + BLIS_CC: 'clang-8' + pylatest_blis_clang_gcc: + PACKAGER: 'conda' + VERSION_PYTHON: '*' + INSTALL_BLIS: 'true' + BLIS_NUM_THREADS: '4' + CC_OUTER_LOOP: 'clang-8' + CC_INNER_LOOP: 'clang-8' + BLIS_CC: 'gcc-8' + pylatest_blis_no_threading: + PACKAGER: 'conda' + VERSION_PYTHON: '*' + INSTALL_BLIS: 'true' + BLIS_NUM_THREADS: '1' + CC_OUTER_LOOP: 'gcc' + CC_INNER_LOOP: 'gcc' + BLIS_CC: 'gcc-8' -- template: continuous_integration/posix.yml - parameters: - name: Linux - vmImage: ubuntu-16.04 - matrix: - # Linux environment to test that packages that comes with Ubuntu Xenial - # 16.04 are correctly handled. - py35_ubuntu_atlas_gcc_gcc: - PACKAGER: 'ubuntu' - APT_BLAS: 'libatlas3-base libatlas-base-dev libatlas-dev' - VERSION_PYTHON: '3.5' - CC_OUTER_LOOP: 'gcc' - CC_INNER_LOOP: 'gcc' - LINT: 'true' - py35_ubuntu_openblas_gcc_gcc: - PACKAGER: 'ubuntu' - APT_BLAS: 'libopenblas-base libopenblas-dev' - VERSION_PYTHON: '3.5' - CC_OUTER_LOOP: 'gcc' - CC_INNER_LOOP: 'gcc' - # Linux + Python 3.6 and homogeneous runtime nesting. - py36_conda_openblas_clang_clang: - PACKAGER: 'conda' - VERSION_PYTHON: '3.6' - NO_MKL: 'true' - CC_OUTER_LOOP: 'clang-8' - CC_INNER_LOOP: 'clang-8' - # Linux environment to test the latest available dependencies and MKL. - # and heterogeneous OpenMP runtime nesting. - pylatest_conda_mkl_clang_gcc: - PACKAGER: 'conda' - VERSION_PYTHON: '*' - CC_OUTER_LOOP: 'clang-8' - CC_INNER_LOOP: 'gcc' - MKL_THREADING_LAYER: 'INTEL' - # Linux + Python 3.7 with numpy / scipy installed with pip from PyPI and - # heterogeneous openmp runtimes. - py37_pip_openblas_gcc_clang: - PACKAGER: 'pip' - VERSION_PYTHON: '3.7' - CC_OUTER_LOOP: 'gcc' - CC_INNER_LOOP: 'clang-8' - # Linux environment with numpy from conda-forge channel - pylatest_conda_forge: - PACKAGER: 'conda-forge' - VERSION_PYTHON: '*' - CC_OUTER_LOOP: 'gcc' - CC_INNER_LOOP: 'gcc' - # Linux environment with no numpy and heterogeneous OpenMP runtime - # nesting. - pylatest_conda_nonumpy_gcc_clang: - PACKAGER: 'conda' - NO_NUMPY: 'true' - VERSION_PYTHON: '*' - CC_OUTER_LOOP: 'gcc' - CC_INNER_LOOP: 'clang-8' - # Linux environment with numpy linked to BLIS - pylatest_blis_gcc_clang: - PACKAGER: 'conda' - VERSION_PYTHON: '*' - INSTALL_BLIS: 'true' - BLIS_NUM_THREADS: '4' - CC_OUTER_LOOP: 'gcc' - CC_INNER_LOOP: 'gcc' - BLIS_CC: 'clang-8' - pylatest_blis_clang_gcc: - PACKAGER: 'conda' - VERSION_PYTHON: '*' - INSTALL_BLIS: 'true' - BLIS_NUM_THREADS: '4' - CC_OUTER_LOOP: 'clang-8' - CC_INNER_LOOP: 'clang-8' - BLIS_CC: 'gcc-8' - pylatest_blis_no_threading: - PACKAGER: 'conda' - VERSION_PYTHON: '*' - INSTALL_BLIS: 'true' - BLIS_NUM_THREADS: '1' - CC_OUTER_LOOP: 'gcc' - CC_INNER_LOOP: 'gcc' - BLIS_CC: 'gcc-8' + - template: continuous_integration/posix.yml + parameters: + name: macOS + vmImage: xcode9-macos10.13 + matrix: + # MacOS environment with OpenMP installed through homebrew + pylatest_conda_homebrew_libomp: + PACKAGER: 'conda' + VERSION_PYTHON: '*' + CC_OUTER_LOOP: 'clang' + CC_INNER_LOOP: 'clang' + INSTALL_LIBOMP: 'homebrew' + # MacOS environment with OpenMP installed through conda-forge compilers + pylatest_conda_conda_forge_clang: + PACKAGER: 'conda' + VERSION_PYTHON: '*' + CC_OUTER_LOOP: 'clang' + CC_INNER_LOOP: 'clang' + INSTALL_LIBOMP: 'conda-forge' -- template: continuous_integration/posix.yml - parameters: - name: macOS - vmImage: xcode9-macos10.13 - matrix: - # MacOS environment with OpenMP installed through homebrew - pylatest_conda_homebrew_libomp: - PACKAGER: 'conda' - VERSION_PYTHON: '*' - CC_OUTER_LOOP: 'clang' - CC_INNER_LOOP: 'clang' - INSTALL_LIBOMP: 'homebrew' - # MacOS environment with OpenMP installed through conda-forge compilers - pylatest_conda_conda_forge_clang: - PACKAGER: 'conda' - VERSION_PYTHON: '*' - CC_OUTER_LOOP: 'clang' - CC_INNER_LOOP: 'clang' - INSTALL_LIBOMP: 'conda-forge' +- stage: + jobs: + # Meta-test to ensure that at least of the above CI configurations had + # the necessary platform settings to execute each test without raising + # skipping. + - job: 'no_test_always_skipped' + displayName: 'No test always skipped' + pool: + vmImage: ubuntu-16.04 + steps: + - download: current + - script: | + python continuous_integration/check_no_test_skipped.py $(Pipeline.Workspace) + displayName: 'No test always skipped' diff --git a/continuous_integration/check_no_test_skipped.py b/continuous_integration/check_no_test_skipped.py new file mode 100644 index 00000000..10fab11a --- /dev/null +++ b/continuous_integration/check_no_test_skipped.py @@ -0,0 +1,38 @@ +"""Check tests are not skipped in every ci job""" + +from __future__ import print_function + +import os +import sys +import xml.etree.ElementTree as ET + + +base_dir = sys.argv[1] + +# dict {test: result} where result is False if the test was skipped in every +# job and True otherwise. +aggregated_results = {} + +for name in os.listdir(base_dir): + # all test result files are in /base_dir/jobs.*/ dirs + if name.startswith("stage1."): + result_file = os.path.join(base_dir, name, "test-data.xml") + root = ET.parse(result_file).getroot() + + # All tests are identified by the xml tag testcase. + for test in root.iter('testcase'): + test_name = test.attrib['name'] + if test_name not in aggregated_results: + # len(test) is > 0 if the test is skipped. + aggregated_results[test_name] = not bool(len(test)) + else: + aggregated_results[test_name] |= not bool(len(test)) + +fail = False +for test, result in aggregated_results.items(): + if not result: + fail = True + print(test, "was skipped in every job") + +if fail: + sys.exit(1) diff --git a/continuous_integration/posix.yml b/continuous_integration/posix.yml index b6cc5b18..40ba20c4 100644 --- a/continuous_integration/posix.yml +++ b/continuous_integration/posix.yml @@ -42,6 +42,7 @@ jobs: testRunTitle: ${{ format('{0}-$(Agent.JobName)', parameters.name) }} displayName: 'Publish Test Results' condition: succeededOrFailed() + - publish: $(JUNITXML) - script: | bash continuous_integration/upload_codecov.sh displayName: 'Upload to codecov' diff --git a/continuous_integration/windows.yml b/continuous_integration/windows.yml index efbe510e..7d6e53da 100644 --- a/continuous_integration/windows.yml +++ b/continuous_integration/windows.yml @@ -28,6 +28,7 @@ jobs: testRunTitle: ${{ format('{0}-$(Agent.JobName)', parameters.name) }} displayName: 'Publish Test Results' condition: succeededOrFailed() + - publish: $(JUNITXML) - script: | bash continuous_integration\\upload_codecov.sh displayName: 'Upload to codecov' diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index 9c54702b..7e427bb6 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -7,7 +7,7 @@ from threadpoolctl import threadpool_limits, threadpool_info from threadpoolctl import _ALL_PREFIXES, _ALL_USER_APIS -from .utils import with_check_openmp_num_threads +from .utils import cython_extensions_compiled from .utils import libopenblas_paths from .utils import scipy @@ -137,7 +137,8 @@ def test_threadpool_limits_bad_input(): threadpool_limits(limits=(1, 2, 3)) -@with_check_openmp_num_threads +@pytest.mark.skipif(not cython_extensions_compiled, + reason='Requires cython extensions to be compiled') @pytest.mark.parametrize('num_threads', [1, 2, 4]) def test_openmp_limit_num_threads(num_threads): # checks that OpenMP effectively uses the number of threads requested by @@ -151,7 +152,8 @@ def test_openmp_limit_num_threads(num_threads): assert check_openmp_num_threads(100) == old_num_threads -@with_check_openmp_num_threads +@pytest.mark.skipif(not cython_extensions_compiled, + reason='Requires cython extensions to be compiled') @pytest.mark.parametrize('nthreads_outer', [None, 1, 2, 4]) def test_openmp_nesting(nthreads_outer): # checks that OpenMP effectively uses the number of threads requested by diff --git a/tests/utils.py b/tests/utils.py index 71aa903f..ce9adfa3 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,19 +1,11 @@ import os -import pytest from glob import glob -def skip_func(msg): - def test_func(*args, **kwargs): - pytest.skip(msg) - return test_func - - # Path to shipped openblas for libraries such as numpy or scipy libopenblas_patterns = [] -# A decorator to run tests only when numpy is available try: # make sure the mkl/blas are loaded for test_threadpool_limits import numpy as np @@ -21,7 +13,6 @@ def test_func(*args, **kwargs): libopenblas_patterns.append(os.path.join(np.__path__[0], ".libs", "libopenblas*")) - except ImportError: pass @@ -40,17 +31,8 @@ def test_func(*args, **kwargs): for path in glob(pattern)) -# A decorator to run tests only when check_openmp_n_threads is available try: - from ._openmp_test_helper import check_openmp_num_threads # noqa: F401 - - def with_check_openmp_num_threads(func): - """A decorator to skip tests if check_openmp_n_threads is not compiled. - """ - return func - + from ._openmp_test_helper import check_openmp_num_threads # noqa: F401 + cython_extensions_compiled = True except ImportError: - def with_check_openmp_num_threads(func): - """A decorator to skip tests if check_openmp_n_threads is not compiled. - """ - return skip_func('Test requires check_openmp_n_threads to be compiled') + cython_extensions_compiled = False From cb97cabed88319e76101d2a4a1d0da5549a21598 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Thu, 21 Nov 2019 11:04:58 +0100 Subject: [PATCH 060/187] class methods --- threadpoolctl.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/threadpoolctl.py b/threadpoolctl.py index ea588a11..a4a6ef39 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -479,35 +479,38 @@ def _check_prefix(self, library_basename, filename_prefixes): return prefix return None - def _get_libc(self): + @classmethod + def _get_libc(cls): """Load the lib-C for unix systems.""" - libc = self._system_libraries.get("libc") + libc = cls._system_libraries.get("libc") if libc is None: libc_name = find_library("c") if libc_name is None: # pragma: no cover return None libc = ctypes.CDLL(libc_name, mode=_RTLD_NOLOAD) - self._system_libraries["libc"] = libc + cls._system_libraries["libc"] = libc return libc - def _get_windll(self, dll_name): + @classmethod + def _get_windll(cls, dll_name): """Load a windows DLL""" - dll = self._system_libraries.get(dll_name) + dll = cls._system_libraries.get(dll_name) if dll is None: dll = ctypes.WinDLL("{}.dll".format(dll_name)) - self._system_libraries[dll_name] = dll + cls._system_libraries[dll_name] = dll return dll - def _realpath(self, filepath, cache_limit=10000): + @classmethod + def _realpath(cls, filepath, cache_limit=10000): """Small caching wrapper around os.path.realpath to limit system calls """ - rpath = self._realpaths.get(filepath) + rpath = cls._realpaths.get(filepath) if rpath is None: rpath = os.path.realpath(filepath) - if len(self._realpaths) < cache_limit: + if len(cls._realpaths) < cache_limit: # If we drop support for Python 2.7, we could use # functools.lru_cache with maxsize=10000 instead. - self._realpaths[filepath] = rpath + cls._realpaths[filepath] = rpath return rpath From 8d06aaf1e7f8915eaea837f550a83063688b472e Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Fri, 22 Nov 2019 10:51:51 +0100 Subject: [PATCH 061/187] from dict subclassing to todict method --- .../nested_prange_blas.pyx | 4 +- tests/test_threadpoolctl.py | 65 +++++---- threadpoolctl.py | 137 ++++++++++++------ 3 files changed, 126 insertions(+), 80 deletions(-) diff --git a/tests/_openmp_test_helper/nested_prange_blas.pyx b/tests/_openmp_test_helper/nested_prange_blas.pyx index a6615fe5..47c445b1 100644 --- a/tests/_openmp_test_helper/nested_prange_blas.pyx +++ b/tests/_openmp_test_helper/nested_prange_blas.pyx @@ -20,7 +20,7 @@ IF USE_BLIS: ELSE: from scipy.linalg.cython_blas cimport dgemm -from threadpoolctl import threadpool_info +from threadpoolctl import _ThreadpoolInfo, _ALL_USER_APIS def check_nested_prange_blas(double[:, ::1] A, double[:, ::1] B, int nthreads): @@ -48,7 +48,7 @@ def check_nested_prange_blas(double[:, ::1] A, double[:, ::1] B, int nthreads): with nogil, parallel(num_threads=nthreads): if openmp.omp_get_thread_num() == 0: with gil: - threadpool_infos[0] = threadpool_info() + threadpool_infos[0] = _ThreadpoolInfo(user_api=_ALL_USER_APIS) prange_num_threads_ptr[0] = openmp.omp_get_num_threads() diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index 21834c1c..db1d6578 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -2,7 +2,7 @@ import re import pytest -from threadpoolctl import threadpool_limits, threadpool_info +from threadpoolctl import threadpool_limits, _ThreadpoolInfo from threadpoolctl import _ALL_PREFIXES, _ALL_USER_APIS from .utils import with_check_openmp_num_threads @@ -13,7 +13,7 @@ def is_old_openblas(module): # Possible bug in getting maximum number of threads with OpenBLAS < 0.2.16 # and OpenBLAS does not expose its version before 0.3.4. - return module["internal_api"] == "openblas" and module["version"] is None + return module.internal_api == "openblas" and module.version is None def effective_num_threads(nthreads, max_threads): @@ -22,11 +22,16 @@ def effective_num_threads(nthreads, max_threads): return nthreads +def _threadpool_info(): + # Like threadpool_info but return the object instead of the list of dicts + return _ThreadpoolInfo(user_api=_ALL_USER_APIS) + + @pytest.mark.parametrize("prefix", _ALL_PREFIXES) @pytest.mark.parametrize("limit", [1, 3]) def test_threadpool_limits_by_prefix(prefix, limit): # Check that the maximum number of threads can be set by prefix - original_infos = threadpool_info() + original_infos = _threadpool_info() modules_matching_prefix = original_infos.get_modules("prefix", prefix) if not modules_matching_prefix: @@ -39,15 +44,14 @@ def test_threadpool_limits_by_prefix(prefix, limit): # threadpool_limits only sets an upper bound on the number of # threads. assert 0 < module.get_num_threads() <= limit - - assert threadpool_info() == original_infos + assert _threadpool_info() == original_infos @pytest.mark.parametrize("user_api", (None, "blas", "openmp")) @pytest.mark.parametrize("limit", [1, 3]) def test_set_threadpool_limits_by_api(user_api, limit): # Check that the maximum number of threads can be set by user_api - original_infos = threadpool_info() + original_infos = _threadpool_info() modules_matching_api = original_infos.get_modules("user_api", user_api) if not modules_matching_api: @@ -62,55 +66,55 @@ def test_set_threadpool_limits_by_api(user_api, limit): # threads. assert 0 < module.get_num_threads() <= limit - assert threadpool_info() == original_infos + assert _threadpool_info() == original_infos def test_threadpool_limits_function_with_side_effect(): # Check that threadpool_limits can be used as a function with # side effects instead of a context manager. - original_infos = threadpool_info() + original_infos = _threadpool_info() threadpool_limits(limits=1) try: - for module in threadpool_info(): + for module in _threadpool_info(): if is_old_openblas(module): continue - assert module["num_threads"] == 1 + assert module.num_threads == 1 finally: # Restore the original limits so that this test does not have any # side-effect. threadpool_limits(limits=original_infos) - assert threadpool_info() == original_infos + assert _threadpool_info() == original_infos def test_set_threadpool_limits_no_limit(): # Check that limits=None does nothing. - original_infos = threadpool_info() + original_infos = _threadpool_info() with threadpool_limits(limits=None): - assert threadpool_info() == original_infos + assert _threadpool_info() == original_infos - assert threadpool_info() == original_infos + assert _threadpool_info() == original_infos def test_threadpool_limits_manual_unregister(): # Check that threadpool_limits can be used as an object which holds the # original state of the threadpools and that can be restored thanks to the # dedicated unregister method - original_infos = threadpool_info() + original_infos = _threadpool_info() limits = threadpool_limits(limits=1) try: - for module in threadpool_info(): + for module in _threadpool_info(): if is_old_openblas(module): continue - assert module["num_threads"] == 1 + assert module.num_threads == 1 finally: # Restore the original limits so that this test does not have any # side-effect. limits.unregister() - assert threadpool_info() == original_infos + assert _threadpool_info() == original_infos def test_threadpool_limits_bad_input(): @@ -153,7 +157,7 @@ def test_openmp_nesting(nthreads_outer): outer_num_threads, inner_num_threads = check_nested_openmp_loops(10) - original_infos = threadpool_info() + original_infos = _threadpool_info() openmp_infos = original_infos.get_modules("user_api", "openmp") if "gcc" in (inner_cc, outer_cc): @@ -182,7 +186,7 @@ def test_openmp_nesting(nthreads_outer): # The state of the original state of all threadpools should have been # restored. - assert threadpool_info() == original_infos + assert _threadpool_info() == original_infos # The number of threads available in the outer loop should not have been # decreased: @@ -202,7 +206,7 @@ def test_openmp_nesting(nthreads_outer): def test_shipped_openblas(): # checks that OpenBLAS effectively uses the number of threads requested by # the context manager - original_info = threadpool_info() + original_info = _threadpool_info() openblas_modules = original_info.get_modules("internal_api", "openblas") @@ -210,7 +214,7 @@ def test_shipped_openblas(): for module in openblas_modules: assert module.get_num_threads() == 1 - assert original_info == threadpool_info() + assert original_info == _threadpool_info() @pytest.mark.skipif(len(libopenblas_paths) < 2, @@ -231,7 +235,7 @@ def test_nested_prange_blas(nthreads_outer): import numpy as np from ._openmp_test_helper import check_nested_prange_blas - original_info = threadpool_info() + original_info = _threadpool_info() blas_info = original_info.get_modules("user_api", "blas") blis_info = original_info.get_modules("internal_api", "blis") @@ -260,9 +264,9 @@ def test_nested_prange_blas(nthreads_outer): nested_blas_info = threadpool_infos.get_modules("user_api", "blas") assert len(nested_blas_info) == len(blas_info) for module in nested_blas_info: - assert module["num_threads"] == 1 + assert module.num_threads == 1 - assert original_info == threadpool_info() + assert original_info == _threadpool_info() @pytest.mark.parametrize("limit", [1, None]) @@ -271,17 +275,18 @@ def test_get_original_num_threads(limit): with threadpool_limits(limits=2, user_api="blas") as ctl: # set different blas num threads to start with (when multiple openblas) if ctl._original_info: - ctl._original_info[0].set_num_threads(1) + ctl._original_info.modules[0].set_num_threads(1) - original_infos = threadpool_info() + original_infos = _threadpool_info() with threadpool_limits(limits=limit, user_api="blas") as threadpoolctx: original_num_threads = threadpoolctx.get_original_num_threads() + print(original_num_threads) assert "openmp" not in original_num_threads blas_infos = original_infos.get_modules("user_api", "blas") if blas_infos: - expected = min(module["num_threads"] for module in blas_infos) + expected = min(module.num_threads for module in blas_infos) assert original_num_threads["blas"] == expected else: assert original_num_threads["blas"] is None @@ -294,12 +299,12 @@ def test_get_original_num_threads(limit): def test_mkl_threading_layer(): # Check that threadpool_info correctly recovers the threading layer used # by mkl - mkl_info = threadpool_info().get_modules("internal_api", "mkl") + mkl_info = _threadpool_info().get_modules("internal_api", "mkl") expected_layer = os.getenv("MKL_THREADING_LAYER") if not (mkl_info and expected_layer): pytest.skip("requires MKL and the environment variable " "MKL_THREADING_LAYER set") - actual_layer = mkl_info[0]["threading_layer"] + actual_layer = mkl_info.modules[0].threading_layer assert actual_layer == expected_layer.lower() diff --git a/threadpoolctl.py b/threadpoolctl.py index a4a6ef39..1c5d0785 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -17,7 +17,6 @@ import warnings from ctypes.util import find_library from abc import ABC, abstractmethod -from collections import UserDict, UserList __version__ = "1.2.0.dev0" __all__ = ["threadpool_limits", "threadpool_info"] @@ -86,7 +85,7 @@ class _dl_phdr_info(ctypes.Structure): } # Helpers for the doc and test names -_ALL_USER_APIS = set(d["user_api"] for lib, d in _SUPPORTED_MODULES.items()) +_ALL_USER_APIS = list(set(d["user_api"] for d in _SUPPORTED_MODULES.values())) _ALL_INTERNAL_APIS = list(_SUPPORTED_MODULES.keys()) _ALL_PREFIXES = [prefix for _, d in _SUPPORTED_MODULES.items() for prefix in d["filename_prefixes"]] @@ -120,7 +119,7 @@ def threadpool_info(): In addition, each module may contain internal_api specific entries. """ - return _ThreadpoolInfo(user_api=_ALL_USER_APIS) + return _ThreadpoolInfo(user_api=_ALL_USER_APIS).todicts() @_format_docstring( @@ -178,20 +177,23 @@ def __exit__(self, type, value, traceback): def unregister(self): if self._original_info is not None: for module in self._original_info: - module.set_num_threads(module["num_threads"]) + module.set_num_threads(module.num_threads) def get_original_num_threads(self): """Original num_threads from before calling threadpool_limits Return a dict `{user_api: num_threads}`. """ - original_limits = self._original_info or threadpool_info() + if self._original_info is not None: + original_limits = self._original_info + else: + original_limits = _ThreadpoolInfo(user_api=self._user_api) num_threads = {} warning_apis = [] for user_api in self._user_api: - limits = [module["num_threads"] for module in + limits = [module.num_threads for module in original_limits.get_modules("user_api", user_api)] limits = set(limits) n_limits = len(limits) @@ -230,11 +232,16 @@ def _check_params(self, limits, user_api): limits = {api: limits for api in user_api} prefixes = [] else: - if isinstance(limits, _ThreadpoolInfo): - # This should be a list of modules, for compatibility with - # the result from threadpool_info. + if isinstance(limits, list): + # This should be a list of dicts of modules, for compatibility + # with the result from threadpool_info. limits = {module["prefix"]: module["num_threads"] for module in limits} + elif isinstance(limits, _ThreadpoolInfo): + # To set the limits from the modules of a _ThreadpoolInfo + # object. + limits = {module.prefix: module.num_threads + for module in limits} if not isinstance(limits, dict): raise TypeError("limits must either be an int, a list or a " @@ -262,10 +269,10 @@ def _set_threadpool_limits(self): # self._limits is a dict {key: num_threads} where key is either # a prefix or a user_api. If a module matches both, the limit # corresponding to the prefix is chosed. - if module["prefix"] in self._limits: - num_threads = self._limits[module["prefix"]] + if module.prefix in self._limits: + num_threads = self._limits[module.prefix] else: - num_threads = self._limits[module["user_api"]] + num_threads = self._limits[module.user_api] if num_threads is not None: module.set_num_threads(num_threads) @@ -273,27 +280,34 @@ def _set_threadpool_limits(self): @_format_docstring( - PREFIXES=", ".join(_ALL_PREFIXES), + PREFIXES=", ".join('"{}"'.format(prefix) for prefix in _ALL_PREFIXES), USER_APIS=", ".join('"{}"'.format(api) for api in _ALL_USER_APIS), BLAS_LIBS=", ".join(_ALL_BLAS_LIRARIES), OPENMP_LIBS=", ".join(_ALL_OPENMP_LIBRARIES)) -class _ThreadpoolInfo(UserList): - """List of all supported modules that have been found +class _ThreadpoolInfo(): + """Collection of all supported modules that have been found Parameters ---------- - prefixes : list of prefixes or None (default=None) - Select libraries matching the requested prefixes. Supported prefixes - are {PREFIXES}. If None, libraries are not selected by their prefix. + user_api : list of user APIs or None (default=None) + Select libraries matching the requested API. Ignored if `modules` is + not None. Supported user APIs are {USER_APIS}. - user_api : {USER_APIS} or None (default=None) - Select libraries matching the requested API. + - "blas" selects all BLAS supported libraries ({BLAS_LIBS}) + - "openmp" selects all OpenMP supported libraries ({OPENMP_LIBS}) - - If "blas", BLAS supported libraries ({BLAS_LIBS}) are selected. + If None, libraries are not selected by their `user_api`. - - If "openmp", OpenMP supported libraries ({OPENMP_LIBS}) are selected. + prefixes : list of prefixes or None (default=None) + Select libraries matching the requested prefixes. Supported prefixes + are {PREFIXES}. + If None, libraries are not selected by their prefix. Ignored if + `modules` is not None. - - If None, libraries are not selected by their `user_api`. + modules : list of _Module objects or None (default=None) + Wraps a list of _Module objects into a _ThreapoolInfo object. Does not + load or reload any shared library. If it is not None, `prefixes` and + `user_api` are ignored. Note ---- @@ -306,11 +320,15 @@ class _ThreadpoolInfo(UserList): # impact of slow system calls (e.g. stat) on slow filesystem _realpaths = dict() - def __init__(self, prefixes=None, user_api=None): - super().__init__() - self.prefixes = [] if prefixes is None else prefixes - self.user_api = [] if user_api is None else user_api - self._load_modules() + def __init__(self, user_api=None, prefixes=None, modules=None): + if modules is None: + self.prefixes = [] if prefixes is None else prefixes + self.user_api = [] if user_api is None else user_api + + self.modules = [] + self._load_modules() + else: + self.modules = modules def get_modules(self, key, values): """Return all modules such that values contains module[key]""" @@ -318,10 +336,25 @@ def get_modules(self, key, values): values = list(_ALL_USER_APIS) if not isinstance(values, list): values = [values] - return [module for module in self if module[key] in values] + modules = [module for module in self.modules + if getattr(module, key) in values] + return _ThreadpoolInfo(modules=modules) + + def todicts(self): + """Return info as a list of dicts""" + return [module.todict() for module in self.modules] + + def __len__(self): + return len(self.modules) + + def __iter__(self): + yield from self.modules + + def __eq__(self, other): + return self.modules == other.modules def _load_modules(self): - """Loop through loaded libraries and store supported ones.""" + """Loop through loaded libraries and store supported ones""" if sys.platform == "darwin": self._find_modules_with_dyld() elif sys.platform == "win32": @@ -467,7 +500,7 @@ def _make_module_from_path(self, filepath): candidate_module["user_api"] in self.user_api): module_class = globals()[candidate_module["module_class"]] module = module_class(filepath, prefix) - self.append(module) + self.modules.append(module) def _check_prefix(self, library_basename, filename_prefixes): """Return the prefix library_basename starts with @@ -514,13 +547,13 @@ def _realpath(cls, filepath, cache_limit=10000): return rpath -class _Module(UserDict, ABC): +class _Module(ABC): """Abstract base class for the modules - A module is represented by a dict with the following information: + A module is represented by the following information: - "user_api" : user API. Possible values are {USER_APIS}. - "internal_api" : internal API. Possible values are {INTERNAL_APIS}. - - "prefix" : filename prefix of the specific implementation. + - "prefix" : prefix of the shared library's name. - "filepath" : path to the loaded module. - "version" : version of the library (if available). - "num_threads" : the current thread limit. @@ -528,15 +561,12 @@ class _Module(UserDict, ABC): In addition, each module may contain internal_api specific entries. """ def __init__(self, filepath=None, prefix=None): - super().__init__() - self["user_api"] = self.user_api - self["internal_api"] = self.internal_api - self["prefix"] = prefix - self["filepath"] = filepath + self.filepath = filepath + self.prefix = prefix self.dynlib = ctypes.CDLL(filepath, mode=_RTLD_NOLOAD) - self["version"] = self.get_version() - self["num_threads"] = self.get_num_threads() - self._add_extra_info() + self.version = self.get_version() + self.num_threads = self.get_num_threads() + self._get_extra_info() @property @abstractmethod @@ -548,6 +578,17 @@ def user_api(self): def internal_api(self): pass # pragma: no cover + def __eq__(self, other): + return self.todict() == other.todict() + + def todict(self): + return {"user_api": self.user_api, + "internal_api": self.internal_api, + "prefix": self.prefix, + "filepath": self.filepath, + "version": self.version, + "num_threads": self.num_threads} + @abstractmethod def get_version(self): """Return the version of the shared library""" @@ -564,7 +605,7 @@ def set_num_threads(self, num_threads): pass # pragma: no cover @abstractmethod - def _add_extra_info(self): + def _get_extra_info(self): """Add additional module specific information""" pass # pragma: no cover @@ -595,7 +636,7 @@ def set_num_threads(self, num_threads): lambda num_threads: None) return set_func(num_threads) - def _add_extra_info(self): + def _get_extra_info(self): pass @@ -623,7 +664,7 @@ def set_num_threads(self, num_threads): lambda num_threads: None) return set_func(num_threads) - def _add_extra_info(self): + def _get_extra_info(self): pass @@ -651,8 +692,8 @@ def set_num_threads(self, num_threads): lambda num_threads: None) return set_func(num_threads) - def _add_extra_info(self): - self["threading_layer"] = self.get_threading_layer() + def _get_extra_info(self): + self.threading_layer = self.get_threading_layer() def get_threading_layer(self): """Return the threading layer of MKL""" @@ -684,5 +725,5 @@ def set_num_threads(self, num_threads): lambda num_threads: None) return set_func(num_threads) - def _add_extra_info(self): + def _get_extra_info(self): pass From b5d6492f18d2c443ce1892a92ddaaa0f5ad84374 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Fri, 22 Nov 2019 10:54:28 +0100 Subject: [PATCH 062/187] cln --- threadpoolctl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/threadpoolctl.py b/threadpoolctl.py index 1c5d0785..4dacc43b 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -582,6 +582,7 @@ def __eq__(self, other): return self.todict() == other.todict() def todict(self): + """Return relevant info wrapped in a dict""" return {"user_api": self.user_api, "internal_api": self.internal_api, "prefix": self.prefix, From 4a89b6c89592dcfba6d482bec30d35cfa05ffa9a Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Fri, 22 Nov 2019 11:27:45 +0100 Subject: [PATCH 063/187] add test for consistency between function and module --- tests/test_threadpoolctl.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index fc67c2e7..4579a6c5 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -2,7 +2,7 @@ import re import pytest -from threadpoolctl import threadpool_limits, _ThreadpoolInfo +from threadpoolctl import threadpool_limits, threadpool_info, _ThreadpoolInfo from threadpoolctl import _ALL_PREFIXES, _ALL_USER_APIS from .utils import cython_extensions_compiled @@ -27,6 +27,15 @@ def _threadpool_info(): return _ThreadpoolInfo(user_api=_ALL_USER_APIS) +def test_threadpool_limits_public_api(): + # Check consistency between threadpool_info and _ThreadpoolInfo + public_info = threadpool_info() + private_info = _threadpool_info() + + for module1, module2 in zip(public_info, private_info): + assert module1 == module2.todict() + + @pytest.mark.parametrize("prefix", _ALL_PREFIXES) @pytest.mark.parametrize("limit", [1, 3]) def test_threadpool_limits_by_prefix(prefix, limit): From 04c6d10ee89f967080d96682b7f237d05fd73370 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Fri, 22 Nov 2019 14:27:02 +0100 Subject: [PATCH 064/187] more verbose meta-test + distinguish between skip and other records --- continuous_integration/check_no_test_skipped.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/continuous_integration/check_no_test_skipped.py b/continuous_integration/check_no_test_skipped.py index 10fab11a..6347929d 100644 --- a/continuous_integration/check_no_test_skipped.py +++ b/continuous_integration/check_no_test_skipped.py @@ -16,18 +16,25 @@ for name in os.listdir(base_dir): # all test result files are in /base_dir/jobs.*/ dirs if name.startswith("stage1."): + print("> processing test result from job", name.replace("stage1", "")) result_file = os.path.join(base_dir, name, "test-data.xml") root = ET.parse(result_file).getroot() # All tests are identified by the xml tag testcase. - for test in root.iter('testcase'): - test_name = test.attrib['name'] + for test in root.iter("testcase"): + test_name = test.attrib["name"] if test_name not in aggregated_results: - # len(test) is > 0 if the test is skipped. - aggregated_results[test_name] = not bool(len(test)) + aggregated_results[test_name] = False + print(" -", test_name) + + for child in test: + print(" -", child.tag) + if child.tag == "skipped": + aggregated_results[test_name] |= False else: - aggregated_results[test_name] |= not bool(len(test)) + aggregated_results[test_name] |= True +print("\n------------------------------------------------------------------\n") fail = False for test, result in aggregated_results.items(): if not result: From 32c033b806ae544181062bf8bbb4a5b5a0ea1d8b Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Fri, 22 Nov 2019 14:34:14 +0100 Subject: [PATCH 065/187] break loop --- continuous_integration/check_no_test_skipped.py | 1 + 1 file changed, 1 insertion(+) diff --git a/continuous_integration/check_no_test_skipped.py b/continuous_integration/check_no_test_skipped.py index 6347929d..69c14dfd 100644 --- a/continuous_integration/check_no_test_skipped.py +++ b/continuous_integration/check_no_test_skipped.py @@ -31,6 +31,7 @@ print(" -", child.tag) if child.tag == "skipped": aggregated_results[test_name] |= False + break else: aggregated_results[test_name] |= True From 1e27bdf92d5934e3544f6c9a4a62a3f808dfcec0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Fri, 22 Nov 2019 14:38:00 +0100 Subject: [PATCH 066/187] [MRG] MNT use new syntax to select blas with conda (#58) --- .azure_pipeline.yml | 17 ++++++++++------- continuous_integration/install.cmd | 2 +- continuous_integration/install.sh | 5 +---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml index fd4f4d22..c84fbb16 100644 --- a/.azure_pipeline.yml +++ b/.azure_pipeline.yml @@ -15,8 +15,8 @@ stages: name: Windows vmImage: vs2017-win2016 matrix: - py37_conda: - VERSION_PYTHON: '3.7' + pylatest_conda: + VERSION_PYTHON: '*' PACKAGER: 'conda' py36_conda: VERSION_PYTHON: '3.6' @@ -50,7 +50,7 @@ stages: py36_conda_openblas_clang_clang: PACKAGER: 'conda' VERSION_PYTHON: '3.6' - NO_MKL: 'true' + BLAS: 'openblas' CC_OUTER_LOOP: 'clang-8' CC_INNER_LOOP: 'clang-8' # Linux environment to test the latest available dependencies and MKL. @@ -58,11 +58,12 @@ stages: pylatest_conda_mkl_clang_gcc: PACKAGER: 'conda' VERSION_PYTHON: '*' + BLAS: 'mkl' CC_OUTER_LOOP: 'clang-8' CC_INNER_LOOP: 'gcc' MKL_THREADING_LAYER: 'INTEL' - # Linux + Python 3.7 with numpy / scipy installed with pip from PyPI and - # heterogeneous openmp runtimes. + # Linux + Python 3.7 with numpy / scipy installed with pip from PyPI + # and heterogeneous openmp runtimes. py37_pip_openblas_gcc_clang: PACKAGER: 'pip' VERSION_PYTHON: '3.7' @@ -114,9 +115,10 @@ stages: vmImage: xcode9-macos10.13 matrix: # MacOS environment with OpenMP installed through homebrew - pylatest_conda_homebrew_libomp: + py35_conda_homebrew_libomp: PACKAGER: 'conda' - VERSION_PYTHON: '*' + VERSION_PYTHON: '3.5' + BLAS: 'mkl' CC_OUTER_LOOP: 'clang' CC_INNER_LOOP: 'clang' INSTALL_LIBOMP: 'homebrew' @@ -124,6 +126,7 @@ stages: pylatest_conda_conda_forge_clang: PACKAGER: 'conda' VERSION_PYTHON: '*' + BLAS: 'mkl' CC_OUTER_LOOP: 'clang' CC_INNER_LOOP: 'clang' INSTALL_LIBOMP: 'conda-forge' diff --git a/continuous_integration/install.cmd b/continuous_integration/install.cmd index 263fdabb..4a0d5970 100644 --- a/continuous_integration/install.cmd +++ b/continuous_integration/install.cmd @@ -18,7 +18,7 @@ python --version pip --version @rem Install dependencies with either conda or pip. -if "%PACKAGER%" == "conda" (%CONDA_INSTALL% numpy=1.15 scipy pytest cython) +if "%PACKAGER%" == "conda" (%CONDA_INSTALL% numpy scipy pytest cython) if "%PACKAGER%" == "pip" (%PIP_INSTALL% numpy scipy pytest cython) @rem Install extra developer dependencies diff --git a/continuous_integration/install.sh b/continuous_integration/install.sh index 920cffe2..707f296f 100755 --- a/continuous_integration/install.sh +++ b/continuous_integration/install.sh @@ -40,10 +40,7 @@ make_conda() { if [[ "$PACKAGER" == "conda" ]]; then TO_INSTALL="python=$VERSION_PYTHON pip" if [[ "$NO_NUMPY" != "true" ]]; then - TO_INSTALL="$TO_INSTALL numpy scipy" - if [[ "$NO_MKL" == "true" ]]; then - TO_INSTALL="$TO_INSTALL nomkl" - fi + TO_INSTALL="$TO_INSTALL numpy scipy blas[build=$BLAS]" fi make_conda $TO_INSTALL From 98f3676d5b8776cdfe27d3e81beff7db324ca541 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Fri, 22 Nov 2019 17:34:18 +0100 Subject: [PATCH 067/187] trigger ci From 5aaa00e695f42dd01e9b723755e7c1191ca07bea Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Fri, 22 Nov 2019 17:52:13 +0100 Subject: [PATCH 068/187] address comments --- tests/test_threadpoolctl.py | 16 ++++++++++++++++ threadpoolctl.py | 23 ++++++++++++++++------- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index 4579a6c5..e886d854 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -36,6 +36,22 @@ def test_threadpool_limits_public_api(): assert module1 == module2.todict() +def test_ThreadpoolInfo_todicts(): + info = _threadpool_info() + + assert info.todicts() == [module.todict() for module in info] + assert info.todicts() == [module.todict() for module in info.modules] + + for module in info: + module_dict = module.todict() + assert "user_api" in module_dict + assert "internal_api" in module_dict + assert "prefix" in module_dict + assert "filepath" in module_dict + assert "version" in module_dict + assert "num_threads" in module_dict + + @pytest.mark.parametrize("prefix", _ALL_PREFIXES) @pytest.mark.parametrize("limit", [1, 3]) def test_threadpool_limits_by_prefix(prefix, limit): diff --git a/threadpoolctl.py b/threadpoolctl.py index 4dacc43b..66f2e151 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -18,7 +18,7 @@ from ctypes.util import find_library from abc import ABC, abstractmethod -__version__ = "1.2.0.dev0" +__version__ = "2.0.0.dev0" __all__ = ["threadpool_limits", "threadpool_info"] @@ -46,7 +46,7 @@ class _dl_phdr_info(ctypes.Structure): ("dlpi_addr", _SYSTEM_UINT), # Base address of object ("dlpi_name", ctypes.c_char_p), # path to the library ("dlpi_phdr", ctypes.c_void_p), # pointer on dlpi_headers - ("dlpi_phnum", _SYSTEM_UINT_HALF) # number of element in dlpi_phdr + ("dlpi_phnum", _SYSTEM_UINT_HALF) # number of elements in dlpi_phdr ] @@ -140,13 +140,13 @@ class threadpool_limits: Parameters ---------- limits : int, dict or None (default=None) - The maximal number of thread that can be used in thread pools + The maximal number of threads that can be used in thread pools - - If int, sets the maximum number of thread to `limits` for each + - If int, sets the maximum number of threads to `limits` for each library selected by `user_api`. - If it is a dictionary `{{key: max_threads}}`, this function sets a - custom maximum number of thread for each `key` which can be either a + custom maximum number of threads for each `key` which can be either a `user_api` or a `prefix` for a specific library. - If None, this function does not do anything. @@ -279,6 +279,9 @@ def _set_threadpool_limits(self): return modules +# The object oriented API of _ThreadpoolInfo and its modules is private. +# The public API (i.e. the "threadpool_info" function) only exposes the +# "list of dicts" representation returned by the .todicts method. @_format_docstring( PREFIXES=", ".join('"{}"'.format(prefix) for prefix in _ALL_PREFIXES), USER_APIS=", ".join('"{}"'.format(api) for api in _ALL_USER_APIS), @@ -314,10 +317,16 @@ class _ThreadpoolInfo(): Is is possible to select libraries both by prefixes and by user_api. All libraries matching one or the other will be selected. """ - # Cache for libc under POSIX and a few system libraries under Windows + # Cache for libc under POSIX and a few system libraries under Windows. + # We use a class level cache instead of an instance level cache because + # it's very unlikely that a shared library will be unloaded and reloaded + # during the lifetime of a program. _system_libraries = dict() # Cache for calls to os.path.realpath on system libraries to reduce the - # impact of slow system calls (e.g. stat) on slow filesystem + # impact of slow system calls (e.g. stat) on slow filesystem. + # We use a class level cache instead of an instance level cache because + # we can safely assume that the filepath of loaded shared libraries will + # never change during the lifetime of a program. _realpaths = dict() def __init__(self, user_api=None, prefixes=None, modules=None): From 8ead8f4b0276964b3086d5e4a74acbb475b2f6b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Fri, 22 Nov 2019 18:02:08 +0100 Subject: [PATCH 069/187] [MRG] MNT Fix meta-test distinguish skipped from other records (#59) --- .../check_no_test_skipped.py | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/continuous_integration/check_no_test_skipped.py b/continuous_integration/check_no_test_skipped.py index 10fab11a..1a8861e1 100644 --- a/continuous_integration/check_no_test_skipped.py +++ b/continuous_integration/check_no_test_skipped.py @@ -11,26 +11,31 @@ # dict {test: result} where result is False if the test was skipped in every # job and True otherwise. -aggregated_results = {} +always_skipped = {} for name in os.listdir(base_dir): # all test result files are in /base_dir/jobs.*/ dirs if name.startswith("stage1."): + print("> processing test result from job", name.replace("stage1", "")) + print(" > tests skipped:") result_file = os.path.join(base_dir, name, "test-data.xml") root = ET.parse(result_file).getroot() # All tests are identified by the xml tag testcase. - for test in root.iter('testcase'): - test_name = test.attrib['name'] - if test_name not in aggregated_results: - # len(test) is > 0 if the test is skipped. - aggregated_results[test_name] = not bool(len(test)) + for test in root.iter("testcase"): + test_name = test.attrib["name"] + skipped = any(child.tag == "skipped" for child in test) + if skipped: + print(" -", test_name) + if test_name in always_skipped: + always_skipped[test_name] &= skipped else: - aggregated_results[test_name] |= not bool(len(test)) + always_skipped[test_name] = skipped +print("\n------------------------------------------------------------------\n") fail = False -for test, result in aggregated_results.items(): - if not result: +for test, skipped in always_skipped.items(): + if skipped: fail = True print(test, "was skipped in every job") From 0fd9071a25aeb769b1d502b65f7b99d364dac8ea Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Mon, 25 Nov 2019 18:07:07 +0100 Subject: [PATCH 070/187] avoid redundant info + remove module class attributes + fix todict --- threadpoolctl.py | 122 +++++++++++++++++++---------------------------- 1 file changed, 50 insertions(+), 72 deletions(-) diff --git a/threadpoolctl.py b/threadpoolctl.py index 66f2e151..bbc125da 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -57,41 +57,42 @@ class _dl_phdr_info(ctypes.Structure): _RTLD_NOLOAD = ctypes.DEFAULT_MODE -# List of the supported libraries. The items hold the possible prefixes of -# loaded shared objects, the name of the internal_api to call, the name of the -# user_api and the name of the class to instanciate to create a module object -# from this library. +# List of the supported libraries. The items are indexed by the name of the +# class to instanciate to create the module objects. The items hold the +# possible prefixes of loaded shared objects, the name of the internal_api to +# call and the name of the user_api. _SUPPORTED_MODULES = { - "openmp": { + "_OpenMPModule": { "user_api": "openmp", - "filename_prefixes": ("libiomp", "libgomp", "libomp", "vcomp",), - "module_class": "_OpenMPModule" + "internal_api": "openmp", + "filename_prefixes": ("libiomp", "libgomp", "libomp", "vcomp") }, - "openblas": { + " _OpenBLASModule": { "user_api": "blas", - "filename_prefixes": ("libopenblas",), - "module_class": "_OpenBLASModule" + "internal_api": "openblas", + "filename_prefixes": ("libopenblas",) }, - "mkl": { + "_MKLModule": { "user_api": "blas", - "filename_prefixes": ("libmkl_rt", "mkl_rt",), - "module_class": "_MKLModule" + "internal_api": "mkl", + "filename_prefixes": ("libmkl_rt", "mkl_rt") }, - "blis": { + "_BLISModule": { "user_api": "blas", - "filename_prefixes": ("libblis",), - "module_class": "_BLISModule" + "internal_api": "blis", + "filename_prefixes": ("libblis",) } } # Helpers for the doc and test names -_ALL_USER_APIS = list(set(d["user_api"] for d in _SUPPORTED_MODULES.values())) -_ALL_INTERNAL_APIS = list(_SUPPORTED_MODULES.keys()) -_ALL_PREFIXES = [prefix for _, d in _SUPPORTED_MODULES.items() - for prefix in d["filename_prefixes"]] -_ALL_BLAS_LIRARIES = [lib for lib, d in _SUPPORTED_MODULES.items() - if d["user_api"] == "blas"] -_ALL_OPENMP_LIBRARIES = list(_SUPPORTED_MODULES["openmp"]["filename_prefixes"]) +_ALL_USER_APIS = list(set(m["user_api"] for m in _SUPPORTED_MODULES.values())) +_ALL_INTERNAL_APIS = [m["internal_api"] for m in _SUPPORTED_MODULES.values()] +_ALL_PREFIXES = [prefix for m in _SUPPORTED_MODULES.values() + for prefix in m["filename_prefixes"]] +_ALL_BLAS_LIBRARIES = [m["internal_api"] for m in _SUPPORTED_MODULES.values() + if m["user_api"] == "blas"] +_ALL_OPENMP_LIBRARIES = list( + _SUPPORTED_MODULES["_OpenMPModule"]["filename_prefixes"]) def _format_docstring(*args, **kwargs): @@ -124,7 +125,7 @@ def threadpool_info(): @_format_docstring( USER_APIS=", ".join('"{}"'.format(api) for api in _ALL_USER_APIS), - BLAS_LIBS=", ".join(_ALL_BLAS_LIRARIES), + BLAS_LIBS=", ".join(_ALL_BLAS_LIBRARIES), OPENMP_LIBS=", ".join(_ALL_OPENMP_LIBRARIES)) class threadpool_limits: """Change the maximal number of threads that can be used in thread pools. @@ -285,7 +286,7 @@ def _set_threadpool_limits(self): @_format_docstring( PREFIXES=", ".join('"{}"'.format(prefix) for prefix in _ALL_PREFIXES), USER_APIS=", ".join('"{}"'.format(api) for api in _ALL_USER_APIS), - BLAS_LIBS=", ".join(_ALL_BLAS_LIRARIES), + BLAS_LIBS=", ".join(_ALL_BLAS_LIBRARIES), OPENMP_LIBS=", ".join(_ALL_OPENMP_LIBRARIES)) class _ThreadpoolInfo(): """Collection of all supported modules that have been found @@ -493,7 +494,7 @@ def _make_module_from_path(self, filepath): # Loop through supported modules to find if this filename corresponds # to a supported module. - for _, candidate_module in _SUPPORTED_MODULES.items(): + for module_class, candidate_module in _SUPPORTED_MODULES.items(): # check if filename matches a supported prefix prefix = self._check_prefix(filename, candidate_module["filename_prefixes"]) @@ -505,10 +506,11 @@ def _make_module_from_path(self, filepath): # filename matches a prefix. Check if it matches the request. If # so, create and store the module. - if (prefix in self.prefixes or - candidate_module["user_api"] in self.user_api): - module_class = globals()[candidate_module["module_class"]] - module = module_class(filepath, prefix) + user_api = candidate_module["user_api"] + internal_api = candidate_module["internal_api"] + if prefix in self.prefixes or user_api in self.user_api: + module_class = globals()[module_class] + module = module_class(filepath, prefix, user_api, internal_api) self.modules.append(module) def _check_prefix(self, library_basename, filename_prefixes): @@ -569,35 +571,23 @@ class _Module(ABC): In addition, each module may contain internal_api specific entries. """ - def __init__(self, filepath=None, prefix=None): + def __init__(self, filepath=None, prefix=None, user_api=None, + internal_api=None): self.filepath = filepath self.prefix = prefix - self.dynlib = ctypes.CDLL(filepath, mode=_RTLD_NOLOAD) + self.user_api = user_api + self.internal_api = internal_api + self._dynlib = ctypes.CDLL(filepath, mode=_RTLD_NOLOAD) self.version = self.get_version() self.num_threads = self.get_num_threads() self._get_extra_info() - @property - @abstractmethod - def user_api(self): - pass # pragma: no cover - - @property - @abstractmethod - def internal_api(self): - pass # pragma: no cover - def __eq__(self, other): return self.todict() == other.todict() def todict(self): """Return relevant info wrapped in a dict""" - return {"user_api": self.user_api, - "internal_api": self.internal_api, - "prefix": self.prefix, - "filepath": self.filepath, - "version": self.version, - "num_threads": self.num_threads} + return {k: v for k, v in vars(self).items() if not k.startswith("_")} @abstractmethod def get_version(self): @@ -622,13 +612,10 @@ def _get_extra_info(self): class _OpenBLASModule(_Module): """Module class for OpenBLAS""" - user_api = "blas" - internal_api = "openblas" - def get_version(self): # None means OpenBLAS is not loaded or version < 0.3.4, since OpenBLAS # did not expose its version before that. - get_config = getattr(self.dynlib, "openblas_get_config", + get_config = getattr(self._dynlib, "openblas_get_config", lambda: None) get_config.restype = ctypes.c_char_p config = get_config().split() @@ -637,12 +624,12 @@ def get_version(self): return None def get_num_threads(self): - get_func = getattr(self.dynlib, "openblas_get_num_threads", + get_func = getattr(self._dynlib, "openblas_get_num_threads", lambda: None) return get_func() def set_num_threads(self, num_threads): - set_func = getattr(self.dynlib, "openblas_set_num_threads", + set_func = getattr(self._dynlib, "openblas_set_num_threads", lambda num_threads: None) return set_func(num_threads) @@ -652,17 +639,14 @@ def _get_extra_info(self): class _BLISModule(_Module): """Module class for BLIS""" - user_api = "blas" - internal_api = "blis" - def get_version(self): - get_version_ = getattr(self.dynlib, "bli_info_get_version_str", + get_version_ = getattr(self._dynlib, "bli_info_get_version_str", lambda: None) get_version_.restype = ctypes.c_char_p return get_version_().decode("utf-8") def get_num_threads(self): - get_func = getattr(self.dynlib, "bli_thread_get_num_threads", + get_func = getattr(self._dynlib, "bli_thread_get_num_threads", lambda: None) num_threads = get_func() # by default BLIS is single-threaded and get_num_threads @@ -670,7 +654,7 @@ def get_num_threads(self): return 1 if num_threads == -1 else num_threads def set_num_threads(self, num_threads): - set_func = getattr(self.dynlib, "bli_thread_set_num_threads", + set_func = getattr(self._dynlib, "bli_thread_set_num_threads", lambda num_threads: None) return set_func(num_threads) @@ -680,12 +664,9 @@ def _get_extra_info(self): class _MKLModule(_Module): """Module class for MKL""" - user_api = "blas" - internal_api = "mkl" - def get_version(self): res = ctypes.create_string_buffer(200) - self.dynlib.mkl_get_version_string(res, 200) + self._dynlib.mkl_get_version_string(res, 200) version = res.value.decode("utf-8") group = re.search(r"Version ([^ ]+) ", version) @@ -694,11 +675,11 @@ def get_version(self): return version.strip() def get_num_threads(self): - get_func = getattr(self.dynlib, "MKL_Get_Max_Threads", lambda: None) + get_func = getattr(self._dynlib, "MKL_Get_Max_Threads", lambda: None) return get_func() def set_num_threads(self, num_threads): - set_func = getattr(self.dynlib, "MKL_Set_Num_Threads", + set_func = getattr(self._dynlib, "MKL_Set_Num_Threads", lambda num_threads: None) return set_func(num_threads) @@ -710,7 +691,7 @@ def get_threading_layer(self): # The function mkl_set_threading_layer returns the current threading # layer. Calling it with an invalid threading layer allows us to safely # get the threading layer - set_threading_layer = getattr(self.dynlib, "MKL_Set_Threading_Layer", + set_threading_layer = getattr(self._dynlib, "MKL_Set_Threading_Layer", lambda layer: -1) layer_map = {0: "intel", 1: "sequential", 2: "pgi", 3: "gnu", 4: "tbb", -1: "not specified"} @@ -719,19 +700,16 @@ def get_threading_layer(self): class _OpenMPModule(_Module): """Module class for OpenMP""" - user_api = "openmp" - internal_api = "openmp" - def get_version(self): # There is no way to get the version number programmatically in OpenMP. return None def get_num_threads(self): - get_func = getattr(self.dynlib, "omp_get_max_threads", lambda: None) + get_func = getattr(self._dynlib, "omp_get_max_threads", lambda: None) return get_func() def set_num_threads(self, num_threads): - set_func = getattr(self.dynlib, "omp_set_num_threads", + set_func = getattr(self._dynlib, "omp_set_num_threads", lambda num_threads: None) return set_func(num_threads) From 94e1a774c55d7d56ec50590ad650e5a49a23e1ea Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Wed, 27 Nov 2019 14:35:25 +0100 Subject: [PATCH 071/187] better tests for todict(s) methods --- tests/test_threadpoolctl.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index e886d854..5d4a850d 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -37,8 +37,11 @@ def test_threadpool_limits_public_api(): def test_ThreadpoolInfo_todicts(): + # Check all keys expected for the public api are in the dicts returned by + # the .todict(s) methods info = _threadpool_info() + assert threadpool_info() == [module.todict() for module in info.modules] assert info.todicts() == [module.todict() for module in info] assert info.todicts() == [module.todict() for module in info.modules] @@ -51,6 +54,9 @@ def test_ThreadpoolInfo_todicts(): assert "version" in module_dict assert "num_threads" in module_dict + if module.internal_api == "mkl": + assert "threading_layer" in module_dict + @pytest.mark.parametrize("prefix", _ALL_PREFIXES) @pytest.mark.parametrize("limit", [1, 3]) From 24958310a8d0ef93165801673485bd099b50f622 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Wed, 27 Nov 2019 14:53:30 +0100 Subject: [PATCH 072/187] infos -> info --- tests/test_threadpoolctl.py | 42 ++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index 5d4a850d..8fd0a65e 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -62,9 +62,9 @@ def test_ThreadpoolInfo_todicts(): @pytest.mark.parametrize("limit", [1, 3]) def test_threadpool_limits_by_prefix(prefix, limit): # Check that the maximum number of threads can be set by prefix - original_infos = _threadpool_info() + original_info = _threadpool_info() - modules_matching_prefix = original_infos.get_modules("prefix", prefix) + modules_matching_prefix = original_info.get_modules("prefix", prefix) if not modules_matching_prefix: pytest.skip("Requires {} runtime".format(prefix)) @@ -75,16 +75,16 @@ def test_threadpool_limits_by_prefix(prefix, limit): # threadpool_limits only sets an upper bound on the number of # threads. assert 0 < module.get_num_threads() <= limit - assert _threadpool_info() == original_infos + assert _threadpool_info() == original_info @pytest.mark.parametrize("user_api", (None, "blas", "openmp")) @pytest.mark.parametrize("limit", [1, 3]) def test_set_threadpool_limits_by_api(user_api, limit): # Check that the maximum number of threads can be set by user_api - original_infos = _threadpool_info() + original_info = _threadpool_info() - modules_matching_api = original_infos.get_modules("user_api", user_api) + modules_matching_api = original_info.get_modules("user_api", user_api) if not modules_matching_api: user_apis = _ALL_USER_APIS if user_api is None else [user_api] pytest.skip("Requires a library which api is in {}".format(user_apis)) @@ -97,13 +97,13 @@ def test_set_threadpool_limits_by_api(user_api, limit): # threads. assert 0 < module.get_num_threads() <= limit - assert _threadpool_info() == original_infos + assert _threadpool_info() == original_info def test_threadpool_limits_function_with_side_effect(): # Check that threadpool_limits can be used as a function with # side effects instead of a context manager. - original_infos = _threadpool_info() + original_info = _threadpool_info() threadpool_limits(limits=1) try: @@ -114,25 +114,25 @@ def test_threadpool_limits_function_with_side_effect(): finally: # Restore the original limits so that this test does not have any # side-effect. - threadpool_limits(limits=original_infos) + threadpool_limits(limits=original_info) - assert _threadpool_info() == original_infos + assert _threadpool_info() == original_info def test_set_threadpool_limits_no_limit(): # Check that limits=None does nothing. - original_infos = _threadpool_info() + original_info = _threadpool_info() with threadpool_limits(limits=None): - assert _threadpool_info() == original_infos + assert _threadpool_info() == original_info - assert _threadpool_info() == original_infos + assert _threadpool_info() == original_info def test_threadpool_limits_manual_unregister(): # Check that threadpool_limits can be used as an object which holds the # original state of the threadpools and that can be restored thanks to the # dedicated unregister method - original_infos = _threadpool_info() + original_info = _threadpool_info() limits = threadpool_limits(limits=1) try: @@ -145,7 +145,7 @@ def test_threadpool_limits_manual_unregister(): # side-effect. limits.unregister() - assert _threadpool_info() == original_infos + assert _threadpool_info() == original_info def test_threadpool_limits_bad_input(): @@ -190,14 +190,14 @@ def test_openmp_nesting(nthreads_outer): outer_num_threads, inner_num_threads = check_nested_openmp_loops(10) - original_infos = _threadpool_info() - openmp_infos = original_infos.get_modules("user_api", "openmp") + original_info = _threadpool_info() + openmp_infos = original_info.get_modules("user_api", "openmp") if "gcc" in (inner_cc, outer_cc): - assert original_infos.get_modules("prefix", "libgomp") + assert original_info.get_modules("prefix", "libgomp") if "clang" in (inner_cc, outer_cc): - assert original_infos.get_modules("prefix", "libomp") + assert original_info.get_modules("prefix", "libomp") if inner_cc == outer_cc: # The openmp runtime should be shared by default, meaning that @@ -219,7 +219,7 @@ def test_openmp_nesting(nthreads_outer): # The state of the original state of all threadpools should have been # restored. - assert _threadpool_info() == original_infos + assert _threadpool_info() == original_info # The number of threads available in the outer loop should not have been # decreased: @@ -310,14 +310,14 @@ def test_get_original_num_threads(limit): if ctl._original_info: ctl._original_info.modules[0].set_num_threads(1) - original_infos = _threadpool_info() + original_info = _threadpool_info() with threadpool_limits(limits=limit, user_api="blas") as threadpoolctx: original_num_threads = threadpoolctx.get_original_num_threads() print(original_num_threads) assert "openmp" not in original_num_threads - blas_infos = original_infos.get_modules("user_api", "blas") + blas_infos = original_info.get_modules("user_api", "blas") if blas_infos: expected = min(module.num_threads for module in blas_infos) assert original_num_threads["blas"] == expected From a61b05ee09d87120f4f286a697eef811e536210e Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Wed, 27 Nov 2019 14:55:56 +0100 Subject: [PATCH 073/187] infos -> info --- tests/_openmp_test_helper/nested_prange_blas.pyx | 6 +++--- tests/test_threadpoolctl.py | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/_openmp_test_helper/nested_prange_blas.pyx b/tests/_openmp_test_helper/nested_prange_blas.pyx index 47c445b1..e327eee0 100644 --- a/tests/_openmp_test_helper/nested_prange_blas.pyx +++ b/tests/_openmp_test_helper/nested_prange_blas.pyx @@ -43,12 +43,12 @@ def check_nested_prange_blas(double[:, ::1] A, double[:, ::1] B, int nthreads): int prange_num_threads int *prange_num_threads_ptr = &prange_num_threads - threadpool_infos = [None] + inner_info = [None] with nogil, parallel(num_threads=nthreads): if openmp.omp_get_thread_num() == 0: with gil: - threadpool_infos[0] = _ThreadpoolInfo(user_api=_ALL_USER_APIS) + inner_info[0] = _ThreadpoolInfo(user_api=_ALL_USER_APIS) prange_num_threads_ptr[0] = openmp.omp_get_num_threads() @@ -62,4 +62,4 @@ def check_nested_prange_blas(double[:, ::1] A, double[:, ::1] B, int nthreads): &alpha, &B[0, 0], &k, &A[i * chunk_size, 0], &k, &beta, &C[i * chunk_size, 0], &n) - return np.asarray(C), prange_num_threads, threadpool_infos[0] + return np.asarray(C), prange_num_threads, inner_info[0] diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index 8fd0a65e..d0f819f1 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -191,7 +191,7 @@ def test_openmp_nesting(nthreads_outer): outer_num_threads, inner_num_threads = check_nested_openmp_loops(10) original_info = _threadpool_info() - openmp_infos = original_info.get_modules("user_api", "openmp") + openmp_info = original_info.get_modules("user_api", "openmp") if "gcc" in (inner_cc, outer_cc): assert original_info.get_modules("prefix", "libgomp") @@ -206,7 +206,7 @@ def test_openmp_nesting(nthreads_outer): assert inner_num_threads == 1 else: # There should be at least 2 OpenMP runtime detected. - assert len(openmp_infos) >= 2 + assert len(openmp_info) >= 2 with threadpool_limits(limits=1) as threadpoolctx: max_threads = threadpoolctx.get_original_num_threads()["openmp"] @@ -289,12 +289,12 @@ def test_nested_prange_blas(nthreads_outer): nthreads = effective_num_threads(nthreads_outer, max_threads) result = check_nested_prange_blas(A, B, nthreads) - C, prange_num_threads, threadpool_infos = result + C, prange_num_threads, inner_info = result assert np.allclose(C, np.dot(A, B.T)) assert prange_num_threads == nthreads - nested_blas_info = threadpool_infos.get_modules("user_api", "blas") + nested_blas_info = inner_info.get_modules("user_api", "blas") assert len(nested_blas_info) == len(blas_info) for module in nested_blas_info: assert module.num_threads == 1 @@ -317,9 +317,9 @@ def test_get_original_num_threads(limit): assert "openmp" not in original_num_threads - blas_infos = original_info.get_modules("user_api", "blas") - if blas_infos: - expected = min(module.num_threads for module in blas_infos) + blas_info = original_info.get_modules("user_api", "blas") + if blas_info: + expected = min(module.num_threads for module in blas_info) assert original_num_threads["blas"] == expected else: assert original_num_threads["blas"] is None From e2beab6046d3f1079e688ab5ba83857abbe5d2f4 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Wed, 27 Nov 2019 15:00:35 +0100 Subject: [PATCH 074/187] address comment --- threadpoolctl.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/threadpoolctl.py b/threadpoolctl.py index bbc125da..cac05a52 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -263,21 +263,21 @@ def _set_threadpool_limits(self): """ if self._limits is None: return None - else: - modules = _ThreadpoolInfo(prefixes=self._prefixes, - user_api=self._user_api) - for module in modules: - # self._limits is a dict {key: num_threads} where key is either - # a prefix or a user_api. If a module matches both, the limit - # corresponding to the prefix is chosed. - if module.prefix in self._limits: - num_threads = self._limits[module.prefix] - else: - num_threads = self._limits[module.user_api] - - if num_threads is not None: - module.set_num_threads(num_threads) - return modules + + modules = _ThreadpoolInfo(prefixes=self._prefixes, + user_api=self._user_api) + for module in modules: + # self._limits is a dict {key: num_threads} where key is either + # a prefix or a user_api. If a module matches both, the limit + # corresponding to the prefix is chosed. + if module.prefix in self._limits: + num_threads = self._limits[module.prefix] + else: + num_threads = self._limits[module.user_api] + + if num_threads is not None: + module.set_num_threads(num_threads) + return modules # The object oriented API of _ThreadpoolInfo and its modules is private. From 6aad6fb25405b8dda777ec0699c47bae21121158 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Wed, 27 Nov 2019 16:42:50 +0100 Subject: [PATCH 075/187] debug --- threadpoolctl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/threadpoolctl.py b/threadpoolctl.py index cac05a52..070eb8ea 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -509,6 +509,7 @@ def _make_module_from_path(self, filepath): user_api = candidate_module["user_api"] internal_api = candidate_module["internal_api"] if prefix in self.prefixes or user_api in self.user_api: + print(globals()) module_class = globals()[module_class] module = module_class(filepath, prefix, user_api, internal_api) self.modules.append(module) From 00928baacec9473eb7edd01b6010602aaceece41 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Wed, 27 Nov 2019 17:00:32 +0100 Subject: [PATCH 076/187] debug --- threadpoolctl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/threadpoolctl.py b/threadpoolctl.py index 070eb8ea..73cb3eff 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -509,9 +509,9 @@ def _make_module_from_path(self, filepath): user_api = candidate_module["user_api"] internal_api = candidate_module["internal_api"] if prefix in self.prefixes or user_api in self.user_api: - print(globals()) module_class = globals()[module_class] module = module_class(filepath, prefix, user_api, internal_api) + print(module) self.modules.append(module) def _check_prefix(self, library_basename, filename_prefixes): From 783fd0ed184a1deb17c9765997505374d6be7d6e Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Wed, 27 Nov 2019 17:39:46 +0100 Subject: [PATCH 077/187] debug finished --- threadpoolctl.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/threadpoolctl.py b/threadpoolctl.py index 73cb3eff..d0749641 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -67,7 +67,7 @@ class _dl_phdr_info(ctypes.Structure): "internal_api": "openmp", "filename_prefixes": ("libiomp", "libgomp", "libomp", "vcomp") }, - " _OpenBLASModule": { + "_OpenBLASModule": { "user_api": "blas", "internal_api": "openblas", "filename_prefixes": ("libopenblas",) @@ -511,7 +511,6 @@ def _make_module_from_path(self, filepath): if prefix in self.prefixes or user_api in self.user_api: module_class = globals()[module_class] module = module_class(filepath, prefix, user_api, internal_api) - print(module) self.modules.append(module) def _check_prefix(self, library_basename, filename_prefixes): From fe0759a9843991638d2f099ab2c523b13d89e393 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Wed, 27 Nov 2019 17:46:52 +0100 Subject: [PATCH 078/187] tst win blas --- .azure_pipeline.yml | 2 ++ continuous_integration/install.cmd | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml index c84fbb16..d312b8e7 100644 --- a/.azure_pipeline.yml +++ b/.azure_pipeline.yml @@ -18,9 +18,11 @@ stages: pylatest_conda: VERSION_PYTHON: '*' PACKAGER: 'conda' + BLAS: 'mkl' py36_conda: VERSION_PYTHON: '3.6' PACKAGER: 'conda' + BLAS: 'mkl' py35_pip: VERSION_PYTHON: '3.5' PACKAGER: 'pip' diff --git a/continuous_integration/install.cmd b/continuous_integration/install.cmd index 4a0d5970..f0642605 100644 --- a/continuous_integration/install.cmd +++ b/continuous_integration/install.cmd @@ -18,7 +18,7 @@ python --version pip --version @rem Install dependencies with either conda or pip. -if "%PACKAGER%" == "conda" (%CONDA_INSTALL% numpy scipy pytest cython) +if "%PACKAGER%" == "conda" (%CONDA_INSTALL% numpy scipy pytest cython blas=*=%BLAS%) if "%PACKAGER%" == "pip" (%PIP_INSTALL% numpy scipy pytest cython) @rem Install extra developer dependencies From 9b80f66cb68873a26b62632419e2438544719216 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Wed, 27 Nov 2019 18:19:44 +0100 Subject: [PATCH 079/187] tst --- continuous_integration/install.cmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/continuous_integration/install.cmd b/continuous_integration/install.cmd index f0642605..51e005e4 100644 --- a/continuous_integration/install.cmd +++ b/continuous_integration/install.cmd @@ -18,7 +18,7 @@ python --version pip --version @rem Install dependencies with either conda or pip. -if "%PACKAGER%" == "conda" (%CONDA_INSTALL% numpy scipy pytest cython blas=*=%BLAS%) +if "%PACKAGER%" == "conda" (%CONDA_INSTALL% numpy scipy pytest cython blas[build=%BLAS%]) if "%PACKAGER%" == "pip" (%PIP_INSTALL% numpy scipy pytest cython) @rem Install extra developer dependencies From 11aa71266a01b53983fc5dd3069862e218b9ce1f Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Thu, 28 Nov 2019 10:17:34 +0100 Subject: [PATCH 080/187] rev tst --- .azure_pipeline.yml | 2 -- continuous_integration/install.cmd | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml index d312b8e7..c84fbb16 100644 --- a/.azure_pipeline.yml +++ b/.azure_pipeline.yml @@ -18,11 +18,9 @@ stages: pylatest_conda: VERSION_PYTHON: '*' PACKAGER: 'conda' - BLAS: 'mkl' py36_conda: VERSION_PYTHON: '3.6' PACKAGER: 'conda' - BLAS: 'mkl' py35_pip: VERSION_PYTHON: '3.5' PACKAGER: 'pip' diff --git a/continuous_integration/install.cmd b/continuous_integration/install.cmd index 51e005e4..4a0d5970 100644 --- a/continuous_integration/install.cmd +++ b/continuous_integration/install.cmd @@ -18,7 +18,7 @@ python --version pip --version @rem Install dependencies with either conda or pip. -if "%PACKAGER%" == "conda" (%CONDA_INSTALL% numpy scipy pytest cython blas[build=%BLAS%]) +if "%PACKAGER%" == "conda" (%CONDA_INSTALL% numpy scipy pytest cython) if "%PACKAGER%" == "pip" (%PIP_INSTALL% numpy scipy pytest cython) @rem Install extra developer dependencies From ad4ef3b858c7e6e24933cc5625fde1b0266478f5 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Thu, 28 Nov 2019 10:35:27 +0100 Subject: [PATCH 081/187] missing docstring format --- threadpoolctl.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/threadpoolctl.py b/threadpoolctl.py index d0749641..aec23726 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -558,6 +558,9 @@ def _realpath(cls, filepath, cache_limit=10000): return rpath +@_format_docstring( + USER_APIS=", ".join('"{}"'.format(api) for api in _ALL_USER_APIS), + INTERNAL_APIS=", ".join('"{}"'.format(api) for api in _ALL_INTERNAL_APIS)) class _Module(ABC): """Abstract base class for the modules From 41db9d16eb6238a91fe884af40d17e0b58c46c29 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Wed, 4 Dec 2019 11:15:24 +0100 Subject: [PATCH 082/187] turn warnings into errors + some clean up --- continuous_integration/test_script.cmd | 2 ++ continuous_integration/test_script.sh | 2 +- tests/test_threadpoolctl.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/continuous_integration/test_script.cmd b/continuous_integration/test_script.cmd index 680f377c..70274f61 100644 --- a/continuous_integration/test_script.cmd +++ b/continuous_integration/test_script.cmd @@ -1,3 +1,5 @@ call activate %VIRTUALENV% +python continuous_integration/display_versions.py + pytest -vlrxXs --junitxml=%JUNITXML% --cov=threadpoolctl diff --git a/continuous_integration/test_script.sh b/continuous_integration/test_script.sh index 2d48fff4..3286fac2 100755 --- a/continuous_integration/test_script.sh +++ b/continuous_integration/test_script.sh @@ -14,4 +14,4 @@ fi set -x PYTHONPATH="." python continuous_integration/display_versions.py -pytest -vlrxXs --junitxml=$JUNITXML --cov=threadpoolctl +pytest -vlrxXs -W error --junitxml=$JUNITXML --cov=threadpoolctl diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index d0f819f1..22e01542 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -302,6 +302,7 @@ def test_nested_prange_blas(nthreads_outer): assert original_info == _threadpool_info() +@pytest.mark.filterwarnings("ignore::UserWarning") @pytest.mark.parametrize("limit", [1, None]) def test_get_original_num_threads(limit): # Tests the method get_original_num_threads of the context manager @@ -313,7 +314,6 @@ def test_get_original_num_threads(limit): original_info = _threadpool_info() with threadpool_limits(limits=limit, user_api="blas") as threadpoolctx: original_num_threads = threadpoolctx.get_original_num_threads() - print(original_num_threads) assert "openmp" not in original_num_threads From febc707b5a7466ba045040d76d46de7fa8865319 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Thu, 5 Dec 2019 12:53:55 +0100 Subject: [PATCH 083/187] ccomment on the warning filter --- tests/test_threadpoolctl.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index 22e01542..b4ab0949 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -302,6 +302,8 @@ def test_nested_prange_blas(nthreads_outer): assert original_info == _threadpool_info() +# the method `get_original_num_threads` raises a UserWarning in one CI job. It +# is expected so we can safely filter it. @pytest.mark.filterwarnings("ignore::UserWarning") @pytest.mark.parametrize("limit", [1, None]) def test_get_original_num_threads(limit): From a0e172b7f6da7d4655cdfabcbaaced6c38ec6858 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Thu, 5 Dec 2019 13:12:04 +0100 Subject: [PATCH 084/187] more precise comment --- tests/test_threadpoolctl.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index b4ab0949..75a37646 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -302,8 +302,10 @@ def test_nested_prange_blas(nthreads_outer): assert original_info == _threadpool_info() -# the method `get_original_num_threads` raises a UserWarning in one CI job. It -# is expected so we can safely filter it. +# the method `get_original_num_threads` raises a UserWarning due to different +# num_threads from libraries with the same `user_api`. It will be raised only +# in the CI job with 2 openblas (py37_pip_openblas_gcc_clang). It is expected +# so we can safely filter it. @pytest.mark.filterwarnings("ignore::UserWarning") @pytest.mark.parametrize("limit", [1, None]) def test_get_original_num_threads(limit): From 95ff6d077fd00c5665040d4fd5bf59852030d56f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Thu, 5 Dec 2019 15:16:40 +0100 Subject: [PATCH 085/187] ENH add threading layer for BLIS and OpenBLAS (#60) --- .azure_pipeline.yml | 7 +++++-- CHANGES.md | 8 +++++--- continuous_integration/install_with_blis.sh | 8 ++------ tests/test_threadpoolctl.py | 18 +++++++++++++++++- threadpoolctl.py | 21 +++++++++++++++++++-- 5 files changed, 48 insertions(+), 14 deletions(-) diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml index c84fbb16..98b752d8 100644 --- a/.azure_pipeline.yml +++ b/.azure_pipeline.yml @@ -84,7 +84,7 @@ stages: CC_OUTER_LOOP: 'gcc' CC_INNER_LOOP: 'clang-8' # Linux environment with numpy linked to BLIS - pylatest_blis_gcc_clang: + pylatest_blis_gcc_clang_openmp: PACKAGER: 'conda' VERSION_PYTHON: '*' INSTALL_BLIS: 'true' @@ -92,7 +92,8 @@ stages: CC_OUTER_LOOP: 'gcc' CC_INNER_LOOP: 'gcc' BLIS_CC: 'clang-8' - pylatest_blis_clang_gcc: + BLIS_ENABLE_THREADING: 'openmp' + pylatest_blis_clang_gcc_pthreads: PACKAGER: 'conda' VERSION_PYTHON: '*' INSTALL_BLIS: 'true' @@ -100,6 +101,7 @@ stages: CC_OUTER_LOOP: 'clang-8' CC_INNER_LOOP: 'clang-8' BLIS_CC: 'gcc-8' + BLIS_ENABLE_THREADING: 'pthreads' pylatest_blis_no_threading: PACKAGER: 'conda' VERSION_PYTHON: '*' @@ -108,6 +110,7 @@ stages: CC_OUTER_LOOP: 'gcc' CC_INNER_LOOP: 'gcc' BLIS_CC: 'gcc-8' + BLIS_ENABLE_THREADING: 'no' - template: continuous_integration/posix.yml parameters: diff --git a/CHANGES.md b/CHANGES.md index 35fe4605..ae034d96 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,9 +1,11 @@ -1.2.0 (TBD) +2.0.0 (TBD) =========== -- Expose MKL threading layer in informations displayed by `threadpool_info`. - This information is referenced in the `threading_layer` field. +- Expose MKL, BLIS and OpenBLAS threading layer in informations displayed by + `threadpool_info`. This information is referenced in the `threading_layer` + field. https://github.com/joblib/threadpoolctl/pull/48 + https://github.com/joblib/threadpoolctl/pull/60 1.1.0 (2019-09-12) diff --git a/continuous_integration/install_with_blis.sh b/continuous_integration/install_with_blis.sh index 356452f1..cd88573a 100755 --- a/continuous_integration/install_with_blis.sh +++ b/continuous_integration/install_with_blis.sh @@ -27,12 +27,8 @@ pushd .. mkdir BLIS_install git clone https://github.com/flame/blis.git pushd blis -if [[ "$BLIS_NUM_THREADS" == "1" ]]; then - THREADING='no' -else - THREADING='openmp' -fi -./configure --prefix=$ABS_PATH/BLIS_install --enable-cblas --enable-threading=$THREADING CC=$BLIS_CC auto + +./configure --prefix=$ABS_PATH/BLIS_install --enable-cblas --enable-threading=$BLIS_ENABLE_THREADING CC=$BLIS_CC auto make -j4 make install popd diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index 75a37646..992fa423 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -54,7 +54,7 @@ def test_ThreadpoolInfo_todicts(): assert "version" in module_dict assert "num_threads" in module_dict - if module.internal_api == "mkl": + if module.internal_api in ("mkl", "blis", "openblas"): assert "threading_layer" in module_dict @@ -345,3 +345,19 @@ def test_mkl_threading_layer(): actual_layer = mkl_info.modules[0].threading_layer assert actual_layer == expected_layer.lower() + + +def test_blis_threading_layer(): + # Check that threadpool_info correctly recovers the threading layer used + # by blis + blis_info = _threadpool_info().get_modules("internal_api", "blis") + expected_layer = os.getenv("BLIS_ENABLE_THREADING") + if expected_layer == "no": + expected_layer = "disabled" + + if not (blis_info and expected_layer): + pytest.skip("requires BLIS and the environment variable " + "BLIS_ENABLE_THREADING set") + + actual_layer = blis_info.modules[0].threading_layer + assert actual_layer == expected_layer diff --git a/threadpoolctl.py b/threadpoolctl.py index aec23726..9bee4cbf 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -637,7 +637,16 @@ def set_num_threads(self, num_threads): return set_func(num_threads) def _get_extra_info(self): - pass + self.threading_layer = self.get_threading_layer() + + def get_threading_layer(self): + """Return the threading layer of OpenBLAS""" + threading_layer = self._dynlib.openblas_get_parallel() + if threading_layer == 2: + return "openmp" + elif threading_layer == 1: + return "pthreads" + return "disabled" class _BLISModule(_Module): @@ -662,7 +671,15 @@ def set_num_threads(self, num_threads): return set_func(num_threads) def _get_extra_info(self): - pass + self.threading_layer = self.get_threading_layer() + + def get_threading_layer(self): + """Return the threading layer of BLIS""" + if self._dynlib.bli_info_get_enable_openmp(): + return "openmp" + elif self._dynlib.bli_info_get_enable_pthreads(): + return "pthreads" + return "disabled" class _MKLModule(_Module): From 59e45f605d270f21ac2538b6dee6a43b815a4318 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Thu, 5 Dec 2019 17:48:01 +0100 Subject: [PATCH 086/187] [MRG] Warn when llvm + intel loaded at the same time (#49) --- .azure_pipeline.yml | 17 ++++++--- CHANGES.md | 4 ++ continuous_integration/test_script.sh | 2 +- multiple_openmp.md | 53 +++++++++++++++++++++++++++ tests/test_threadpoolctl.py | 27 ++++++++++++++ threadpoolctl.py | 22 +++++++++++ 6 files changed, 119 insertions(+), 6 deletions(-) create mode 100644 multiple_openmp.md diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml index 98b752d8..4dd26299 100644 --- a/.azure_pipeline.yml +++ b/.azure_pipeline.yml @@ -53,17 +53,25 @@ stages: BLAS: 'openblas' CC_OUTER_LOOP: 'clang-8' CC_INNER_LOOP: 'clang-8' - # Linux environment to test the latest available dependencies and MKL. - # and heterogeneous OpenMP runtime nesting. + # Linux environment with MKL and Clang (known to be unsafe for + # threadpoolctl) to only test the warning from multiple OpenMP. pylatest_conda_mkl_clang_gcc: PACKAGER: 'conda' VERSION_PYTHON: '*' BLAS: 'mkl' CC_OUTER_LOOP: 'clang-8' CC_INNER_LOOP: 'gcc' + TESTS: 'libomp_libiomp_warning' + # Linux environment with MKL, safe for threadpoolctl. + pylatest_conda_mkl_gcc_gcc: + PACKAGER: 'conda' + VERSION_PYTHON: '*' + BLAS: 'mkl' + CC_OUTER_LOOP: 'gcc' + CC_INNER_LOOP: 'gcc' MKL_THREADING_LAYER: 'INTEL' # Linux + Python 3.7 with numpy / scipy installed with pip from PyPI - # and heterogeneous openmp runtimes. + # and heterogeneous OpenMP runtimes. py37_pip_openblas_gcc_clang: PACKAGER: 'pip' VERSION_PYTHON: '3.7' @@ -75,8 +83,7 @@ stages: VERSION_PYTHON: '*' CC_OUTER_LOOP: 'gcc' CC_INNER_LOOP: 'gcc' - # Linux environment with no numpy and heterogeneous OpenMP runtime - # nesting. + # Linux environment with no numpy and heterogeneous OpenMP runtimes. pylatest_conda_nonumpy_gcc_clang: PACKAGER: 'conda' NO_NUMPY: 'true' diff --git a/CHANGES.md b/CHANGES.md index ae034d96..985be483 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,10 @@ https://github.com/joblib/threadpoolctl/pull/48 https://github.com/joblib/threadpoolctl/pull/60 +- When threadpoolctl finds libomp (LLVM OpenMP) and libiomp (Intel OpenMP) + both loaded, a warning is raised to recall that using threadpoolctl with + this mix of OpenMP libraries may cause crashes or deadlocks. + https://github.com/joblib/threadpoolctl/pull/49 1.1.0 (2019-09-12) ================== diff --git a/continuous_integration/test_script.sh b/continuous_integration/test_script.sh index 3286fac2..e08a750f 100755 --- a/continuous_integration/test_script.sh +++ b/continuous_integration/test_script.sh @@ -14,4 +14,4 @@ fi set -x PYTHONPATH="." python continuous_integration/display_versions.py -pytest -vlrxXs -W error --junitxml=$JUNITXML --cov=threadpoolctl +pytest -vlrxXs -W error -k "$TESTS" --junitxml=$JUNITXML --cov=threadpoolctl diff --git a/multiple_openmp.md b/multiple_openmp.md new file mode 100644 index 00000000..87bc3395 --- /dev/null +++ b/multiple_openmp.md @@ -0,0 +1,53 @@ +# Multiple OpenMP runtimes + +OpenMP is an API specification for parallel programming. There are many +implementations of it, tied to a compiler most of the time: the libgomp library +for GCC, libomp for LLVM/Clang, libiomp for ICC and vcomp for MSVC for example. +In the following, we refer to OpenMP without distinction between the +specification and an implementation. + +In general, it's not advised to have different OpenMP libraries (or even +different copies of the same library) loaded at the same time in a program. +It's considered an undefined behavior. Fortunately it's not as bad as it sounds +in most situations. + +However it can easily happen in the Python ecosystem since you can install +packages compiled with different compilers (hence linked to different OpenMP +implementations) and import them in a same Python program. + +A typical example is installing numpy from Anaconda which ships MKL +(Intel's math library) and a package using multi-threading with OpenMP +(Scikit-learn, LightGBM, XGBoost, ...). + +From our experience, most OpenMP libraries can seamlessly coexist in a same +program. For instance, on Linux, we never observed any issue between libgomp +and libiomp, which is the most common mix (numpy with MKL + a package compiled +with GCC, the most widely used C compiler). + +The only unrecoverable incompatibility we encountered is between libomp +(LLVM/Clang) and libiomp (ICC), on Linux, manifested by crashes or deadlocks. +It can happen even with the simplest OpenMP calls like getting the maximum +number of threads that will be used in a subsequent parallel region. A possible +explanation is that libomp is actually a fork of libiomp causing name colliding +for instance. Using threadpoolctl may crash your program in such a setting. + +Surprisingly, we never encountered this kind of issue on macOS, where this mix +is the most frequent (Clang being the default C compiler on macOS). + +As far as we know, the only workaround consists in getting rid of one of the +OpenMP libraries. For example: + +- Build your OpenMP-enabled extensions with GCC (or ICC) instead of Clang. + +- Install a build of Numpy linked against OpenBLAS instead of MKL. This can be + done for instance by installing Numpy from PyPI:: + + pip install numpy + + from the conda-forge conda channel:: + + conda install -c conda-forge numpy + + or from the default conda channel:: + + conda install numpy blas[build=openblas] diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index 992fa423..01076e51 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -1,5 +1,6 @@ import os import re +import sys import pytest from threadpoolctl import threadpool_limits, threadpool_info, _ThreadpoolInfo @@ -361,3 +362,29 @@ def test_blis_threading_layer(): actual_layer = blis_info.modules[0].threading_layer assert actual_layer == expected_layer + + +@pytest.mark.skipif(not cython_extensions_compiled, + reason='Requires cython extensions to be compiled') +def test_libomp_libiomp_warning(recwarn): + # Trigger the import of a potentially clang-compiled extension: + from ._openmp_test_helper import check_nested_openmp_loops # noqa + + # Trigger the import of numpy to potentially import Intel OpenMP via MKL + import numpy.linalg # noqa + + # Check that a warning is raised when both libomp and libiomp are loaded + # It should happen in one CI job (pylatest_conda_mkl_clang_gcc). + info = _threadpool_info() + prefixes = [module.prefix for module in info] + + if not ("libomp" in prefixes and "libiomp" in prefixes + and sys.platform == "linux"): + pytest.skip("Requires both libomp and libiomp loaded, on Linux") + + assert len(recwarn) == 1 + wm = recwarn[0] + assert wm.category == RuntimeWarning + assert "Found Intel" in str(wm.message) + assert "LLVM" in str(wm.message) + assert "multiple_openmp.md" in str(wm.message) diff --git a/threadpoolctl.py b/threadpoolctl.py index 9bee4cbf..d7b80d74 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -14,6 +14,7 @@ import re import sys import ctypes +import textwrap import warnings from ctypes.util import find_library from abc import ABC, abstractmethod @@ -337,6 +338,7 @@ def __init__(self, user_api=None, prefixes=None, modules=None): self.modules = [] self._load_modules() + self._warn_if_incompatible_openmp() else: self.modules = modules @@ -523,6 +525,26 @@ def _check_prefix(self, library_basename, filename_prefixes): return prefix return None + def _warn_if_incompatible_openmp(self): + """Raise a warning if llvm-OpenMP and intel-OpenMP are both loaded""" + if sys.platform != 'linux': + # Only raise the warning on linux + return + + prefixes = [module.prefix for module in self.modules] + msg = textwrap.dedent( + """ + Found Intel OpenMP ('libiomp') and LLVM OpenMP ('libomp') loaded at + the same time. Both libraries are known to be incompatible and this + can cause random crashes or deadlocks on Linux when loaded in the + same Python program. + Using threadpoolctl may cause crashes or deadlocks. For more + information and possible workarounds, please see + https://github.com/joblib/threadpoolctl/blob/master/multiple_openmp.md + """) + if 'libomp' in prefixes and 'libiomp' in prefixes: + warnings.warn(msg, RuntimeWarning) + @classmethod def _get_libc(cls): """Load the lib-C for unix systems.""" From c7b958d9edd6a0c4e60280364588fe67b9d9701c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Thu, 5 Dec 2019 18:32:07 +0100 Subject: [PATCH 087/187] bump version (#62) --- CHANGES.md | 2 +- threadpoolctl.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 985be483..6efec3a6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,4 @@ -2.0.0 (TBD) +2.0.0 (2019-12-05) =========== - Expose MKL, BLIS and OpenBLAS threading layer in informations displayed by diff --git a/threadpoolctl.py b/threadpoolctl.py index d7b80d74..8f3c756b 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -19,7 +19,7 @@ from ctypes.util import find_library from abc import ABC, abstractmethod -__version__ = "2.0.0.dev0" +__version__ = "2.0.0" __all__ = ["threadpool_limits", "threadpool_info"] From 61bc9433882aafa0abd34652cd6d6c46da878942 Mon Sep 17 00:00:00 2001 From: jeremie du boisberranger Date: Thu, 5 Dec 2019 18:39:47 +0100 Subject: [PATCH 088/187] bump version --- threadpoolctl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/threadpoolctl.py b/threadpoolctl.py index 8f3c756b..3efbaa57 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -19,7 +19,7 @@ from ctypes.util import find_library from abc import ABC, abstractmethod -__version__ = "2.0.0" +__version__ = "2.1.0.dev0" __all__ = ["threadpool_limits", "threadpool_info"] From 90ac5ff7b8892e220628ff78c322d9d02523503d Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Fri, 6 Dec 2019 11:04:11 +0100 Subject: [PATCH 089/187] DOC improve the multiple OpenMP runtimes doc (#63) --- multiple_openmp.md | 112 +++++++++++++++++++++++++++++---------------- 1 file changed, 72 insertions(+), 40 deletions(-) diff --git a/multiple_openmp.md b/multiple_openmp.md index 87bc3395..07d13aa3 100644 --- a/multiple_openmp.md +++ b/multiple_openmp.md @@ -1,53 +1,85 @@ -# Multiple OpenMP runtimes +# Multiple OpenMP Runtimes + +## Context OpenMP is an API specification for parallel programming. There are many -implementations of it, tied to a compiler most of the time: the libgomp library -for GCC, libomp for LLVM/Clang, libiomp for ICC and vcomp for MSVC for example. -In the following, we refer to OpenMP without distinction between the -specification and an implementation. - -In general, it's not advised to have different OpenMP libraries (or even -different copies of the same library) loaded at the same time in a program. -It's considered an undefined behavior. Fortunately it's not as bad as it sounds -in most situations. - -However it can easily happen in the Python ecosystem since you can install -packages compiled with different compilers (hence linked to different OpenMP -implementations) and import them in a same Python program. - -A typical example is installing numpy from Anaconda which ships MKL -(Intel's math library) and a package using multi-threading with OpenMP -(Scikit-learn, LightGBM, XGBoost, ...). - -From our experience, most OpenMP libraries can seamlessly coexist in a same -program. For instance, on Linux, we never observed any issue between libgomp -and libiomp, which is the most common mix (numpy with MKL + a package compiled -with GCC, the most widely used C compiler). - -The only unrecoverable incompatibility we encountered is between libomp -(LLVM/Clang) and libiomp (ICC), on Linux, manifested by crashes or deadlocks. -It can happen even with the simplest OpenMP calls like getting the maximum -number of threads that will be used in a subsequent parallel region. A possible -explanation is that libomp is actually a fork of libiomp causing name colliding -for instance. Using threadpoolctl may crash your program in such a setting. +implementations of it, tied to a compiler most of the time: + +- `libgomp` for GCC (GNU C/C++ Compiler), +- `libomp` for Clang (LLVM C/C++ Compiler), +- `libiomp` for ICC (Intel C/C++ Compiler), +- `vcomp` for MSVC (Microsoft Visual Studio C/C++ Compiler). + +In general, it is not advised to have different OpenMP runtime libraries (or +even different copies of the same library) loaded at the same time in a +program. It's considered an undefined behavior. Fortunately it is not as bad as +it sounds in most situations. + +However this situation is frequent in the Python ecosystem since you can +install packages compiled with different compilers (hence linked to different +OpenMP implementations) and import them together in a Python program. + +A typical example is installing NumPy from Anaconda which is linked against MKL +(Intel's math library) and another package that uses multi-threading with OpenMP +directly in a compiled extension, as is the case in Scikit-learn (via Cython +`prange`), LightGBM and XGBoost (via pragmas in the C++ source code). + +From our experience, **most OpenMP libraries can seamlessly coexist in a same +program**. For instance, on Linux, we never observed any issue between +`libgomp` and `libiomp`, which is the most common mix (NumPy with MKL + a +package compiled with GCC, the most widely used C compiler on that platform). + +## Incompatibility between Intel OpenMP and LLVM OpenMP under Linux + +The only unrecoverable incompatibility we encountered happens when loading a +mix of compiled extensions linked with **`libomp` (LLVM/Clang) and `libiomp` +(ICC), on Linux**, manifested by crashes or deadlocks. It can happen even with +the simplest OpenMP calls like getting the maximum number of threads that will +be used in a subsequent parallel region. A possible explanation is that +`libomp` is actually a fork of `libiomp` causing name colliding for instance. +Using `threadpoolctl` may crash your program in such a setting. + +**Fortunately this problem is very rare**: at the time of writing, all major +binary distributions of Python packages for Linux use either GCC or ICC to +build the Python scientific packages. Therefore this problem would only happen +if some packagers decide to start shipping Python packages built with +LLVM/Clang instead of GCC. Surprisingly, we never encountered this kind of issue on macOS, where this mix is the most frequent (Clang being the default C compiler on macOS). -As far as we know, the only workaround consists in getting rid of one of the -OpenMP libraries. For example: +## Workarounds for Intel OpenMP and LLVM OpenMP case + +As far as we know, the only workaround consists in making sure only of one of +the two incompatible OpenMP libraries is loaded. For example: + +- Tell MKL (used by NumPy) to use the GNU OpenMP runtime instead of the Intel + OpenMP runtime by setting the following environment variable: + + export MKL_THREADING_LAYER=GNU + +- Install a build of NumPy and SciPy linked against OpenBLAS instead of MKL. + This can be done for instance by installing NumPy and SciPy from PyPI: + + pip install numpy scipy + + from the conda-forge conda channel: -- Build your OpenMP-enabled extensions with GCC (or ICC) instead of Clang. + conda install -c conda-forge numpy scipy -- Install a build of Numpy linked against OpenBLAS instead of MKL. This can be - done for instance by installing Numpy from PyPI:: + or from the default conda channel: - pip install numpy + conda install numpy scipy blas[build=openblas] - from the conda-forge conda channel:: +- Re-build your OpenMP-enabled extensions from source with GCC (or ICC) instead + of Clang if you want to keep on using NumPy/SciPy linked against MKL with the + default `libiomp`-based threading layer. - conda install -c conda-forge numpy +## References - or from the default conda channel:: +The above incompatibility has been reported upstream to the LLVM and Intel +developers on the following public issue trackers/forums along with a minimal +reproducer written in C: - conda install numpy blas[build=openblas] +- https://bugs.llvm.org/show_bug.cgi?id=43565 +- https://software.intel.com/en-us/forums/intel-c-compiler/topic/827607 From 210925279a8048932e95214b20acc37f7e0b0db0 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 15 Jan 2020 15:59:08 +0100 Subject: [PATCH 090/187] [MRG] Simple Command Line Interface (#65) --- CHANGES.md | 14 ++++++- README.md | 80 ++++++++++++++++++++++++++++--------- tests/test_threadpoolctl.py | 46 ++++++++++++++++++++- threadpoolctl.py | 37 +++++++++++++++++ 4 files changed, 157 insertions(+), 20 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6efec3a6..90bc6e21 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,17 @@ +2.1.0 (Under development) +========================= + +- New commandline interface: + + python -m threadpoolctl -i numpy + + will try to import the `numpy` package and then return the output of + `threadpoolctl.threadpool_info()` on STDOUT formatted using the JSON + syntax. This makes it easier to quickly introspect a Python environment. + + 2.0.0 (2019-12-05) -=========== +================== - Expose MKL, BLIS and OpenBLAS threading layer in informations displayed by `threadpool_info`. This information is referenced in the `threading_layer` diff --git a/README.md b/README.md index dcef5638..e1339567 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,38 @@ oversubscription issues. ## Usage -### Runtime Introspection +### Command Line Interface + +Get a JSON description of thread-pools initialized when importing python +packages such as numpy or scipy for instance: + +``` +python -m threadpoolctl -i numpy scipy.linalg +[ + { + "filepath": "/home/ogrisel/miniconda3/envs/tmp/lib/libmkl_rt.so", + "prefix": "libmkl_rt", + "user_api": "blas", + "internal_api": "mkl", + "version": "2019.0.4", + "num_threads": 2, + "threading_layer": "intel" + }, + { + "filepath": "/home/ogrisel/miniconda3/envs/tmp/lib/libiomp5.so", + "prefix": "libiomp", + "user_api": "openmp", + "internal_api": "openmp", + "version": null, + "num_threads": 4 + } +] +``` + +The JSON information is written on STDOUT. If some of the packages are missing, +a warning message is displayed on STDERR. + +### Python Runtime Programmatic Introspection Introspect the current state of the threadpool-enabled runtime libraries that are loaded when importing Python packages: @@ -45,28 +76,36 @@ that are loaded when importing Python packages: >>> import numpy >>> pprint(threadpool_info()) -[{'filepath': '/opt/venvs/py37/lib/python3.7/site-packages/numpy/.libs/libopenblasp-r0-382c8f3a.3.5.dev.so', - 'internal_api': 'openblas', - 'num_threads': 4, - 'prefix': 'libopenblas', +[{'filepath': '/home/ogrisel/miniconda3/envs/tmp/lib/libmkl_rt.so', + 'internal_api': 'mkl', + 'num_threads': 2, + 'prefix': 'libmkl_rt', + 'threading_layer': 'intel', 'user_api': 'blas', - 'version': '0.3.5.dev'}] + 'version': '2019.0.4'}, + {'filepath': '/home/ogrisel/miniconda3/envs/tmp/lib/libiomp5.so', + 'internal_api': 'openmp', + 'num_threads': 4, + 'prefix': 'libiomp', + 'user_api': 'openmp', + 'version': None}] >>> import xgboost >>> pprint(threadpool_info()) -[{'filepath': '/opt/venvs/py37/lib/python3.7/site-packages/numpy/.libs/libopenblasp-r0-382c8f3a.3.5.dev.so', - 'internal_api': 'openblas', - 'num_threads': 4, - 'prefix': 'libopenblas', +[{'filepath': '/home/ogrisel/miniconda3/envs/tmp/lib/libmkl_rt.so', + 'internal_api': 'mkl', + 'num_threads': 2, + 'prefix': 'libmkl_rt', + 'threading_layer': 'intel', 'user_api': 'blas', - 'version': '0.3.5.dev'}, - {'filepath': '/opt/venvs/py37/lib/python3.7/site-packages/scipy/.libs/libopenblasp-r0-8dca6697.3.0.dev.so', - 'internal_api': 'openblas', + 'version': '2019.0.4'}, + {'filepath': '/home/ogrisel/miniconda3/envs/tmp/lib/libiomp5.so', + 'internal_api': 'openmp', 'num_threads': 4, - 'prefix': 'libopenblas', - 'user_api': 'blas', + 'prefix': 'libiomp', + 'user_api': 'openmp', 'version': None}, - {'filepath': '/usr/lib/x86_64-linux-gnu/libgomp.so.1', + {'filepath': '/home/ogrisel/miniconda3/envs/tmp/lib/libgomp.so.1.0.0', 'internal_api': 'openmp', 'num_threads': 4, 'prefix': 'libgomp', @@ -74,7 +113,12 @@ that are loaded when importing Python packages: 'version': None}] ``` -### Set the maximum size of thread-pools +In the above example, `numpy` was installed from the default anaconda channel and +comes with the MKL and its Intel OpenMP (`libiomp5`) implementation while +`xgboost` was installed from pypi.org and links against GNU OpenMP (`libgomp`) +so both OpenMP runtimes are loaded in the same Python program. + +### Setting the Maximum Size of Thread-Pools Control the number of threads used by the underlying runtime libraries in specific sections of your Python program: @@ -92,7 +136,7 @@ with threadpool_limits(limits=1, user_api='blas'): a_squared = a @ a ``` -### Known limitation +### Known Limitations `threadpool_limits` does not act as expected in nested parallel loops managed by distinct OpenMP runtime implementations (for instance libgomp diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index 01076e51..dd6c9d80 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -1,7 +1,9 @@ +import json import os +import pytest import re +import subprocess import sys -import pytest from threadpoolctl import threadpool_limits, threadpool_info, _ThreadpoolInfo from threadpoolctl import _ALL_PREFIXES, _ALL_USER_APIS @@ -388,3 +390,45 @@ def test_libomp_libiomp_warning(recwarn): assert "Found Intel" in str(wm.message) assert "LLVM" in str(wm.message) assert "multiple_openmp.md" in str(wm.message) + + +def test_command_line_empty(): + output = subprocess.check_output( + "python -m threadpoolctl".split()) + assert json.loads(output.decode("utf-8")) == [] + + +def test_command_line_command_flag(): + pytest.importorskip("numpy") + output = subprocess.check_output( + ["python", "-m", "threadpoolctl", "-c", "import numpy"]) + cli_info = json.loads(output.decode("utf-8")) + + this_process_info = threadpool_info() + for module in cli_info: + assert module in this_process_info + + +@pytest.mark.skipif(sys.version_info < (3, 7), + reason="need recent subprocess.run options") +def test_command_line_import_flag(): + result = subprocess.run([ + "python", "-m", "threadpoolctl", "-i", + "numpy", + "scipy.linalg", + "invalid_package", + "numpy.invalid_sumodule", + ], capture_output=True, check=True, encoding="utf-8") + cli_info = json.loads(result.stdout) + + this_process_info = threadpool_info() + for module in cli_info: + assert module in this_process_info + + warnings = [w.strip() for w in result.stderr.splitlines()] + assert "WARNING: could not import invalid_package" in warnings + assert "WARNING: could not import numpy.invalid_sumodule" in warnings + if scipy is None: + assert "WARNING: could not import scipy.linalg" in warnings + else: + assert "WARNING: could not import scipy.linalg" not in warnings diff --git a/threadpoolctl.py b/threadpoolctl.py index 3efbaa57..2e8f344f 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -757,3 +757,40 @@ def set_num_threads(self, num_threads): def _get_extra_info(self): pass + + +def _main(): + """Commandline interface to display thread-pool information and exit.""" + import argparse + import importlib + import json + import sys + + parser = argparse.ArgumentParser( + usage="python -m threadpoolctl -i numpy scipy.linalg xgboost", + description="Display thread-pool information and exit.", + ) + parser.add_argument( + "-i", "--import", dest="modules", nargs="*", default=(), + help="Python modules to import before introspecting thread-pools." + ) + parser.add_argument( + "-c", "--command", + help="a Python statement to execute before introspecting" + " thread-pools.") + + options = parser.parse_args(sys.argv[1:]) + for module in options.modules: + try: + importlib.import_module(module, package=None) + except ImportError: + print("WARNING: could not import", module, file=sys.stderr) + + if options.command: + exec(options.command) + + print(json.dumps(threadpool_info(), indent=2)) + + +if __name__ == "__main__": + _main() From 9d83dfc23500eb089e6ddd3acab38b183085f798 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Thu, 16 Jan 2020 15:05:02 +0100 Subject: [PATCH 091/187] Use the CLI instead of continuous_integration/display_versions.py (#66) --- README.md | 24 +++++++++++++++++----- continuous_integration/display_versions.py | 23 --------------------- continuous_integration/test_script.cmd | 4 +++- continuous_integration/test_script.sh | 5 ++++- tests/_openmp_test_helper/__init__.py | 7 ++++++- tests/test_threadpoolctl.py | 4 +++- 6 files changed, 35 insertions(+), 32 deletions(-) delete mode 100644 continuous_integration/display_versions.py diff --git a/README.md b/README.md index e1339567..ddb20fba 100644 --- a/README.md +++ b/README.md @@ -138,12 +138,26 @@ with threadpool_limits(limits=1, user_api='blas'): ### Known Limitations -`threadpool_limits` does not act as expected in nested parallel loops -managed by distinct OpenMP runtime implementations (for instance libgomp -from GCC and libomp from clang/llvm or libiomp from ICC). +- `threadpool_limits` can fail to limit the number of inner threads when nesting + parallel loops managed by distinct OpenMP runtime implementations (for instance + libgomp from GCC and libomp from clang/llvm or libiomp from ICC). + + See the `test_openmp_nesting` function in [tests/test_threadpoolctl.py]( + https://github.com/joblib/threadpoolctl/blob/master/tests/test_threadpoolctl.py) + for an example. More information can be found at: + https://github.com/jeremiedbb/Nested_OpenMP + + Note however that this problem does not happen when `threadpool_limits` is + used to limit the number of threads used internally by BLAS calls that are + themselves nested under OpenMP parallel loops. `threadpool_limits` works as + expected, even if the inner BLAS implementation relies on a distinct OpenMP + implementation. + +- Using Intel OpenMP (ICC) and LLVM OpenMP (clang) in the same Python program + under Linux is known to cause problems. See the following guide for more details + and workarounds: + https://github.com/joblib/threadpoolctl/blob/master/multiple_openmp.md -See the `test_openmp_nesting()` function in `tests/test_threadpoolctl.py` -for an example. ## Maintainers diff --git a/continuous_integration/display_versions.py b/continuous_integration/display_versions.py deleted file mode 100644 index aae3ebb7..00000000 --- a/continuous_integration/display_versions.py +++ /dev/null @@ -1,23 +0,0 @@ -from threadpoolctl import threadpool_info -from pprint import pprint - -try: - import numpy as np - print("numpy", np.__version__) -except ImportError: - pass - -try: - import scipy - import scipy.linalg - print("scipy", scipy.__version__) -except ImportError: - pass - -try: - from tests._openmp_test_helper import * # noqa -except ImportError: - pass - - -pprint(threadpool_info()) diff --git a/continuous_integration/test_script.cmd b/continuous_integration/test_script.cmd index 70274f61..e6952f5e 100644 --- a/continuous_integration/test_script.cmd +++ b/continuous_integration/test_script.cmd @@ -1,5 +1,7 @@ call activate %VIRTUALENV% -python continuous_integration/display_versions.py +# Use the CLI to display the effective runtime environment prior to +# launching the tests: +python -m threadpoolctl -i numpy scipy.linalg tests._openmp_test_helper pytest -vlrxXs --junitxml=%JUNITXML% --cov=threadpoolctl diff --git a/continuous_integration/test_script.sh b/continuous_integration/test_script.sh index e08a750f..b59023f6 100755 --- a/continuous_integration/test_script.sh +++ b/continuous_integration/test_script.sh @@ -12,6 +12,9 @@ elif [[ "$PACKAGER" == "ubuntu" ]]; then fi set -x -PYTHONPATH="." python continuous_integration/display_versions.py + +# Use the CLI to display the effective runtime environment prior to +# launching the tests: +python -m threadpoolctl -i numpy scipy.linalg tests._openmp_test_helper pytest -vlrxXs -W error -k "$TESTS" --junitxml=$JUNITXML --cov=threadpoolctl diff --git a/tests/_openmp_test_helper/__init__.py b/tests/_openmp_test_helper/__init__.py index 192c8e21..88d48ef1 100644 --- a/tests/_openmp_test_helper/__init__.py +++ b/tests/_openmp_test_helper/__init__.py @@ -2,7 +2,12 @@ from .openmp_helpers_inner import get_compiler as get_inner_compiler from .openmp_helpers_outer import check_nested_openmp_loops from .openmp_helpers_outer import get_compiler as get_outer_compiler -from .nested_prange_blas import check_nested_prange_blas + +try: + from .nested_prange_blas import check_nested_prange_blas +except ImportError: + # Can happen if numpy and scipy are missing. + check_nested_prange_blas = None __all__ = ["check_openmp_num_threads", diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index dd6c9d80..1d0068c0 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -263,6 +263,8 @@ def test_multiple_shipped_openblas(): @pytest.mark.skipif(scipy is None, reason="requires scipy") +@pytest.mark.skipif(not cython_extensions_compiled, + reason='Requires cython extensions to be compiled') @pytest.mark.parametrize("nthreads_outer", [None, 1, 2, 4]) def test_nested_prange_blas(nthreads_outer): # Check that the BLAS linked to scipy effectively uses the number of @@ -373,7 +375,7 @@ def test_libomp_libiomp_warning(recwarn): from ._openmp_test_helper import check_nested_openmp_loops # noqa # Trigger the import of numpy to potentially import Intel OpenMP via MKL - import numpy.linalg # noqa + pytest.importorskip("numpy.linalg") # Check that a warning is raised when both libomp and libiomp are loaded # It should happen in one CI job (pylatest_conda_mkl_clang_gcc). From 006bad66af53066a21615389d78bb2c6de7337a5 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Fri, 29 May 2020 11:16:18 +0200 Subject: [PATCH 092/187] Release 2.1.0 --- CHANGES.md | 4 ++-- threadpoolctl.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 90bc6e21..00a81956 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,5 @@ -2.1.0 (Under development) -========================= +2.1.0 (2020-05-29) +================== - New commandline interface: diff --git a/threadpoolctl.py b/threadpoolctl.py index 2e8f344f..b94d7a15 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -19,7 +19,7 @@ from ctypes.util import find_library from abc import ABC, abstractmethod -__version__ = "2.1.0.dev0" +__version__ = "2.1.0" __all__ = ["threadpool_limits", "threadpool_info"] From 701d6913daf944a6e06cb79160f5c74c42001221 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Fri, 29 May 2020 11:21:46 +0200 Subject: [PATCH 093/187] Back to dev mode --- CHANGES.md | 4 ++++ threadpoolctl.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 00a81956..2f71bf64 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,7 @@ +2.2.0 (In development) +====================== + + 2.1.0 (2020-05-29) ================== diff --git a/threadpoolctl.py b/threadpoolctl.py index b94d7a15..9157660b 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -19,7 +19,7 @@ from ctypes.util import find_library from abc import ABC, abstractmethod -__version__ = "2.1.0" +__version__ = "2.2.0.dev0" __all__ = ["threadpool_limits", "threadpool_info"] From a581aebad26853012a1afd061e1f2f4d2bc25314 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Wed, 3 Jun 2020 11:20:50 +0200 Subject: [PATCH 094/187] MNT Update macOS Image on azure pipelines (#72) --- .azure_pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml index 4dd26299..3de506c9 100644 --- a/.azure_pipeline.yml +++ b/.azure_pipeline.yml @@ -122,7 +122,7 @@ stages: - template: continuous_integration/posix.yml parameters: name: macOS - vmImage: xcode9-macos10.13 + vmImage: macOS-10.14 matrix: # MacOS environment with OpenMP installed through homebrew py35_conda_homebrew_libomp: From aa3ea23d52ab8af043a7714da039e1fe53dcff7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Wed, 3 Jun 2020 11:28:38 +0200 Subject: [PATCH 095/187] MNT upgrade pip in CI (#73) --- continuous_integration/posix.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/continuous_integration/posix.yml b/continuous_integration/posix.yml index 40ba20c4..753cf4af 100644 --- a/continuous_integration/posix.yml +++ b/continuous_integration/posix.yml @@ -13,7 +13,8 @@ jobs: steps: - script: | - pip3 install flake8 + python3 -m pip install --upgrade pip + python3 -m pip install flake8 python3 -m flake8 . displayName: Lint condition: eq(variables['LINT'], 'true') From ce2a2d91851447a47aa045f9c372e58e4432adf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Fri, 5 Jun 2020 11:08:12 +0200 Subject: [PATCH 096/187] [MRG] Rework test_openmp_nesting to be conda-forge friendly (#74) --- tests/_openmp_test_helper/__init__.py | 17 ----- .../openmp_helpers_inner.pxd | 2 +- .../openmp_helpers_inner.pyx | 4 -- .../openmp_helpers_outer.pyx | 4 -- tests/test_threadpoolctl.py | 62 ++++++++++--------- tests/utils.py | 27 +++++++- 6 files changed, 61 insertions(+), 55 deletions(-) diff --git a/tests/_openmp_test_helper/__init__.py b/tests/_openmp_test_helper/__init__.py index 88d48ef1..e69de29b 100644 --- a/tests/_openmp_test_helper/__init__.py +++ b/tests/_openmp_test_helper/__init__.py @@ -1,17 +0,0 @@ -from .openmp_helpers_inner import check_openmp_num_threads -from .openmp_helpers_inner import get_compiler as get_inner_compiler -from .openmp_helpers_outer import check_nested_openmp_loops -from .openmp_helpers_outer import get_compiler as get_outer_compiler - -try: - from .nested_prange_blas import check_nested_prange_blas -except ImportError: - # Can happen if numpy and scipy are missing. - check_nested_prange_blas = None - - -__all__ = ["check_openmp_num_threads", - "check_nested_openmp_loops", - "check_nested_prange_blas", - "get_inner_compiler", - "get_outer_compiler"] diff --git a/tests/_openmp_test_helper/openmp_helpers_inner.pxd b/tests/_openmp_test_helper/openmp_helpers_inner.pxd index 50005710..52522ad7 100644 --- a/tests/_openmp_test_helper/openmp_helpers_inner.pxd +++ b/tests/_openmp_test_helper/openmp_helpers_inner.pxd @@ -1 +1 @@ -cdef int inner_openmp_loop(int) nogil \ No newline at end of file +cdef int inner_openmp_loop(int) nogil diff --git a/tests/_openmp_test_helper/openmp_helpers_inner.pyx b/tests/_openmp_test_helper/openmp_helpers_inner.pyx index 9fdffca0..78d65303 100644 --- a/tests/_openmp_test_helper/openmp_helpers_inner.pyx +++ b/tests/_openmp_test_helper/openmp_helpers_inner.pyx @@ -36,7 +36,3 @@ cdef int inner_openmp_loop(int n) nogil: return -1 return num_threads - - -def get_compiler(): - return CC_INNER_LOOP diff --git a/tests/_openmp_test_helper/openmp_helpers_outer.pyx b/tests/_openmp_test_helper/openmp_helpers_outer.pyx index ffed13ef..2c8a383c 100644 --- a/tests/_openmp_test_helper/openmp_helpers_outer.pyx +++ b/tests/_openmp_test_helper/openmp_helpers_outer.pyx @@ -20,7 +20,3 @@ def check_nested_openmp_loops(int n, nthreads=None): outer_num_threads = openmp.omp_get_num_threads() return outer_num_threads, inner_num_threads - - -def get_compiler(): - return CC_OUTER_LOOP diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index 1d0068c0..09fe1a4d 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -11,6 +11,7 @@ from .utils import cython_extensions_compiled from .utils import libopenblas_paths from .utils import scipy +from .utils import threadpool_info_from_subprocess def is_old_openblas(module): @@ -169,7 +170,8 @@ def test_threadpool_limits_bad_input(): def test_openmp_limit_num_threads(num_threads): # checks that OpenMP effectively uses the number of threads requested by # the context manager - from ._openmp_test_helper import check_openmp_num_threads + import tests._openmp_test_helper.openmp_helpers_inner as omp_inner + check_openmp_num_threads = omp_inner.check_openmp_num_threads old_num_threads = check_openmp_num_threads(100) @@ -179,37 +181,40 @@ def test_openmp_limit_num_threads(num_threads): @pytest.mark.skipif(not cython_extensions_compiled, - reason='Requires cython extensions to be compiled') -@pytest.mark.parametrize('nthreads_outer', [None, 1, 2, 4]) + reason="Requires cython extensions to be compiled") +@pytest.mark.parametrize("nthreads_outer", [None, 1, 2, 4]) def test_openmp_nesting(nthreads_outer): # checks that OpenMP effectively uses the number of threads requested by # the context manager when nested in an outer OpenMP loop. - from ._openmp_test_helper import check_nested_openmp_loops - from ._openmp_test_helper import get_inner_compiler - from ._openmp_test_helper import get_outer_compiler - - inner_cc = get_inner_compiler() - outer_cc = get_outer_compiler() + import tests._openmp_test_helper.openmp_helpers_outer as omp_outer + check_nested_openmp_loops = omp_outer.check_nested_openmp_loops + + # Find which OpenMP lib is used at runtime for inner loop + inner_info = threadpool_info_from_subprocess( + "tests._openmp_test_helper.openmp_helpers_inner") + assert len(inner_info) == 1 + inner_omp = inner_info[0]["prefix"] + + # Find which OpenMP lib is used at runtime for outer loop + outer_info = threadpool_info_from_subprocess( + "tests._openmp_test_helper.openmp_helpers_outer") + if len(outer_info) == 1: + # Only 1 openmp loaded. It has to be this one. + outer_omp = outer_info[0]["prefix"] + else: + # There are 2 openmp, the one from inner and the one from outer. + assert len(outer_info) == 2 + # We already know the one from inner. It has to be the other one. + prefixes = {module["prefix"] for module in outer_info} + outer_omp = prefixes - {inner_omp} outer_num_threads, inner_num_threads = check_nested_openmp_loops(10) - original_info = _threadpool_info() - openmp_info = original_info.get_modules("user_api", "openmp") - - if "gcc" in (inner_cc, outer_cc): - assert original_info.get_modules("prefix", "libgomp") - if "clang" in (inner_cc, outer_cc): - assert original_info.get_modules("prefix", "libomp") - - if inner_cc == outer_cc: - # The openmp runtime should be shared by default, meaning that - # the inner loop should automatically be run serially by the OpenMP - # runtime. + if inner_omp == outer_omp: + # The OpenMP runtime should be shared by default, meaning that the + # inner loop should automatically be run serially by the OpenMP runtime assert inner_num_threads == 1 - else: - # There should be at least 2 OpenMP runtime detected. - assert len(openmp_info) >= 2 with threadpool_limits(limits=1) as threadpoolctx: max_threads = threadpoolctx.get_original_num_threads()["openmp"] @@ -229,8 +234,8 @@ def test_openmp_nesting(nthreads_outer): assert outer_num_threads == nthreads # The number of threads available in the inner loop should have been - # set to 1 so avoid oversubscription and preserve performance: - if inner_cc != outer_cc: + # set to 1 to avoid oversubscription and preserve performance: + if inner_omp != outer_omp: if inner_num_threads != 1: # XXX: this does not always work when nesting independent openmp # implementations. See: https://github.com/jeremiedbb/Nested_OpenMP @@ -271,7 +276,8 @@ def test_nested_prange_blas(nthreads_outer): # threads requested by the context manager when nested in an outer OpenMP # loop. import numpy as np - from ._openmp_test_helper import check_nested_prange_blas + import tests._openmp_test_helper.nested_prange_blas as prange_blas + check_nested_prange_blas = prange_blas.check_nested_prange_blas original_info = _threadpool_info() @@ -372,7 +378,7 @@ def test_blis_threading_layer(): reason='Requires cython extensions to be compiled') def test_libomp_libiomp_warning(recwarn): # Trigger the import of a potentially clang-compiled extension: - from ._openmp_test_helper import check_nested_openmp_loops # noqa + import tests._openmp_test_helper.openmp_helpers_outer # noqa # Trigger the import of numpy to potentially import Intel OpenMP via MKL pytest.importorskip("numpy.linalg") diff --git a/tests/utils.py b/tests/utils.py index ce9adfa3..a7efeb02 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,5 +1,10 @@ import os +import json +import sys +import threadpoolctl from glob import glob +from os.path import dirname, normpath +from subprocess import check_output # Path to shipped openblas for libraries such as numpy or scipy @@ -32,7 +37,27 @@ try: - from ._openmp_test_helper import check_openmp_num_threads # noqa: F401 + import tests._openmp_test_helper.openmp_helpers_inner # noqa: F401 cython_extensions_compiled = True except ImportError: cython_extensions_compiled = False + + +def threadpool_info_from_subprocess(module): + """Utility to call threadpool_info in a subprocess + + `module` is imported before calling threadpool_info + """ + # set PYTHONPATH to import from non sub-modules + path1 = normpath(dirname(threadpoolctl.__file__)) + path2 = os.path.join(path1, "tests", "_openmp_test_helper") + pythonpath = os.pathsep.join([path1, path2]) + env = os.environ.copy() + try: + env["PYTHONPATH"] = os.pathsep.join([pythonpath, env["PYTHONPATH"]]) + except KeyError: + env["PYTHONPATH"] = pythonpath + + cmd = [sys.executable, "-m", "threadpoolctl", "-i", module] + out = check_output(cmd, env=env).decode("utf-8") + return json.loads(out) From 7ba7e0984f8c8685ede1bdb1a89f0aca252546e0 Mon Sep 17 00:00:00 2001 From: MeggyCal Date: Thu, 3 Sep 2020 13:41:02 +0200 Subject: [PATCH 097/187] fix python executable in tests (#75) --- tests/test_threadpoolctl.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index 09fe1a4d..160be643 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -402,14 +402,14 @@ def test_libomp_libiomp_warning(recwarn): def test_command_line_empty(): output = subprocess.check_output( - "python -m threadpoolctl".split()) + (sys.executable + " -m threadpoolctl").split()) assert json.loads(output.decode("utf-8")) == [] def test_command_line_command_flag(): pytest.importorskip("numpy") output = subprocess.check_output( - ["python", "-m", "threadpoolctl", "-c", "import numpy"]) + [sys.executable, "-m", "threadpoolctl", "-c", "import numpy"]) cli_info = json.loads(output.decode("utf-8")) this_process_info = threadpool_info() @@ -421,7 +421,7 @@ def test_command_line_command_flag(): reason="need recent subprocess.run options") def test_command_line_import_flag(): result = subprocess.run([ - "python", "-m", "threadpoolctl", "-i", + sys.executable, "-m", "threadpoolctl", "-i", "numpy", "scipy.linalg", "invalid_package", From 32037cf43a61909282b5d07e6b21d9621fa03e25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Thu, 1 Oct 2020 15:02:26 +0200 Subject: [PATCH 098/187] add license to metadata (#79) --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 4e76e43d..ef319a08 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ author-email = "thomas.moreau.2010@gmail.com" home-page = "https://github.com/joblib/threadpoolctl" description-file = "README.md" requires-python = ">=3.5" +license = "BSD-3-Clause" classifiers = [ "Intended Audience :: Developers", "License :: OSI Approved :: BSD License", From befb5caa3290ba2f0d7d32e7b06a605b7081f2e7 Mon Sep 17 00:00:00 2001 From: Ashutosh Varma Date: Wed, 6 Jan 2021 01:01:49 +0530 Subject: [PATCH 099/187] FIX typo operationg -> operating (#80) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ddb20fba..5a40d799 100644 --- a/README.md +++ b/README.md @@ -186,7 +186,7 @@ flit publish The initial dynamic library introspection code was written by @anton-malakhov for the smp package available at https://github.com/IntelPython/smp . -threadpoolctl extends this for other operationg systems. Contrary to smp, +threadpoolctl extends this for other operating systems. Contrary to smp, threadpoolctl does not attempt to limit the size of Python multiprocessing pools (threads or processes) or set operating system-level CPU affinity constraints: threadpoolctl only interacts with native libraries via their From 16807d229a62f1c72cb411d816a808c4f9984fc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Thu, 15 Apr 2021 19:21:36 +0200 Subject: [PATCH 100/187] [MRG] Fix compilation with clang and flake8 install in CI (#83) Co-authored-by: Olivier Grisel --- .azure_pipeline.yml | 16 ++++++++-------- continuous_integration/install.sh | 10 ++++------ continuous_integration/install_with_blis.sh | 10 ++++------ continuous_integration/posix.yml | 3 +-- tests/test_threadpoolctl.py | 4 ++-- threadpoolctl.py | 4 ++-- 6 files changed, 21 insertions(+), 26 deletions(-) diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml index 3de506c9..5a26a572 100644 --- a/.azure_pipeline.yml +++ b/.azure_pipeline.yml @@ -51,15 +51,15 @@ stages: PACKAGER: 'conda' VERSION_PYTHON: '3.6' BLAS: 'openblas' - CC_OUTER_LOOP: 'clang-8' - CC_INNER_LOOP: 'clang-8' + CC_OUTER_LOOP: 'clang-10' + CC_INNER_LOOP: 'clang-10' # Linux environment with MKL and Clang (known to be unsafe for # threadpoolctl) to only test the warning from multiple OpenMP. pylatest_conda_mkl_clang_gcc: PACKAGER: 'conda' VERSION_PYTHON: '*' BLAS: 'mkl' - CC_OUTER_LOOP: 'clang-8' + CC_OUTER_LOOP: 'clang-10' CC_INNER_LOOP: 'gcc' TESTS: 'libomp_libiomp_warning' # Linux environment with MKL, safe for threadpoolctl. @@ -76,7 +76,7 @@ stages: PACKAGER: 'pip' VERSION_PYTHON: '3.7' CC_OUTER_LOOP: 'gcc' - CC_INNER_LOOP: 'clang-8' + CC_INNER_LOOP: 'clang-10' # Linux environment with numpy from conda-forge channel pylatest_conda_forge: PACKAGER: 'conda-forge' @@ -89,7 +89,7 @@ stages: NO_NUMPY: 'true' VERSION_PYTHON: '*' CC_OUTER_LOOP: 'gcc' - CC_INNER_LOOP: 'clang-8' + CC_INNER_LOOP: 'clang-10' # Linux environment with numpy linked to BLIS pylatest_blis_gcc_clang_openmp: PACKAGER: 'conda' @@ -98,15 +98,15 @@ stages: BLIS_NUM_THREADS: '4' CC_OUTER_LOOP: 'gcc' CC_INNER_LOOP: 'gcc' - BLIS_CC: 'clang-8' + BLIS_CC: 'clang-10' BLIS_ENABLE_THREADING: 'openmp' pylatest_blis_clang_gcc_pthreads: PACKAGER: 'conda' VERSION_PYTHON: '*' INSTALL_BLIS: 'true' BLIS_NUM_THREADS: '4' - CC_OUTER_LOOP: 'clang-8' - CC_INNER_LOOP: 'clang-8' + CC_OUTER_LOOP: 'clang-10' + CC_INNER_LOOP: 'clang-10' BLIS_CC: 'gcc-8' BLIS_ENABLE_THREADING: 'pthreads' pylatest_blis_no_threading: diff --git a/continuous_integration/install.sh b/continuous_integration/install.sh index 707f296f..16fd5134 100755 --- a/continuous_integration/install.sh +++ b/continuous_integration/install.sh @@ -4,13 +4,11 @@ set -e UNAMESTR=`uname` -if [[ "$CC_OUTER_LOOP" == "clang-8" || "$CC_INNER_LOOP" == "clang-8" ]]; then +if [[ "$CC_OUTER_LOOP" == "clang-10" || "$CC_INNER_LOOP" == "clang-10" ]]; then # Assume Ubuntu: install a recent version of clang and libomp - echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main" | sudo tee -a /etc/apt/sources.list.d/llvm.list - echo "deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main" | sudo tee -a /etc/apt/sources.list.d/llvm.list - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - - sudo apt update - sudo apt install clang-8 libomp-8-dev + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 10 fi make_conda() { diff --git a/continuous_integration/install_with_blis.sh b/continuous_integration/install_with_blis.sh index cd88573a..c9a9796b 100755 --- a/continuous_integration/install_with_blis.sh +++ b/continuous_integration/install_with_blis.sh @@ -6,12 +6,10 @@ pushd .. ABS_PATH=$(pwd) popd -# Assume Ubuntu: install a recent version of clang and libomp -echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main" | sudo tee -a /etc/apt/sources.list.d/llvm.list -echo "deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main" | sudo tee -a /etc/apt/sources.list.d/llvm.list -wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - -sudo apt update -sudo apt install clang-8 libomp-8-dev +# Install a recent version of clang and libomp +wget https://apt.llvm.org/llvm.sh +chmod +x llvm.sh +sudo ./llvm.sh 10 # create conda env conda create -n $VIRTUALENV -q --yes python=$VERSION_PYTHON pip cython diff --git a/continuous_integration/posix.yml b/continuous_integration/posix.yml index 753cf4af..a3d6b754 100644 --- a/continuous_integration/posix.yml +++ b/continuous_integration/posix.yml @@ -13,8 +13,7 @@ jobs: steps: - script: | - python3 -m pip install --upgrade pip - python3 -m pip install flake8 + sudo apt-get install python3-flake8 python3 -m flake8 . displayName: Lint condition: eq(variables['LINT'], 'true') diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index 160be643..b68dcba8 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -388,8 +388,8 @@ def test_libomp_libiomp_warning(recwarn): info = _threadpool_info() prefixes = [module.prefix for module in info] - if not ("libomp" in prefixes and "libiomp" in prefixes - and sys.platform == "linux"): + if not ("libomp" in prefixes and "libiomp" in prefixes and + sys.platform == "linux"): pytest.skip("Requires both libomp and libiomp loaded, on Linux") assert len(recwarn) == 1 diff --git a/threadpoolctl.py b/threadpoolctl.py index 9157660b..caac05c0 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -212,8 +212,8 @@ def get_original_num_threads(self): if warning_apis: warnings.warn( - "Multiple value possible for following user apis: " - + ", ".join(warning_apis) + ". Returning the minimum.") + "Multiple value possible for following user apis: " + + ", ".join(warning_apis) + ". Returning the minimum.") return num_threads From 875a0fd6035000b6e91d38ae0926f4025e8c882d Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Fri, 9 Jul 2021 11:32:46 +0200 Subject: [PATCH 101/187] Update Python versions (3.6 min) (#86) --- .azure_pipeline.yml | 44 ++++++++++----------- continuous_integration/install.sh | 1 + continuous_integration/install_with_blis.sh | 1 + pyproject.toml | 5 ++- 4 files changed, 27 insertions(+), 24 deletions(-) diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml index 5a26a572..60db4076 100644 --- a/.azure_pipeline.yml +++ b/.azure_pipeline.yml @@ -18,38 +18,38 @@ stages: pylatest_conda: VERSION_PYTHON: '*' PACKAGER: 'conda' - py36_conda: - VERSION_PYTHON: '3.6' + py37_conda: + VERSION_PYTHON: '3.7' PACKAGER: 'conda' - py35_pip: - VERSION_PYTHON: '3.5' + py36_pip: + VERSION_PYTHON: '3.6' PACKAGER: 'pip' - template: continuous_integration/posix.yml parameters: name: Linux - vmImage: ubuntu-16.04 + vmImage: ubuntu-20.04 matrix: - # Linux environment to test that packages that comes with Ubuntu Xenial - # 16.04 are correctly handled. - py35_ubuntu_atlas_gcc_gcc: + # Linux environment to test that packages that comes with Ubuntu 20.04 + # are correctly handled. + py38_ubuntu_atlas_gcc_gcc: PACKAGER: 'ubuntu' - APT_BLAS: 'libatlas3-base libatlas-base-dev libatlas-dev' - VERSION_PYTHON: '3.5' + APT_BLAS: 'libatlas3-base libatlas-base-dev' + VERSION_PYTHON: '3.8' CC_OUTER_LOOP: 'gcc' CC_INNER_LOOP: 'gcc' LINT: 'true' - py35_ubuntu_openblas_gcc_gcc: + py36_ubuntu_openblas_gcc_gcc: PACKAGER: 'ubuntu' APT_BLAS: 'libopenblas-base libopenblas-dev' - VERSION_PYTHON: '3.5' + VERSION_PYTHON: '3.8' CC_OUTER_LOOP: 'gcc' CC_INNER_LOOP: 'gcc' - # Linux + Python 3.6 and homogeneous runtime nesting. - py36_conda_openblas_clang_clang: + # Linux + Python 3.7 and homogeneous runtime nesting. + py37_conda_openblas_clang_clang: PACKAGER: 'conda' - VERSION_PYTHON: '3.6' + VERSION_PYTHON: '3.7' BLAS: 'openblas' CC_OUTER_LOOP: 'clang-10' CC_INNER_LOOP: 'clang-10' @@ -70,11 +70,11 @@ stages: CC_OUTER_LOOP: 'gcc' CC_INNER_LOOP: 'gcc' MKL_THREADING_LAYER: 'INTEL' - # Linux + Python 3.7 with numpy / scipy installed with pip from PyPI + # Linux + Python 3.8 with numpy / scipy installed with pip from PyPI # and heterogeneous OpenMP runtimes. - py37_pip_openblas_gcc_clang: + py38_pip_openblas_gcc_clang: PACKAGER: 'pip' - VERSION_PYTHON: '3.7' + VERSION_PYTHON: '3.8' CC_OUTER_LOOP: 'gcc' CC_INNER_LOOP: 'clang-10' # Linux environment with numpy from conda-forge channel @@ -125,10 +125,10 @@ stages: vmImage: macOS-10.14 matrix: # MacOS environment with OpenMP installed through homebrew - py35_conda_homebrew_libomp: + py36_conda_homebrew_libomp: PACKAGER: 'conda' - VERSION_PYTHON: '3.5' - BLAS: 'mkl' + VERSION_PYTHON: '3.6' + BLAS: 'openblas' CC_OUTER_LOOP: 'clang' CC_INNER_LOOP: 'clang' INSTALL_LIBOMP: 'homebrew' @@ -149,7 +149,7 @@ stages: - job: 'no_test_always_skipped' displayName: 'No test always skipped' pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-20.04 steps: - download: current - script: | diff --git a/continuous_integration/install.sh b/continuous_integration/install.sh index 16fd5134..02779a3b 100755 --- a/continuous_integration/install.sh +++ b/continuous_integration/install.sh @@ -9,6 +9,7 @@ if [[ "$CC_OUTER_LOOP" == "clang-10" || "$CC_INNER_LOOP" == "clang-10" ]]; then wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh sudo ./llvm.sh 10 + sudo apt-get install libomp-dev fi make_conda() { diff --git a/continuous_integration/install_with_blis.sh b/continuous_integration/install_with_blis.sh index c9a9796b..4a10d95c 100755 --- a/continuous_integration/install_with_blis.sh +++ b/continuous_integration/install_with_blis.sh @@ -10,6 +10,7 @@ popd wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh sudo ./llvm.sh 10 +sudo apt-get install libomp-dev # create conda env conda create -n $VIRTUALENV -q --yes python=$VERSION_PYTHON pip cython diff --git a/pyproject.toml b/pyproject.toml index ef319a08..58a4d75f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,14 +8,15 @@ author = "Thomas Moreau" author-email = "thomas.moreau.2010@gmail.com" home-page = "https://github.com/joblib/threadpoolctl" description-file = "README.md" -requires-python = ">=3.5" +requires-python = ">=3.6" license = "BSD-3-Clause" classifiers = [ "Intended Audience :: Developers", "License :: OSI Approved :: BSD License", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", "Topic :: Software Development :: Libraries :: Python Modules", ] From 172628234f4a61465274c7c687f47319d5ca7ba8 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Fri, 9 Jul 2021 15:45:51 +0200 Subject: [PATCH 102/187] Add OpenBLAS corename detection (#85) --- CHANGES.md | 3 +++ tests/test_threadpoolctl.py | 23 +++++++++++++++++++++++ threadpoolctl.py | 22 ++++++++++++++++++++++ 3 files changed, 48 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 2f71bf64..47053b10 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,9 @@ 2.2.0 (In development) ====================== +- Report the architecture of the CPU cores detected by OpenBLAS + (`openblas_get_corename`) in `threadpool_info()` and BLIS + (`bli_arch_query_id` and `bli_arch_string`). 2.1.0 (2020-05-29) ================== diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index b68dcba8..544d86e3 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -440,3 +440,26 @@ def test_command_line_import_flag(): assert "WARNING: could not import scipy.linalg" in warnings else: assert "WARNING: could not import scipy.linalg" not in warnings + + +def test_architecture(): + expected_openblas_architectures = ( + # XXX: add more as needed by CI or developer laptops + "armv8", + "Haswell", + "SkylakeX", + "Sandybridge", + ) + expected_blis_architectures = ( + # XXX: add more as needed by CI or developer laptops + "skx", + "haswell", + ) + for module in threadpool_info(): + if module["internal_api"] == "openblas": + assert module["architecture"] in expected_openblas_architectures + elif module["internal_api"] == "blis": + assert module["architecture"] in expected_blis_architectures + else: + # Not supported for other modules + assert "architecture" not in module diff --git a/threadpoolctl.py b/threadpoolctl.py index caac05c0..66d3572d 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -660,6 +660,7 @@ def set_num_threads(self, num_threads): def _get_extra_info(self): self.threading_layer = self.get_threading_layer() + self.architecture = self.get_architecture() def get_threading_layer(self): """Return the threading layer of OpenBLAS""" @@ -670,6 +671,14 @@ def get_threading_layer(self): return "pthreads" return "disabled" + def get_architecture(self): + get_corename = getattr(self._dynlib, "openblas_get_corename", None) + if get_corename is None: + return None + + get_corename.restype = ctypes.c_char_p + return get_corename().decode("utf-8") + class _BLISModule(_Module): """Module class for BLIS""" @@ -694,6 +703,7 @@ def set_num_threads(self, num_threads): def _get_extra_info(self): self.threading_layer = self.get_threading_layer() + self.architecture = self.get_architecture() def get_threading_layer(self): """Return the threading layer of BLIS""" @@ -703,6 +713,18 @@ def get_threading_layer(self): return "pthreads" return "disabled" + def get_architecture(self): + bli_arch_query_id = getattr(self._dynlib, "bli_arch_query_id", None) + bli_arch_string = getattr(self._dynlib, "bli_arch_string", None) + if bli_arch_query_id is None or bli_arch_string is None: + return None + + # the true restype should be BLIS' arch_t (enum) but int should work + # for us: + bli_arch_query_id.restype = ctypes.c_int + bli_arch_string.restype = ctypes.c_char_p + return bli_arch_string(bli_arch_query_id()).decode("utf-8") + class _MKLModule(_Module): """Module class for MKL""" From 9171dceca5d5474062c0c527d2ceac9275e348f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Fri, 9 Jul 2021 17:05:39 +0200 Subject: [PATCH 103/187] FIX use more reliable naming syntax for MKL version (#82) --- CHANGES.md | 4 ++++ threadpoolctl.py | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 47053b10..a7440377 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,10 @@ (`openblas_get_corename`) in `threadpool_info()` and BLIS (`bli_arch_query_id` and `bli_arch_string`). +- Fixed a bug when the version of MKL was not found. The + "version" field is now set to None in that case. + https://github.com/joblib/threadpoolctl/pull/82 + 2.1.0 (2020-05-29) ================== diff --git a/threadpoolctl.py b/threadpoolctl.py index 66d3572d..d0a9e0a5 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -729,8 +729,11 @@ def get_architecture(self): class _MKLModule(_Module): """Module class for MKL""" def get_version(self): + if not hasattr(self._dynlib, "MKL_Get_Version_String"): + return None + res = ctypes.create_string_buffer(200) - self._dynlib.mkl_get_version_string(res, 200) + self._dynlib.MKL_Get_Version_String(res, 200) version = res.value.decode("utf-8") group = re.search(r"Version ([^ ]+) ", version) From ee7fad84615fbdf846a69a915530542b3f7efee9 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Fri, 9 Jul 2021 17:08:32 +0200 Subject: [PATCH 104/187] Release 2.2.0 --- CHANGES.md | 4 ++-- threadpoolctl.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a7440377..66806b5d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,5 @@ -2.2.0 (In development) -====================== +2.2.0 (2021-07-09) +================== - Report the architecture of the CPU cores detected by OpenBLAS (`openblas_get_corename`) in `threadpool_info()` and BLIS diff --git a/threadpoolctl.py b/threadpoolctl.py index d0a9e0a5..93868824 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -19,7 +19,7 @@ from ctypes.util import find_library from abc import ABC, abstractmethod -__version__ = "2.2.0.dev0" +__version__ = "2.2.0" __all__ = ["threadpool_limits", "threadpool_info"] From b14f1e87130365ffc8fa7570ee448f119c8b64d2 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Fri, 9 Jul 2021 17:30:27 +0200 Subject: [PATCH 105/187] Back to dev mode --- CHANGES.md | 5 +++++ threadpoolctl.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 66806b5d..045956d2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,8 @@ +2.3.0 (in development) +====================== + + + 2.2.0 (2021-07-09) ================== diff --git a/threadpoolctl.py b/threadpoolctl.py index 93868824..cc850370 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -19,7 +19,7 @@ from ctypes.util import find_library from abc import ABC, abstractmethod -__version__ = "2.2.0" +__version__ = "2.3.0.dev0" __all__ = ["threadpool_limits", "threadpool_info"] From 5ca893e022295b6f65886bab408cbdc5afe63f6c Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Fri, 9 Jul 2021 17:33:57 +0200 Subject: [PATCH 106/187] Fix phrasing in changelog --- CHANGES.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 045956d2..161231f3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,9 +6,9 @@ 2.2.0 (2021-07-09) ================== -- Report the architecture of the CPU cores detected by OpenBLAS - (`openblas_get_corename`) in `threadpool_info()` and BLIS - (`bli_arch_query_id` and `bli_arch_string`). +- `threadpoolctl.threadpool_info()` now reports the architecture of the CPU + cores detected by OpenBLAS (via `openblas_get_corename`) and BLIS (via + `bli_arch_query_id` and `bli_arch_string`). - Fixed a bug when the version of MKL was not found. The "version" field is now set to None in that case. From 4a31c658af2066df5f8c7bfec76716eefe5fea37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Thu, 22 Jul 2021 17:21:45 +0200 Subject: [PATCH 107/187] CI pin llvm-openmp version (#89) --- continuous_integration/install.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/continuous_integration/install.sh b/continuous_integration/install.sh index 02779a3b..604c3175 100755 --- a/continuous_integration/install.sh +++ b/continuous_integration/install.sh @@ -17,7 +17,10 @@ make_conda() { if [[ "$UNAMESTR" == "Darwin" ]]; then if [[ "$INSTALL_LIBOMP" == "conda-forge" ]]; then # Install an OpenMP-enabled clang/llvm from conda-forge - TO_INSTALL="$TO_INSTALL conda-forge::compilers conda-forge::llvm-openmp" + + # temporary pin llvm-openmp version. version 12 + mkl segfaults. + TO_INSTALL="$TO_INSTALL conda-forge::compilers conda-forge::llvm-openmp<=11.1.0" + export CFLAGS="$CFLAGS -I$CONDA/envs/$VIRTUALENV/include" export LDFLAGS="$LDFLAGS -Wl,-rpath,$CONDA/envs/$VIRTUALENV/lib -L$CONDA/envs/$VIRTUALENV/lib" From 50eab9284471a4ad45ddfdc2c15077fe7f47efe7 Mon Sep 17 00:00:00 2001 From: peterbell10 Date: Fri, 23 Jul 2021 00:55:23 +0100 Subject: [PATCH 108/187] Fix getattr uses with invalid default values (#88) --- CHANGES.md | 3 +++ threadpoolctl.py | 12 ++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 161231f3..fab8f653 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,9 @@ 2.3.0 (in development) ====================== +- Fixed an attribute error when using old versions of OpenBLAS or BLIS that are + missing version query functions. + https://github.com/joblib/threadpoolctl/pull/88 2.2.0 (2021-07-09) diff --git a/threadpoolctl.py b/threadpoolctl.py index cc850370..94db2989 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -640,8 +640,10 @@ class _OpenBLASModule(_Module): def get_version(self): # None means OpenBLAS is not loaded or version < 0.3.4, since OpenBLAS # did not expose its version before that. - get_config = getattr(self._dynlib, "openblas_get_config", - lambda: None) + get_config = getattr(self._dynlib, "openblas_get_config", None) + if get_config is None: + return None + get_config.restype = ctypes.c_char_p config = get_config().split() if config[0] == b"OpenBLAS": @@ -683,8 +685,10 @@ def get_architecture(self): class _BLISModule(_Module): """Module class for BLIS""" def get_version(self): - get_version_ = getattr(self._dynlib, "bli_info_get_version_str", - lambda: None) + get_version_ = getattr(self._dynlib, "bli_info_get_version_str", None) + if get_version_ is None: + return None + get_version_.restype = ctypes.c_char_p return get_version_().decode("utf-8") From 480443e437f50b7af429e3b5b93a5c8dec15b5e2 Mon Sep 17 00:00:00 2001 From: peterbell10 Date: Fri, 23 Jul 2021 09:37:43 +0100 Subject: [PATCH 109/187] Fix import with -OO (#87) --- CHANGES.md | 2 ++ threadpoolctl.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index fab8f653..ed3b208d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,8 @@ missing version query functions. https://github.com/joblib/threadpoolctl/pull/88 +- Fixed an attribute error when python is run with -OO. + https://github.com/joblib/threadpoolctl/pull/87 2.2.0 (2021-07-09) ================== diff --git a/threadpoolctl.py b/threadpoolctl.py index 94db2989..d32d6697 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -98,7 +98,8 @@ class _dl_phdr_info(ctypes.Structure): def _format_docstring(*args, **kwargs): def decorator(o): - o.__doc__ = o.__doc__.format(*args, **kwargs) + if o.__doc__ is not None: + o.__doc__ = o.__doc__.format(*args, **kwargs) return o return decorator From 4bb941e92817855e6751c14b23df7745b31288c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Thu, 2 Sep 2021 18:01:21 +0200 Subject: [PATCH 110/187] CI use all packages from the conda-forge channel on macos (#90) --- .azure_pipeline.yml | 5 +++-- continuous_integration/install.sh | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml index 60db4076..78545df5 100644 --- a/.azure_pipeline.yml +++ b/.azure_pipeline.yml @@ -81,6 +81,7 @@ stages: pylatest_conda_forge: PACKAGER: 'conda-forge' VERSION_PYTHON: '*' + BLAS: 'openblas' CC_OUTER_LOOP: 'gcc' CC_INNER_LOOP: 'gcc' # Linux environment with no numpy and heterogeneous OpenMP runtimes. @@ -133,8 +134,8 @@ stages: CC_INNER_LOOP: 'clang' INSTALL_LIBOMP: 'homebrew' # MacOS environment with OpenMP installed through conda-forge compilers - pylatest_conda_conda_forge_clang: - PACKAGER: 'conda' + pylatest_conda_forge_clang: + PACKAGER: 'conda-forge' VERSION_PYTHON: '*' BLAS: 'mkl' CC_OUTER_LOOP: 'clang' diff --git a/continuous_integration/install.sh b/continuous_integration/install.sh index 604c3175..fb643256 100755 --- a/continuous_integration/install.sh +++ b/continuous_integration/install.sh @@ -18,8 +18,8 @@ make_conda() { if [[ "$INSTALL_LIBOMP" == "conda-forge" ]]; then # Install an OpenMP-enabled clang/llvm from conda-forge - # temporary pin llvm-openmp version. version 12 + mkl segfaults. - TO_INSTALL="$TO_INSTALL conda-forge::compilers conda-forge::llvm-openmp<=11.1.0" + # assumes conda-forge is set on priority channel + TO_INSTALL="$TO_INSTALL compilers llvm-openmp" export CFLAGS="$CFLAGS -I$CONDA/envs/$VIRTUALENV/include" export LDFLAGS="$LDFLAGS -Wl,-rpath,$CONDA/envs/$VIRTUALENV/lib -L$CONDA/envs/$VIRTUALENV/lib" @@ -49,7 +49,7 @@ if [[ "$PACKAGER" == "conda" ]]; then elif [[ "$PACKAGER" == "conda-forge" ]]; then conda config --prepend channels conda-forge conda config --set channel_priority strict - TO_INSTALL="python=$VERSION_PYTHON numpy scipy" + TO_INSTALL="python=$VERSION_PYTHON numpy scipy blas[build=$BLAS]" make_conda $TO_INSTALL elif [[ "$PACKAGER" == "pip" ]]; then From 9b5b41f67946912b0cc9ccb516081750171c9432 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Fri, 3 Sep 2021 00:45:26 +0200 Subject: [PATCH 111/187] Make threadpoolctl robust to missing openblas_get_parallel symbol (#91) --- CHANGES.md | 1 + threadpoolctl.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index ed3b208d..0d34112b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,7 @@ - Fixed an attribute error when using old versions of OpenBLAS or BLIS that are missing version query functions. https://github.com/joblib/threadpoolctl/pull/88 + https://github.com/joblib/threadpoolctl/pull/91 - Fixed an attribute error when python is run with -OO. https://github.com/joblib/threadpoolctl/pull/87 diff --git a/threadpoolctl.py b/threadpoolctl.py index d32d6697..a62e33a0 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -667,7 +667,11 @@ def _get_extra_info(self): def get_threading_layer(self): """Return the threading layer of OpenBLAS""" - threading_layer = self._dynlib.openblas_get_parallel() + openblas_get_parallel = getattr(self._dynlib, "openblas_get_parallel", + None) + if openblas_get_parallel is None: + return "unknown" + threading_layer = openblas_get_parallel if threading_layer == 2: return "openmp" elif threading_layer == 1: From 2a778839b2aa6df27ff365fe5d20ef0ad8f1c2be Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Fri, 3 Sep 2021 09:55:46 +0200 Subject: [PATCH 112/187] Fix typo in get_threading_layer (#93) --- threadpoolctl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/threadpoolctl.py b/threadpoolctl.py index a62e33a0..9aefe515 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -671,7 +671,7 @@ def get_threading_layer(self): None) if openblas_get_parallel is None: return "unknown" - threading_layer = openblas_get_parallel + threading_layer = openblas_get_parallel() if threading_layer == 2: return "openmp" elif threading_layer == 1: From 708aabf4fcef09d1b987fc43e571fe9c99d51c9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Thu, 23 Sep 2021 16:13:30 +0200 Subject: [PATCH 113/187] use lru_cache for realpath (#97) --- threadpoolctl.py | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/threadpoolctl.py b/threadpoolctl.py index 9aefe515..8bc1798e 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -18,6 +18,7 @@ import warnings from ctypes.util import find_library from abc import ABC, abstractmethod +from functools import lru_cache __version__ = "2.3.0.dev0" __all__ = ["threadpool_limits", "threadpool_info"] @@ -105,6 +106,12 @@ def decorator(o): return decorator +@lru_cache(maxsize=10000) +def _realpath(filepath): + """Small caching wrapper around os.path.realpath to limit system calls""" + return os.path.realpath(filepath) + + @_format_docstring(USER_APIS=list(_ALL_USER_APIS), INTERNAL_APIS=_ALL_INTERNAL_APIS) def threadpool_info(): @@ -325,12 +332,6 @@ class _ThreadpoolInfo(): # it's very unlikely that a shared library will be unloaded and reloaded # during the lifetime of a program. _system_libraries = dict() - # Cache for calls to os.path.realpath on system libraries to reduce the - # impact of slow system calls (e.g. stat) on slow filesystem. - # We use a class level cache instead of an instance level cache because - # we can safely assume that the filepath of loaded shared libraries will - # never change during the lifetime of a program. - _realpaths = dict() def __init__(self, user_api=None, prefixes=None, modules=None): if modules is None: @@ -490,7 +491,7 @@ def _find_modules_with_enum_process_module_ex(self): def _make_module_from_path(self, filepath): """Store a module if it is supported and selected""" # Required to resolve symlinks - filepath = self._realpath(filepath) + filepath = _realpath(filepath) # `lower` required to take account of OpenMP dll case on Windows # (vcomp, VCOMP, Vcomp, ...) filename = os.path.basename(filepath).lower() @@ -567,19 +568,6 @@ def _get_windll(cls, dll_name): cls._system_libraries[dll_name] = dll return dll - @classmethod - def _realpath(cls, filepath, cache_limit=10000): - """Small caching wrapper around os.path.realpath to limit system calls - """ - rpath = cls._realpaths.get(filepath) - if rpath is None: - rpath = os.path.realpath(filepath) - if len(cls._realpaths) < cache_limit: - # If we drop support for Python 2.7, we could use - # functools.lru_cache with maxsize=10000 instead. - cls._realpaths[filepath] = rpath - return rpath - @_format_docstring( USER_APIS=", ".join('"{}"'.format(api) for api in _ALL_USER_APIS), From 05ba3b4d371cd3d89a15dc18a038a25aafed2086 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Fri, 24 Sep 2021 10:17:51 +0200 Subject: [PATCH 114/187] MNT Blackify the code base (#96) --- .azure_pipeline.yml | 2 +- benchmarks/bench_context_manager_overhead.py | 22 +- continuous_integration/posix.yml | 12 +- pyproject.toml | 5 + tests/_openmp_test_helper/build_utils.py | 5 +- tests/_openmp_test_helper/setup_inner.py | 17 +- .../setup_nested_prange_blas.py | 21 +- tests/_openmp_test_helper/setup_outer.py | 15 +- tests/test_threadpoolctl.py | 96 +++++---- tests/utils.py | 16 +- threadpoolctl.py | 196 ++++++++++-------- 11 files changed, 241 insertions(+), 166 deletions(-) diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml index 78545df5..b7d963a8 100644 --- a/.azure_pipeline.yml +++ b/.azure_pipeline.yml @@ -39,7 +39,6 @@ stages: VERSION_PYTHON: '3.8' CC_OUTER_LOOP: 'gcc' CC_INNER_LOOP: 'gcc' - LINT: 'true' py36_ubuntu_openblas_gcc_gcc: PACKAGER: 'ubuntu' APT_BLAS: 'libopenblas-base libopenblas-dev' @@ -84,6 +83,7 @@ stages: BLAS: 'openblas' CC_OUTER_LOOP: 'gcc' CC_INNER_LOOP: 'gcc' + LINT: 'true' # Linux environment with no numpy and heterogeneous OpenMP runtimes. pylatest_conda_nonumpy_gcc_clang: PACKAGER: 'conda' diff --git a/benchmarks/bench_context_manager_overhead.py b/benchmarks/bench_context_manager_overhead.py index 34c1d92e..c96da9aa 100644 --- a/benchmarks/bench_context_manager_overhead.py +++ b/benchmarks/bench_context_manager_overhead.py @@ -4,12 +4,15 @@ from statistics import mean, stdev from threadpoolctl import threadpool_info, threadpool_limits -parser = ArgumentParser(description='Measure threadpool_limits call overhead.') -parser.add_argument('--import', dest="packages", default=[], nargs='+', - help='Python packages to import to load threadpool enabled' - ' libraries.') -parser.add_argument("--n-calls", type=int, default=100, - help="Number of iterations") +parser = ArgumentParser(description="Measure threadpool_limits call overhead.") +parser.add_argument( + "--import", + dest="packages", + default=[], + nargs="+", + help="Python packages to import to load threadpool enabled libraries.", +) +parser.add_argument("--n-calls", type=int, default=100, help="Number of iterations") args = parser.parse_args() for package_name in args.packages: @@ -24,5 +27,8 @@ pass timings.append(time.time() - t) -print("Overhead per call: {:.3f} +/-{:.3f} ms" - .format(mean(timings) * 1e3, stdev(timings) * 1e3)) +print( + "Overhead per call: {:.3f} +/-{:.3f} ms".format( + mean(timings) * 1e3, stdev(timings) * 1e3 + ) +) diff --git a/continuous_integration/posix.yml b/continuous_integration/posix.yml index a3d6b754..3d3bc963 100644 --- a/continuous_integration/posix.yml +++ b/continuous_integration/posix.yml @@ -12,11 +12,6 @@ jobs: ${{ insert }}: ${{ parameters.matrix }} steps: - - script: | - sudo apt-get install python3-flake8 - python3 -m flake8 . - displayName: Lint - condition: eq(variables['LINT'], 'true') - bash: echo "##vso[task.prependpath]$CONDA/bin" displayName: Add conda to PATH condition: or(startsWith(variables['PACKAGER'], 'conda'), eq(variables['PACKAGER'], 'pip')) @@ -25,6 +20,13 @@ jobs: # We need to take ownership if we want to update conda or install packages globally displayName: Take ownership of conda installation condition: eq('${{ parameters.name }}', 'macOS') + - script: | + conda create -n tmp -y -c conda-forge python black + source activate tmp + black --check . + conda deactivate + displayName: Lint + condition: eq(variables['LINT'], 'true') - script: | continuous_integration/install.sh displayName: 'Install without BLIS' diff --git a/pyproject.toml b/pyproject.toml index 58a4d75f..f6955703 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,3 +20,8 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Topic :: Software Development :: Libraries :: Python Modules", ] + +[tool.black] +line-length = 88 +target_version = ['py36', 'py37', 'py38', 'py39'] +experimental_string_processing = true diff --git a/tests/_openmp_test_helper/build_utils.py b/tests/_openmp_test_helper/build_utils.py index 06cc52bb..90356b33 100644 --- a/tests/_openmp_test_helper/build_utils.py +++ b/tests/_openmp_test_helper/build_utils.py @@ -7,8 +7,7 @@ def set_cc_variables(var_name="CC"): if cc_var is not None: os.environ["CC"] = cc_var if sys.platform == "darwin": - os.environ["LDSHARED"] = ( - cc_var + " -bundle -undefined dynamic_lookup") + os.environ["LDSHARED"] = cc_var + " -bundle -undefined dynamic_lookup" else: os.environ["LDSHARED"] = cc_var + " -shared" @@ -18,6 +17,6 @@ def set_cc_variables(var_name="CC"): def get_openmp_flag(): if sys.platform == "win32": return ["/openmp"] - elif sys.platform == "darwin" and 'openmp' in os.getenv('CPPFLAGS', ''): + elif sys.platform == "darwin" and "openmp" in os.getenv("CPPFLAGS", ""): return [] return ["-fopenmp"] diff --git a/tests/_openmp_test_helper/setup_inner.py b/tests/_openmp_test_helper/setup_inner.py index af3e40df..e6103141 100644 --- a/tests/_openmp_test_helper/setup_inner.py +++ b/tests/_openmp_test_helper/setup_inner.py @@ -19,18 +19,21 @@ "openmp_helpers_inner", ["openmp_helpers_inner.pyx"], extra_compile_args=openmp_flag, - extra_link_args=openmp_flag - ) + extra_link_args=openmp_flag, + ) ] setup( - name='_openmp_test_helper_inner', + name="_openmp_test_helper_inner", ext_modules=cythonize( ext_modules, - compiler_directives={'language_level': 3, - 'boundscheck': False, - 'wraparound': False}, - compile_time_env={"CC_INNER_LOOP": inner_loop_cc_var or "unknown"}) + compiler_directives={ + "language_level": 3, + "boundscheck": False, + "wraparound": False, + }, + compile_time_env={"CC_INNER_LOOP": inner_loop_cc_var or "unknown"}, + ), ) finally: diff --git a/tests/_openmp_test_helper/setup_nested_prange_blas.py b/tests/_openmp_test_helper/setup_nested_prange_blas.py index 54da058f..275a92ce 100644 --- a/tests/_openmp_test_helper/setup_nested_prange_blas.py +++ b/tests/_openmp_test_helper/setup_nested_prange_blas.py @@ -12,8 +12,8 @@ set_cc_variables("CC_OUTER_LOOP") openmp_flag = get_openmp_flag() - use_blis = os.getenv('INSTALL_BLIS', False) - libraries = ['blis'] if use_blis else [] + use_blis = os.getenv("INSTALL_BLIS", False) + libraries = ["blis"] if use_blis else [] ext_modules = [ Extension( @@ -21,18 +21,21 @@ ["nested_prange_blas.pyx"], extra_compile_args=openmp_flag, extra_link_args=openmp_flag, - libraries=libraries - ) + libraries=libraries, + ) ] setup( - name='_openmp_test_helper_nested_prange_blas', + name="_openmp_test_helper_nested_prange_blas", ext_modules=cythonize( ext_modules, - compile_time_env={'USE_BLIS': use_blis}, - compiler_directives={'language_level': 3, - 'boundscheck': False, - 'wraparound': False}) + compile_time_env={"USE_BLIS": use_blis}, + compiler_directives={ + "language_level": 3, + "boundscheck": False, + "wraparound": False, + }, + ), ) finally: diff --git a/tests/_openmp_test_helper/setup_outer.py b/tests/_openmp_test_helper/setup_outer.py index 56c9bd2a..8f875bdf 100644 --- a/tests/_openmp_test_helper/setup_outer.py +++ b/tests/_openmp_test_helper/setup_outer.py @@ -19,18 +19,21 @@ "openmp_helpers_outer", ["openmp_helpers_outer.pyx"], extra_compile_args=openmp_flag, - extra_link_args=openmp_flag - ) + extra_link_args=openmp_flag, + ) ] setup( name="_openmp_test_helper_outer", ext_modules=cythonize( ext_modules, - compiler_directives={'language_level': 3, - 'boundscheck': False, - 'wraparound': False}, - compile_time_env={"CC_OUTER_LOOP": outer_loop_cc_var or "unknown"}) + compiler_directives={ + "language_level": 3, + "boundscheck": False, + "wraparound": False, + }, + compile_time_env={"CC_OUTER_LOOP": outer_loop_cc_var or "unknown"}, + ), ) finally: diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index 544d86e3..3fed30c4 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -154,23 +154,25 @@ def test_threadpool_limits_manual_unregister(): def test_threadpool_limits_bad_input(): # Check that appropriate errors are raised for invalid arguments - match = re.escape("user_api must be either in {} or None." - .format(_ALL_USER_APIS)) + match = re.escape("user_api must be either in {} or None.".format(_ALL_USER_APIS)) with pytest.raises(ValueError, match=match): threadpool_limits(limits=1, user_api="wrong") - with pytest.raises(TypeError, - match="limits must either be an int, a list or a dict"): + with pytest.raises( + TypeError, match="limits must either be an int, a list or a dict" + ): threadpool_limits(limits=(1, 2, 3)) -@pytest.mark.skipif(not cython_extensions_compiled, - reason='Requires cython extensions to be compiled') -@pytest.mark.parametrize('num_threads', [1, 2, 4]) +@pytest.mark.skipif( + not cython_extensions_compiled, reason="Requires cython extensions to be compiled" +) +@pytest.mark.parametrize("num_threads", [1, 2, 4]) def test_openmp_limit_num_threads(num_threads): # checks that OpenMP effectively uses the number of threads requested by # the context manager import tests._openmp_test_helper.openmp_helpers_inner as omp_inner + check_openmp_num_threads = omp_inner.check_openmp_num_threads old_num_threads = check_openmp_num_threads(100) @@ -180,24 +182,28 @@ def test_openmp_limit_num_threads(num_threads): assert check_openmp_num_threads(100) == old_num_threads -@pytest.mark.skipif(not cython_extensions_compiled, - reason="Requires cython extensions to be compiled") +@pytest.mark.skipif( + not cython_extensions_compiled, reason="Requires cython extensions to be compiled" +) @pytest.mark.parametrize("nthreads_outer", [None, 1, 2, 4]) def test_openmp_nesting(nthreads_outer): # checks that OpenMP effectively uses the number of threads requested by # the context manager when nested in an outer OpenMP loop. import tests._openmp_test_helper.openmp_helpers_outer as omp_outer + check_nested_openmp_loops = omp_outer.check_nested_openmp_loops # Find which OpenMP lib is used at runtime for inner loop inner_info = threadpool_info_from_subprocess( - "tests._openmp_test_helper.openmp_helpers_inner") + "tests._openmp_test_helper.openmp_helpers_inner" + ) assert len(inner_info) == 1 inner_omp = inner_info[0]["prefix"] # Find which OpenMP lib is used at runtime for outer loop outer_info = threadpool_info_from_subprocess( - "tests._openmp_test_helper.openmp_helpers_outer") + "tests._openmp_test_helper.openmp_helpers_outer" + ) if len(outer_info) == 1: # Only 1 openmp loaded. It has to be this one. outer_omp = outer_info[0]["prefix"] @@ -222,8 +228,7 @@ def test_openmp_nesting(nthreads_outer): # Ask outer loop to run on nthreads threads and inner loop run on 1 # thread - outer_num_threads, inner_num_threads = \ - check_nested_openmp_loops(10, nthreads) + outer_num_threads, inner_num_threads = check_nested_openmp_loops(10, nthreads) # The state of the original state of all threadpools should have been # restored. @@ -239,8 +244,9 @@ def test_openmp_nesting(nthreads_outer): if inner_num_threads != 1: # XXX: this does not always work when nesting independent openmp # implementations. See: https://github.com/jeremiedbb/Nested_OpenMP - pytest.xfail("Inner OpenMP num threads was %d instead of 1" - % inner_num_threads) + pytest.xfail( + "Inner OpenMP num threads was %d instead of 1" % inner_num_threads + ) assert inner_num_threads == 1 @@ -258,8 +264,9 @@ def test_shipped_openblas(): assert original_info == _threadpool_info() -@pytest.mark.skipif(len(libopenblas_paths) < 2, - reason="need at least 2 shipped openblas library") +@pytest.mark.skipif( + len(libopenblas_paths) < 2, reason="need at least 2 shipped openblas library" +) def test_multiple_shipped_openblas(): # This redundant test is meant to make it easier to see if the system # has 2 or more active openblas runtimes available just be reading the @@ -268,8 +275,9 @@ def test_multiple_shipped_openblas(): @pytest.mark.skipif(scipy is None, reason="requires scipy") -@pytest.mark.skipif(not cython_extensions_compiled, - reason='Requires cython extensions to be compiled') +@pytest.mark.skipif( + not cython_extensions_compiled, reason="Requires cython extensions to be compiled" +) @pytest.mark.parametrize("nthreads_outer", [None, 1, 2, 4]) def test_nested_prange_blas(nthreads_outer): # Check that the BLAS linked to scipy effectively uses the number of @@ -277,6 +285,7 @@ def test_nested_prange_blas(nthreads_outer): # loop. import numpy as np import tests._openmp_test_helper.nested_prange_blas as prange_blas + check_nested_prange_blas = prange_blas.check_nested_prange_blas original_info = _threadpool_info() @@ -351,8 +360,7 @@ def test_mkl_threading_layer(): expected_layer = os.getenv("MKL_THREADING_LAYER") if not (mkl_info and expected_layer): - pytest.skip("requires MKL and the environment variable " - "MKL_THREADING_LAYER set") + pytest.skip("requires MKL and the environment variable MKL_THREADING_LAYER set") actual_layer = mkl_info.modules[0].threading_layer assert actual_layer == expected_layer.lower() @@ -367,15 +375,17 @@ def test_blis_threading_layer(): expected_layer = "disabled" if not (blis_info and expected_layer): - pytest.skip("requires BLIS and the environment variable " - "BLIS_ENABLE_THREADING set") + pytest.skip( + "requires BLIS and the environment variable BLIS_ENABLE_THREADING set" + ) actual_layer = blis_info.modules[0].threading_layer assert actual_layer == expected_layer -@pytest.mark.skipif(not cython_extensions_compiled, - reason='Requires cython extensions to be compiled') +@pytest.mark.skipif( + not cython_extensions_compiled, reason="Requires cython extensions to be compiled" +) def test_libomp_libiomp_warning(recwarn): # Trigger the import of a potentially clang-compiled extension: import tests._openmp_test_helper.openmp_helpers_outer # noqa @@ -388,8 +398,7 @@ def test_libomp_libiomp_warning(recwarn): info = _threadpool_info() prefixes = [module.prefix for module in info] - if not ("libomp" in prefixes and "libiomp" in prefixes and - sys.platform == "linux"): + if not ("libomp" in prefixes and "libiomp" in prefixes and sys.platform == "linux"): pytest.skip("Requires both libomp and libiomp loaded, on Linux") assert len(recwarn) == 1 @@ -401,15 +410,15 @@ def test_libomp_libiomp_warning(recwarn): def test_command_line_empty(): - output = subprocess.check_output( - (sys.executable + " -m threadpoolctl").split()) + output = subprocess.check_output((sys.executable + " -m threadpoolctl").split()) assert json.loads(output.decode("utf-8")) == [] def test_command_line_command_flag(): pytest.importorskip("numpy") output = subprocess.check_output( - [sys.executable, "-m", "threadpoolctl", "-c", "import numpy"]) + [sys.executable, "-m", "threadpoolctl", "-c", "import numpy"] + ) cli_info = json.loads(output.decode("utf-8")) this_process_info = threadpool_info() @@ -417,16 +426,25 @@ def test_command_line_command_flag(): assert module in this_process_info -@pytest.mark.skipif(sys.version_info < (3, 7), - reason="need recent subprocess.run options") +@pytest.mark.skipif( + sys.version_info < (3, 7), reason="need recent subprocess.run options" +) def test_command_line_import_flag(): - result = subprocess.run([ - sys.executable, "-m", "threadpoolctl", "-i", - "numpy", - "scipy.linalg", - "invalid_package", - "numpy.invalid_sumodule", - ], capture_output=True, check=True, encoding="utf-8") + result = subprocess.run( + [ + sys.executable, + "-m", + "threadpoolctl", + "-i", + "numpy", + "scipy.linalg", + "invalid_package", + "numpy.invalid_sumodule", + ], + capture_output=True, + check=True, + encoding="utf-8", + ) cli_info = json.loads(result.stdout) this_process_info = threadpool_info() diff --git a/tests/utils.py b/tests/utils.py index a7efeb02..a0043da3 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -14,10 +14,10 @@ try: # make sure the mkl/blas are loaded for test_threadpool_limits import numpy as np + np.dot(np.ones(1000), np.ones(1000)) - libopenblas_patterns.append(os.path.join(np.__path__[0], ".libs", - "libopenblas*")) + libopenblas_patterns.append(os.path.join(np.__path__[0], ".libs", "libopenblas*")) except ImportError: pass @@ -25,19 +25,23 @@ try: import scipy import scipy.linalg # noqa: F401 + scipy.linalg.svd([[1, 2], [3, 4]]) - libopenblas_patterns.append(os.path.join(scipy.__path__[0], ".libs", - "libopenblas*")) + libopenblas_patterns.append( + os.path.join(scipy.__path__[0], ".libs", "libopenblas*") + ) except ImportError: scipy = None -libopenblas_paths = set(path for pattern in libopenblas_patterns - for path in glob(pattern)) +libopenblas_paths = set( + path for pattern in libopenblas_patterns for path in glob(pattern) +) try: import tests._openmp_test_helper.openmp_helpers_inner # noqa: F401 + cython_extensions_compiled = True except ImportError: cython_extensions_compiled = False diff --git a/threadpoolctl.py b/threadpoolctl.py index 8bc1798e..a8c1c5f8 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -39,16 +39,16 @@ # Structure to cast the info on dynamically loaded library. See # https://linux.die.net/man/3/dl_iterate_phdr for more details. -_SYSTEM_UINT = ctypes.c_uint64 if sys.maxsize > 2**32 else ctypes.c_uint32 -_SYSTEM_UINT_HALF = ctypes.c_uint32 if sys.maxsize > 2**32 else ctypes.c_uint16 +_SYSTEM_UINT = ctypes.c_uint64 if sys.maxsize > 2 ** 32 else ctypes.c_uint32 +_SYSTEM_UINT_HALF = ctypes.c_uint32 if sys.maxsize > 2 ** 32 else ctypes.c_uint16 class _dl_phdr_info(ctypes.Structure): _fields_ = [ - ("dlpi_addr", _SYSTEM_UINT), # Base address of object - ("dlpi_name", ctypes.c_char_p), # path to the library - ("dlpi_phdr", ctypes.c_void_p), # pointer on dlpi_headers - ("dlpi_phnum", _SYSTEM_UINT_HALF) # number of elements in dlpi_phdr + ("dlpi_addr", _SYSTEM_UINT), # Base address of object + ("dlpi_name", ctypes.c_char_p), # path to the library + ("dlpi_phdr", ctypes.c_void_p), # pointer on dlpi_headers + ("dlpi_phnum", _SYSTEM_UINT_HALF), # number of elements in dlpi_phdr ] @@ -67,34 +67,35 @@ class _dl_phdr_info(ctypes.Structure): "_OpenMPModule": { "user_api": "openmp", "internal_api": "openmp", - "filename_prefixes": ("libiomp", "libgomp", "libomp", "vcomp") + "filename_prefixes": ("libiomp", "libgomp", "libomp", "vcomp"), }, "_OpenBLASModule": { "user_api": "blas", "internal_api": "openblas", - "filename_prefixes": ("libopenblas",) + "filename_prefixes": ("libopenblas",), }, "_MKLModule": { "user_api": "blas", "internal_api": "mkl", - "filename_prefixes": ("libmkl_rt", "mkl_rt") + "filename_prefixes": ("libmkl_rt", "mkl_rt"), }, "_BLISModule": { "user_api": "blas", "internal_api": "blis", - "filename_prefixes": ("libblis",) - } + "filename_prefixes": ("libblis",), + }, } # Helpers for the doc and test names _ALL_USER_APIS = list(set(m["user_api"] for m in _SUPPORTED_MODULES.values())) _ALL_INTERNAL_APIS = [m["internal_api"] for m in _SUPPORTED_MODULES.values()] -_ALL_PREFIXES = [prefix for m in _SUPPORTED_MODULES.values() - for prefix in m["filename_prefixes"]] -_ALL_BLAS_LIBRARIES = [m["internal_api"] for m in _SUPPORTED_MODULES.values() - if m["user_api"] == "blas"] -_ALL_OPENMP_LIBRARIES = list( - _SUPPORTED_MODULES["_OpenMPModule"]["filename_prefixes"]) +_ALL_PREFIXES = [ + prefix for m in _SUPPORTED_MODULES.values() for prefix in m["filename_prefixes"] +] +_ALL_BLAS_LIBRARIES = [ + m["internal_api"] for m in _SUPPORTED_MODULES.values() if m["user_api"] == "blas" +] +_ALL_OPENMP_LIBRARIES = list(_SUPPORTED_MODULES["_OpenMPModule"]["filename_prefixes"]) def _format_docstring(*args, **kwargs): @@ -106,14 +107,12 @@ def decorator(o): return decorator -@lru_cache(maxsize=10000) def _realpath(filepath): """Small caching wrapper around os.path.realpath to limit system calls""" return os.path.realpath(filepath) -@_format_docstring(USER_APIS=list(_ALL_USER_APIS), - INTERNAL_APIS=_ALL_INTERNAL_APIS) +@_format_docstring(USER_APIS=list(_ALL_USER_APIS), INTERNAL_APIS=_ALL_INTERNAL_APIS) def threadpool_info(): """Return the maximal number of threads for each detected library. @@ -135,7 +134,8 @@ def threadpool_info(): @_format_docstring( USER_APIS=", ".join('"{}"'.format(api) for api in _ALL_USER_APIS), BLAS_LIBS=", ".join(_ALL_BLAS_LIBRARIES), - OPENMP_LIBS=", ".join(_ALL_OPENMP_LIBRARIES)) + OPENMP_LIBS=", ".join(_ALL_OPENMP_LIBRARIES), +) class threadpool_limits: """Change the maximal number of threads that can be used in thread pools. @@ -172,9 +172,11 @@ class threadpool_limits: - If None, this function will apply to all supported libraries. """ + def __init__(self, limits=None, user_api=None): - self._limits, self._user_api, self._prefixes = \ - self._check_params(limits, user_api) + self._limits, self._user_api, self._prefixes = self._check_params( + limits, user_api + ) self._original_info = self._set_threadpool_limits() @@ -203,8 +205,10 @@ def get_original_num_threads(self): warning_apis = [] for user_api in self._user_api: - limits = [module.num_threads for module in - original_limits.get_modules("user_api", user_api)] + limits = [ + module.num_threads + for module in original_limits.get_modules("user_api", user_api) + ] limits = set(limits) n_limits = len(limits) @@ -220,14 +224,15 @@ def get_original_num_threads(self): if warning_apis: warnings.warn( - "Multiple value possible for following user apis: " + - ", ".join(warning_apis) + ". Returning the minimum.") + "Multiple value possible for following user apis: " + + ", ".join(warning_apis) + + ". Returning the minimum." + ) return num_threads def _check_params(self, limits, user_api): - """Suitable values for the _limits, _user_api and _prefixes attributes - """ + """Suitable values for the _limits, _user_api and _prefixes attributes""" if limits is None or isinstance(limits, int): if user_api is None: user_api = _ALL_USER_APIS @@ -235,8 +240,10 @@ def _check_params(self, limits, user_api): user_api = [user_api] else: raise ValueError( - "user_api must be either in {} or None. Got " - "{} instead.".format(_ALL_USER_APIS, user_api)) + "user_api must be either in {} or None. Got {} instead.".format( + _ALL_USER_APIS, user_api + ) + ) if limits is not None: limits = {api: limits for api in user_api} @@ -245,17 +252,17 @@ def _check_params(self, limits, user_api): if isinstance(limits, list): # This should be a list of dicts of modules, for compatibility # with the result from threadpool_info. - limits = {module["prefix"]: module["num_threads"] - for module in limits} + limits = {module["prefix"]: module["num_threads"] for module in limits} elif isinstance(limits, _ThreadpoolInfo): # To set the limits from the modules of a _ThreadpoolInfo # object. - limits = {module.prefix: module.num_threads - for module in limits} + limits = {module.prefix: module.num_threads for module in limits} if not isinstance(limits, dict): - raise TypeError("limits must either be an int, a list or a " - "dict. Got {} instead".format(type(limits))) + raise TypeError( + "limits must either be an int, a list or a " + "dict. Got {} instead".format(type(limits)) + ) # With a dictionary, can set both specific limit for given modules # and global limit for user_api. Fetch each separately. @@ -273,8 +280,7 @@ def _set_threadpool_limits(self): if self._limits is None: return None - modules = _ThreadpoolInfo(prefixes=self._prefixes, - user_api=self._user_api) + modules = _ThreadpoolInfo(prefixes=self._prefixes, user_api=self._user_api) for module in modules: # self._limits is a dict {key: num_threads} where key is either # a prefix or a user_api. If a module matches both, the limit @@ -296,8 +302,9 @@ def _set_threadpool_limits(self): PREFIXES=", ".join('"{}"'.format(prefix) for prefix in _ALL_PREFIXES), USER_APIS=", ".join('"{}"'.format(api) for api in _ALL_USER_APIS), BLAS_LIBS=", ".join(_ALL_BLAS_LIBRARIES), - OPENMP_LIBS=", ".join(_ALL_OPENMP_LIBRARIES)) -class _ThreadpoolInfo(): + OPENMP_LIBS=", ".join(_ALL_OPENMP_LIBRARIES), +) +class _ThreadpoolInfo: """Collection of all supported modules that have been found Parameters @@ -327,13 +334,14 @@ class _ThreadpoolInfo(): Is is possible to select libraries both by prefixes and by user_api. All libraries matching one or the other will be selected. """ + # Cache for libc under POSIX and a few system libraries under Windows. # We use a class level cache instead of an instance level cache because # it's very unlikely that a shared library will be unloaded and reloaded # during the lifetime of a program. _system_libraries = dict() - def __init__(self, user_api=None, prefixes=None, modules=None): + def __init__(self, user_api=None, prefixes=None, modules=None): if modules is None: self.prefixes = [] if prefixes is None else prefixes self.user_api = [] if user_api is None else user_api @@ -350,8 +358,7 @@ def get_modules(self, key, values): values = list(_ALL_USER_APIS) if not isinstance(values, list): values = [values] - modules = [module for module in self.modules - if getattr(module, key) in values] + modules = [module for module in self.modules if getattr(module, key) in values] return _ThreadpoolInfo(modules=modules) def todicts(self): @@ -404,7 +411,10 @@ def match_module_callback(info, size, data): c_func_signature = ctypes.CFUNCTYPE( ctypes.c_int, # Return type - ctypes.POINTER(_dl_phdr_info), ctypes.c_size_t, ctypes.c_char_p) + ctypes.POINTER(_dl_phdr_info), + ctypes.c_size_t, + ctypes.c_char_p, + ) c_match_module_callback = c_func_signature(match_module_callback) data = ctypes.c_char_p(b"") @@ -447,8 +457,8 @@ def _find_modules_with_enum_process_module_ex(self): kernel_32 = self._get_windll("kernel32") h_process = kernel_32.OpenProcess( - PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, - False, os.getpid()) + PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, False, os.getpid() + ) if not h_process: # pragma: no cover raise OSError("Could not open PID %s" % os.getpid()) @@ -461,8 +471,12 @@ def _find_modules_with_enum_process_module_ex(self): buf = (HMODULE * buf_count)() buf_size = ctypes.sizeof(buf) if not ps_api.EnumProcessModulesEx( - h_process, ctypes.byref(buf), buf_size, - ctypes.byref(needed), LIST_MODULES_ALL): + h_process, + ctypes.byref(buf), + buf_size, + ctypes.byref(needed), + LIST_MODULES_ALL, + ): raise OSError("EnumProcessModulesEx failed") if buf_size >= needed.value: break @@ -478,8 +492,8 @@ def _find_modules_with_enum_process_module_ex(self): # Get the path of the current module if not ps_api.GetModuleFileNameExW( - h_process, h_module, ctypes.byref(buf), - ctypes.byref(n_size)): + h_process, h_module, ctypes.byref(buf), ctypes.byref(n_size) + ): raise OSError("GetModuleFileNameEx failed") filepath = buf.value @@ -500,8 +514,7 @@ def _make_module_from_path(self, filepath): # to a supported module. for module_class, candidate_module in _SUPPORTED_MODULES.items(): # check if filename matches a supported prefix - prefix = self._check_prefix(filename, - candidate_module["filename_prefixes"]) + prefix = self._check_prefix(filename, candidate_module["filename_prefixes"]) # filename does not match any of the prefixes of the candidate # module. move to next module. @@ -529,7 +542,7 @@ def _check_prefix(self, library_basename, filename_prefixes): def _warn_if_incompatible_openmp(self): """Raise a warning if llvm-OpenMP and intel-OpenMP are both loaded""" - if sys.platform != 'linux': + if sys.platform != "linux": # Only raise the warning on linux return @@ -543,8 +556,9 @@ def _warn_if_incompatible_openmp(self): Using threadpoolctl may cause crashes or deadlocks. For more information and possible workarounds, please see https://github.com/joblib/threadpoolctl/blob/master/multiple_openmp.md - """) - if 'libomp' in prefixes and 'libiomp' in prefixes: + """ + ) + if "libomp" in prefixes and "libiomp" in prefixes: warnings.warn(msg, RuntimeWarning) @classmethod @@ -571,7 +585,8 @@ def _get_windll(cls, dll_name): @_format_docstring( USER_APIS=", ".join('"{}"'.format(api) for api in _ALL_USER_APIS), - INTERNAL_APIS=", ".join('"{}"'.format(api) for api in _ALL_INTERNAL_APIS)) + INTERNAL_APIS=", ".join('"{}"'.format(api) for api in _ALL_INTERNAL_APIS), +) class _Module(ABC): """Abstract base class for the modules @@ -585,8 +600,8 @@ class _Module(ABC): In addition, each module may contain internal_api specific entries. """ - def __init__(self, filepath=None, prefix=None, user_api=None, - internal_api=None): + + def __init__(self, filepath=None, prefix=None, user_api=None, internal_api=None): self.filepath = filepath self.prefix = prefix self.user_api = user_api @@ -626,6 +641,7 @@ def _get_extra_info(self): class _OpenBLASModule(_Module): """Module class for OpenBLAS""" + def get_version(self): # None means OpenBLAS is not loaded or version < 0.3.4, since OpenBLAS # did not expose its version before that. @@ -640,13 +656,13 @@ def get_version(self): return None def get_num_threads(self): - get_func = getattr(self._dynlib, "openblas_get_num_threads", - lambda: None) + get_func = getattr(self._dynlib, "openblas_get_num_threads", lambda: None) return get_func() def set_num_threads(self, num_threads): - set_func = getattr(self._dynlib, "openblas_set_num_threads", - lambda num_threads: None) + set_func = getattr( + self._dynlib, "openblas_set_num_threads", lambda num_threads: None + ) return set_func(num_threads) def _get_extra_info(self): @@ -655,8 +671,7 @@ def _get_extra_info(self): def get_threading_layer(self): """Return the threading layer of OpenBLAS""" - openblas_get_parallel = getattr(self._dynlib, "openblas_get_parallel", - None) + openblas_get_parallel = getattr(self._dynlib, "openblas_get_parallel", None) if openblas_get_parallel is None: return "unknown" threading_layer = openblas_get_parallel() @@ -677,6 +692,7 @@ def get_architecture(self): class _BLISModule(_Module): """Module class for BLIS""" + def get_version(self): get_version_ = getattr(self._dynlib, "bli_info_get_version_str", None) if get_version_ is None: @@ -686,16 +702,16 @@ def get_version(self): return get_version_().decode("utf-8") def get_num_threads(self): - get_func = getattr(self._dynlib, "bli_thread_get_num_threads", - lambda: None) + get_func = getattr(self._dynlib, "bli_thread_get_num_threads", lambda: None) num_threads = get_func() # by default BLIS is single-threaded and get_num_threads # returns -1. We map it to 1 for consistency with other libraries. return 1 if num_threads == -1 else num_threads def set_num_threads(self, num_threads): - set_func = getattr(self._dynlib, "bli_thread_set_num_threads", - lambda num_threads: None) + set_func = getattr( + self._dynlib, "bli_thread_set_num_threads", lambda num_threads: None + ) return set_func(num_threads) def _get_extra_info(self): @@ -725,6 +741,7 @@ def get_architecture(self): class _MKLModule(_Module): """Module class for MKL""" + def get_version(self): if not hasattr(self._dynlib, "MKL_Get_Version_String"): return None @@ -743,8 +760,9 @@ def get_num_threads(self): return get_func() def set_num_threads(self, num_threads): - set_func = getattr(self._dynlib, "MKL_Set_Num_Threads", - lambda num_threads: None) + set_func = getattr( + self._dynlib, "MKL_Set_Num_Threads", lambda num_threads: None + ) return set_func(num_threads) def _get_extra_info(self): @@ -755,15 +773,23 @@ def get_threading_layer(self): # The function mkl_set_threading_layer returns the current threading # layer. Calling it with an invalid threading layer allows us to safely # get the threading layer - set_threading_layer = getattr(self._dynlib, "MKL_Set_Threading_Layer", - lambda layer: -1) - layer_map = {0: "intel", 1: "sequential", 2: "pgi", - 3: "gnu", 4: "tbb", -1: "not specified"} + set_threading_layer = getattr( + self._dynlib, "MKL_Set_Threading_Layer", lambda layer: -1 + ) + layer_map = { + 0: "intel", + 1: "sequential", + 2: "pgi", + 3: "gnu", + 4: "tbb", + -1: "not specified", + } return layer_map[set_threading_layer(-1)] class _OpenMPModule(_Module): """Module class for OpenMP""" + def get_version(self): # There is no way to get the version number programmatically in OpenMP. return None @@ -773,8 +799,9 @@ def get_num_threads(self): return get_func() def set_num_threads(self, num_threads): - set_func = getattr(self._dynlib, "omp_set_num_threads", - lambda num_threads: None) + set_func = getattr( + self._dynlib, "omp_set_num_threads", lambda num_threads: None + ) return set_func(num_threads) def _get_extra_info(self): @@ -793,13 +820,18 @@ def _main(): description="Display thread-pool information and exit.", ) parser.add_argument( - "-i", "--import", dest="modules", nargs="*", default=(), - help="Python modules to import before introspecting thread-pools." + "-i", + "--import", + dest="modules", + nargs="*", + default=(), + help="Python modules to import before introspecting thread-pools.", ) parser.add_argument( - "-c", "--command", - help="a Python statement to execute before introspecting" - " thread-pools.") + "-c", + "--command", + help="a Python statement to execute before introspecting thread-pools.", + ) options = parser.parse_args(sys.argv[1:]) for module in options.modules: From 3f95895a13c773ed2d7f952edf98a319b043a384 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Fri, 24 Sep 2021 10:35:07 +0200 Subject: [PATCH 115/187] put back the decorator (#100) --- threadpoolctl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/threadpoolctl.py b/threadpoolctl.py index a8c1c5f8..f39f4115 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -107,6 +107,7 @@ def decorator(o): return decorator +@lru_cache(maxsize=10000) def _realpath(filepath): """Small caching wrapper around os.path.realpath to limit system calls""" return os.path.realpath(filepath) From 63a22f93b9d4d2e078c2a1f262105c48284431b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Fri, 24 Sep 2021 10:56:20 +0200 Subject: [PATCH 116/187] MNT use fstrings when possible (#98) --- benchmarks/bench_context_manager_overhead.py | 6 +----- tests/test_threadpoolctl.py | 6 +++--- threadpoolctl.py | 15 +++++++-------- 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/benchmarks/bench_context_manager_overhead.py b/benchmarks/bench_context_manager_overhead.py index c96da9aa..d3b69c15 100644 --- a/benchmarks/bench_context_manager_overhead.py +++ b/benchmarks/bench_context_manager_overhead.py @@ -27,8 +27,4 @@ pass timings.append(time.time() - t) -print( - "Overhead per call: {:.3f} +/-{:.3f} ms".format( - mean(timings) * 1e3, stdev(timings) * 1e3 - ) -) +print(f"Overhead per call: {mean(timings) * 1e3:.3f} +/-{stdev(timings) * 1e3:.3f} ms") diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index 3fed30c4..c4f5e18c 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -70,7 +70,7 @@ def test_threadpool_limits_by_prefix(prefix, limit): modules_matching_prefix = original_info.get_modules("prefix", prefix) if not modules_matching_prefix: - pytest.skip("Requires {} runtime".format(prefix)) + pytest.skip(f"Requires {prefix} runtime") with threadpool_limits(limits={prefix: limit}): for module in modules_matching_prefix: @@ -91,7 +91,7 @@ def test_set_threadpool_limits_by_api(user_api, limit): modules_matching_api = original_info.get_modules("user_api", user_api) if not modules_matching_api: user_apis = _ALL_USER_APIS if user_api is None else [user_api] - pytest.skip("Requires a library which api is in {}".format(user_apis)) + pytest.skip(f"Requires a library which api is in {user_apis}") with threadpool_limits(limits=limit, user_api=user_api): for module in modules_matching_api: @@ -154,7 +154,7 @@ def test_threadpool_limits_manual_unregister(): def test_threadpool_limits_bad_input(): # Check that appropriate errors are raised for invalid arguments - match = re.escape("user_api must be either in {} or None.".format(_ALL_USER_APIS)) + match = re.escape(f"user_api must be either in {_ALL_USER_APIS} or None.") with pytest.raises(ValueError, match=match): threadpool_limits(limits=1, user_api="wrong") diff --git a/threadpoolctl.py b/threadpoolctl.py index f39f4115..205db9e8 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -133,7 +133,7 @@ def threadpool_info(): @_format_docstring( - USER_APIS=", ".join('"{}"'.format(api) for api in _ALL_USER_APIS), + USER_APIS=", ".join(f'"{api}"' for api in _ALL_USER_APIS), BLAS_LIBS=", ".join(_ALL_BLAS_LIBRARIES), OPENMP_LIBS=", ".join(_ALL_OPENMP_LIBRARIES), ) @@ -241,9 +241,8 @@ def _check_params(self, limits, user_api): user_api = [user_api] else: raise ValueError( - "user_api must be either in {} or None. Got {} instead.".format( - _ALL_USER_APIS, user_api - ) + f"user_api must be either in {_ALL_USER_APIS} or None. Got " + f"{user_api} instead." ) if limits is not None: @@ -262,7 +261,7 @@ def _check_params(self, limits, user_api): if not isinstance(limits, dict): raise TypeError( "limits must either be an int, a list or a " - "dict. Got {} instead".format(type(limits)) + f"dict. Got {type(limits)} instead." ) # With a dictionary, can set both specific limit for given modules @@ -300,8 +299,8 @@ def _set_threadpool_limits(self): # The public API (i.e. the "threadpool_info" function) only exposes the # "list of dicts" representation returned by the .todicts method. @_format_docstring( - PREFIXES=", ".join('"{}"'.format(prefix) for prefix in _ALL_PREFIXES), - USER_APIS=", ".join('"{}"'.format(api) for api in _ALL_USER_APIS), + PREFIXES=", ".join(f'"{prefix}"' for prefix in _ALL_PREFIXES), + USER_APIS=", ".join(f'"{api}"' for api in _ALL_USER_APIS), BLAS_LIBS=", ".join(_ALL_BLAS_LIBRARIES), OPENMP_LIBS=", ".join(_ALL_OPENMP_LIBRARIES), ) @@ -579,7 +578,7 @@ def _get_windll(cls, dll_name): """Load a windows DLL""" dll = cls._system_libraries.get(dll_name) if dll is None: - dll = ctypes.WinDLL("{}.dll".format(dll_name)) + dll = ctypes.WinDLL(f"{dll_name}.dll") cls._system_libraries[dll_name] = dll return dll From 0ef9667c06bd000ac6c8a30e422921708e3d8082 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Fri, 24 Sep 2021 11:58:50 +0200 Subject: [PATCH 117/187] [MRG] FEA Make it possible to pass pre-inspected modules to threadpool_limit (#95) --- CHANGES.md | 9 +- README.md | 59 +- continuous_integration/install.sh | 4 +- continuous_integration/install_with_blis.sh | 4 +- .../nested_prange_blas.pyx | 8 +- tests/test_threadpoolctl.py | 274 ++++++---- threadpoolctl.py | 512 ++++++++++-------- 7 files changed, 520 insertions(+), 350 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 0d34112b..8219ddd5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,13 @@ -2.3.0 (in development) +3.0.0 (in development) ====================== +- New object `threadpooctl.ThreadpoolController` which holds controllers for all the + supported native libraries. The states of these libraries is accessible through the + `info` method (equivalent to `threadpoolctl.threadpool_info()`) and their number of + threads can be limited with the `limit` method which can be used as a context + manager (equivalent to `threadpoolctl.threadpool_limits()`). This is especially useful + to avoid searching through all loaded shared libraries each time. + - Fixed an attribute error when using old versions of OpenBLAS or BLIS that are missing version query functions. https://github.com/joblib/threadpoolctl/pull/88 diff --git a/README.md b/README.md index 5a40d799..624c4922 100644 --- a/README.md +++ b/README.md @@ -113,10 +113,31 @@ that are loaded when importing Python packages: 'version': None}] ``` -In the above example, `numpy` was installed from the default anaconda channel and -comes with the MKL and its Intel OpenMP (`libiomp5`) implementation while -`xgboost` was installed from pypi.org and links against GNU OpenMP (`libgomp`) -so both OpenMP runtimes are loaded in the same Python program. +In the above example, `numpy` was installed from the default anaconda channel and comes +with MKL and its Intel OpenMP (`libiomp5`) implementation while `xgboost` was installed +from pypi.org and links against GNU OpenMP (`libgomp`) so both OpenMP runtimes are +loaded in the same Python program. + +The state of these libraries is also accessible through the object oriented API: + +```python +>>> from threadpoolctl import ThreadpoolController, threadpool_info +>>> from pprint import pprint +>>> import numpy +>>> controller = ThreadpoolController() +>>> pprint(controller.info()) +[{'architecture': 'Haswell', + 'filepath': '/home/jeremie/miniconda/envs/dev/lib/libopenblasp-r0.3.17.so', + 'internal_api': 'openblas', + 'num_threads': 4, + 'prefix': 'libopenblas', + 'threading_layer': 'pthreads', + 'user_api': 'blas', + 'version': '0.3.17'}] + +>>> controller.info() == threadpool_info() +True +``` ### Setting the Maximum Size of Thread-Pools @@ -124,16 +145,30 @@ Control the number of threads used by the underlying runtime libraries in specific sections of your Python program: ```python -from threadpoolctl import threadpool_limits -import numpy as np +>>> from threadpoolctl import threadpool_limits +>>> import numpy as np + +>>> with threadpool_limits(limits=1, user_api='blas'): +... # In this block, calls to blas implementation (like openblas or MKL) +... # will be limited to use only one thread. They can thus be used jointly +... # with thread-parallelism. +... a = np.random.randn(1000, 1000) +... a_squared = a @ a +``` + +The threadpools can also be controlled via the object oriented API, which is especially +useful to avoid searching through all the loaded shared libraries each time. It will +however not act on libraries loaded after the instanciation of the +``ThreadpoolController``: +```python +>>> from threadpoolctl import ThreadpoolController +>>> import numpy as np +>>> controller = ThreadpoolController() -with threadpool_limits(limits=1, user_api='blas'): - # In this block, calls to blas implementation (like openblas or MKL) - # will be limited to use only one thread. They can thus be used jointly - # with thread-parallelism. - a = np.random.randn(1000, 1000) - a_squared = a @ a +>>> with controller.limit(limits=1, user_api='blas'): +... a = np.random.randn(1000, 1000) +... a_squared = a @ a ``` ### Known Limitations diff --git a/continuous_integration/install.sh b/continuous_integration/install.sh index fb643256..8b20fef0 100755 --- a/continuous_integration/install.sh +++ b/continuous_integration/install.sh @@ -76,7 +76,7 @@ python -m pip install -q -r dev-requirements.txt bash ./continuous_integration/build_test_ext.sh python --version -python -c "import numpy; print('numpy %s' % numpy.__version__)" || echo "no numpy" -python -c "import scipy; print('scipy %s' % scipy.__version__)" || echo "no scipy" +python -c "import numpy; print(f'numpy {numpy.__version__}')" || echo "no numpy" +python -c "import scipy; print(f'scipy {scipy.__version__}')" || echo "no scipy" python -m flit install --symlink diff --git a/continuous_integration/install_with_blis.sh b/continuous_integration/install_with_blis.sh index 4a10d95c..a2d6c962 100755 --- a/continuous_integration/install_with_blis.sh +++ b/continuous_integration/install_with_blis.sh @@ -51,7 +51,7 @@ CFLAGS=-I$ABS_PATH/BLIS_install/include/blis LDFLAGS=-L$ABS_PATH/BLIS_install/li bash ./continuous_integration/build_test_ext.sh python --version -python -c "import numpy; print('numpy %s' % numpy.__version__)" || echo "no numpy" -python -c "import scipy; print('scipy %s' % scipy.__version__)" || echo "no scipy" +python -c "import numpy; print(f'numpy {numpy.__version__}')" || echo "no numpy" +python -c "import scipy; print(f'scipy {scipy.__version__}')" || echo "no scipy" python -m flit install --symlink diff --git a/tests/_openmp_test_helper/nested_prange_blas.pyx b/tests/_openmp_test_helper/nested_prange_blas.pyx index e327eee0..aec7f815 100644 --- a/tests/_openmp_test_helper/nested_prange_blas.pyx +++ b/tests/_openmp_test_helper/nested_prange_blas.pyx @@ -20,7 +20,7 @@ IF USE_BLIS: ELSE: from scipy.linalg.cython_blas cimport dgemm -from threadpoolctl import _ThreadpoolInfo, _ALL_USER_APIS +from threadpoolctl import ThreadpoolController def check_nested_prange_blas(double[:, ::1] A, double[:, ::1] B, int nthreads): @@ -43,12 +43,12 @@ def check_nested_prange_blas(double[:, ::1] A, double[:, ::1] B, int nthreads): int prange_num_threads int *prange_num_threads_ptr = &prange_num_threads - inner_info = [None] + inner_controller = [None] with nogil, parallel(num_threads=nthreads): if openmp.omp_get_thread_num() == 0: with gil: - inner_info[0] = _ThreadpoolInfo(user_api=_ALL_USER_APIS) + inner_controller[0] = ThreadpoolController() prange_num_threads_ptr[0] = openmp.omp_get_num_threads() @@ -62,4 +62,4 @@ def check_nested_prange_blas(double[:, ::1] A, double[:, ::1] B, int nthreads): &alpha, &B[0, 0], &k, &A[i * chunk_size, 0], &k, &beta, &C[i * chunk_size, 0], &n) - return np.asarray(C), prange_num_threads, inner_info[0] + return np.asarray(C), prange_num_threads, inner_controller[0] diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index c4f5e18c..cc7fe06a 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -5,7 +5,8 @@ import subprocess import sys -from threadpoolctl import threadpool_limits, threadpool_info, _ThreadpoolInfo +from threadpoolctl import threadpool_limits, threadpool_info +from threadpoolctl import ThreadpoolController from threadpoolctl import _ALL_PREFIXES, _ALL_USER_APIS from .utils import cython_extensions_compiled @@ -14,10 +15,10 @@ from .utils import threadpool_info_from_subprocess -def is_old_openblas(module): +def is_old_openblas(lib_controller): # Possible bug in getting maximum number of threads with OpenBLAS < 0.2.16 # and OpenBLAS does not expose its version before 0.3.4. - return module.internal_api == "openblas" and module.version is None + return lib_controller.internal_api == "openblas" and lib_controller.version is None def effective_num_threads(nthreads, max_threads): @@ -26,130 +27,191 @@ def effective_num_threads(nthreads, max_threads): return nthreads -def _threadpool_info(): - # Like threadpool_info but return the object instead of the list of dicts - return _ThreadpoolInfo(user_api=_ALL_USER_APIS) +def test_threadpool_info(): + # Check consistency between threadpool_info and ThreadpoolController + function_info = threadpool_info() + object_info = ThreadpoolController().lib_controllers + for lib_info, lib_controller in zip(function_info, object_info): + assert lib_info == lib_controller.info() -def test_threadpool_limits_public_api(): - # Check consistency between threadpool_info and _ThreadpoolInfo - public_info = threadpool_info() - private_info = _threadpool_info() - for module1, module2 in zip(public_info, private_info): - assert module1 == module2.todict() +def test_threadpool_controller_info(): + # Check that all keys expected for the private api are in the dicts + # returned by the `info` methods + controller = ThreadpoolController() + assert threadpool_info() == [ + lib_controller.info() for lib_controller in controller.lib_controllers + ] + assert controller.info() == [ + lib_controller.info() for lib_controller in controller.lib_controllers + ] -def test_ThreadpoolInfo_todicts(): - # Check all keys expected for the public api are in the dicts returned by - # the .todict(s) methods - info = _threadpool_info() + for lib_controller_dict in controller.info(): + assert "user_api" in lib_controller_dict + assert "internal_api" in lib_controller_dict + assert "prefix" in lib_controller_dict + assert "filepath" in lib_controller_dict + assert "version" in lib_controller_dict + assert "num_threads" in lib_controller_dict - assert threadpool_info() == [module.todict() for module in info.modules] - assert info.todicts() == [module.todict() for module in info] - assert info.todicts() == [module.todict() for module in info.modules] + if lib_controller_dict["internal_api"] in ("mkl", "blis", "openblas"): + assert "threading_layer" in lib_controller_dict - for module in info: - module_dict = module.todict() - assert "user_api" in module_dict - assert "internal_api" in module_dict - assert "prefix" in module_dict - assert "filepath" in module_dict - assert "version" in module_dict - assert "num_threads" in module_dict - if module.internal_api in ("mkl", "blis", "openblas"): - assert "threading_layer" in module_dict +@pytest.mark.parametrize( + "kwargs", + [ + {"user_api": "blas"}, + {"prefix": "libgomp"}, + {"internal_api": "openblas", "prefix": "libomp"}, + {"prefix": ["libgomp", "libomp", "libiomp"]}, + ], +) +def test_threadpool_controller_select(kwargs): + # Check the behior of the select method of ThreadpoolController + controller = ThreadpoolController().select(**kwargs) + if not controller: + pytest.skip(f"Requires at least one of {list(kwargs.values())}.") + + for lib_controller in controller.lib_controllers: + assert any( + getattr(lib_controller, key) in (val if isinstance(val, list) else [val]) + for key, val in kwargs.items() + ) @pytest.mark.parametrize("prefix", _ALL_PREFIXES) @pytest.mark.parametrize("limit", [1, 3]) def test_threadpool_limits_by_prefix(prefix, limit): # Check that the maximum number of threads can be set by prefix - original_info = _threadpool_info() + original_controller = ThreadpoolController() - modules_matching_prefix = original_info.get_modules("prefix", prefix) - if not modules_matching_prefix: + controller_matching_prefix = original_controller.select(prefix=prefix) + if not controller_matching_prefix: pytest.skip(f"Requires {prefix} runtime") with threadpool_limits(limits={prefix: limit}): - for module in modules_matching_prefix: - if is_old_openblas(module): + for lib_controller in controller_matching_prefix.lib_controllers: + if is_old_openblas(lib_controller): continue # threadpool_limits only sets an upper bound on the number of # threads. - assert 0 < module.get_num_threads() <= limit - assert _threadpool_info() == original_info + assert 0 < lib_controller.get_num_threads() <= limit + assert ThreadpoolController().info() == original_controller.info() @pytest.mark.parametrize("user_api", (None, "blas", "openmp")) @pytest.mark.parametrize("limit", [1, 3]) def test_set_threadpool_limits_by_api(user_api, limit): # Check that the maximum number of threads can be set by user_api - original_info = _threadpool_info() + original_controller = ThreadpoolController() - modules_matching_api = original_info.get_modules("user_api", user_api) - if not modules_matching_api: + if user_api is None: + controller_matching_api = original_controller + else: + controller_matching_api = original_controller.select(user_api=user_api) + if not controller_matching_api: user_apis = _ALL_USER_APIS if user_api is None else [user_api] pytest.skip(f"Requires a library which api is in {user_apis}") with threadpool_limits(limits=limit, user_api=user_api): - for module in modules_matching_api: - if is_old_openblas(module): + for lib_controller in controller_matching_api.lib_controllers: + if is_old_openblas(lib_controller): continue # threadpool_limits only sets an upper bound on the number of # threads. - assert 0 < module.get_num_threads() <= limit + assert 0 < lib_controller.get_num_threads() <= limit - assert _threadpool_info() == original_info + assert ThreadpoolController().info() == original_controller.info() def test_threadpool_limits_function_with_side_effect(): # Check that threadpool_limits can be used as a function with # side effects instead of a context manager. - original_info = _threadpool_info() + original_controller = ThreadpoolController() threadpool_limits(limits=1) try: - for module in _threadpool_info(): - if is_old_openblas(module): + for lib_controller in ThreadpoolController().lib_controllers: + if is_old_openblas(lib_controller): continue - assert module.num_threads == 1 + assert lib_controller.num_threads == 1 finally: # Restore the original limits so that this test does not have any # side-effect. - threadpool_limits(limits=original_info) + threadpool_limits(limits=original_controller) - assert _threadpool_info() == original_info + assert ThreadpoolController().info() == original_controller.info() def test_set_threadpool_limits_no_limit(): # Check that limits=None does nothing. - original_info = _threadpool_info() + original_controller = ThreadpoolController() with threadpool_limits(limits=None): - assert _threadpool_info() == original_info + assert ThreadpoolController().info() == original_controller.info() - assert _threadpool_info() == original_info + assert ThreadpoolController().info() == original_controller.info() def test_threadpool_limits_manual_unregister(): # Check that threadpool_limits can be used as an object which holds the # original state of the threadpools and that can be restored thanks to the # dedicated unregister method - original_info = _threadpool_info() + original_controller = ThreadpoolController() limits = threadpool_limits(limits=1) try: - for module in _threadpool_info(): - if is_old_openblas(module): + for lib_controller in ThreadpoolController().lib_controllers: + if is_old_openblas(lib_controller): continue - assert module.num_threads == 1 + assert lib_controller.num_threads == 1 finally: # Restore the original limits so that this test does not have any # side-effect. limits.unregister() - assert _threadpool_info() == original_info + assert ThreadpoolController().info() == original_controller.info() + + +def test_threadpool_controller_limit(): + # Check that using the limit method of ThreadpoolController only impact its + # library controllers. + original_blas_controller = ThreadpoolController().select(user_api="blas") + original_openmp_controller = ThreadpoolController().select(user_api="openmp") + + with original_blas_controller.limit(limits=1): + blas_controller = ThreadpoolController().select(user_api="blas") + openmp_controller = ThreadpoolController().select(user_api="openmp") + + assert all( + lib_controller.num_threads == 1 + for lib_controller in blas_controller.lib_controllers + ) + # original_blas_controller contains only blas libraries so no opemp library + # should be impacted. + assert openmp_controller.info() == original_openmp_controller.info() + + +def test_threadpool_controller_restore(): + # Check that the restore_limits method of ThreadpoolController is able to set the + # limits back to their original values. Similar to + # test_threadpool_limits_function_with_side_effect but with the object api + controller = ThreadpoolController() + + controller.limit(limits=1) + try: + for lib_controller in ThreadpoolController().lib_controllers: + if is_old_openblas(lib_controller): + continue + assert lib_controller.num_threads == 1 + finally: + # Restore the original limits so that this test does not have any + # side-effect. + controller.restore_limits() + + assert ThreadpoolController().info() == controller.info() def test_threadpool_limits_bad_input(): @@ -211,11 +273,11 @@ def test_openmp_nesting(nthreads_outer): # There are 2 openmp, the one from inner and the one from outer. assert len(outer_info) == 2 # We already know the one from inner. It has to be the other one. - prefixes = {module["prefix"] for module in outer_info} + prefixes = {lib_info["prefix"] for lib_info in outer_info} outer_omp = prefixes - {inner_omp} outer_num_threads, inner_num_threads = check_nested_openmp_loops(10) - original_info = _threadpool_info() + original_controller = ThreadpoolController() if inner_omp == outer_omp: # The OpenMP runtime should be shared by default, meaning that the @@ -232,7 +294,7 @@ def test_openmp_nesting(nthreads_outer): # The state of the original state of all threadpools should have been # restored. - assert _threadpool_info() == original_info + assert ThreadpoolController().info() == original_controller.info() # The number of threads available in the outer loop should not have been # decreased: @@ -245,7 +307,7 @@ def test_openmp_nesting(nthreads_outer): # XXX: this does not always work when nesting independent openmp # implementations. See: https://github.com/jeremiedbb/Nested_OpenMP pytest.xfail( - "Inner OpenMP num threads was %d instead of 1" % inner_num_threads + f"Inner OpenMP num threads was {inner_num_threads} instead of 1" ) assert inner_num_threads == 1 @@ -253,15 +315,15 @@ def test_openmp_nesting(nthreads_outer): def test_shipped_openblas(): # checks that OpenBLAS effectively uses the number of threads requested by # the context manager - original_info = _threadpool_info() + original_controller = ThreadpoolController() - openblas_modules = original_info.get_modules("internal_api", "openblas") + openblas_controllers = original_controller.select(internal_api="openblas") with threadpool_limits(1): - for module in openblas_modules: - assert module.get_num_threads() == 1 + for lib_controller in openblas_controllers.lib_controllers: + assert lib_controller.get_num_threads() == 1 - assert original_info == _threadpool_info() + assert original_controller.info() == ThreadpoolController().info() @pytest.mark.skipif( @@ -288,17 +350,20 @@ def test_nested_prange_blas(nthreads_outer): check_nested_prange_blas = prange_blas.check_nested_prange_blas - original_info = _threadpool_info() + original_controller = ThreadpoolController() - blas_info = original_info.get_modules("user_api", "blas") - blis_info = original_info.get_modules("internal_api", "blis") + blas_controllers = original_controller.select(user_api="blas") + blis_controllers = original_controller.select(internal_api="blis") # skip if the BLAS used by numpy is an old openblas. OpenBLAS 0.3.3 and # older are known to cause an unrecoverable deadlock at process shutdown # time (after pytest has exited). # numpy can be linked to BLIS for CBLAS and OpenBLAS for LAPACK. In that # case this test will run BLIS gemm so no need to skip. - if not blis_info and any(is_old_openblas(module) for module in blas_info): + if not blis_controllers and any( + is_old_openblas(lib_controller) + for lib_controller in blas_controllers.lib_controllers + ): pytest.skip("Old OpenBLAS: skipping test to avoid deadlock") A = np.ones((1000, 10)) @@ -309,17 +374,19 @@ def test_nested_prange_blas(nthreads_outer): nthreads = effective_num_threads(nthreads_outer, max_threads) result = check_nested_prange_blas(A, B, nthreads) - C, prange_num_threads, inner_info = result + C, prange_num_threads, inner_controller = result assert np.allclose(C, np.dot(A, B.T)) assert prange_num_threads == nthreads - nested_blas_info = inner_info.get_modules("user_api", "blas") - assert len(nested_blas_info) == len(blas_info) - for module in nested_blas_info: - assert module.num_threads == 1 + nested_blas_controllers = inner_controller.select(user_api="blas") + assert len(nested_blas_controllers.lib_controllers) == len( + blas_controllers.lib_controllers + ) + for lib_controller in nested_blas_controllers.lib_controllers: + assert lib_controller.num_threads == 1 - assert original_info == _threadpool_info() + assert original_controller.info() == ThreadpoolController().info() # the method `get_original_num_threads` raises a UserWarning due to different @@ -330,20 +397,23 @@ def test_nested_prange_blas(nthreads_outer): @pytest.mark.parametrize("limit", [1, None]) def test_get_original_num_threads(limit): # Tests the method get_original_num_threads of the context manager - with threadpool_limits(limits=2, user_api="blas") as ctl: + with threadpool_limits(limits=2, user_api="blas") as ctx: # set different blas num threads to start with (when multiple openblas) - if ctl._original_info: - ctl._original_info.modules[0].set_num_threads(1) + if ctx._controller: + ctx._controller.lib_controllers[0].set_num_threads(1) - original_info = _threadpool_info() + original_controller = ThreadpoolController() with threadpool_limits(limits=limit, user_api="blas") as threadpoolctx: original_num_threads = threadpoolctx.get_original_num_threads() assert "openmp" not in original_num_threads - blas_info = original_info.get_modules("user_api", "blas") - if blas_info: - expected = min(module.num_threads for module in blas_info) + blas_controller = original_controller.select(user_api="blas") + if blas_controller: + expected = min( + lib_controller.num_threads + for lib_controller in blas_controller.lib_controllers + ) assert original_num_threads["blas"] == expected else: assert original_num_threads["blas"] is None @@ -356,30 +426,30 @@ def test_get_original_num_threads(limit): def test_mkl_threading_layer(): # Check that threadpool_info correctly recovers the threading layer used # by mkl - mkl_info = _threadpool_info().get_modules("internal_api", "mkl") + mkl_controller = ThreadpoolController().select(internal_api="mkl") expected_layer = os.getenv("MKL_THREADING_LAYER") - if not (mkl_info and expected_layer): + if not (mkl_controller and expected_layer): pytest.skip("requires MKL and the environment variable MKL_THREADING_LAYER set") - actual_layer = mkl_info.modules[0].threading_layer + actual_layer = mkl_controller.lib_controllers[0].threading_layer assert actual_layer == expected_layer.lower() def test_blis_threading_layer(): # Check that threadpool_info correctly recovers the threading layer used # by blis - blis_info = _threadpool_info().get_modules("internal_api", "blis") + blis_controller = ThreadpoolController().select(internal_api="blis") expected_layer = os.getenv("BLIS_ENABLE_THREADING") if expected_layer == "no": expected_layer = "disabled" - if not (blis_info and expected_layer): + if not (blis_controller and expected_layer): pytest.skip( "requires BLIS and the environment variable BLIS_ENABLE_THREADING set" ) - actual_layer = blis_info.modules[0].threading_layer + actual_layer = blis_controller.lib_controllers[0].threading_layer assert actual_layer == expected_layer @@ -395,8 +465,8 @@ def test_libomp_libiomp_warning(recwarn): # Check that a warning is raised when both libomp and libiomp are loaded # It should happen in one CI job (pylatest_conda_mkl_clang_gcc). - info = _threadpool_info() - prefixes = [module.prefix for module in info] + controller = ThreadpoolController() + prefixes = [lib_controller.prefix for lib_controller in controller.lib_controllers] if not ("libomp" in prefixes and "libiomp" in prefixes and sys.platform == "linux"): pytest.skip("Requires both libomp and libiomp loaded, on Linux") @@ -422,8 +492,8 @@ def test_command_line_command_flag(): cli_info = json.loads(output.decode("utf-8")) this_process_info = threadpool_info() - for module in cli_info: - assert module in this_process_info + for lib_info in cli_info: + assert lib_info in this_process_info @pytest.mark.skipif( @@ -448,8 +518,8 @@ def test_command_line_import_flag(): cli_info = json.loads(result.stdout) this_process_info = threadpool_info() - for module in cli_info: - assert module in this_process_info + for lib_info in cli_info: + assert lib_info in this_process_info warnings = [w.strip() for w in result.stderr.splitlines()] assert "WARNING: could not import invalid_package" in warnings @@ -473,11 +543,11 @@ def test_architecture(): "skx", "haswell", ) - for module in threadpool_info(): - if module["internal_api"] == "openblas": - assert module["architecture"] in expected_openblas_architectures - elif module["internal_api"] == "blis": - assert module["architecture"] in expected_blis_architectures + for lib_info in threadpool_info(): + if lib_info["internal_api"] == "openblas": + assert lib_info["architecture"] in expected_openblas_architectures + elif lib_info["internal_api"] == "blis": + assert lib_info["architecture"] in expected_blis_architectures else: - # Not supported for other modules - assert "architecture" not in module + # Not supported for other libraries + assert "architecture" not in lib_info diff --git a/threadpoolctl.py b/threadpoolctl.py index 205db9e8..a7e368c3 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -21,7 +21,7 @@ from functools import lru_cache __version__ = "2.3.0.dev0" -__all__ = ["threadpool_limits", "threadpool_info"] +__all__ = ["threadpool_limits", "threadpool_info", "ThreadpoolController"] # One can get runtime errors or even segfaults due to multiple OpenMP libraries @@ -60,26 +60,26 @@ class _dl_phdr_info(ctypes.Structure): # List of the supported libraries. The items are indexed by the name of the -# class to instanciate to create the module objects. The items hold the -# possible prefixes of loaded shared objects, the name of the internal_api to -# call and the name of the user_api. -_SUPPORTED_MODULES = { - "_OpenMPModule": { +# class to instanciate to create the library controller objects. The items hold +# the possible prefixes of loaded shared objects, the name of the internal_api +# to call and the name of the user_api. +_SUPPORTED_LIBRARIES = { + "OpenMPController": { "user_api": "openmp", "internal_api": "openmp", "filename_prefixes": ("libiomp", "libgomp", "libomp", "vcomp"), }, - "_OpenBLASModule": { + "OpenBLASController": { "user_api": "blas", "internal_api": "openblas", "filename_prefixes": ("libopenblas",), }, - "_MKLModule": { + "MKLController": { "user_api": "blas", "internal_api": "mkl", "filename_prefixes": ("libmkl_rt", "mkl_rt"), }, - "_BLISModule": { + "BLISController": { "user_api": "blas", "internal_api": "blis", "filename_prefixes": ("libblis",), @@ -87,15 +87,21 @@ class _dl_phdr_info(ctypes.Structure): } # Helpers for the doc and test names -_ALL_USER_APIS = list(set(m["user_api"] for m in _SUPPORTED_MODULES.values())) -_ALL_INTERNAL_APIS = [m["internal_api"] for m in _SUPPORTED_MODULES.values()] +_ALL_USER_APIS = list(set(lib["user_api"] for lib in _SUPPORTED_LIBRARIES.values())) +_ALL_INTERNAL_APIS = [lib["internal_api"] for lib in _SUPPORTED_LIBRARIES.values()] _ALL_PREFIXES = [ - prefix for m in _SUPPORTED_MODULES.values() for prefix in m["filename_prefixes"] + prefix + for lib in _SUPPORTED_LIBRARIES.values() + for prefix in lib["filename_prefixes"] ] _ALL_BLAS_LIBRARIES = [ - m["internal_api"] for m in _SUPPORTED_MODULES.values() if m["user_api"] == "blas" + lib["internal_api"] + for lib in _SUPPORTED_LIBRARIES.values() + if lib["user_api"] == "blas" ] -_ALL_OPENMP_LIBRARIES = list(_SUPPORTED_MODULES["_OpenMPModule"]["filename_prefixes"]) +_ALL_OPENMP_LIBRARIES = list( + _SUPPORTED_LIBRARIES["OpenMPController"]["filename_prefixes"] +) def _format_docstring(*args, **kwargs): @@ -117,19 +123,19 @@ def _realpath(filepath): def threadpool_info(): """Return the maximal number of threads for each detected library. - Return a list with all the supported modules that have been found. Each - module is represented by a dict with the following information: + Return a list with all the supported libraries that have been found. Each + library is represented by a dict with the following information: - "user_api" : user API. Possible values are {USER_APIS}. - "internal_api": internal API. Possible values are {INTERNAL_APIS}. - "prefix" : filename prefix of the specific implementation. - - "filepath": path to the loaded module. + - "filepath": path to the loaded library. - "version": version of the library (if available). - "num_threads": the current thread limit. - In addition, each module may contain internal_api specific entries. + In addition, each library may contain internal_api specific entries. """ - return _ThreadpoolInfo(user_api=_ALL_USER_APIS).todicts() + return ThreadpoolController().info() @_format_docstring( @@ -137,12 +143,13 @@ def threadpool_info(): BLAS_LIBS=", ".join(_ALL_BLAS_LIBRARIES), OPENMP_LIBS=", ".join(_ALL_OPENMP_LIBRARIES), ) -class threadpool_limits: +def threadpool_limits(limits=None, user_api=None): """Change the maximal number of threads that can be used in thread pools. - This class can be used either as a function (the construction of this - object limits the number of threads) or as a context manager, in a `with` - block. + This function returns an object that can be used either as a callable (the + construction of this object limits the number of threads) or as a context manager, + in a `with` block to automatically restore the original state of the controlled + libraries when exiting the block. Set the maximal number of threads that can be used in thread pools used in the supported libraries to `limit`. This function works for libraries that @@ -173,13 +180,23 @@ class threadpool_limits: - If None, this function will apply to all supported libraries. """ + return ThreadpoolController().limit(limits=limits, user_api=user_api) + + +class _threadpool_limits: + """The guts of ThreadpoolController.limit + + Refer to the docstring of ThreadpoolController.limit for more details. - def __init__(self, limits=None, user_api=None): + It will only act on the library controllers held by the provided `controller`. + """ + + def __init__(self, controller, *, limits=None, user_api=None): self._limits, self._user_api, self._prefixes = self._check_params( limits, user_api ) - - self._original_info = self._set_threadpool_limits() + self._controller = controller + self._set_threadpool_limits() def __enter__(self): return self @@ -188,27 +205,25 @@ def __exit__(self, type, value, traceback): self.unregister() def unregister(self): - if self._original_info is not None: - for module in self._original_info: - module.set_num_threads(module.num_threads) + for lib_controller in self._controller.lib_controllers: + # Since we never call get_num_threads after instanciation of + # ThreadpoolController, num_threads holds the original value. + lib_controller.set_num_threads(lib_controller.num_threads) def get_original_num_threads(self): """Original num_threads from before calling threadpool_limits Return a dict `{user_api: num_threads}`. """ - if self._original_info is not None: - original_limits = self._original_info - else: - original_limits = _ThreadpoolInfo(user_api=self._user_api) - num_threads = {} warning_apis = [] for user_api in self._user_api: limits = [ - module.num_threads - for module in original_limits.get_modules("user_api", user_api) + lib_controller.num_threads + for lib_controller in self._controller.select( + user_api=user_api + ).lib_controllers ] limits = set(limits) n_limits = len(limits) @@ -250,22 +265,27 @@ def _check_params(self, limits, user_api): prefixes = [] else: if isinstance(limits, list): - # This should be a list of dicts of modules, for compatibility - # with the result from threadpool_info. - limits = {module["prefix"]: module["num_threads"] for module in limits} - elif isinstance(limits, _ThreadpoolInfo): - # To set the limits from the modules of a _ThreadpoolInfo - # object. - limits = {module.prefix: module.num_threads for module in limits} + # This should be a list of dicts of library info, for + # compatibility with the result from threadpool_info. + limits = { + lib_info["prefix"]: lib_info["num_threads"] for lib_info in limits + } + elif isinstance(limits, ThreadpoolController): + # To set the limits from the library controllers of a + # ThreadpoolController object. + limits = { + lib_controller.prefix: lib_controller.num_threads + for lib_controller in limits.lib_controllers + } if not isinstance(limits, dict): raise TypeError( "limits must either be an int, a list or a " - f"dict. Got {type(limits)} instead." + f"dict. Got {type(limits)} instead" ) - # With a dictionary, can set both specific limit for given modules - # and global limit for user_api. Fetch each separately. + # With a dictionary, can set both specific limit for given + # libraries and global limit for user_api. Fetch each separately. prefixes = [prefix for prefix in limits if prefix in _ALL_PREFIXES] user_api = [api for api in limits if api in _ALL_USER_APIS] @@ -274,65 +294,40 @@ def _check_params(self, limits, user_api): def _set_threadpool_limits(self): """Change the maximal number of threads in selected thread pools. - Return a list with all the supported modules that have been found + Return a list with all the supported libraries that have been found matching `self._prefixes` and `self._user_api`. """ if self._limits is None: return None - modules = _ThreadpoolInfo(prefixes=self._prefixes, user_api=self._user_api) - for module in modules: + for lib_controller in self._controller.lib_controllers: # self._limits is a dict {key: num_threads} where key is either - # a prefix or a user_api. If a module matches both, the limit - # corresponding to the prefix is chosed. - if module.prefix in self._limits: - num_threads = self._limits[module.prefix] + # a prefix or a user_api. If a library matches both, the limit + # corresponding to the prefix is chosen. + if lib_controller.prefix in self._limits: + num_threads = self._limits[lib_controller.prefix] + elif lib_controller.user_api in self._limits: + num_threads = self._limits[lib_controller.user_api] else: - num_threads = self._limits[module.user_api] + continue if num_threads is not None: - module.set_num_threads(num_threads) - return modules + lib_controller.set_num_threads(num_threads) -# The object oriented API of _ThreadpoolInfo and its modules is private. -# The public API (i.e. the "threadpool_info" function) only exposes the -# "list of dicts" representation returned by the .todicts method. @_format_docstring( PREFIXES=", ".join(f'"{prefix}"' for prefix in _ALL_PREFIXES), USER_APIS=", ".join(f'"{api}"' for api in _ALL_USER_APIS), BLAS_LIBS=", ".join(_ALL_BLAS_LIBRARIES), OPENMP_LIBS=", ".join(_ALL_OPENMP_LIBRARIES), ) -class _ThreadpoolInfo: - """Collection of all supported modules that have been found +class ThreadpoolController: + """Collection of LibController objects for all loaded supported libraries - Parameters + Attributes ---------- - user_api : list of user APIs or None (default=None) - Select libraries matching the requested API. Ignored if `modules` is - not None. Supported user APIs are {USER_APIS}. - - - "blas" selects all BLAS supported libraries ({BLAS_LIBS}) - - "openmp" selects all OpenMP supported libraries ({OPENMP_LIBS}) - - If None, libraries are not selected by their `user_api`. - - prefixes : list of prefixes or None (default=None) - Select libraries matching the requested prefixes. Supported prefixes - are {PREFIXES}. - If None, libraries are not selected by their prefix. Ignored if - `modules` is not None. - - modules : list of _Module objects or None (default=None) - Wraps a list of _Module objects into a _ThreapoolInfo object. Does not - load or reload any shared library. If it is not None, `prefixes` and - `user_api` are ignored. - - Note - ---- - Is is possible to select libraries both by prefixes and by user_api. All - libraries matching one or the other will be selected. + lib_controllers : list of `LibController` objects + The list of library controllers of all loaded supported libraries. """ # Cache for libc under POSIX and a few system libraries under Windows. @@ -341,49 +336,113 @@ class _ThreadpoolInfo: # during the lifetime of a program. _system_libraries = dict() - def __init__(self, user_api=None, prefixes=None, modules=None): - if modules is None: - self.prefixes = [] if prefixes is None else prefixes - self.user_api = [] if user_api is None else user_api + def __init__(self): + self.lib_controllers = [] + self._load_libraries() + self._warn_if_incompatible_openmp() - self.modules = [] - self._load_modules() - self._warn_if_incompatible_openmp() - else: - self.modules = modules + @classmethod + def _from_controllers(cls, lib_controllers): + new_controller = cls.__new__(cls) + new_controller.lib_controllers = lib_controllers + return new_controller + + def info(self): + """Return lib_controllers info as a list of dicts""" + return [lib_controller.info() for lib_controller in self.lib_controllers] + + def select(self, **kwargs): + """Return a ThreadpoolController containing a subset of its current + library controllers + + It will select all libraries matching at least one pair (key, value) from kwargs + where key is an entry of the library info dict (like "user_api", "internal_api", + "prefix", ...) and value is the value or a list of acceptable values for that + entry. + + For instance, `ThreadpoolController().select(internal_api=["blis", "openblas"])` + will select all library controllers whose internal_api is either "blis" or + "openblas". + """ + for key, vals in kwargs.items(): + kwargs[key] = [vals] if not isinstance(vals, list) else vals + + lib_controllers = [ + lib_controller + for lib_controller in self.lib_controllers + if any( + getattr(lib_controller, key, None) in vals + for key, vals in kwargs.items() + ) + ] + + return ThreadpoolController._from_controllers(lib_controllers) - def get_modules(self, key, values): - """Return all modules such that values contains module[key]""" - if key == "user_api" and values is None: - values = list(_ALL_USER_APIS) - if not isinstance(values, list): - values = [values] - modules = [module for module in self.modules if getattr(module, key) in values] - return _ThreadpoolInfo(modules=modules) + @_format_docstring( + USER_APIS=", ".join('"{}"'.format(api) for api in _ALL_USER_APIS), + BLAS_LIBS=", ".join(_ALL_BLAS_LIBRARIES), + OPENMP_LIBS=", ".join(_ALL_OPENMP_LIBRARIES), + ) + def limit(self, *, limits=None, user_api=None): + """Change the maximal number of threads that can be used in thread pools. - def todicts(self): - """Return info as a list of dicts""" - return [module.todict() for module in self.modules] + This function returns an object that can be used either as a callable (the + construction of this object limits the number of threads) or as a context + manager, in a `with` block to automatically restore the original state of the + controlled libraries when exiting the block. - def __len__(self): - return len(self.modules) + Set the maximal number of threads that can be used in thread pools used in + the supported libraries to `limits`. This function works for libraries that + are already loaded in the interpreter and can be changed dynamically. + + Parameters + ---------- + limits : int, dict or None (default=None) + The maximal number of threads that can be used in thread pools + + - If int, sets the maximum number of threads to `limits` for each + library selected by `user_api`. + + - If it is a dictionary `{{key: max_threads}}`, this function sets a + custom maximum number of threads for each `key` which can be either a + `user_api` or a `prefix` for a specific library. + + - If None, this function does not do anything. + + user_api : {USER_APIS} or None (default=None) + APIs of libraries to limit. Used only if `limits` is an int. + + - If "blas", it will only limit BLAS supported libraries ({BLAS_LIBS}). + + - If "openmp", it will only limit OpenMP supported libraries + ({OPENMP_LIBS}). Note that it can affect the number of threads used + by the BLAS libraries if they rely on OpenMP. + + - If None, this function will apply to all supported libraries. + """ + return _threadpool_limits(self, limits=limits, user_api=user_api) - def __iter__(self): - yield from self.modules + def restore_limits(self): + """Set the limits back to their original values - def __eq__(self, other): - return self.modules == other.modules + Since get_num_threads is only called once at initialization, the instance keeps + the original num_threads during its whole lifetime. + """ + self.limit(limits=self) + + def __len__(self): + return len(self.lib_controllers) - def _load_modules(self): - """Loop through loaded libraries and store supported ones""" + def _load_libraries(self): + """Loop through loaded shared libraries and store the supported ones""" if sys.platform == "darwin": - self._find_modules_with_dyld() + self._find_libraries_with_dyld() elif sys.platform == "win32": - self._find_modules_with_enum_process_module_ex() + self._find_libraries_with_enum_process_module_ex() else: - self._find_modules_with_dl_iterate_phdr() + self._find_libraries_with_dl_iterate_phdr() - def _find_modules_with_dl_iterate_phdr(self): + def _find_libraries_with_dl_iterate_phdr(self): """Loop through loaded libraries and return binders on supported ones This function is expected to work on POSIX system only. @@ -398,15 +457,15 @@ def _find_modules_with_dl_iterate_phdr(self): return [] # Callback function for `dl_iterate_phdr` which is called for every - # module loaded in the current process until it returns 1. - def match_module_callback(info, size, data): - # Get the path of the current module + # library loaded in the current process until it returns 1. + def match_library_callback(info, size, data): + # Get the path of the current library filepath = info.contents.dlpi_name if filepath: filepath = filepath.decode("utf-8") - # Store the module if it is supported and selected - self._make_module_from_path(filepath) + # Store the library controller if it is supported and selected + self._make_controller_from_path(filepath) return 0 c_func_signature = ctypes.CFUNCTYPE( @@ -415,12 +474,12 @@ def match_module_callback(info, size, data): ctypes.c_size_t, ctypes.c_char_p, ) - c_match_module_callback = c_func_signature(match_module_callback) + c_match_library_callback = c_func_signature(match_library_callback) data = ctypes.c_char_p(b"") - libc.dl_iterate_phdr(c_match_module_callback, data) + libc.dl_iterate_phdr(c_match_library_callback, data) - def _find_modules_with_dyld(self): + def _find_libraries_with_dyld(self): """Loop through loaded libraries and return binders on supported ones This function is expected to work on OSX system only @@ -436,10 +495,10 @@ def _find_modules_with_dyld(self): filepath = ctypes.string_at(libc._dyld_get_image_name(i)) filepath = filepath.decode("utf-8") - # Store the module if it is supported and selected - self._make_module_from_path(filepath) + # Store the library controller if it is supported and selected + self._make_controller_from_path(filepath) - def _find_modules_with_enum_process_module_ex(self): + def _find_libraries_with_enum_process_module_ex(self): """Loop through loaded libraries and return binders on supported ones This function is expected to work on windows system only. @@ -451,7 +510,7 @@ def _find_modules_with_enum_process_module_ex(self): PROCESS_QUERY_INFORMATION = 0x0400 PROCESS_VM_READ = 0x0010 - LIST_MODULES_ALL = 0x03 + LIST_LIBRARIES_ALL = 0x03 ps_api = self._get_windll("Psapi") kernel_32 = self._get_windll("kernel32") @@ -460,7 +519,7 @@ def _find_modules_with_enum_process_module_ex(self): PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, False, os.getpid() ) if not h_process: # pragma: no cover - raise OSError("Could not open PID %s" % os.getpid()) + raise OSError(f"Could not open PID {os.getpid()}") try: buf_count = 256 @@ -475,7 +534,7 @@ def _find_modules_with_enum_process_module_ex(self): ctypes.byref(buf), buf_size, ctypes.byref(needed), - LIST_MODULES_ALL, + LIST_LIBRARIES_ALL, ): raise OSError("EnumProcessModulesEx failed") if buf_size >= needed.value: @@ -485,7 +544,7 @@ def _find_modules_with_enum_process_module_ex(self): count = needed.value // (buf_size // buf_count) h_modules = map(HMODULE, buf[:count]) - # Loop through all the module headers and get the module path + # Loop through all the module headers and get the library path buf = ctypes.create_unicode_buffer(MAX_PATH) n_size = DWORD() for h_module in h_modules: @@ -497,38 +556,43 @@ def _find_modules_with_enum_process_module_ex(self): raise OSError("GetModuleFileNameEx failed") filepath = buf.value - # Store the module if it is supported and selected - self._make_module_from_path(filepath) + # Store the library controller if it is supported and selected + self._make_controller_from_path(filepath) finally: kernel_32.CloseHandle(h_process) - def _make_module_from_path(self, filepath): - """Store a module if it is supported and selected""" + def _make_controller_from_path(self, filepath): + """Store a library controller if it is supported and selected""" # Required to resolve symlinks filepath = _realpath(filepath) # `lower` required to take account of OpenMP dll case on Windows # (vcomp, VCOMP, Vcomp, ...) filename = os.path.basename(filepath).lower() - # Loop through supported modules to find if this filename corresponds - # to a supported module. - for module_class, candidate_module in _SUPPORTED_MODULES.items(): + # Loop through supported libraries to find if this filename corresponds + # to a supported one. + for controller_class, candidate_lib in _SUPPORTED_LIBRARIES.items(): # check if filename matches a supported prefix - prefix = self._check_prefix(filename, candidate_module["filename_prefixes"]) + prefix = self._check_prefix(filename, candidate_lib["filename_prefixes"]) # filename does not match any of the prefixes of the candidate - # module. move to next module. + # library. move to next library. if prefix is None: continue - # filename matches a prefix. Check if it matches the request. If - # so, create and store the module. - user_api = candidate_module["user_api"] - internal_api = candidate_module["internal_api"] - if prefix in self.prefixes or user_api in self.user_api: - module_class = globals()[module_class] - module = module_class(filepath, prefix, user_api, internal_api) - self.modules.append(module) + # filename matches a prefix. Create and store the library + # controller. + user_api = candidate_lib["user_api"] + internal_api = candidate_lib["internal_api"] + + lib_controller_class = globals()[controller_class] + lib_controller = lib_controller_class( + filepath=filepath, + prefix=prefix, + user_api=user_api, + internal_api=internal_api, + ) + self.lib_controllers.append(lib_controller) def _check_prefix(self, library_basename, filename_prefixes): """Return the prefix library_basename starts with @@ -546,7 +610,7 @@ def _warn_if_incompatible_openmp(self): # Only raise the warning on linux return - prefixes = [module.prefix for module in self.modules] + prefixes = [lib_controller.prefix for lib_controller in self.lib_controllers] msg = textwrap.dedent( """ Found Intel OpenMP ('libiomp') and LLVM OpenMP ('libomp') loaded at @@ -587,42 +651,34 @@ def _get_windll(cls, dll_name): USER_APIS=", ".join('"{}"'.format(api) for api in _ALL_USER_APIS), INTERNAL_APIS=", ".join('"{}"'.format(api) for api in _ALL_INTERNAL_APIS), ) -class _Module(ABC): - """Abstract base class for the modules +class LibController(ABC): + """Abstract base class for the individual library controllers - A module is represented by the following information: + A library controller is represented by the following information: - "user_api" : user API. Possible values are {USER_APIS}. - "internal_api" : internal API. Possible values are {INTERNAL_APIS}. - "prefix" : prefix of the shared library's name. - - "filepath" : path to the loaded module. + - "filepath" : path to the loaded library. - "version" : version of the library (if available). - "num_threads" : the current thread limit. - In addition, each module may contain internal_api specific entries. + In addition, each library controller may contain internal_api specific + entries. """ - def __init__(self, filepath=None, prefix=None, user_api=None, internal_api=None): - self.filepath = filepath - self.prefix = prefix + def __init__(self, *, filepath=None, prefix=None, user_api=None, internal_api=None): self.user_api = user_api self.internal_api = internal_api + self.prefix = prefix + self.filepath = filepath self._dynlib = ctypes.CDLL(filepath, mode=_RTLD_NOLOAD) self.version = self.get_version() self.num_threads = self.get_num_threads() - self._get_extra_info() - - def __eq__(self, other): - return self.todict() == other.todict() - def todict(self): + def info(self): """Return relevant info wrapped in a dict""" return {k: v for k, v in vars(self).items() if not k.startswith("_")} - @abstractmethod - def get_version(self): - """Return the version of the shared library""" - pass # pragma: no cover - @abstractmethod def get_num_threads(self): """Return the maximum number of threads available to use""" @@ -634,13 +690,28 @@ def set_num_threads(self, num_threads): pass # pragma: no cover @abstractmethod - def _get_extra_info(self): - """Add additional module specific information""" + def get_version(self): + """Return the version of the shared library""" pass # pragma: no cover -class _OpenBLASModule(_Module): - """Module class for OpenBLAS""" +class OpenBLASController(LibController): + """Controller class for OpenBLAS""" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.threading_layer = self._get_threading_layer() + self.architecture = self._get_architecture() + + def get_num_threads(self): + get_func = getattr(self._dynlib, "openblas_get_num_threads", lambda: None) + return get_func() + + def set_num_threads(self, num_threads): + set_func = getattr( + self._dynlib, "openblas_set_num_threads", lambda num_threads: None + ) + return set_func(num_threads) def get_version(self): # None means OpenBLAS is not loaded or version < 0.3.4, since OpenBLAS @@ -655,21 +726,7 @@ def get_version(self): return config[1].decode("utf-8") return None - def get_num_threads(self): - get_func = getattr(self._dynlib, "openblas_get_num_threads", lambda: None) - return get_func() - - def set_num_threads(self, num_threads): - set_func = getattr( - self._dynlib, "openblas_set_num_threads", lambda num_threads: None - ) - return set_func(num_threads) - - def _get_extra_info(self): - self.threading_layer = self.get_threading_layer() - self.architecture = self.get_architecture() - - def get_threading_layer(self): + def _get_threading_layer(self): """Return the threading layer of OpenBLAS""" openblas_get_parallel = getattr(self._dynlib, "openblas_get_parallel", None) if openblas_get_parallel is None: @@ -681,7 +738,8 @@ def get_threading_layer(self): return "pthreads" return "disabled" - def get_architecture(self): + def _get_architecture(self): + """Return the architecture detected by OpenBLAS""" get_corename = getattr(self._dynlib, "openblas_get_corename", None) if get_corename is None: return None @@ -690,16 +748,13 @@ def get_architecture(self): return get_corename().decode("utf-8") -class _BLISModule(_Module): - """Module class for BLIS""" +class BLISController(LibController): + """Controller class for BLIS""" - def get_version(self): - get_version_ = getattr(self._dynlib, "bli_info_get_version_str", None) - if get_version_ is None: - return None - - get_version_.restype = ctypes.c_char_p - return get_version_().decode("utf-8") + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.threading_layer = self._get_threading_layer() + self.architecture = self._get_architecture() def get_num_threads(self): get_func = getattr(self._dynlib, "bli_thread_get_num_threads", lambda: None) @@ -714,11 +769,15 @@ def set_num_threads(self, num_threads): ) return set_func(num_threads) - def _get_extra_info(self): - self.threading_layer = self.get_threading_layer() - self.architecture = self.get_architecture() + def get_version(self): + get_version_ = getattr(self._dynlib, "bli_info_get_version_str", None) + if get_version_ is None: + return None + + get_version_.restype = ctypes.c_char_p + return get_version_().decode("utf-8") - def get_threading_layer(self): + def _get_threading_layer(self): """Return the threading layer of BLIS""" if self._dynlib.bli_info_get_enable_openmp(): return "openmp" @@ -726,7 +785,8 @@ def get_threading_layer(self): return "pthreads" return "disabled" - def get_architecture(self): + def _get_architecture(self): + """Return the architecture detected by BLIS""" bli_arch_query_id = getattr(self._dynlib, "bli_arch_query_id", None) bli_arch_string = getattr(self._dynlib, "bli_arch_string", None) if bli_arch_query_id is None or bli_arch_string is None: @@ -739,8 +799,22 @@ def get_architecture(self): return bli_arch_string(bli_arch_query_id()).decode("utf-8") -class _MKLModule(_Module): - """Module class for MKL""" +class MKLController(LibController): + """Controller class for MKL""" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.threading_layer = self._get_threading_layer() + + def get_num_threads(self): + get_func = getattr(self._dynlib, "MKL_Get_Max_Threads", lambda: None) + return get_func() + + def set_num_threads(self, num_threads): + set_func = getattr( + self._dynlib, "MKL_Set_Num_Threads", lambda num_threads: None + ) + return set_func(num_threads) def get_version(self): if not hasattr(self._dynlib, "MKL_Get_Version_String"): @@ -755,20 +829,7 @@ def get_version(self): version = group.groups()[0] return version.strip() - def get_num_threads(self): - get_func = getattr(self._dynlib, "MKL_Get_Max_Threads", lambda: None) - return get_func() - - def set_num_threads(self, num_threads): - set_func = getattr( - self._dynlib, "MKL_Set_Num_Threads", lambda num_threads: None - ) - return set_func(num_threads) - - def _get_extra_info(self): - self.threading_layer = self.get_threading_layer() - - def get_threading_layer(self): + def _get_threading_layer(self): """Return the threading layer of MKL""" # The function mkl_set_threading_layer returns the current threading # layer. Calling it with an invalid threading layer allows us to safely @@ -787,12 +848,8 @@ def get_threading_layer(self): return layer_map[set_threading_layer(-1)] -class _OpenMPModule(_Module): - """Module class for OpenMP""" - - def get_version(self): - # There is no way to get the version number programmatically in OpenMP. - return None +class OpenMPController(LibController): + """Controller class for OpenMP""" def get_num_threads(self): get_func = getattr(self._dynlib, "omp_get_max_threads", lambda: None) @@ -804,8 +861,9 @@ def set_num_threads(self, num_threads): ) return set_func(num_threads) - def _get_extra_info(self): - pass + def get_version(self): + # There is no way to get the version number programmatically in OpenMP. + return None def _main(): From d14aea199853b05f24dfec79cc9daf13c7809ed1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Fri, 24 Sep 2021 14:30:14 +0200 Subject: [PATCH 118/187] ENH Add support for OpenBLAS built for 64bit integers in Fortran (#101) --- CHANGES.md | 4 ++++ threadpoolctl.py | 33 ++++++++++++++++++++++++++++----- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 8219ddd5..bcfeddd7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,10 @@ threads can be limited with the `limit` method which can be used as a context manager (equivalent to `threadpoolctl.threadpool_limits()`). This is especially useful to avoid searching through all loaded shared libraries each time. + https://github.com/joblib/threadpoolctl/pull/95 + +- Added support for OpenBLAS built for 64bit integers in Fortran. + https://github.com/joblib/threadpoolctl/pull/101 - Fixed an attribute error when using old versions of OpenBLAS or BLIS that are missing version query functions. diff --git a/threadpoolctl.py b/threadpoolctl.py index a7e368c3..05a4216c 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -704,19 +704,34 @@ def __init__(self, **kwargs): self.architecture = self._get_architecture() def get_num_threads(self): - get_func = getattr(self._dynlib, "openblas_get_num_threads", lambda: None) + get_func = getattr( + self._dynlib, + "openblas_get_num_threads", + # Symbols differ when built for 64bit integers in Fortran + getattr(self._dynlib, "openblas_get_num_threads64_", lambda: None), + ) + return get_func() def set_num_threads(self, num_threads): set_func = getattr( - self._dynlib, "openblas_set_num_threads", lambda num_threads: None + self._dynlib, + "openblas_set_num_threads", + # Symbols differ when built for 64bit integers in Fortran + getattr( + self._dynlib, "openblas_set_num_threads64_", lambda num_threads: None + ), ) return set_func(num_threads) def get_version(self): # None means OpenBLAS is not loaded or version < 0.3.4, since OpenBLAS # did not expose its version before that. - get_config = getattr(self._dynlib, "openblas_get_config", None) + get_config = getattr( + self._dynlib, + "openblas_get_config", + getattr(self._dynlib, "openblas_get_config64_", None), + ) if get_config is None: return None @@ -728,7 +743,11 @@ def get_version(self): def _get_threading_layer(self): """Return the threading layer of OpenBLAS""" - openblas_get_parallel = getattr(self._dynlib, "openblas_get_parallel", None) + openblas_get_parallel = getattr( + self._dynlib, + "openblas_get_parallel", + getattr(self._dynlib, "openblas_get_parallel64_", None), + ) if openblas_get_parallel is None: return "unknown" threading_layer = openblas_get_parallel() @@ -740,7 +759,11 @@ def _get_threading_layer(self): def _get_architecture(self): """Return the architecture detected by OpenBLAS""" - get_corename = getattr(self._dynlib, "openblas_get_corename", None) + get_corename = getattr( + self._dynlib, + "openblas_get_corename", + getattr(self._dynlib, "openblas_get_corename64_", None), + ) if get_corename is None: return None From 75cff9631bddd5c02846ccb6b034fba7513b56ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Mon, 27 Sep 2021 17:55:52 +0200 Subject: [PATCH 119/187] add test for openblas threading layer (#103) --- tests/test_threadpoolctl.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index cc7fe06a..e2293ccf 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -551,3 +551,22 @@ def test_architecture(): else: # Not supported for other libraries assert "architecture" not in lib_info + + +def test_openblas_threading_layer(): + # Check that threadpool_info correctly recovers the threading layer used by openblas + openblas_controller = ThreadpoolController().select(internal_api="openblas") + + if not (openblas_controller): + pytest.skip("requires OpenBLAS.") + + expected_openblas_threading_layers = ("openmp", "pthreads", "disabled") + + threading_layer = openblas_controller.lib_controllers[0].threading_layer + + if threading_layer == "unknown": + # If we never recover an acceptable value for the threading layer, it will be + # always skipped and caught by check_no_test_always_skipped. + pytest.skip("Unknown OpenBLAS threading layer.") + + assert threading_layer in expected_openblas_threading_layers From d09d20fcfbf1b5da30f22a4e2a48f7467b243781 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Wed, 29 Sep 2021 18:14:11 +0200 Subject: [PATCH 120/187] FIX make the num_threads attribute dynamic (#104) --- .../nested_prange_blas.pyx | 6 +- tests/test_threadpoolctl.py | 157 ++++++++++-------- tests/utils.py | 16 ++ threadpoolctl.py | 40 ++--- 4 files changed, 126 insertions(+), 93 deletions(-) diff --git a/tests/_openmp_test_helper/nested_prange_blas.pyx b/tests/_openmp_test_helper/nested_prange_blas.pyx index aec7f815..879fe5c2 100644 --- a/tests/_openmp_test_helper/nested_prange_blas.pyx +++ b/tests/_openmp_test_helper/nested_prange_blas.pyx @@ -43,12 +43,12 @@ def check_nested_prange_blas(double[:, ::1] A, double[:, ::1] B, int nthreads): int prange_num_threads int *prange_num_threads_ptr = &prange_num_threads - inner_controller = [None] + inner_info = [None] with nogil, parallel(num_threads=nthreads): if openmp.omp_get_thread_num() == 0: with gil: - inner_controller[0] = ThreadpoolController() + inner_info[0] = ThreadpoolController().info() prange_num_threads_ptr[0] = openmp.omp_get_num_threads() @@ -62,4 +62,4 @@ def check_nested_prange_blas(double[:, ::1] A, double[:, ::1] B, int nthreads): &alpha, &B[0, 0], &k, &A[i * chunk_size, 0], &k, &beta, &C[i * chunk_size, 0], &n) - return np.asarray(C), prange_num_threads, inner_controller[0] + return np.asarray(C), prange_num_threads, inner_info[0] diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index e2293ccf..6702b0d9 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -13,6 +13,7 @@ from .utils import libopenblas_paths from .utils import scipy from .utils import threadpool_info_from_subprocess +from .utils import select def is_old_openblas(lib_controller): @@ -60,6 +61,20 @@ def test_threadpool_controller_info(): assert "threading_layer" in lib_controller_dict +def test_controller_info_actualized(): + # Check that the num_threads attribute reflects the actual state of the threadpools + controller = ThreadpoolController() + original_info = controller.info() + + with threadpool_limits(limits=1): + assert all( + lib_controller.num_threads == 1 + for lib_controller in controller.lib_controllers + ) + + assert controller.info() == original_info + + @pytest.mark.parametrize( "kwargs", [ @@ -86,9 +101,10 @@ def test_threadpool_controller_select(kwargs): @pytest.mark.parametrize("limit", [1, 3]) def test_threadpool_limits_by_prefix(prefix, limit): # Check that the maximum number of threads can be set by prefix - original_controller = ThreadpoolController() + controller = ThreadpoolController() + original_info = controller.info() - controller_matching_prefix = original_controller.select(prefix=prefix) + controller_matching_prefix = controller.select(prefix=prefix) if not controller_matching_prefix: pytest.skip(f"Requires {prefix} runtime") @@ -98,20 +114,21 @@ def test_threadpool_limits_by_prefix(prefix, limit): continue # threadpool_limits only sets an upper bound on the number of # threads. - assert 0 < lib_controller.get_num_threads() <= limit - assert ThreadpoolController().info() == original_controller.info() + assert 0 < lib_controller.num_threads <= limit + assert ThreadpoolController().info() == original_info @pytest.mark.parametrize("user_api", (None, "blas", "openmp")) @pytest.mark.parametrize("limit", [1, 3]) def test_set_threadpool_limits_by_api(user_api, limit): # Check that the maximum number of threads can be set by user_api - original_controller = ThreadpoolController() + controller = ThreadpoolController() + original_info = controller.info() if user_api is None: - controller_matching_api = original_controller + controller_matching_api = controller else: - controller_matching_api = original_controller.select(user_api=user_api) + controller_matching_api = controller.select(user_api=user_api) if not controller_matching_api: user_apis = _ALL_USER_APIS if user_api is None else [user_api] pytest.skip(f"Requires a library which api is in {user_apis}") @@ -122,15 +139,15 @@ def test_set_threadpool_limits_by_api(user_api, limit): continue # threadpool_limits only sets an upper bound on the number of # threads. - assert 0 < lib_controller.get_num_threads() <= limit + assert 0 < lib_controller.num_threads <= limit - assert ThreadpoolController().info() == original_controller.info() + assert ThreadpoolController().info() == original_info def test_threadpool_limits_function_with_side_effect(): # Check that threadpool_limits can be used as a function with # side effects instead of a context manager. - original_controller = ThreadpoolController() + original_info = ThreadpoolController().info() threadpool_limits(limits=1) try: @@ -141,25 +158,26 @@ def test_threadpool_limits_function_with_side_effect(): finally: # Restore the original limits so that this test does not have any # side-effect. - threadpool_limits(limits=original_controller) + threadpool_limits(limits=original_info) - assert ThreadpoolController().info() == original_controller.info() + assert ThreadpoolController().info() == original_info def test_set_threadpool_limits_no_limit(): # Check that limits=None does nothing. - original_controller = ThreadpoolController() + original_info = ThreadpoolController().info() + with threadpool_limits(limits=None): - assert ThreadpoolController().info() == original_controller.info() + assert ThreadpoolController().info() == original_info - assert ThreadpoolController().info() == original_controller.info() + assert ThreadpoolController().info() == original_info -def test_threadpool_limits_manual_unregister(): +def test_threadpool_limits_manual_restore(): # Check that threadpool_limits can be used as an object which holds the # original state of the threadpools and that can be restored thanks to the - # dedicated unregister method - original_controller = ThreadpoolController() + # dedicated restore_original_limits method + original_info = ThreadpoolController().info() limits = threadpool_limits(limits=1) try: @@ -170,20 +188,20 @@ def test_threadpool_limits_manual_unregister(): finally: # Restore the original limits so that this test does not have any # side-effect. - limits.unregister() + limits.restore_original_limits() - assert ThreadpoolController().info() == original_controller.info() + assert ThreadpoolController().info() == original_info def test_threadpool_controller_limit(): # Check that using the limit method of ThreadpoolController only impact its # library controllers. - original_blas_controller = ThreadpoolController().select(user_api="blas") - original_openmp_controller = ThreadpoolController().select(user_api="openmp") + blas_controller = ThreadpoolController().select(user_api="blas") + original_openmp_info = ThreadpoolController().select(user_api="openmp").info() - with original_blas_controller.limit(limits=1): + with blas_controller.limit(limits=1): blas_controller = ThreadpoolController().select(user_api="blas") - openmp_controller = ThreadpoolController().select(user_api="openmp") + openmp_info = ThreadpoolController().select(user_api="openmp").info() assert all( lib_controller.num_threads == 1 @@ -191,27 +209,33 @@ def test_threadpool_controller_limit(): ) # original_blas_controller contains only blas libraries so no opemp library # should be impacted. - assert openmp_controller.info() == original_openmp_controller.info() + assert openmp_info == original_openmp_info -def test_threadpool_controller_restore(): - # Check that the restore_limits method of ThreadpoolController is able to set the - # limits back to their original values. Similar to - # test_threadpool_limits_function_with_side_effect but with the object api +def test_nested_limits(): + # Check that exiting the context manager properly restores the original limits even + # when nested. controller = ThreadpoolController() + original_info = controller.info() - controller.limit(limits=1) - try: - for lib_controller in ThreadpoolController().lib_controllers: - if is_old_openblas(lib_controller): - continue - assert lib_controller.num_threads == 1 - finally: - # Restore the original limits so that this test does not have any - # side-effect. - controller.restore_limits() + if any(info["num_threads"] < 2 for info in original_info): + pytest.skip("Test requires at least 2 CPUs on host machine") + + def check_num_threads(expected_num_threads): + assert all( + lib_controller.num_threads == expected_num_threads + for lib_controller in ThreadpoolController().lib_controllers + ) + + with controller.limit(limits=1): + check_num_threads(expected_num_threads=1) - assert ThreadpoolController().info() == controller.info() + with controller.limit(limits=2): + check_num_threads(expected_num_threads=2) + + check_num_threads(expected_num_threads=1) + + assert ThreadpoolController().info() == original_info def test_threadpool_limits_bad_input(): @@ -277,7 +301,7 @@ def test_openmp_nesting(nthreads_outer): outer_omp = prefixes - {inner_omp} outer_num_threads, inner_num_threads = check_nested_openmp_loops(10) - original_controller = ThreadpoolController() + original_info = ThreadpoolController().info() if inner_omp == outer_omp: # The OpenMP runtime should be shared by default, meaning that the @@ -294,7 +318,7 @@ def test_openmp_nesting(nthreads_outer): # The state of the original state of all threadpools should have been # restored. - assert ThreadpoolController().info() == original_controller.info() + assert ThreadpoolController().info() == original_info # The number of threads available in the outer loop should not have been # decreased: @@ -315,15 +339,14 @@ def test_openmp_nesting(nthreads_outer): def test_shipped_openblas(): # checks that OpenBLAS effectively uses the number of threads requested by # the context manager - original_controller = ThreadpoolController() - - openblas_controllers = original_controller.select(internal_api="openblas") + original_info = ThreadpoolController().info() + openblas_controller = ThreadpoolController().select(internal_api="openblas") with threadpool_limits(1): - for lib_controller in openblas_controllers.lib_controllers: - assert lib_controller.get_num_threads() == 1 + for lib_controller in openblas_controller.lib_controllers: + assert lib_controller.num_threads == 1 - assert original_controller.info() == ThreadpoolController().info() + assert ThreadpoolController().info() == original_info @pytest.mark.skipif( @@ -331,7 +354,7 @@ def test_shipped_openblas(): ) def test_multiple_shipped_openblas(): # This redundant test is meant to make it easier to see if the system - # has 2 or more active openblas runtimes available just be reading the + # has 2 or more active openblas runtimes available just by reading the # pytest report (whether or not this test has been skipped). test_shipped_openblas() @@ -350,19 +373,19 @@ def test_nested_prange_blas(nthreads_outer): check_nested_prange_blas = prange_blas.check_nested_prange_blas - original_controller = ThreadpoolController() + original_info = ThreadpoolController().info() - blas_controllers = original_controller.select(user_api="blas") - blis_controllers = original_controller.select(internal_api="blis") + blas_controller = ThreadpoolController().select(user_api="blas") + blis_controller = ThreadpoolController().select(internal_api="blis") # skip if the BLAS used by numpy is an old openblas. OpenBLAS 0.3.3 and # older are known to cause an unrecoverable deadlock at process shutdown # time (after pytest has exited). # numpy can be linked to BLIS for CBLAS and OpenBLAS for LAPACK. In that # case this test will run BLIS gemm so no need to skip. - if not blis_controllers and any( + if not blis_controller and any( is_old_openblas(lib_controller) - for lib_controller in blas_controllers.lib_controllers + for lib_controller in blas_controller.lib_controllers ): pytest.skip("Old OpenBLAS: skipping test to avoid deadlock") @@ -374,19 +397,16 @@ def test_nested_prange_blas(nthreads_outer): nthreads = effective_num_threads(nthreads_outer, max_threads) result = check_nested_prange_blas(A, B, nthreads) - C, prange_num_threads, inner_controller = result + C, prange_num_threads, inner_info = result assert np.allclose(C, np.dot(A, B.T)) assert prange_num_threads == nthreads - nested_blas_controllers = inner_controller.select(user_api="blas") - assert len(nested_blas_controllers.lib_controllers) == len( - blas_controllers.lib_controllers - ) - for lib_controller in nested_blas_controllers.lib_controllers: - assert lib_controller.num_threads == 1 + nested_blas_info = select(inner_info, user_api="blas") + assert len(nested_blas_info) == len(blas_controller.lib_controllers) + assert all(lib_info["num_threads"] == 1 for lib_info in nested_blas_info) - assert original_controller.info() == ThreadpoolController().info() + assert ThreadpoolController().info() == original_info # the method `get_original_num_threads` raises a UserWarning due to different @@ -399,21 +419,18 @@ def test_get_original_num_threads(limit): # Tests the method get_original_num_threads of the context manager with threadpool_limits(limits=2, user_api="blas") as ctx: # set different blas num threads to start with (when multiple openblas) - if ctx._controller: + if len(ctx._controller.select(user_api="blas")) > 1: ctx._controller.lib_controllers[0].set_num_threads(1) - original_controller = ThreadpoolController() + original_info = ThreadpoolController().info() with threadpool_limits(limits=limit, user_api="blas") as threadpoolctx: original_num_threads = threadpoolctx.get_original_num_threads() assert "openmp" not in original_num_threads - blas_controller = original_controller.select(user_api="blas") - if blas_controller: - expected = min( - lib_controller.num_threads - for lib_controller in blas_controller.lib_controllers - ) + blas_info = select(original_info, user_api="blas") + if blas_info: + expected = min(lib_info["num_threads"] for lib_info in blas_info) assert original_num_threads["blas"] == expected else: assert original_num_threads["blas"] is None diff --git a/tests/utils.py b/tests/utils.py index a0043da3..e8d778ff 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -65,3 +65,19 @@ def threadpool_info_from_subprocess(module): cmd = [sys.executable, "-m", "threadpoolctl", "-i", module] out = check_output(cmd, env=env).decode("utf-8") return json.loads(out) + + +def select(info, **kwargs): + """Select a subset of the list of library info matching the request""" + # It's just a utility function to avoid repeating the pattern + # [lib_info for lib_info in info if lib_info[""] == key] + for key, vals in kwargs.items(): + kwargs[key] = [vals] if not isinstance(vals, list) else vals + + selected_info = [ + lib_info + for lib_info in info + if any(lib_info.get(key, None) in vals for key, vals in kwargs.items()) + ] + + return selected_info diff --git a/threadpoolctl.py b/threadpoolctl.py index 05a4216c..93fe3983 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -196,19 +196,24 @@ def __init__(self, controller, *, limits=None, user_api=None): limits, user_api ) self._controller = controller + self._original_info = self._controller.info() self._set_threadpool_limits() def __enter__(self): return self def __exit__(self, type, value, traceback): - self.unregister() + self.restore_original_limits() - def unregister(self): - for lib_controller in self._controller.lib_controllers: - # Since we never call get_num_threads after instanciation of - # ThreadpoolController, num_threads holds the original value. - lib_controller.set_num_threads(lib_controller.num_threads) + def restore_original_limits(self): + """Set the limits back to their original values""" + for lib_controller, original_info in zip( + self._controller.lib_controllers, self._original_info + ): + lib_controller.set_num_threads(original_info["num_threads"]) + + # Alias of `restore_original_limits` for backward compatibility + unregister = restore_original_limits def get_original_num_threads(self): """Original num_threads from before calling threadpool_limits @@ -220,10 +225,9 @@ def get_original_num_threads(self): for user_api in self._user_api: limits = [ - lib_controller.num_threads - for lib_controller in self._controller.select( - user_api=user_api - ).lib_controllers + lib_info["num_threads"] + for lib_info in self._original_info + if lib_info["user_api"] == user_api ] limits = set(limits) n_limits = len(limits) @@ -422,14 +426,6 @@ def limit(self, *, limits=None, user_api=None): """ return _threadpool_limits(self, limits=limits, user_api=user_api) - def restore_limits(self): - """Set the limits back to their original values - - Since get_num_threads is only called once at initialization, the instance keeps - the original num_threads during its whole lifetime. - """ - self.limit(limits=self) - def __len__(self): return len(self.lib_controllers) @@ -673,11 +669,15 @@ def __init__(self, *, filepath=None, prefix=None, user_api=None, internal_api=No self.filepath = filepath self._dynlib = ctypes.CDLL(filepath, mode=_RTLD_NOLOAD) self.version = self.get_version() - self.num_threads = self.get_num_threads() def info(self): """Return relevant info wrapped in a dict""" - return {k: v for k, v in vars(self).items() if not k.startswith("_")} + all_attrs = dict(vars(self), **{"num_threads": self.num_threads}) + return {k: v for k, v in all_attrs.items() if not k.startswith("_")} + + @property + def num_threads(self): + return self.get_num_threads() @abstractmethod def get_num_threads(self): From 74bdf10ffcb766a961213c27f8595ad64d4cf11e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Fri, 1 Oct 2021 10:31:27 +0200 Subject: [PATCH 121/187] FEA Add possibility to use threadpool_limits as a decorator (#102) --- CHANGES.md | 4 + README.md | 28 +++++- tests/test_threadpoolctl.py | 33 +++++++ threadpoolctl.py | 180 ++++++++++++++++++++++++++---------- 4 files changed, 196 insertions(+), 49 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index bcfeddd7..be1e4fe0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -12,6 +12,10 @@ - Added support for OpenBLAS built for 64bit integers in Fortran. https://github.com/joblib/threadpoolctl/pull/101 +- Added the possibility to use `threadpoolctl.threadpool_limits` and + `threadpooctl.ThreadpoolController` as decorators through their `wrap` method. + https://github.com/joblib/threadpoolctl/pull/102 + - Fixed an attribute error when using old versions of OpenBLAS or BLIS that are missing version query functions. https://github.com/joblib/threadpoolctl/pull/88 diff --git a/README.md b/README.md index 624c4922..6f2565ca 100644 --- a/README.md +++ b/README.md @@ -159,7 +159,7 @@ in specific sections of your Python program: The threadpools can also be controlled via the object oriented API, which is especially useful to avoid searching through all the loaded shared libraries each time. It will however not act on libraries loaded after the instanciation of the -``ThreadpoolController``: +`ThreadpoolController`: ```python >>> from threadpoolctl import ThreadpoolController @@ -171,6 +171,27 @@ however not act on libraries loaded after the instanciation of the ... a_squared = a @ a ``` +### Restricting the limits to the scope of a function + +`threadpool_limits` and `ThreadpoolController` can also be used as decorators to set +the maximum number of threads used by the supported libraries at a function level. The +decorators are accessible through their `wrap` method: + +```python +>>> from threadpoolctl import ThreadpoolController, threadpool_limits +>>> import numpy as np +>>> controller = ThreadpoolController() + +>>> @controller.wrap(limits=1, user_api='blas') +... # or @threadpool_limits.wrap(limits=1, user_api='blas') +... def my_func(): +... # Inside this function, calls to blas implementation (like openblas or MKL) +... # will be limited to use only one thread. +... a = np.random.randn(1000, 1000) +... a_squared = a @ a +... +``` + ### Known Limitations - `threadpool_limits` can fail to limit the number of inner threads when nesting @@ -193,6 +214,11 @@ however not act on libraries loaded after the instanciation of the and workarounds: https://github.com/joblib/threadpoolctl/blob/master/multiple_openmp.md +- Setting the maximum number of threads of the OpenMP and BLAS libraries has a global + effect and impacts the whole Python process. There is no thread level isolation as + these libraries do not offer thread-local APIs to configure the number of threads to + use in nested parallel calls. + ## Maintainers diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index 6702b0d9..7bfffc03 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -587,3 +587,36 @@ def test_openblas_threading_layer(): pytest.skip("Unknown OpenBLAS threading layer.") assert threading_layer in expected_openblas_threading_layers + + +def test_threadpool_controller_as_decorator(): + # Check that using the decorator can be nested and is restricted to the scope of + # the decorated function. + controller = ThreadpoolController() + original_info = controller.info() + + if any(info["num_threads"] < 2 for info in original_info): + pytest.skip("Test requires at least 2 CPUs on host machine") + if not controller.select(user_api="blas"): + pytest.skip("Requires a blas runtime.") + + def check_blas_num_threads(expected_num_threads): + blas_controller = ThreadpoolController().select(user_api="blas") + assert all( + lib_controller.num_threads == expected_num_threads + for lib_controller in blas_controller.lib_controllers + ) + + @controller.wrap(limits=1, user_api="blas") + def outer_func(): + check_blas_num_threads(expected_num_threads=1) + inner_func() + check_blas_num_threads(expected_num_threads=1) + + @controller.wrap(limits=2, user_api="blas") + def inner_func(): + check_blas_num_threads(expected_num_threads=2) + + outer_func() + + assert ThreadpoolController().info() == original_info diff --git a/threadpoolctl.py b/threadpoolctl.py index 93fe3983..d700963e 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -19,6 +19,7 @@ from ctypes.util import find_library from abc import ABC, abstractmethod from functools import lru_cache +from contextlib import ContextDecorator __version__ = "2.3.0.dev0" __all__ = ["threadpool_limits", "threadpool_info", "ThreadpoolController"] @@ -138,57 +139,15 @@ def threadpool_info(): return ThreadpoolController().info() -@_format_docstring( - USER_APIS=", ".join(f'"{api}"' for api in _ALL_USER_APIS), - BLAS_LIBS=", ".join(_ALL_BLAS_LIBRARIES), - OPENMP_LIBS=", ".join(_ALL_OPENMP_LIBRARIES), -) -def threadpool_limits(limits=None, user_api=None): - """Change the maximal number of threads that can be used in thread pools. - - This function returns an object that can be used either as a callable (the - construction of this object limits the number of threads) or as a context manager, - in a `with` block to automatically restore the original state of the controlled - libraries when exiting the block. - - Set the maximal number of threads that can be used in thread pools used in - the supported libraries to `limit`. This function works for libraries that - are already loaded in the interpreter and can be changed dynamically. - - Parameters - ---------- - limits : int, dict or None (default=None) - The maximal number of threads that can be used in thread pools - - - If int, sets the maximum number of threads to `limits` for each - library selected by `user_api`. - - - If it is a dictionary `{{key: max_threads}}`, this function sets a - custom maximum number of threads for each `key` which can be either a - `user_api` or a `prefix` for a specific library. - - - If None, this function does not do anything. - - user_api : {USER_APIS} or None (default=None) - APIs of libraries to limit. Used only if `limits` is an int. - - - If "blas", it will only limit BLAS supported libraries ({BLAS_LIBS}). - - - If "openmp", it will only limit OpenMP supported libraries - ({OPENMP_LIBS}). Note that it can affect the number of threads used - by the BLAS libraries if they rely on OpenMP. - - - If None, this function will apply to all supported libraries. - """ - return ThreadpoolController().limit(limits=limits, user_api=user_api) - - -class _threadpool_limits: +class _ThreadpoolLimiter: """The guts of ThreadpoolController.limit Refer to the docstring of ThreadpoolController.limit for more details. It will only act on the library controllers held by the provided `controller`. + Using the default constructor sets the limits right away such that it can be used as + a callable. Setting the limits can be delayed by using the `wrap` class method such + that it can be used as a decorator. """ def __init__(self, controller, *, limits=None, user_api=None): @@ -205,6 +164,13 @@ def __enter__(self): def __exit__(self, type, value, traceback): self.restore_original_limits() + @classmethod + def wrap(cls, controller, *, limits=None, user_api=None): + """Return an instance of this class that can be used as a decorator""" + return _ThreadpoolLimiterDecorator( + controller=controller, limits=limits, user_api=user_api + ) + def restore_original_limits(self): """Set the limits back to their original values""" for lib_controller, original_info in zip( @@ -302,7 +268,7 @@ def _set_threadpool_limits(self): matching `self._prefixes` and `self._user_api`. """ if self._limits is None: - return None + return for lib_controller in self._controller.lib_controllers: # self._limits is a dict {key: num_threads} where key is either @@ -319,6 +285,79 @@ def _set_threadpool_limits(self): lib_controller.set_num_threads(num_threads) +class _ThreadpoolLimiterDecorator(_ThreadpoolLimiter, ContextDecorator): + """Same as _ThreadpoolLimiter but to be used as a decorator""" + + def __init__(self, controller, *, limits=None, user_api=None): + self._limits, self._user_api, self._prefixes = self._check_params( + limits, user_api + ) + self._controller = controller + + def __enter__(self): + # we need to set the limits here and not in the __init__ because we want the + # limits to be set when calling the decorated function, not when creating the + # decorator. + self._original_info = self._controller.info() + self._set_threadpool_limits() + return self + + +@_format_docstring( + USER_APIS=", ".join(f'"{api}"' for api in _ALL_USER_APIS), + BLAS_LIBS=", ".join(_ALL_BLAS_LIBRARIES), + OPENMP_LIBS=", ".join(_ALL_OPENMP_LIBRARIES), +) +class threadpool_limits(_ThreadpoolLimiter): + """Change the maximal number of threads that can be used in thread pools. + + This object can be used either as a callable (the construction of this object + limits the number of threads), as a context manager in a `with` block to + automatically restore the original state of the controlled libraries when exiting + the block, or as a decorator through its `wrap` method. + + Set the maximal number of threads that can be used in thread pools used in + the supported libraries to `limit`. This function works for libraries that + are already loaded in the interpreter and can be changed dynamically. + + This effect is global and impacts the whole Python process. There is no thread level + isolation as these libraries do not offer thread-local APIs to configure the number + of threads to use in nested parallel calls. + + Parameters + ---------- + limits : int, dict or None (default=None) + The maximal number of threads that can be used in thread pools + + - If int, sets the maximum number of threads to `limits` for each + library selected by `user_api`. + + - If it is a dictionary `{{key: max_threads}}`, this function sets a + custom maximum number of threads for each `key` which can be either a + `user_api` or a `prefix` for a specific library. + + - If None, this function does not do anything. + + user_api : {USER_APIS} or None (default=None) + APIs of libraries to limit. Used only if `limits` is an int. + + - If "blas", it will only limit BLAS supported libraries ({BLAS_LIBS}). + + - If "openmp", it will only limit OpenMP supported libraries + ({OPENMP_LIBS}). Note that it can affect the number of threads used + by the BLAS libraries if they rely on OpenMP. + + - If None, this function will apply to all supported libraries. + """ + + def __init__(self, limits=None, user_api=None): + super().__init__(ThreadpoolController(), limits=limits, user_api=user_api) + + @classmethod + def wrap(cls, limits=None, user_api=None): + return super().wrap(ThreadpoolController(), limits=limits, user_api=user_api) + + @_format_docstring( PREFIXES=", ".join(f'"{prefix}"' for prefix in _ALL_PREFIXES), USER_APIS=", ".join(f'"{api}"' for api in _ALL_USER_APIS), @@ -399,6 +438,51 @@ def limit(self, *, limits=None, user_api=None): the supported libraries to `limits`. This function works for libraries that are already loaded in the interpreter and can be changed dynamically. + This effect is global and impacts the whole Python process. There is no thread + level isolation as these libraries do not offer thread-local APIs to configure + the number of threads to use in nested parallel calls. + + Parameters + ---------- + limits : int, dict or None (default=None) + The maximal number of threads that can be used in thread pools + + - If int, sets the maximum number of threads to `limits` for each + library selected by `user_api`. + + - If it is a dictionary `{{key: max_threads}}`, this function sets a + custom maximum number of threads for each `key` which can be either a + `user_api` or a `prefix` for a specific library. + + - If None, this function does not do anything. + + user_api : {USER_APIS} or None (default=None) + APIs of libraries to limit. Used only if `limits` is an int. + + - If "blas", it will only limit BLAS supported libraries ({BLAS_LIBS}). + + - If "openmp", it will only limit OpenMP supported libraries + ({OPENMP_LIBS}). Note that it can affect the number of threads used + by the BLAS libraries if they rely on OpenMP. + + - If None, this function will apply to all supported libraries. + """ + return _ThreadpoolLimiter(self, limits=limits, user_api=user_api) + + @_format_docstring( + USER_APIS=", ".join('"{}"'.format(api) for api in _ALL_USER_APIS), + BLAS_LIBS=", ".join(_ALL_BLAS_LIBRARIES), + OPENMP_LIBS=", ".join(_ALL_OPENMP_LIBRARIES), + ) + def wrap(self, *, limits=None, user_api=None): + """Change the maximal number of threads that can be used in thread pools. + + This function returns an object that can be used as a decorator. + + Set the maximal number of threads that can be used in thread pools used in + the supported libraries to `limits`. This function works for libraries that + are already loaded in the interpreter and can be changed dynamically. + Parameters ---------- limits : int, dict or None (default=None) @@ -424,7 +508,7 @@ def limit(self, *, limits=None, user_api=None): - If None, this function will apply to all supported libraries. """ - return _threadpool_limits(self, limits=limits, user_api=user_api) + return _ThreadpoolLimiter.wrap(self, limits=limits, user_api=user_api) def __len__(self): return len(self.lib_controllers) From 2e5e67e5779ff32075082275b9ecac1fb375011f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Fri, 1 Oct 2021 13:56:08 +0200 Subject: [PATCH 122/187] Release 3.0.0 (#106) --- CHANGES.md | 4 ++-- threadpoolctl.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index be1e4fe0..e7d813c9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,5 @@ -3.0.0 (in development) -====================== +3.0.0 (2021-10-01) +================== - New object `threadpooctl.ThreadpoolController` which holds controllers for all the supported native libraries. The states of these libraries is accessible through the diff --git a/threadpoolctl.py b/threadpoolctl.py index d700963e..c21b20dc 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -21,7 +21,7 @@ from functools import lru_cache from contextlib import ContextDecorator -__version__ = "2.3.0.dev0" +__version__ = "3.0.0" __all__ = ["threadpool_limits", "threadpool_info", "ThreadpoolController"] From cb7f2b5d5d9b4ed32b5eb509137a409af9df294e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Fri, 1 Oct 2021 15:29:58 +0200 Subject: [PATCH 123/187] MNT back to dev mode 3.1.0.dev0 (#107) --- threadpoolctl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/threadpoolctl.py b/threadpoolctl.py index c21b20dc..bf4c7185 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -21,7 +21,7 @@ from functools import lru_cache from contextlib import ContextDecorator -__version__ = "3.0.0" +__version__ = "3.1.0.dev0" __all__ = ["threadpool_limits", "threadpool_info", "ThreadpoolController"] From ab57822dbd593ac40ce8d1cd5f465c6ece0b4f6e Mon Sep 17 00:00:00 2001 From: Kian Meng Ang Date: Thu, 25 Nov 2021 20:06:25 +0800 Subject: [PATCH 124/187] CLN correct typos (#109) --- CHANGES.md | 2 +- README.md | 2 +- tests/_openmp_test_helper/openmp_helpers_inner.pyx | 2 +- threadpoolctl.py | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e7d813c9..a3c5fb2e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -50,7 +50,7 @@ 2.0.0 (2019-12-05) ================== -- Expose MKL, BLIS and OpenBLAS threading layer in informations displayed by +- Expose MKL, BLIS and OpenBLAS threading layer in information displayed by `threadpool_info`. This information is referenced in the `threading_layer` field. https://github.com/joblib/threadpoolctl/pull/48 diff --git a/README.md b/README.md index 6f2565ca..e40b6d22 100644 --- a/README.md +++ b/README.md @@ -158,7 +158,7 @@ in specific sections of your Python program: The threadpools can also be controlled via the object oriented API, which is especially useful to avoid searching through all the loaded shared libraries each time. It will -however not act on libraries loaded after the instanciation of the +however not act on libraries loaded after the instantiation of the `ThreadpoolController`: ```python diff --git a/tests/_openmp_test_helper/openmp_helpers_inner.pyx b/tests/_openmp_test_helper/openmp_helpers_inner.pyx index 78d65303..eca797b9 100644 --- a/tests/_openmp_test_helper/openmp_helpers_inner.pyx +++ b/tests/_openmp_test_helper/openmp_helpers_inner.pyx @@ -22,7 +22,7 @@ cdef int inner_openmp_loop(int n) nogil: OpenMP runtime. This function is expected to run without the GIL and can be called - by an outer OpenMP / prange loop written in Cython in anoter file. + by an outer OpenMP / prange loop written in Cython in another file. """ cdef long n_sum = 0 cdef int i, num_threads diff --git a/threadpoolctl.py b/threadpoolctl.py index bf4c7185..e30c498f 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -7,7 +7,7 @@ # License: BSD 3-Clause # The code to introspect dynamically loaded libraries on POSIX systems is -# adapted from code by Intel developper @anton-malakhov available at +# adapted from code by Intel developer @anton-malakhov available at # https://github.com/IntelPython/smp (Copyright (c) 2017, Intel Corporation) # and also published under the BSD 3-Clause license import os @@ -61,7 +61,7 @@ class _dl_phdr_info(ctypes.Structure): # List of the supported libraries. The items are indexed by the name of the -# class to instanciate to create the library controller objects. The items hold +# class to instantiate to create the library controller objects. The items hold # the possible prefixes of loaded shared objects, the name of the internal_api # to call and the name of the user_api. _SUPPORTED_LIBRARIES = { @@ -526,7 +526,7 @@ def _find_libraries_with_dl_iterate_phdr(self): """Loop through loaded libraries and return binders on supported ones This function is expected to work on POSIX system only. - This code is adapted from code by Intel developper @anton-malakhov + This code is adapted from code by Intel developer @anton-malakhov available at https://github.com/IntelPython/smp Copyright (c) 2017, Intel Corporation published under the BSD 3-Clause From 47bc5868f2790b71d446450ca076e3978b179758 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Wed, 5 Jan 2022 09:34:30 +0100 Subject: [PATCH 125/187] add Zen openblas arch (#111) --- tests/test_threadpoolctl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index 7bfffc03..0245ed52 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -554,6 +554,7 @@ def test_architecture(): "Haswell", "SkylakeX", "Sandybridge", + "Zen", ) expected_blis_architectures = ( # XXX: add more as needed by CI or developer laptops From 3279d27cbc6c96c0c1a078af16fc477814059b2c Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 5 Jan 2022 13:45:13 +0100 Subject: [PATCH 126/187] Add VORTEX to test_architecture (#113) --- tests/test_threadpoolctl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index 0245ed52..c6b483b2 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -554,6 +554,7 @@ def test_architecture(): "Haswell", "SkylakeX", "Sandybridge", + "VORTEX", "Zen", ) expected_blis_architectures = ( From 4420d82b5e11a11111bdbcddbd84b5e4082def3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Wed, 5 Jan 2022 15:38:34 +0100 Subject: [PATCH 127/187] FIX BLAS detection conda-forge + Windows (#112) Co-authored-by: Olivier Grisel --- .azure_pipeline.yml | 11 +++-- CHANGES.md | 6 +++ continuous_integration/install.cmd | 7 +++- continuous_integration/install_with_blis.sh | 1 + continuous_integration/test_script.cmd | 2 +- continuous_integration/test_script.sh | 2 +- threadpoolctl.py | 45 ++++++++++++++++----- 7 files changed, 58 insertions(+), 16 deletions(-) diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml index b7d963a8..e6111f2b 100644 --- a/.azure_pipeline.yml +++ b/.azure_pipeline.yml @@ -13,11 +13,16 @@ stages: - template: continuous_integration/windows.yml parameters: name: Windows - vmImage: vs2017-win2016 + vmImage: windows-latest matrix: - pylatest_conda: + pylatest_conda_forge_mkl: VERSION_PYTHON: '*' - PACKAGER: 'conda' + PACKAGER: 'conda-forge' + BLAS: 'mkl' + py39_conda_forge_openblas: + VERSION_PYTHON: '3.9' + PACKAGER: 'conda-forge' + BLAS: 'openblas' py37_conda: VERSION_PYTHON: '3.7' PACKAGER: 'conda' diff --git a/CHANGES.md b/CHANGES.md index a3c5fb2e..8c79f56a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,9 @@ +3.1.0 (TBD) +=========== + +- Fixed a detection issue of the BLAS libraires packaged by conda-forge on Windows. + https://github.com/joblib/threadpoolctl/pull/112 + 3.0.0 (2021-10-01) ================== diff --git a/continuous_integration/install.cmd b/continuous_integration/install.cmd index 4a0d5970..4da530ff 100644 --- a/continuous_integration/install.cmd +++ b/continuous_integration/install.cmd @@ -18,8 +18,11 @@ python --version pip --version @rem Install dependencies with either conda or pip. -if "%PACKAGER%" == "conda" (%CONDA_INSTALL% numpy scipy pytest cython) -if "%PACKAGER%" == "pip" (%PIP_INSTALL% numpy scipy pytest cython) +set TO_INSTALL=numpy scipy cython pytest + +if "%PACKAGER%" == "conda" (%CONDA_INSTALL% %TO_INSTALL%) +if "%PACKAGER%" == "conda-forge" (%CONDA_INSTALL% -c conda-forge %TO_INSTALL% blas[build=%BLAS%]) +if "%PACKAGER%" == "pip" (%PIP_INSTALL% %TO_INSTALL%) @rem Install extra developer dependencies pip install -q -r dev-requirements.txt diff --git a/continuous_integration/install_with_blis.sh b/continuous_integration/install_with_blis.sh index a2d6c962..c42e6aa3 100755 --- a/continuous_integration/install_with_blis.sh +++ b/continuous_integration/install_with_blis.sh @@ -35,6 +35,7 @@ popd # build & install numpy git clone https://github.com/numpy/numpy.git pushd numpy +git submodule update --init echo "[blis] libraries = blis library_dirs = $ABS_PATH/BLIS_install/lib diff --git a/continuous_integration/test_script.cmd b/continuous_integration/test_script.cmd index e6952f5e..c3c3da05 100644 --- a/continuous_integration/test_script.cmd +++ b/continuous_integration/test_script.cmd @@ -2,6 +2,6 @@ call activate %VIRTUALENV% # Use the CLI to display the effective runtime environment prior to # launching the tests: -python -m threadpoolctl -i numpy scipy.linalg tests._openmp_test_helper +python -m threadpoolctl -i numpy scipy.linalg tests._openmp_test_helper.openmp_helpers_inner pytest -vlrxXs --junitxml=%JUNITXML% --cov=threadpoolctl diff --git a/continuous_integration/test_script.sh b/continuous_integration/test_script.sh index b59023f6..b1d3a64c 100755 --- a/continuous_integration/test_script.sh +++ b/continuous_integration/test_script.sh @@ -15,6 +15,6 @@ set -x # Use the CLI to display the effective runtime environment prior to # launching the tests: -python -m threadpoolctl -i numpy scipy.linalg tests._openmp_test_helper +python -m threadpoolctl -i numpy scipy.linalg tests._openmp_test_helper.openmp_helpers_inner pytest -vlrxXs -W error -k "$TESTS" --junitxml=$JUNITXML --cov=threadpoolctl diff --git a/threadpoolctl.py b/threadpoolctl.py index e30c498f..5ec08f20 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -63,7 +63,9 @@ class _dl_phdr_info(ctypes.Structure): # List of the supported libraries. The items are indexed by the name of the # class to instantiate to create the library controller objects. The items hold # the possible prefixes of loaded shared objects, the name of the internal_api -# to call and the name of the user_api. +# to call, the name of the user_api and potentially some symbols that the library is +# expected to have (this is necessary to distinguish between the blas implementations +# when they are all renamed "libblas.dll" on conda-forge on windows). _SUPPORTED_LIBRARIES = { "OpenMPController": { "user_api": "openmp", @@ -73,28 +75,33 @@ class _dl_phdr_info(ctypes.Structure): "OpenBLASController": { "user_api": "blas", "internal_api": "openblas", - "filename_prefixes": ("libopenblas",), + "filename_prefixes": ("libopenblas", "libblas"), + "check_symbols": ("openblas_get_num_threads", "openblas_get_num_threads64_"), }, "MKLController": { "user_api": "blas", "internal_api": "mkl", - "filename_prefixes": ("libmkl_rt", "mkl_rt"), + "filename_prefixes": ("libmkl_rt", "mkl_rt", "libblas"), + "check_symbols": ("MKL_Get_Max_Threads",), }, "BLISController": { "user_api": "blas", "internal_api": "blis", - "filename_prefixes": ("libblis",), + "filename_prefixes": ("libblis", "libblas"), + "check_symbols": ("bli_thread_get_num_threads",), }, } # Helpers for the doc and test names _ALL_USER_APIS = list(set(lib["user_api"] for lib in _SUPPORTED_LIBRARIES.values())) _ALL_INTERNAL_APIS = [lib["internal_api"] for lib in _SUPPORTED_LIBRARIES.values()] -_ALL_PREFIXES = [ - prefix - for lib in _SUPPORTED_LIBRARIES.values() - for prefix in lib["filename_prefixes"] -] +_ALL_PREFIXES = list( + set( + prefix + for lib in _SUPPORTED_LIBRARIES.values() + for prefix in lib["filename_prefixes"] + ) +) _ALL_BLAS_LIBRARIES = [ lib["internal_api"] for lib in _SUPPORTED_LIBRARIES.values() @@ -660,6 +667,26 @@ def _make_controller_from_path(self, filepath): if prefix is None: continue + # workaround for BLAS libraries packaged by conda-forge on windows, which + # are all renamed "libblas.dll". We thus have to check to which BLAS + # implementation it actually corresponds looking for implementation + # specific symbols. + if prefix == "libblas": + if filename.endswith(".dll"): + libblas = ctypes.CDLL(filepath, _RTLD_NOLOAD) + if not any( + hasattr(libblas, func) + for func in candidate_lib["check_symbols"] + ): + continue + else: + # We ignore libblas on other platforms than windows because there + # might be a libblas dso comming with openblas for instance that + # can't be used to instantiate a pertinent LibController (many + # symbols are missing) and would create confusion by making a + # duplicate entry in threadpool_info. + continue + # filename matches a prefix. Create and store the library # controller. user_api = candidate_lib["user_api"] From 07c6f66aa2db112b0b3449b3f720d4156ebf8a77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Thu, 20 Jan 2022 18:04:54 +0100 Subject: [PATCH 128/187] CI upgrade macos vm image (#115) --- .azure_pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml index e6111f2b..9420f5eb 100644 --- a/.azure_pipeline.yml +++ b/.azure_pipeline.yml @@ -128,7 +128,7 @@ stages: - template: continuous_integration/posix.yml parameters: name: macOS - vmImage: macOS-10.14 + vmImage: macOS-10.15 matrix: # MacOS environment with OpenMP installed through homebrew py36_conda_homebrew_libomp: From 41cef8753c5c8e3230c16f406c613267a82097ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Fri, 28 Jan 2022 14:25:54 +0100 Subject: [PATCH 129/187] MTN add cron job to run CI (#118) --- .azure_pipeline.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml index 9420f5eb..42fbd0a2 100644 --- a/.azure_pipeline.yml +++ b/.azure_pipeline.yml @@ -6,6 +6,14 @@ variables: JUNITXML: 'test-data.xml' CODECOV_TOKEN: 'cee0e505-c12e-4139-aa43-621fb16a2347' +schedules: +- cron: "0 1 * * *" # 1am UTC + displayName: Run nightly build + branches: + include: + - master + always: true + stages: - stage: jobs: From 9fac583a5551d08a208f216fe2246b712ac59249 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Fri, 28 Jan 2022 15:55:54 +0100 Subject: [PATCH 130/187] MNT Use flit_core build-backend (#116) Use flit_core build-backend instead of flit per upstream recommendations. flit_core provides the core package building functionality without the complete package manager flit provides. Since it has fewer dependencies, it makes package building much faster. --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f6955703..6e8673b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [build-system] -requires = ["flit"] -build-backend = "flit.buildapi" +requires = ["flit_core"] +build-backend = "flit_core.buildapi" [tool.flit.metadata] module = "threadpoolctl" From 77a9170c2d47779e29b0b654c95b328afe5b3145 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Fri, 28 Jan 2022 16:20:15 +0100 Subject: [PATCH 131/187] MNT Add Prescott to the expected openblas archs (#117) --- tests/test_threadpoolctl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index c6b483b2..9bdd7bda 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -552,6 +552,7 @@ def test_architecture(): # XXX: add more as needed by CI or developer laptops "armv8", "Haswell", + "Prescott", # see: https://github.com/xianyi/OpenBLAS/pull/3485 "SkylakeX", "Sandybridge", "VORTEX", From 9dad5136e74e5ba7f95ea71ad2c27c3bf32b6201 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Fri, 28 Jan 2022 18:20:16 +0100 Subject: [PATCH 132/187] Workaround for openblas + openmp threading layer (#114) --- .azure_pipeline.yml | 3 ++- CHANGES.md | 7 ++++++ README.md | 7 ++++++ continuous_integration/install.sh | 3 +++ tests/test_threadpoolctl.py | 40 ++++++++++++++++++++++++++++--- threadpoolctl.py | 39 ++++++++++++++++++++++++++---- 6 files changed, 90 insertions(+), 9 deletions(-) diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml index 42fbd0a2..0d171bf4 100644 --- a/.azure_pipeline.yml +++ b/.azure_pipeline.yml @@ -89,11 +89,12 @@ stages: VERSION_PYTHON: '3.8' CC_OUTER_LOOP: 'gcc' CC_INNER_LOOP: 'clang-10' - # Linux environment with numpy from conda-forge channel + # Linux environment with numpy from conda-forge channel and openblas-openmp pylatest_conda_forge: PACKAGER: 'conda-forge' VERSION_PYTHON: '*' BLAS: 'openblas' + OPENBLAS_THREADING_LAYER: 'openmp' CC_OUTER_LOOP: 'gcc' CC_INNER_LOOP: 'gcc' LINT: 'true' diff --git a/CHANGES.md b/CHANGES.md index 8c79f56a..4842c859 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,13 @@ - Fixed a detection issue of the BLAS libraires packaged by conda-forge on Windows. https://github.com/joblib/threadpoolctl/pull/112 +- `threadpool_limits` and `ThreadpoolController.limit` now accept the string + "sequential_blas_under_openmp" for the `limits` parameter. It should only be used for + the specific case when one wants to have sequential BLAS calls within an OpenMP + parallel region. It takes into account the unexpected behavior of OpenBLAS with the + OpenMP threading layer. + https://github.com/joblib/threadpoolctl/pull/114 + 3.0.0 (2021-10-01) ================== diff --git a/README.md b/README.md index e40b6d22..f41905e1 100644 --- a/README.md +++ b/README.md @@ -192,6 +192,13 @@ decorators are accessible through their `wrap` method: ... ``` +### Sequential BLAS within OpenMP parallel region + +When one wants to have sequential BLAS calls within an OpenMP parallel region, it's +safer to set `limits="sequential_blas_under_openmp"` since setting `limits=1` and `user_api="blas"` might not lead to the expected behavior in some configurations +(e.g. OpenBLAS with the OpenMP threading layer +https://github.com/xianyi/OpenBLAS/issues/2985). + ### Known Limitations - `threadpool_limits` can fail to limit the number of inner threads when nesting diff --git a/continuous_integration/install.sh b/continuous_integration/install.sh index 8b20fef0..46d96b35 100755 --- a/continuous_integration/install.sh +++ b/continuous_integration/install.sh @@ -50,6 +50,9 @@ elif [[ "$PACKAGER" == "conda-forge" ]]; then conda config --prepend channels conda-forge conda config --set channel_priority strict TO_INSTALL="python=$VERSION_PYTHON numpy scipy blas[build=$BLAS]" + if [[ "$BLAS" == "openblas" && "$OPENBLAS_THREADING_LAYER" == "openmp" ]]; then + TO_INSTALL="$TO_INSTALL libopenblas=*=*openmp*" + fi make_conda $TO_INSTALL elif [[ "$PACKAGER" == "pip" ]]; then diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index 9bdd7bda..250cf8ed 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -208,8 +208,42 @@ def test_threadpool_controller_limit(): for lib_controller in blas_controller.lib_controllers ) # original_blas_controller contains only blas libraries so no opemp library - # should be impacted. - assert openmp_info == original_openmp_info + # should be impacted. This is not True for OpenBLAS with the OpenMP threading + # layer. + if not any( + lib_controller.internal_api == "openblas" + and lib_controller.threading_layer == "openmp" + for lib_controller in blas_controller.lib_controllers + ): + assert openmp_info == original_openmp_info + + +def test_get_params_for_sequential_blas_under_openmp(): + # Test for the behavior of get_params_for_sequential_blas_under_openmp. + controller = ThreadpoolController() + original_info = controller.info() + + params = controller._get_params_for_sequential_blas_under_openmp() + + if controller.select( + internal_api="openblas", threading_layer="openmp" + ).lib_controllers: + assert params["limits"] is None + assert params["user_api"] is None + + with controller.limit(limits="sequential_blas_under_openmp"): + assert controller.info() == original_info + + else: + assert params["limits"] == 1 + assert params["user_api"] == "blas" + + with controller.limit(limits="sequential_blas_under_openmp"): + assert all( + lib_info["num_threads"] == 1 + for lib_info in controller.info() + if lib_info["user_api"] == "blas" + ) def test_nested_limits(): @@ -245,7 +279,7 @@ def test_threadpool_limits_bad_input(): threadpool_limits(limits=1, user_api="wrong") with pytest.raises( - TypeError, match="limits must either be an int, a list or a dict" + TypeError, match="limits must either be an int, a list, a dict, or" ): threadpool_limits(limits=(1, 2, 3)) diff --git a/threadpoolctl.py b/threadpoolctl.py index 5ec08f20..9e0ee40b 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -158,10 +158,10 @@ class _ThreadpoolLimiter: """ def __init__(self, controller, *, limits=None, user_api=None): + self._controller = controller self._limits, self._user_api, self._prefixes = self._check_params( limits, user_api ) - self._controller = controller self._original_info = self._controller.info() self._set_threadpool_limits() @@ -226,6 +226,13 @@ def get_original_num_threads(self): def _check_params(self, limits, user_api): """Suitable values for the _limits, _user_api and _prefixes attributes""" + + if isinstance(limits, str) and limits == "sequential_blas_under_openmp": + ( + limits, + user_api, + ) = self._controller._get_params_for_sequential_blas_under_openmp().values() + if limits is None or isinstance(limits, int): if user_api is None: user_api = _ALL_USER_APIS @@ -257,8 +264,8 @@ def _check_params(self, limits, user_api): if not isinstance(limits, dict): raise TypeError( - "limits must either be an int, a list or a " - f"dict. Got {type(limits)} instead" + "limits must either be an int, a list, a dict, or " + f"'sequential_blas_under_openmp'. Got {type(limits)} instead" ) # With a dictionary, can set both specific limit for given @@ -333,7 +340,7 @@ class threadpool_limits(_ThreadpoolLimiter): Parameters ---------- - limits : int, dict or None (default=None) + limits : int, dict, 'sequential_blas_under_openmp' or None (default=None) The maximal number of threads that can be used in thread pools - If int, sets the maximum number of threads to `limits` for each @@ -343,6 +350,11 @@ class threadpool_limits(_ThreadpoolLimiter): custom maximum number of threads for each `key` which can be either a `user_api` or a `prefix` for a specific library. + - If 'sequential_blas_under_openmp', it will chose the appropriate `limits` + and `user_api` parameters for the specific use case of sequential BLAS + calls within an OpenMP parallel region. The `user_api` parameter is + ignored. + - If None, this function does not do anything. user_api : {USER_APIS} or None (default=None) @@ -428,6 +440,18 @@ def select(self, **kwargs): return ThreadpoolController._from_controllers(lib_controllers) + def _get_params_for_sequential_blas_under_openmp(self): + """Return appropriate params to use for a sequential BLAS call in an OpenMP loop + + This function takes into account the unexpected behavior of OpenBLAS with the + OpenMP threading layer. + """ + if self.select( + internal_api="openblas", threading_layer="openmp" + ).lib_controllers: + return {"limits": None, "user_api": None} + return {"limits": 1, "user_api": "blas"} + @_format_docstring( USER_APIS=", ".join('"{}"'.format(api) for api in _ALL_USER_APIS), BLAS_LIBS=", ".join(_ALL_BLAS_LIBRARIES), @@ -451,7 +475,7 @@ def limit(self, *, limits=None, user_api=None): Parameters ---------- - limits : int, dict or None (default=None) + limits : int, dict, 'sequential_blas_under_openmp' or None (default=None) The maximal number of threads that can be used in thread pools - If int, sets the maximum number of threads to `limits` for each @@ -461,6 +485,11 @@ def limit(self, *, limits=None, user_api=None): custom maximum number of threads for each `key` which can be either a `user_api` or a `prefix` for a specific library. + - If 'sequential_blas_under_openmp', it will chose the appropriate `limits` + and `user_api` parameters for the specific use case of sequential BLAS + calls within an OpenMP parallel region. The `user_api` parameter is + ignored. + - If None, this function does not do anything. user_api : {USER_APIS} or None (default=None) From 0b554d6968cc345a0fad15338d2d4abd40e994a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Mon, 31 Jan 2022 17:23:29 +0100 Subject: [PATCH 133/187] Release 3.1.0 (#119) --- CHANGES.md | 4 ++-- threadpoolctl.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 4842c859..aafeba7f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,5 @@ -3.1.0 (TBD) -=========== +3.1.0 (2022-01-31) +================== - Fixed a detection issue of the BLAS libraires packaged by conda-forge on Windows. https://github.com/joblib/threadpoolctl/pull/112 diff --git a/threadpoolctl.py b/threadpoolctl.py index 9e0ee40b..86bfc392 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -21,7 +21,7 @@ from functools import lru_cache from contextlib import ContextDecorator -__version__ = "3.1.0.dev0" +__version__ = "3.1.0" __all__ = ["threadpool_limits", "threadpool_info", "ThreadpoolController"] From 86a709da5759d7e5c084eead191134929d1f5815 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Mon, 31 Jan 2022 18:16:48 +0100 Subject: [PATCH 134/187] MNT back to dev mode 3.2.0.dev0 (#120) --- threadpoolctl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/threadpoolctl.py b/threadpoolctl.py index 86bfc392..489b9150 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -21,7 +21,7 @@ from functools import lru_cache from contextlib import ContextDecorator -__version__ = "3.1.0" +__version__ = "3.2.0.dev0" __all__ = ["threadpool_limits", "threadpool_info", "ThreadpoolController"] From a33819e4d48ad154edad1cc5665939f27abbca0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Wed, 20 Apr 2022 18:46:28 +0200 Subject: [PATCH 135/187] TST Avoid side effects in test suite (#124) --- tests/test_threadpoolctl.py | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index 250cf8ed..ee197a2d 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -22,6 +22,21 @@ def is_old_openblas(lib_controller): return lib_controller.internal_api == "openblas" and lib_controller.version is None +def skip_if_openblas_openmp(): + """Helper to skip tests with side effects when OpenBLAS has the OpenMP + threading layer. + """ + if any( + lib_controller.internal_api == "openblas" + and lib_controller.threading_layer == "openmp" + for lib_controller in ThreadpoolController().lib_controllers + ): + pytest.skip( + "Setting a limit on OpenBLAS when using the OpenMP threading layer also " + "impact the OpenMP library. They can't be controlled independently." + ) + + def effective_num_threads(nthreads, max_threads): if nthreads is None or nthreads > max_threads: return max_threads @@ -196,6 +211,10 @@ def test_threadpool_limits_manual_restore(): def test_threadpool_controller_limit(): # Check that using the limit method of ThreadpoolController only impact its # library controllers. + + # This is not True for OpenBLAS with the OpenMP threading layer. + skip_if_openblas_openmp() + blas_controller = ThreadpoolController().select(user_api="blas") original_openmp_info = ThreadpoolController().select(user_api="openmp").info() @@ -208,14 +227,8 @@ def test_threadpool_controller_limit(): for lib_controller in blas_controller.lib_controllers ) # original_blas_controller contains only blas libraries so no opemp library - # should be impacted. This is not True for OpenBLAS with the OpenMP threading - # layer. - if not any( - lib_controller.internal_api == "openblas" - and lib_controller.threading_layer == "openmp" - for lib_controller in blas_controller.lib_controllers - ): - assert openmp_info == original_openmp_info + # should be impacted. + assert openmp_info == original_openmp_info def test_get_params_for_sequential_blas_under_openmp(): @@ -407,6 +420,8 @@ def test_nested_prange_blas(nthreads_outer): check_nested_prange_blas = prange_blas.check_nested_prange_blas + skip_if_openblas_openmp() + original_info = ThreadpoolController().info() blas_controller = ThreadpoolController().select(user_api="blas") From a39c6a49a297d0ef941269fc655670b63edab84c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Wed, 25 May 2022 11:30:09 +0200 Subject: [PATCH 136/187] CI Use packages from conda-forge to get more up to date versions in install_with_blis.sh (#126) --- continuous_integration/install_with_blis.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/continuous_integration/install_with_blis.sh b/continuous_integration/install_with_blis.sh index c42e6aa3..ee4e9645 100755 --- a/continuous_integration/install_with_blis.sh +++ b/continuous_integration/install_with_blis.sh @@ -13,7 +13,7 @@ sudo ./llvm.sh 10 sudo apt-get install libomp-dev # create conda env -conda create -n $VIRTUALENV -q --yes python=$VERSION_PYTHON pip cython +conda create -n $VIRTUALENV -q --yes -c conda-forge python=$VERSION_PYTHON pip cython source activate $VIRTUALENV if [[ "$BLIS_CC" == "gcc-8" ]]; then From b9897ce6ed2f295f1fd1c4f27a2906c70f4321f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= Date: Tue, 13 Sep 2022 15:51:00 +0200 Subject: [PATCH 137/187] DOC Fix link to Intel Community forum (#129) --- multiple_openmp.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multiple_openmp.md b/multiple_openmp.md index 07d13aa3..b8da5a1a 100644 --- a/multiple_openmp.md +++ b/multiple_openmp.md @@ -82,4 +82,4 @@ developers on the following public issue trackers/forums along with a minimal reproducer written in C: - https://bugs.llvm.org/show_bug.cgi?id=43565 -- https://software.intel.com/en-us/forums/intel-c-compiler/topic/827607 +- https://community.intel.com/t5/Intel-C-Compiler/Cannot-call-OpenMP-functions-from-libiomp-after-calling-from/m-p/1176406 From 8826b8bde1f9f772254731995844d62b12017c65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Fri, 23 Sep 2022 18:14:01 +0200 Subject: [PATCH 138/187] MAINT drop support for python 3.6/3.7 (#130) --- .azure_pipeline.yml | 34 +++++++++++++++++++++------------- CHANGES.md | 5 +++++ README.md | 3 ++- pyproject.toml | 9 ++++----- tests/test_threadpoolctl.py | 2 +- threadpoolctl.py | 10 ++++++++-- 6 files changed, 41 insertions(+), 22 deletions(-) diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml index 0d171bf4..3ea8f7a9 100644 --- a/.azure_pipeline.yml +++ b/.azure_pipeline.yml @@ -27,15 +27,15 @@ stages: VERSION_PYTHON: '*' PACKAGER: 'conda-forge' BLAS: 'mkl' - py39_conda_forge_openblas: - VERSION_PYTHON: '3.9' + py310_conda_forge_openblas: + VERSION_PYTHON: '3.10' PACKAGER: 'conda-forge' BLAS: 'openblas' - py37_conda: - VERSION_PYTHON: '3.7' + py39_conda: + VERSION_PYTHON: '3.9' PACKAGER: 'conda' - py36_pip: - VERSION_PYTHON: '3.6' + py38_pip: + VERSION_PYTHON: '3.8' PACKAGER: 'pip' @@ -52,16 +52,16 @@ stages: VERSION_PYTHON: '3.8' CC_OUTER_LOOP: 'gcc' CC_INNER_LOOP: 'gcc' - py36_ubuntu_openblas_gcc_gcc: + py38_ubuntu_openblas_gcc_gcc: PACKAGER: 'ubuntu' APT_BLAS: 'libopenblas-base libopenblas-dev' VERSION_PYTHON: '3.8' CC_OUTER_LOOP: 'gcc' CC_INNER_LOOP: 'gcc' - # Linux + Python 3.7 and homogeneous runtime nesting. - py37_conda_openblas_clang_clang: + # Linux + Python 3.9 and homogeneous runtime nesting. + py39_conda_openblas_clang_clang: PACKAGER: 'conda' - VERSION_PYTHON: '3.7' + VERSION_PYTHON: '3.9' BLAS: 'openblas' CC_OUTER_LOOP: 'clang-10' CC_INNER_LOOP: 'clang-10' @@ -137,16 +137,24 @@ stages: - template: continuous_integration/posix.yml parameters: name: macOS - vmImage: macOS-10.15 + vmImage: macOS-11 matrix: # MacOS environment with OpenMP installed through homebrew - py36_conda_homebrew_libomp: + py38_conda_homebrew_libomp: PACKAGER: 'conda' - VERSION_PYTHON: '3.6' + VERSION_PYTHON: '3.8' BLAS: 'openblas' CC_OUTER_LOOP: 'clang' CC_INNER_LOOP: 'clang' INSTALL_LIBOMP: 'homebrew' + # MacOS env with OpenBLAS and OpenMP installed through conda-forge compilers + py39_conda_forge_clang_openblas: + PACKAGER: 'conda-forge' + VERSION_PYTHON: '*' + BLAS: 'openblas' + CC_OUTER_LOOP: 'clang' + CC_INNER_LOOP: 'clang' + INSTALL_LIBOMP: 'conda-forge' # MacOS environment with OpenMP installed through conda-forge compilers pylatest_conda_forge_clang: PACKAGER: 'conda-forge' diff --git a/CHANGES.md b/CHANGES.md index aafeba7f..d7ff6719 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,8 @@ +3.2.0 (TBD) +=========== + +- Dropped support for Python 3.6 and 3.7. + 3.1.0 (2022-01-31) ================== diff --git a/README.md b/README.md index f41905e1..387c70c5 100644 --- a/README.md +++ b/README.md @@ -195,7 +195,8 @@ decorators are accessible through their `wrap` method: ### Sequential BLAS within OpenMP parallel region When one wants to have sequential BLAS calls within an OpenMP parallel region, it's -safer to set `limits="sequential_blas_under_openmp"` since setting `limits=1` and `user_api="blas"` might not lead to the expected behavior in some configurations +safer to set `limits="sequential_blas_under_openmp"` since setting `limits=1` and +`user_api="blas"` might not lead to the expected behavior in some configurations (e.g. OpenBLAS with the OpenMP threading layer https://github.com/xianyi/OpenBLAS/issues/2985). diff --git a/pyproject.toml b/pyproject.toml index 6e8673b5..af44de37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,20 +8,19 @@ author = "Thomas Moreau" author-email = "thomas.moreau.2010@gmail.com" home-page = "https://github.com/joblib/threadpoolctl" description-file = "README.md" -requires-python = ">=3.6" +requires-python = ">=3.8" license = "BSD-3-Clause" classifiers = [ "Intended Audience :: Developers", "License :: OSI Approved :: BSD License", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "Topic :: Software Development :: Libraries :: Python Modules", ] [tool.black] line-length = 88 -target_version = ['py36', 'py37', 'py38', 'py39'] -experimental_string_processing = true +target_version = ['py38', 'py39', 'py310'] +preview = true diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index ee197a2d..4e15db4a 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -460,7 +460,7 @@ def test_nested_prange_blas(nthreads_outer): # the method `get_original_num_threads` raises a UserWarning due to different # num_threads from libraries with the same `user_api`. It will be raised only -# in the CI job with 2 openblas (py37_pip_openblas_gcc_clang). It is expected +# in the CI job with 2 openblas (py38_pip_openblas_gcc_clang). It is expected # so we can safely filter it. @pytest.mark.filterwarnings("ignore::UserWarning") @pytest.mark.parametrize("limit", [1, None]) diff --git a/threadpoolctl.py b/threadpoolctl.py index 489b9150..b20356cc 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -40,8 +40,8 @@ # Structure to cast the info on dynamically loaded library. See # https://linux.die.net/man/3/dl_iterate_phdr for more details. -_SYSTEM_UINT = ctypes.c_uint64 if sys.maxsize > 2 ** 32 else ctypes.c_uint32 -_SYSTEM_UINT_HALF = ctypes.c_uint32 if sys.maxsize > 2 ** 32 else ctypes.c_uint16 +_SYSTEM_UINT = ctypes.c_uint64 if sys.maxsize > 2**32 else ctypes.c_uint32 +_SYSTEM_UINT_HALF = ctypes.c_uint32 if sys.maxsize > 2**32 else ctypes.c_uint16 class _dl_phdr_info(ctypes.Structure): @@ -768,6 +768,12 @@ def _get_libc(cls): if libc is None: libc_name = find_library("c") if libc_name is None: # pragma: no cover + warnings.warn( + "libc not found. The ctypes module in Python " + f"{sys.version_info.major}.{sys.version_info.minor} is maybe too " + "old for this OS.", + RuntimeWarning, + ) return None libc = ctypes.CDLL(libc_name, mode=_RTLD_NOLOAD) cls._system_libraries["libc"] = libc From 009e5b93d1cfd41677efafb1d213989497b4e67f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Tue, 21 Feb 2023 16:01:45 +0100 Subject: [PATCH 139/187] whitelist test_multiple_openmp (#133) --- continuous_integration/check_no_test_skipped.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/continuous_integration/check_no_test_skipped.py b/continuous_integration/check_no_test_skipped.py index 1a8861e1..7744aae3 100644 --- a/continuous_integration/check_no_test_skipped.py +++ b/continuous_integration/check_no_test_skipped.py @@ -33,11 +33,20 @@ always_skipped[test_name] = skipped print("\n------------------------------------------------------------------\n") + +# List of tests that we don't want to fail the CI if they are skipped in +# every job. This is useful for tests that depend on specific versions of +# numpy or scipy and we don't want to pin old versions of these libraries. +SAFE_SKIPPED_TESTS = ["test_multiple_shipped_openblas"] + fail = False for test, skipped in always_skipped.items(): if skipped: - fail = True - print(test, "was skipped in every job") + if test in SAFE_SKIPPED_TESTS: + print(test, "was skipped in every job but it's fine to skip it") + else: + fail = True + print(test, "was skipped in every job") if fail: sys.exit(1) From 36aebce43b7e2941884c2d5d660d7cab1c215585 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Fri, 7 Jul 2023 17:58:40 +0200 Subject: [PATCH 140/187] MAINT Fix BLIS install (#139) --- continuous_integration/install_with_blis.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/continuous_integration/install_with_blis.sh b/continuous_integration/install_with_blis.sh index ee4e9645..6463ac86 100755 --- a/continuous_integration/install_with_blis.sh +++ b/continuous_integration/install_with_blis.sh @@ -41,8 +41,7 @@ libraries = blis library_dirs = $ABS_PATH/BLIS_install/lib include_dirs = $ABS_PATH/BLIS_install/include/blis runtime_library_dirs = $ABS_PATH/BLIS_install/lib" > site.cfg -python setup.py build_ext -i -pip install -e . +python setup.py develop popd popd From 7b7969324bbf947786e01ea472f9442c53c6223d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Fri, 7 Jul 2023 18:47:31 +0200 Subject: [PATCH 141/187] add python 3.11 in pyproject.toml (#141) --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index af44de37..e53f6814 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,10 +17,11 @@ classifiers = [ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Topic :: Software Development :: Libraries :: Python Modules", ] [tool.black] line-length = 88 -target_version = ['py38', 'py39', 'py310'] +target_version = ['py38', 'py39', 'py310', 'py311'] preview = true From 57a59e4efa9335d787473896bfaa4468dfbbe84b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Tue, 11 Jul 2023 10:14:57 +0200 Subject: [PATCH 142/187] DOC Warn if coexisting libomp / libiomp on MacOS (#142) Co-authored-by: Olivier Grisel --- CHANGES.md | 5 +++++ multiple_openmp.md | 22 ++++++++++++---------- threadpoolctl.py | 4 ---- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d7ff6719..7ae8a673 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,6 +3,11 @@ - Dropped support for Python 3.6 and 3.7. +- A warning is raised on macOS when threadpoolctl finds both Intel OpenMP and LLVM + OpenMP runtimes loaded simultaneously by the same Python program. See details and + workarounds at https://github.com/joblib/threadpoolctl/blob/master/multiple_openmp.md. + https://github.com/joblib/threadpoolctl/pull/142 + 3.1.0 (2022-01-31) ================== diff --git a/multiple_openmp.md b/multiple_openmp.md index b8da5a1a..8da2cb91 100644 --- a/multiple_openmp.md +++ b/multiple_openmp.md @@ -29,13 +29,13 @@ program**. For instance, on Linux, we never observed any issue between `libgomp` and `libiomp`, which is the most common mix (NumPy with MKL + a package compiled with GCC, the most widely used C compiler on that platform). -## Incompatibility between Intel OpenMP and LLVM OpenMP under Linux +## Incompatibility between Intel OpenMP and LLVM OpenMP The only unrecoverable incompatibility we encountered happens when loading a mix of compiled extensions linked with **`libomp` (LLVM/Clang) and `libiomp` -(ICC), on Linux**, manifested by crashes or deadlocks. It can happen even with -the simplest OpenMP calls like getting the maximum number of threads that will -be used in a subsequent parallel region. A possible explanation is that +(ICC), on Linux and macOS**, manifested by crashes or deadlocks. It can happen +even with the simplest OpenMP calls like getting the maximum number of threads +that will be used in a subsequent parallel region. A possible explanation is that `libomp` is actually a fork of `libiomp` causing name colliding for instance. Using `threadpoolctl` may crash your program in such a setting. @@ -43,21 +43,23 @@ Using `threadpoolctl` may crash your program in such a setting. binary distributions of Python packages for Linux use either GCC or ICC to build the Python scientific packages. Therefore this problem would only happen if some packagers decide to start shipping Python packages built with -LLVM/Clang instead of GCC. - -Surprisingly, we never encountered this kind of issue on macOS, where this mix -is the most frequent (Clang being the default C compiler on macOS). +LLVM/Clang instead of GCC (this is the case for instance with conda's default channel). ## Workarounds for Intel OpenMP and LLVM OpenMP case As far as we know, the only workaround consists in making sure only of one of the two incompatible OpenMP libraries is loaded. For example: -- Tell MKL (used by NumPy) to use the GNU OpenMP runtime instead of the Intel - OpenMP runtime by setting the following environment variable: +- Tell MKL (used by NumPy) to use another threading implementation instead of the Intel + OpenMP runtime. It can be the GNU OpenMP runtime on Linux or TBB on Linux and MacOS + for instance. This is done by setting the following environment variable: export MKL_THREADING_LAYER=GNU + or, if TBB is installed: + + export MKL_THREADING_LAYER=TBB + - Install a build of NumPy and SciPy linked against OpenBLAS instead of MKL. This can be done for instance by installing NumPy and SciPy from PyPI: diff --git a/threadpoolctl.py b/threadpoolctl.py index b20356cc..0fb857ae 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -742,10 +742,6 @@ def _check_prefix(self, library_basename, filename_prefixes): def _warn_if_incompatible_openmp(self): """Raise a warning if llvm-OpenMP and intel-OpenMP are both loaded""" - if sys.platform != "linux": - # Only raise the warning on linux - return - prefixes = [lib_controller.prefix for lib_controller in self.lib_controllers] msg = textwrap.dedent( """ From c2f4b0ad24970739e43618d4e49253c8b1decf4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Tue, 11 Jul 2023 16:14:39 +0200 Subject: [PATCH 143/187] FEA Support custom lib controllers (#138) Co-authored-by: Olivier Grisel --- .gitignore | 7 +- CHANGES.md | 5 + README.md | 14 + continuous_integration/build_test_ext.sh | 8 + tests/_pyMylib/__init__.py | 46 ++ tests/_pyMylib/my_threaded_lib.c | 17 + tests/test_threadpoolctl.py | 30 ++ threadpoolctl.py | 644 ++++++++++++----------- 8 files changed, 451 insertions(+), 320 deletions(-) create mode 100644 tests/_pyMylib/__init__.py create mode 100644 tests/_pyMylib/my_threaded_lib.c diff --git a/.gitignore b/.gitignore index a0afdb17..387f3b4c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,14 @@ -# Python and Cython generated files +# Python generated files *.pyc __pycache__ .cache .pytest_cache + +# Cython/C generated files +*.o *.so *.dylib -*.c +tests/_openmp_test_helper/*.c # Python install files, build and release artifacts *.egg-info/ diff --git a/CHANGES.md b/CHANGES.md index 7ae8a673..3a78b49f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,6 +3,11 @@ - Dropped support for Python 3.6 and 3.7. +- Added support for custom library controllers. Custom controllers must inherit from + the `threadpoolctl.LibController` class and be registered to threadpoolctl using the + `threadpoolctl.register` function. + https://github.com/joblib/threadpoolctl/pull/138 + - A warning is raised on macOS when threadpoolctl finds both Intel OpenMP and LLVM OpenMP runtimes loaded simultaneously by the same Python program. See details and workarounds at https://github.com/joblib/threadpoolctl/blob/master/multiple_openmp.md. diff --git a/README.md b/README.md index 387c70c5..3b22b45c 100644 --- a/README.md +++ b/README.md @@ -192,6 +192,20 @@ decorators are accessible through their `wrap` method: ... ``` +### Writing a custom library controller + +Currently, `threadpoolctl` has support for `OpenMP` and the main `BLAS` libraries. +However it can also be used to control the threadpool of other native libraries, +provided that they expose an API to get and set the limit on the number of threads. +For that, one must implement a controller for this library and register it to +`threadpoolctl`. + +A custom controller must be a subclass of the `LibController` class and implement +the attributes and methods described in the docstring of `LibController`. Then this +new controller class must be registered using the `threadpoolctl.register` function. +An complete example can be found [here]( + https://github.com/joblib/threadpoolctl/blob/master/tests/_pyMylib/__init__.py). + ### Sequential BLAS within OpenMP parallel region When one wants to have sequential BLAS calls within an OpenMP parallel region, it's diff --git a/continuous_integration/build_test_ext.sh b/continuous_integration/build_test_ext.sh index 7b91c7e1..bd0e1ae7 100755 --- a/continuous_integration/build_test_ext.sh +++ b/continuous_integration/build_test_ext.sh @@ -2,6 +2,14 @@ set -e +if [[ "$OSTYPE" == "linux-gnu"* ]]; then + pushd tests/_pyMylib + rm -rf *.so *.o + gcc -c -Wall -Werror -fpic -o my_threaded_lib.o my_threaded_lib.c + gcc -shared -o my_threaded_lib.so my_threaded_lib.o + popd +fi + pushd tests/_openmp_test_helper rm -rf *.c *.so *.dylib build/ python setup_inner.py build_ext -i diff --git a/tests/_pyMylib/__init__.py b/tests/_pyMylib/__init__.py new file mode 100644 index 00000000..d9b60c3e --- /dev/null +++ b/tests/_pyMylib/__init__.py @@ -0,0 +1,46 @@ +import ctypes +from pathlib import Path + +from threadpoolctl import LibController, register + + +path = Path(__file__).parent / "my_threaded_lib.so" +ctypes.CDLL(path) + + +class MyThreadedLibController(LibController): + # names for threadpoolctl's context filtering + user_api = "my_threaded_lib" + internal_api = "my_threaded_lib" + + # Patterns to identify the name of the linked library to load. + # If a dynamic library with a matching filename is linked to the python + # process, it will be loaded as the `dynlib` attribute of the LibController + # instance. + filename_prefixes = ("my_threaded_lib",) + + def get_num_threads(self): + # This function should return the current maximum number of threads, + # which is reported as "num_threads" by `ThreadpoolController.info`. + return getattr(self.dynlib, "mylib_get_num_threads")() + + def set_num_threads(self, num_threads): + # This function limits the maximum number of threads, + # when `ThreadpoolController.limit` is called. + getattr(self.dynlib, "mylib_set_num_threads")(num_threads) + + def get_version(self): + # This function returns the version of the linked library if it is exposed, + # which is reported as "version" by `ThreadpoolController.info`. + get_version = getattr(self.dynlib, "mylib_get_version") + get_version.restype = ctypes.c_char_p + return get_version().decode("utf-8") + + def set_additional_attributes(self): + # This function is called during the initialization of the LibController. + # Additional information meant to be exposed by `ThreadpoolController.info` + # should be set here as attributes of the LibController instance. + self.some_attr = "some_value" + + +register(MyThreadedLibController) diff --git a/tests/_pyMylib/my_threaded_lib.c b/tests/_pyMylib/my_threaded_lib.c new file mode 100644 index 00000000..d38dfe70 --- /dev/null +++ b/tests/_pyMylib/my_threaded_lib.c @@ -0,0 +1,17 @@ +int NUM_THREADS = 42; +int* NUM_THREADS_p = &NUM_THREADS; + + +int mylib_get_num_threads(){ + return *NUM_THREADS_p; +} + + +void mylib_set_num_threads(int num_threads){ + *NUM_THREADS_p = num_threads; +} + + +char* mylib_get_version(){ + return "2.0"; +} diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index 4e15db4a..0a14ddf8 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -672,3 +672,33 @@ def inner_func(): outer_func() assert ThreadpoolController().info() == original_info + + +def test_custom_controller(): + # Check that a custom controller can be used to change the number of threads + # used by a library. + try: + import tests._pyMylib # noqa + except: + pytest.skip("requires my_thread_lib to be compiled") + + controller = ThreadpoolController() + original_info = controller.info() + + mylib_controller = controller.select(user_api="my_threaded_lib") + + # my_threaded_lib has been found and there's 1 matching shared library + assert len(mylib_controller.lib_controllers) == 1 + mylib_controller = mylib_controller.lib_controllers[0] + + # we linked against my_threaded_lib v2.0 and by default it uses 42 thread + assert mylib_controller.version == "2.0" + assert mylib_controller.num_threads == 42 + + # my_threaded_lib exposes an additional info "some_attr": + assert mylib_controller.info()["some_attr"] == "some_value" + + with controller.limit(limits=1, user_api="my_threaded_lib"): + assert mylib_controller.num_threads == 1 + + assert ThreadpoolController().info() == original_info diff --git a/threadpoolctl.py b/threadpoolctl.py index 0fb857ae..95698452 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -15,6 +15,7 @@ import sys import ctypes import textwrap +from typing import final import warnings from ctypes.util import find_library from abc import ABC, abstractmethod @@ -22,7 +23,13 @@ from contextlib import ContextDecorator __version__ = "3.2.0.dev0" -__all__ = ["threadpool_limits", "threadpool_info", "ThreadpoolController"] +__all__ = [ + "threadpool_limits", + "threadpool_info", + "ThreadpoolController", + "LibController", + "register", +] # One can get runtime errors or even segfaults due to multiple OpenMP libraries @@ -60,56 +67,318 @@ class _dl_phdr_info(ctypes.Structure): _RTLD_NOLOAD = ctypes.DEFAULT_MODE -# List of the supported libraries. The items are indexed by the name of the -# class to instantiate to create the library controller objects. The items hold -# the possible prefixes of loaded shared objects, the name of the internal_api -# to call, the name of the user_api and potentially some symbols that the library is -# expected to have (this is necessary to distinguish between the blas implementations -# when they are all renamed "libblas.dll" on conda-forge on windows). -_SUPPORTED_LIBRARIES = { - "OpenMPController": { - "user_api": "openmp", - "internal_api": "openmp", - "filename_prefixes": ("libiomp", "libgomp", "libomp", "vcomp"), - }, - "OpenBLASController": { - "user_api": "blas", - "internal_api": "openblas", - "filename_prefixes": ("libopenblas", "libblas"), - "check_symbols": ("openblas_get_num_threads", "openblas_get_num_threads64_"), - }, - "MKLController": { - "user_api": "blas", - "internal_api": "mkl", - "filename_prefixes": ("libmkl_rt", "mkl_rt", "libblas"), - "check_symbols": ("MKL_Get_Max_Threads",), - }, - "BLISController": { - "user_api": "blas", - "internal_api": "blis", - "filename_prefixes": ("libblis", "libblas"), - "check_symbols": ("bli_thread_get_num_threads",), - }, -} +class LibController(ABC): + """Abstract base class for the individual library controllers + + A library controller must expose the following class attributes: + - user_api : str + Usually the name of the library or generic specification the library + implements, e.g. "blas" is a specification with different implementations. + - internal_api : str + Usually the name of the library or concrete implementation of some + specification, e.g. "openblas" is an implementation of the "blas" + specification. + - filename_prefixes : tuple + Possible prefixes of the shared library's filename that allow to + identify the library. e.g. "libopenblas" for libopenblas.so. + + and implement the following methods: `get_num_threads`, `set_num_threads` and + `get_version`. + + Threadpoolctl loops through all the loaded shared libraries and tries to match + the filename of each library with the `filename_prefixes`. If a match is found, a + controller is instantiated and a handler to the library is stored in the `dynlib` + attribute as a `ctypes.CDLL` object. It can be used to access the necessary symbols + of the shared library to implement the above methods. + + The following information will be exposed in the info dictionary: + - user_api : standardized API, if any, or a copy of internal_api. + - internal_api : implementation-specific API. + - num_threads : the current thread limit. + - prefix : prefix of the shared library's filename. + - filepath : path to the loaded shared library. + - version : version of the library (if available). + + In addition, each library controller may expose internal API specific entries. They + must be set as attributes in the `set_additional_attributes` method. + """ + + @final + def __init__(self, *, filepath=None, prefix=None): + """This is not meant to be overriden by subclasses.""" + self.prefix = prefix + self.filepath = filepath + self.dynlib = ctypes.CDLL(filepath, mode=_RTLD_NOLOAD) + self.version = self.get_version() + self.set_additional_attributes() + + @final + def info(self): + """Return relevant info wrapped in a dict + + This is not meant to be overriden by subclasses. + """ + exposed_attrs = { + "user_api": self.user_api, + "internal_api": self.internal_api, + "num_threads": self.num_threads, + **vars(self), + } + exposed_attrs.pop("dynlib") + return exposed_attrs + + def set_additional_attributes(self): + """Set additional attributes meant to be exposed in the info dict""" + + @property + def num_threads(self): + """Exposes the current thread limit as a dynamic property + + This is not meant to be used or overriden by subclasses. + """ + return self.get_num_threads() + + @abstractmethod + def get_num_threads(self): + """Return the maximum number of threads available to use""" + + @abstractmethod + def set_num_threads(self, num_threads): + """Set the maximum number of threads to use""" + + @abstractmethod + def get_version(self): + """Return the version of the shared library""" + + +class OpenBLASController(LibController): + """Controller class for OpenBLAS""" + + user_api = "blas" + internal_api = "openblas" + filename_prefixes = ("libopenblas", "libblas") + check_symbols = ("openblas_get_num_threads", "openblas_get_num_threads64_") + + def set_additional_attributes(self): + self.threading_layer = self._get_threading_layer() + self.architecture = self._get_architecture() + + def get_num_threads(self): + get_func = getattr( + self.dynlib, + "openblas_get_num_threads", + # Symbols differ when built for 64bit integers in Fortran + getattr(self.dynlib, "openblas_get_num_threads64_", lambda: None), + ) + + return get_func() + + def set_num_threads(self, num_threads): + set_func = getattr( + self.dynlib, + "openblas_set_num_threads", + # Symbols differ when built for 64bit integers in Fortran + getattr( + self.dynlib, "openblas_set_num_threads64_", lambda num_threads: None + ), + ) + return set_func(num_threads) + + def get_version(self): + # None means OpenBLAS is not loaded or version < 0.3.4, since OpenBLAS + # did not expose its version before that. + get_config = getattr( + self.dynlib, + "openblas_get_config", + getattr(self.dynlib, "openblas_get_config64_", None), + ) + if get_config is None: + return None + + get_config.restype = ctypes.c_char_p + config = get_config().split() + if config[0] == b"OpenBLAS": + return config[1].decode("utf-8") + return None + + def _get_threading_layer(self): + """Return the threading layer of OpenBLAS""" + openblas_get_parallel = getattr( + self.dynlib, + "openblas_get_parallel", + getattr(self.dynlib, "openblas_get_parallel64_", None), + ) + if openblas_get_parallel is None: + return "unknown" + threading_layer = openblas_get_parallel() + if threading_layer == 2: + return "openmp" + elif threading_layer == 1: + return "pthreads" + return "disabled" + + def _get_architecture(self): + """Return the architecture detected by OpenBLAS""" + get_corename = getattr( + self.dynlib, + "openblas_get_corename", + getattr(self.dynlib, "openblas_get_corename64_", None), + ) + if get_corename is None: + return None + + get_corename.restype = ctypes.c_char_p + return get_corename().decode("utf-8") + + +class BLISController(LibController): + """Controller class for BLIS""" + + user_api = "blas" + internal_api = "blis" + filename_prefixes = ("libblis", "libblas") + check_symbols = ("bli_thread_get_num_threads",) + + def set_additional_attributes(self): + self.threading_layer = self._get_threading_layer() + self.architecture = self._get_architecture() + + def get_num_threads(self): + get_func = getattr(self.dynlib, "bli_thread_get_num_threads", lambda: None) + num_threads = get_func() + # by default BLIS is single-threaded and get_num_threads + # returns -1. We map it to 1 for consistency with other libraries. + return 1 if num_threads == -1 else num_threads + + def set_num_threads(self, num_threads): + set_func = getattr( + self.dynlib, "bli_thread_set_num_threads", lambda num_threads: None + ) + return set_func(num_threads) + + def get_version(self): + get_version_ = getattr(self.dynlib, "bli_info_get_version_str", None) + if get_version_ is None: + return None + + get_version_.restype = ctypes.c_char_p + return get_version_().decode("utf-8") + + def _get_threading_layer(self): + """Return the threading layer of BLIS""" + if self.dynlib.bli_info_get_enable_openmp(): + return "openmp" + elif self.dynlib.bli_info_get_enable_pthreads(): + return "pthreads" + return "disabled" + + def _get_architecture(self): + """Return the architecture detected by BLIS""" + bli_arch_query_id = getattr(self.dynlib, "bli_arch_query_id", None) + bli_arch_string = getattr(self.dynlib, "bli_arch_string", None) + if bli_arch_query_id is None or bli_arch_string is None: + return None + + # the true restype should be BLIS' arch_t (enum) but int should work + # for us: + bli_arch_query_id.restype = ctypes.c_int + bli_arch_string.restype = ctypes.c_char_p + return bli_arch_string(bli_arch_query_id()).decode("utf-8") + + +class MKLController(LibController): + """Controller class for MKL""" + + user_api = "blas" + internal_api = "mkl" + filename_prefixes = ("libmkl_rt", "mkl_rt", "libblas") + check_symbols = ("MKL_Get_Max_Threads",) + + def set_additional_attributes(self): + self.threading_layer = self._get_threading_layer() + + def get_num_threads(self): + get_func = getattr(self.dynlib, "MKL_Get_Max_Threads", lambda: None) + return get_func() + + def set_num_threads(self, num_threads): + set_func = getattr(self.dynlib, "MKL_Set_Num_Threads", lambda num_threads: None) + return set_func(num_threads) + + def get_version(self): + if not hasattr(self.dynlib, "MKL_Get_Version_String"): + return None + + res = ctypes.create_string_buffer(200) + self.dynlib.MKL_Get_Version_String(res, 200) + + version = res.value.decode("utf-8") + group = re.search(r"Version ([^ ]+) ", version) + if group is not None: + version = group.groups()[0] + return version.strip() + + def _get_threading_layer(self): + """Return the threading layer of MKL""" + # The function mkl_set_threading_layer returns the current threading + # layer. Calling it with an invalid threading layer allows us to safely + # get the threading layer + set_threading_layer = getattr( + self.dynlib, "MKL_Set_Threading_Layer", lambda layer: -1 + ) + layer_map = { + 0: "intel", + 1: "sequential", + 2: "pgi", + 3: "gnu", + 4: "tbb", + -1: "not specified", + } + return layer_map[set_threading_layer(-1)] + + +class OpenMPController(LibController): + """Controller class for OpenMP""" + + user_api = "openmp" + internal_api = "openmp" + filename_prefixes = ("libiomp", "libgomp", "libomp", "vcomp") + + def get_num_threads(self): + get_func = getattr(self.dynlib, "omp_get_max_threads", lambda: None) + return get_func() + + def set_num_threads(self, num_threads): + set_func = getattr(self.dynlib, "omp_set_num_threads", lambda num_threads: None) + return set_func(num_threads) + + def get_version(self): + # There is no way to get the version number programmatically in OpenMP. + return None + + +# Controllers for the libraries that we'll look for in the loaded libraries. +# Third party libraries can register their own controllers. +_ALL_CONTROLLERS = [OpenBLASController, BLISController, MKLController, OpenMPController] # Helpers for the doc and test names -_ALL_USER_APIS = list(set(lib["user_api"] for lib in _SUPPORTED_LIBRARIES.values())) -_ALL_INTERNAL_APIS = [lib["internal_api"] for lib in _SUPPORTED_LIBRARIES.values()] +_ALL_USER_APIS = list(set(lib.user_api for lib in _ALL_CONTROLLERS)) +_ALL_INTERNAL_APIS = [lib.internal_api for lib in _ALL_CONTROLLERS] _ALL_PREFIXES = list( - set( - prefix - for lib in _SUPPORTED_LIBRARIES.values() - for prefix in lib["filename_prefixes"] - ) + set(prefix for lib in _ALL_CONTROLLERS for prefix in lib.filename_prefixes) ) _ALL_BLAS_LIBRARIES = [ - lib["internal_api"] - for lib in _SUPPORTED_LIBRARIES.values() - if lib["user_api"] == "blas" + lib.internal_api for lib in _ALL_CONTROLLERS if lib.user_api == "blas" ] -_ALL_OPENMP_LIBRARIES = list( - _SUPPORTED_LIBRARIES["OpenMPController"]["filename_prefixes"] -) +_ALL_OPENMP_LIBRARIES = OpenMPController.filename_prefixes + + +def register(controller): + """Register a new controller""" + _ALL_CONTROLLERS.append(controller) + _ALL_USER_APIS.append(controller.user_api) + _ALL_INTERNAL_APIS.append(controller.internal_api) + _ALL_PREFIXES.extend(controller.filename_prefixes) def _format_docstring(*args, **kwargs): @@ -377,12 +646,6 @@ def wrap(cls, limits=None, user_api=None): return super().wrap(ThreadpoolController(), limits=limits, user_api=user_api) -@_format_docstring( - PREFIXES=", ".join(f'"{prefix}"' for prefix in _ALL_PREFIXES), - USER_APIS=", ".join(f'"{api}"' for api in _ALL_USER_APIS), - BLAS_LIBS=", ".join(_ALL_BLAS_LIBRARIES), - OPENMP_LIBS=", ".join(_ALL_OPENMP_LIBRARIES), -) class ThreadpoolController: """Collection of LibController objects for all loaded supported libraries @@ -664,7 +927,6 @@ def _find_libraries_with_enum_process_module_ex(self): buf = ctypes.create_unicode_buffer(MAX_PATH) n_size = DWORD() for h_module in h_modules: - # Get the path of the current module if not ps_api.GetModuleFileNameExW( h_process, h_module, ctypes.byref(buf), ctypes.byref(n_size) @@ -687,9 +949,9 @@ def _make_controller_from_path(self, filepath): # Loop through supported libraries to find if this filename corresponds # to a supported one. - for controller_class, candidate_lib in _SUPPORTED_LIBRARIES.items(): + for controller_class in _ALL_CONTROLLERS: # check if filename matches a supported prefix - prefix = self._check_prefix(filename, candidate_lib["filename_prefixes"]) + prefix = self._check_prefix(filename, controller_class.filename_prefixes) # filename does not match any of the prefixes of the candidate # library. move to next library. @@ -705,7 +967,7 @@ def _make_controller_from_path(self, filepath): libblas = ctypes.CDLL(filepath, _RTLD_NOLOAD) if not any( hasattr(libblas, func) - for func in candidate_lib["check_symbols"] + for func in controller_class.check_symbols ): continue else: @@ -718,16 +980,8 @@ def _make_controller_from_path(self, filepath): # filename matches a prefix. Create and store the library # controller. - user_api = candidate_lib["user_api"] - internal_api = candidate_lib["internal_api"] - - lib_controller_class = globals()[controller_class] - lib_controller = lib_controller_class( - filepath=filepath, - prefix=prefix, - user_api=user_api, - internal_api=internal_api, - ) + + lib_controller = controller_class(filepath=filepath, prefix=prefix) self.lib_controllers.append(lib_controller) def _check_prefix(self, library_basename, filename_prefixes): @@ -743,8 +997,7 @@ def _check_prefix(self, library_basename, filename_prefixes): def _warn_if_incompatible_openmp(self): """Raise a warning if llvm-OpenMP and intel-OpenMP are both loaded""" prefixes = [lib_controller.prefix for lib_controller in self.lib_controllers] - msg = textwrap.dedent( - """ + msg = textwrap.dedent(""" Found Intel OpenMP ('libiomp') and LLVM OpenMP ('libomp') loaded at the same time. Both libraries are known to be incompatible and this can cause random crashes or deadlocks on Linux when loaded in the @@ -752,8 +1005,7 @@ def _warn_if_incompatible_openmp(self): Using threadpoolctl may cause crashes or deadlocks. For more information and possible workarounds, please see https://github.com/joblib/threadpoolctl/blob/master/multiple_openmp.md - """ - ) + """) if "libomp" in prefixes and "libiomp" in prefixes: warnings.warn(msg, RuntimeWarning) @@ -765,9 +1017,11 @@ def _get_libc(cls): libc_name = find_library("c") if libc_name is None: # pragma: no cover warnings.warn( - "libc not found. The ctypes module in Python " - f"{sys.version_info.major}.{sys.version_info.minor} is maybe too " - "old for this OS.", + ( + "libc not found. The ctypes module in Python" + f" {sys.version_info.major}.{sys.version_info.minor} is maybe" + " too old for this OS." + ), RuntimeWarning, ) return None @@ -785,252 +1039,6 @@ def _get_windll(cls, dll_name): return dll -@_format_docstring( - USER_APIS=", ".join('"{}"'.format(api) for api in _ALL_USER_APIS), - INTERNAL_APIS=", ".join('"{}"'.format(api) for api in _ALL_INTERNAL_APIS), -) -class LibController(ABC): - """Abstract base class for the individual library controllers - - A library controller is represented by the following information: - - "user_api" : user API. Possible values are {USER_APIS}. - - "internal_api" : internal API. Possible values are {INTERNAL_APIS}. - - "prefix" : prefix of the shared library's name. - - "filepath" : path to the loaded library. - - "version" : version of the library (if available). - - "num_threads" : the current thread limit. - - In addition, each library controller may contain internal_api specific - entries. - """ - - def __init__(self, *, filepath=None, prefix=None, user_api=None, internal_api=None): - self.user_api = user_api - self.internal_api = internal_api - self.prefix = prefix - self.filepath = filepath - self._dynlib = ctypes.CDLL(filepath, mode=_RTLD_NOLOAD) - self.version = self.get_version() - - def info(self): - """Return relevant info wrapped in a dict""" - all_attrs = dict(vars(self), **{"num_threads": self.num_threads}) - return {k: v for k, v in all_attrs.items() if not k.startswith("_")} - - @property - def num_threads(self): - return self.get_num_threads() - - @abstractmethod - def get_num_threads(self): - """Return the maximum number of threads available to use""" - pass # pragma: no cover - - @abstractmethod - def set_num_threads(self, num_threads): - """Set the maximum number of threads to use""" - pass # pragma: no cover - - @abstractmethod - def get_version(self): - """Return the version of the shared library""" - pass # pragma: no cover - - -class OpenBLASController(LibController): - """Controller class for OpenBLAS""" - - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.threading_layer = self._get_threading_layer() - self.architecture = self._get_architecture() - - def get_num_threads(self): - get_func = getattr( - self._dynlib, - "openblas_get_num_threads", - # Symbols differ when built for 64bit integers in Fortran - getattr(self._dynlib, "openblas_get_num_threads64_", lambda: None), - ) - - return get_func() - - def set_num_threads(self, num_threads): - set_func = getattr( - self._dynlib, - "openblas_set_num_threads", - # Symbols differ when built for 64bit integers in Fortran - getattr( - self._dynlib, "openblas_set_num_threads64_", lambda num_threads: None - ), - ) - return set_func(num_threads) - - def get_version(self): - # None means OpenBLAS is not loaded or version < 0.3.4, since OpenBLAS - # did not expose its version before that. - get_config = getattr( - self._dynlib, - "openblas_get_config", - getattr(self._dynlib, "openblas_get_config64_", None), - ) - if get_config is None: - return None - - get_config.restype = ctypes.c_char_p - config = get_config().split() - if config[0] == b"OpenBLAS": - return config[1].decode("utf-8") - return None - - def _get_threading_layer(self): - """Return the threading layer of OpenBLAS""" - openblas_get_parallel = getattr( - self._dynlib, - "openblas_get_parallel", - getattr(self._dynlib, "openblas_get_parallel64_", None), - ) - if openblas_get_parallel is None: - return "unknown" - threading_layer = openblas_get_parallel() - if threading_layer == 2: - return "openmp" - elif threading_layer == 1: - return "pthreads" - return "disabled" - - def _get_architecture(self): - """Return the architecture detected by OpenBLAS""" - get_corename = getattr( - self._dynlib, - "openblas_get_corename", - getattr(self._dynlib, "openblas_get_corename64_", None), - ) - if get_corename is None: - return None - - get_corename.restype = ctypes.c_char_p - return get_corename().decode("utf-8") - - -class BLISController(LibController): - """Controller class for BLIS""" - - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.threading_layer = self._get_threading_layer() - self.architecture = self._get_architecture() - - def get_num_threads(self): - get_func = getattr(self._dynlib, "bli_thread_get_num_threads", lambda: None) - num_threads = get_func() - # by default BLIS is single-threaded and get_num_threads - # returns -1. We map it to 1 for consistency with other libraries. - return 1 if num_threads == -1 else num_threads - - def set_num_threads(self, num_threads): - set_func = getattr( - self._dynlib, "bli_thread_set_num_threads", lambda num_threads: None - ) - return set_func(num_threads) - - def get_version(self): - get_version_ = getattr(self._dynlib, "bli_info_get_version_str", None) - if get_version_ is None: - return None - - get_version_.restype = ctypes.c_char_p - return get_version_().decode("utf-8") - - def _get_threading_layer(self): - """Return the threading layer of BLIS""" - if self._dynlib.bli_info_get_enable_openmp(): - return "openmp" - elif self._dynlib.bli_info_get_enable_pthreads(): - return "pthreads" - return "disabled" - - def _get_architecture(self): - """Return the architecture detected by BLIS""" - bli_arch_query_id = getattr(self._dynlib, "bli_arch_query_id", None) - bli_arch_string = getattr(self._dynlib, "bli_arch_string", None) - if bli_arch_query_id is None or bli_arch_string is None: - return None - - # the true restype should be BLIS' arch_t (enum) but int should work - # for us: - bli_arch_query_id.restype = ctypes.c_int - bli_arch_string.restype = ctypes.c_char_p - return bli_arch_string(bli_arch_query_id()).decode("utf-8") - - -class MKLController(LibController): - """Controller class for MKL""" - - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.threading_layer = self._get_threading_layer() - - def get_num_threads(self): - get_func = getattr(self._dynlib, "MKL_Get_Max_Threads", lambda: None) - return get_func() - - def set_num_threads(self, num_threads): - set_func = getattr( - self._dynlib, "MKL_Set_Num_Threads", lambda num_threads: None - ) - return set_func(num_threads) - - def get_version(self): - if not hasattr(self._dynlib, "MKL_Get_Version_String"): - return None - - res = ctypes.create_string_buffer(200) - self._dynlib.MKL_Get_Version_String(res, 200) - - version = res.value.decode("utf-8") - group = re.search(r"Version ([^ ]+) ", version) - if group is not None: - version = group.groups()[0] - return version.strip() - - def _get_threading_layer(self): - """Return the threading layer of MKL""" - # The function mkl_set_threading_layer returns the current threading - # layer. Calling it with an invalid threading layer allows us to safely - # get the threading layer - set_threading_layer = getattr( - self._dynlib, "MKL_Set_Threading_Layer", lambda layer: -1 - ) - layer_map = { - 0: "intel", - 1: "sequential", - 2: "pgi", - 3: "gnu", - 4: "tbb", - -1: "not specified", - } - return layer_map[set_threading_layer(-1)] - - -class OpenMPController(LibController): - """Controller class for OpenMP""" - - def get_num_threads(self): - get_func = getattr(self._dynlib, "omp_get_max_threads", lambda: None) - return get_func() - - def set_num_threads(self, num_threads): - set_func = getattr( - self._dynlib, "omp_set_num_threads", lambda num_threads: None - ) - return set_func(num_threads) - - def get_version(self): - # There is no way to get the version number programmatically in OpenMP. - return None - - def _main(): """Commandline interface to display thread-pool information and exit.""" import argparse From 0172dfa5e8f11bc3d5fd3d338e342ef896e6585e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Thu, 13 Jul 2023 16:18:00 +0200 Subject: [PATCH 144/187] MAINT make CI fail when linting fails (#143) --- .azure_pipeline.yml | 18 ++++++++++++++++-- continuous_integration/posix.yml | 7 ------- threadpoolctl.py | 8 +++----- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml index 3ea8f7a9..15975a42 100644 --- a/.azure_pipeline.yml +++ b/.azure_pipeline.yml @@ -18,6 +18,21 @@ stages: - stage: jobs: + - job: 'linting' + displayName: Linting + pool: + vmImage: ubuntu-latest + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.11' + - script: | + pip install black + displayName: install black + - script: | + black --check --diff . + displayName: Run black + - template: continuous_integration/windows.yml parameters: name: Windows @@ -97,7 +112,6 @@ stages: OPENBLAS_THREADING_LAYER: 'openmp' CC_OUTER_LOOP: 'gcc' CC_INNER_LOOP: 'gcc' - LINT: 'true' # Linux environment with no numpy and heterogeneous OpenMP runtimes. pylatest_conda_nonumpy_gcc_clang: PACKAGER: 'conda' @@ -172,7 +186,7 @@ stages: - job: 'no_test_always_skipped' displayName: 'No test always skipped' pool: - vmImage: ubuntu-20.04 + vmImage: ubuntu-latest steps: - download: current - script: | diff --git a/continuous_integration/posix.yml b/continuous_integration/posix.yml index 3d3bc963..5edb0ac7 100644 --- a/continuous_integration/posix.yml +++ b/continuous_integration/posix.yml @@ -20,13 +20,6 @@ jobs: # We need to take ownership if we want to update conda or install packages globally displayName: Take ownership of conda installation condition: eq('${{ parameters.name }}', 'macOS') - - script: | - conda create -n tmp -y -c conda-forge python black - source activate tmp - black --check . - conda deactivate - displayName: Lint - condition: eq(variables['LINT'], 'true') - script: | continuous_integration/install.sh displayName: 'Install without BLIS' diff --git a/threadpoolctl.py b/threadpoolctl.py index 95698452..c04af0a1 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -1017,11 +1017,9 @@ def _get_libc(cls): libc_name = find_library("c") if libc_name is None: # pragma: no cover warnings.warn( - ( - "libc not found. The ctypes module in Python" - f" {sys.version_info.major}.{sys.version_info.minor} is maybe" - " too old for this OS." - ), + "libc not found. The ctypes module in Python" + f" {sys.version_info.major}.{sys.version_info.minor} is maybe" + " too old for this OS.", RuntimeWarning, ) return None From be1bf998c3bc8d8c3628f35700f842d066e430bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Thu, 13 Jul 2023 16:43:49 +0200 Subject: [PATCH 145/187] Release 3.2.0 (#144) --- CHANGES.md | 4 ++-- threadpoolctl.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 3a78b49f..579cbdff 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,5 @@ -3.2.0 (TBD) -=========== +3.2.0 (2023-07-13) +================== - Dropped support for Python 3.6 and 3.7. diff --git a/threadpoolctl.py b/threadpoolctl.py index c04af0a1..f425a541 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -22,7 +22,7 @@ from functools import lru_cache from contextlib import ContextDecorator -__version__ = "3.2.0.dev0" +__version__ = "3.2.0" __all__ = [ "threadpool_limits", "threadpool_info", From d9736780c09cff75de5c7a3b295aeddd6b8cbf0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Thu, 13 Jul 2023 17:28:26 +0200 Subject: [PATCH 146/187] MAINT start 3.3.0 dev (#145) --- threadpoolctl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/threadpoolctl.py b/threadpoolctl.py index f425a541..55b4decc 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -22,7 +22,7 @@ from functools import lru_cache from contextlib import ContextDecorator -__version__ = "3.2.0" +__version__ = "3.3.0.dev0" __all__ = [ "threadpool_limits", "threadpool_info", From 85d329a6223d17beb92641f937caa22b2a8ca3dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Tue, 10 Oct 2023 15:49:15 +0200 Subject: [PATCH 147/187] FIX check symbols to better identify supported libraries (#151) --- CHANGES.md | 8 ++++ continuous_integration/install_with_blis.sh | 1 + tests/_pyMylib/__init__.py | 8 ++++ threadpoolctl.py | 50 +++++++++++++++++---- 4 files changed, 59 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 579cbdff..3dbb7e03 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,11 @@ +3.2.1 (TBD) +=========== + +- Fixed a bug where an unsupported library would be detected because it shares a common + prefix with one of the supported libraries. Now the symbols are also checked to + identify the supported libraries. + https://github.com/joblib/threadpoolctl/pull/151 + 3.2.0 (2023-07-13) ================== diff --git a/continuous_integration/install_with_blis.sh b/continuous_integration/install_with_blis.sh index 6463ac86..89984f1e 100755 --- a/continuous_integration/install_with_blis.sh +++ b/continuous_integration/install_with_blis.sh @@ -35,6 +35,7 @@ popd # build & install numpy git clone https://github.com/numpy/numpy.git pushd numpy +git checkout v1.26.0 # pin numpy < 2 for now git submodule update --init echo "[blis] libraries = blis diff --git a/tests/_pyMylib/__init__.py b/tests/_pyMylib/__init__.py index d9b60c3e..af2867e5 100644 --- a/tests/_pyMylib/__init__.py +++ b/tests/_pyMylib/__init__.py @@ -19,6 +19,14 @@ class MyThreadedLibController(LibController): # instance. filename_prefixes = ("my_threaded_lib",) + # (Optional) Symbols that the linked library is expected to expose. It is used along + # with the `filename_prefixes` to make sure that the correct library is identified. + check_symbols = ( + "mylib_get_num_threads", + "mylib_set_num_threads", + "mylib_get_version", + ) + def get_num_threads(self): # This function should return the current maximum number of threads, # which is reported as "num_threads" by `ThreadpoolController.info`. diff --git a/threadpoolctl.py b/threadpoolctl.py index 55b4decc..2e231101 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -157,7 +157,18 @@ class OpenBLASController(LibController): user_api = "blas" internal_api = "openblas" filename_prefixes = ("libopenblas", "libblas") - check_symbols = ("openblas_get_num_threads", "openblas_get_num_threads64_") + check_symbols = ( + "openblas_get_num_threads", + "openblas_get_num_threads64_", + "openblas_set_num_threads", + "openblas_set_num_threads64_", + "openblas_get_config", + "openblas_get_config64_", + "openblas_get_parallel", + "openblas_get_parallel64_", + "openblas_get_corename", + "openblas_get_corename64_", + ) def set_additional_attributes(self): self.threading_layer = self._get_threading_layer() @@ -237,7 +248,15 @@ class BLISController(LibController): user_api = "blas" internal_api = "blis" filename_prefixes = ("libblis", "libblas") - check_symbols = ("bli_thread_get_num_threads",) + check_symbols = ( + "bli_thread_get_num_threads", + "bli_thread_set_num_threads", + "bli_info_get_version_str", + "bli_info_get_enable_openmp", + "bli_info_get_enable_pthreads", + "bli_arch_query_id", + "bli_arch_string", + ) def set_additional_attributes(self): self.threading_layer = self._get_threading_layer() @@ -266,9 +285,9 @@ def get_version(self): def _get_threading_layer(self): """Return the threading layer of BLIS""" - if self.dynlib.bli_info_get_enable_openmp(): + if getattr(self.dynlib, "bli_info_get_enable_openmp", lambda: False)(): return "openmp" - elif self.dynlib.bli_info_get_enable_pthreads(): + elif getattr(self.dynlib, "bli_info_get_enable_pthreads", lambda: False)(): return "pthreads" return "disabled" @@ -292,7 +311,12 @@ class MKLController(LibController): user_api = "blas" internal_api = "mkl" filename_prefixes = ("libmkl_rt", "mkl_rt", "libblas") - check_symbols = ("MKL_Get_Max_Threads",) + check_symbols = ( + "MKL_Get_Max_Threads", + "MKL_Set_Num_Threads", + "MKL_Get_Version_String", + "MKL_Set_Threading_Layer", + ) def set_additional_attributes(self): self.threading_layer = self._get_threading_layer() @@ -343,6 +367,10 @@ class OpenMPController(LibController): user_api = "openmp" internal_api = "openmp" filename_prefixes = ("libiomp", "libgomp", "libomp", "vcomp") + check_symbols = ( + "omp_get_max_threads", + "omp_get_num_threads", + ) def get_num_threads(self): get_func = getattr(self.dynlib, "omp_get_max_threads", lambda: None) @@ -978,11 +1006,17 @@ def _make_controller_from_path(self, filepath): # duplicate entry in threadpool_info. continue - # filename matches a prefix. Create and store the library + # filename matches a prefix. Now we check if the library has the symbols we + # are looking for. If none of the symbols exists, it's very likely not the + # expected library (e.g. a library having a common prefix with one of the + # our supported libraries). Otherwise, create and store the library # controller. - lib_controller = controller_class(filepath=filepath, prefix=prefix) - self.lib_controllers.append(lib_controller) + if not hasattr(controller_class, "check_symbols") or any( + hasattr(lib_controller.dynlib, func) + for func in controller_class.check_symbols + ): + self.lib_controllers.append(lib_controller) def _check_prefix(self, library_basename, filename_prefixes): """Return the prefix library_basename starts with From 1644199c3e5bf56b0c2b7390d9cce948a12bdeae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Tue, 17 Oct 2023 18:41:37 +0200 Subject: [PATCH 148/187] TST Remove cython conditional compilation (#152) Co-authored-by: Olivier Grisel --- .../nested_prange_blas.pyx | 29 ++-------- .../nested_prange_blas_blis.pyx | 55 +++++++++++++++++++ .../setup_nested_prange_blas.py | 5 +- 3 files changed, 62 insertions(+), 27 deletions(-) create mode 100644 tests/_openmp_test_helper/nested_prange_blas_blis.pyx diff --git a/tests/_openmp_test_helper/nested_prange_blas.pyx b/tests/_openmp_test_helper/nested_prange_blas.pyx index 879fe5c2..af2a6693 100644 --- a/tests/_openmp_test_helper/nested_prange_blas.pyx +++ b/tests/_openmp_test_helper/nested_prange_blas.pyx @@ -2,23 +2,7 @@ cimport openmp from cython.parallel import parallel, prange import numpy as np - -IF USE_BLIS: - cdef extern from 'cblas.h' nogil: - ctypedef enum CBLAS_ORDER: - CblasRowMajor=101 - CblasColMajor=102 - ctypedef enum CBLAS_TRANSPOSE: - CblasNoTrans=111 - CblasTrans=112 - CblasConjTrans=113 - void dgemm 'cblas_dgemm' ( - CBLAS_ORDER Order, CBLAS_TRANSPOSE TransA, - CBLAS_TRANSPOSE TransB, int M, int N, - int K, double alpha, double *A, int lda, - double *B, int ldb, double beta, double *C, int ldc) -ELSE: - from scipy.linalg.cython_blas cimport dgemm +from scipy.linalg.cython_blas cimport dgemm from threadpoolctl import ThreadpoolController @@ -53,13 +37,8 @@ def check_nested_prange_blas(double[:, ::1] A, double[:, ::1] B, int nthreads): prange_num_threads_ptr[0] = openmp.omp_get_num_threads() for i in prange(n_chunks): - IF USE_BLIS: - dgemm(CblasRowMajor, CblasNoTrans, CblasTrans, - chunk_size, n, k, alpha, &A[i * chunk_size, 0], k, - &B[0, 0], k, beta, &C[i * chunk_size, 0], n) - ELSE: - dgemm(trans, no_trans, &n, &chunk_size, &k, - &alpha, &B[0, 0], &k, &A[i * chunk_size, 0], &k, - &beta, &C[i * chunk_size, 0], &n) + dgemm(trans, no_trans, &n, &chunk_size, &k, + &alpha, &B[0, 0], &k, &A[i * chunk_size, 0], &k, + &beta, &C[i * chunk_size, 0], &n) return np.asarray(C), prange_num_threads, inner_info[0] diff --git a/tests/_openmp_test_helper/nested_prange_blas_blis.pyx b/tests/_openmp_test_helper/nested_prange_blas_blis.pyx new file mode 100644 index 00000000..6c0a9771 --- /dev/null +++ b/tests/_openmp_test_helper/nested_prange_blas_blis.pyx @@ -0,0 +1,55 @@ +cimport openmp +from cython.parallel import parallel, prange + +import numpy as np + +cdef extern from 'cblas.h' nogil: + ctypedef enum CBLAS_ORDER: + CblasRowMajor=101 + CblasColMajor=102 + ctypedef enum CBLAS_TRANSPOSE: + CblasNoTrans=111 + CblasTrans=112 + CblasConjTrans=113 + void dgemm 'cblas_dgemm' ( + CBLAS_ORDER Order, CBLAS_TRANSPOSE TransA, + CBLAS_TRANSPOSE TransB, int M, int N, + int K, double alpha, double *A, int lda, + double *B, int ldb, double beta, double *C, int ldc) + +from threadpoolctl import ThreadpoolController + + +def check_nested_prange_blas(double[:, ::1] A, double[:, ::1] B, int nthreads): + """Run multithreaded BLAS calls within OpenMP parallel loop""" + cdef: + int m = A.shape[0] + int n = B.shape[0] + int k = A.shape[1] + + double[:, ::1] C = np.empty((m, n)) + int n_chunks = 100 + int chunk_size = A.shape[0] // n_chunks + + double alpha = 1.0 + double beta = 0.0 + + int i + int prange_num_threads + int *prange_num_threads_ptr = &prange_num_threads + + inner_info = [None] + + with nogil, parallel(num_threads=nthreads): + if openmp.omp_get_thread_num() == 0: + with gil: + inner_info[0] = ThreadpoolController().info() + + prange_num_threads_ptr[0] = openmp.omp_get_num_threads() + + for i in prange(n_chunks): + dgemm(CblasRowMajor, CblasNoTrans, CblasTrans, + chunk_size, n, k, alpha, &A[i * chunk_size, 0], k, + &B[0, 0], k, beta, &C[i * chunk_size, 0], n) + + return np.asarray(C), prange_num_threads, inner_info[0] diff --git a/tests/_openmp_test_helper/setup_nested_prange_blas.py b/tests/_openmp_test_helper/setup_nested_prange_blas.py index 275a92ce..88b900b3 100644 --- a/tests/_openmp_test_helper/setup_nested_prange_blas.py +++ b/tests/_openmp_test_helper/setup_nested_prange_blas.py @@ -14,11 +14,13 @@ use_blis = os.getenv("INSTALL_BLIS", False) libraries = ["blis"] if use_blis else [] + blis_suffix = "_blis" if use_blis else "" + filename = f"nested_prange_blas{blis_suffix}.pyx" ext_modules = [ Extension( "nested_prange_blas", - ["nested_prange_blas.pyx"], + [filename], extra_compile_args=openmp_flag, extra_link_args=openmp_flag, libraries=libraries, @@ -29,7 +31,6 @@ name="_openmp_test_helper_nested_prange_blas", ext_modules=cythonize( ext_modules, - compile_time_env={"USE_BLIS": use_blis}, compiler_directives={ "language_level": 3, "boundscheck": False, From 767e37dd38b888775228f1db833c91ee5ff94deb Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 18 Oct 2023 10:08:14 +0200 Subject: [PATCH 149/187] MAINT Python 3.12 version support and make tests tolerate the presence of OpenMP for system Python (#155) --- .azure_pipeline.yml | 38 ++++++++++----------- continuous_integration/install.cmd | 4 ++- continuous_integration/install.sh | 9 ++--- continuous_integration/install_with_blis.sh | 4 ++- continuous_integration/test_script.cmd | 3 ++ continuous_integration/test_script.sh | 3 ++ pyproject.toml | 3 +- tests/test_threadpoolctl.py | 15 ++++++-- 8 files changed, 51 insertions(+), 28 deletions(-) diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml index 15975a42..b88abe1c 100644 --- a/.azure_pipeline.yml +++ b/.azure_pipeline.yml @@ -25,7 +25,7 @@ stages: steps: - task: UsePythonVersion@0 inputs: - versionSpec: '3.11' + versionSpec: '3.12' - script: | pip install black displayName: install black @@ -39,18 +39,18 @@ stages: vmImage: windows-latest matrix: pylatest_conda_forge_mkl: - VERSION_PYTHON: '*' + PYTHON_VERSION: '*' PACKAGER: 'conda-forge' BLAS: 'mkl' py310_conda_forge_openblas: - VERSION_PYTHON: '3.10' + PYTHON_VERSION: '3.10' PACKAGER: 'conda-forge' BLAS: 'openblas' py39_conda: - VERSION_PYTHON: '3.9' + PYTHON_VERSION: '3.9' PACKAGER: 'conda' py38_pip: - VERSION_PYTHON: '3.8' + PYTHON_VERSION: '3.8' PACKAGER: 'pip' @@ -64,19 +64,19 @@ stages: py38_ubuntu_atlas_gcc_gcc: PACKAGER: 'ubuntu' APT_BLAS: 'libatlas3-base libatlas-base-dev' - VERSION_PYTHON: '3.8' + PYTHON_VERSION: '3.8' CC_OUTER_LOOP: 'gcc' CC_INNER_LOOP: 'gcc' py38_ubuntu_openblas_gcc_gcc: PACKAGER: 'ubuntu' APT_BLAS: 'libopenblas-base libopenblas-dev' - VERSION_PYTHON: '3.8' + PYTHON_VERSION: '3.8' CC_OUTER_LOOP: 'gcc' CC_INNER_LOOP: 'gcc' # Linux + Python 3.9 and homogeneous runtime nesting. py39_conda_openblas_clang_clang: PACKAGER: 'conda' - VERSION_PYTHON: '3.9' + PYTHON_VERSION: '3.9' BLAS: 'openblas' CC_OUTER_LOOP: 'clang-10' CC_INNER_LOOP: 'clang-10' @@ -84,7 +84,7 @@ stages: # threadpoolctl) to only test the warning from multiple OpenMP. pylatest_conda_mkl_clang_gcc: PACKAGER: 'conda' - VERSION_PYTHON: '*' + PYTHON_VERSION: '*' BLAS: 'mkl' CC_OUTER_LOOP: 'clang-10' CC_INNER_LOOP: 'gcc' @@ -92,7 +92,7 @@ stages: # Linux environment with MKL, safe for threadpoolctl. pylatest_conda_mkl_gcc_gcc: PACKAGER: 'conda' - VERSION_PYTHON: '*' + PYTHON_VERSION: '*' BLAS: 'mkl' CC_OUTER_LOOP: 'gcc' CC_INNER_LOOP: 'gcc' @@ -101,13 +101,13 @@ stages: # and heterogeneous OpenMP runtimes. py38_pip_openblas_gcc_clang: PACKAGER: 'pip' - VERSION_PYTHON: '3.8' + PYTHON_VERSION: '3.8' CC_OUTER_LOOP: 'gcc' CC_INNER_LOOP: 'clang-10' # Linux environment with numpy from conda-forge channel and openblas-openmp pylatest_conda_forge: PACKAGER: 'conda-forge' - VERSION_PYTHON: '*' + PYTHON_VERSION: '*' BLAS: 'openblas' OPENBLAS_THREADING_LAYER: 'openmp' CC_OUTER_LOOP: 'gcc' @@ -116,13 +116,13 @@ stages: pylatest_conda_nonumpy_gcc_clang: PACKAGER: 'conda' NO_NUMPY: 'true' - VERSION_PYTHON: '*' + PYTHON_VERSION: '*' CC_OUTER_LOOP: 'gcc' CC_INNER_LOOP: 'clang-10' # Linux environment with numpy linked to BLIS pylatest_blis_gcc_clang_openmp: PACKAGER: 'conda' - VERSION_PYTHON: '*' + PYTHON_VERSION: '*' INSTALL_BLIS: 'true' BLIS_NUM_THREADS: '4' CC_OUTER_LOOP: 'gcc' @@ -131,7 +131,7 @@ stages: BLIS_ENABLE_THREADING: 'openmp' pylatest_blis_clang_gcc_pthreads: PACKAGER: 'conda' - VERSION_PYTHON: '*' + PYTHON_VERSION: '*' INSTALL_BLIS: 'true' BLIS_NUM_THREADS: '4' CC_OUTER_LOOP: 'clang-10' @@ -140,7 +140,7 @@ stages: BLIS_ENABLE_THREADING: 'pthreads' pylatest_blis_no_threading: PACKAGER: 'conda' - VERSION_PYTHON: '*' + PYTHON_VERSION: '*' INSTALL_BLIS: 'true' BLIS_NUM_THREADS: '1' CC_OUTER_LOOP: 'gcc' @@ -156,7 +156,7 @@ stages: # MacOS environment with OpenMP installed through homebrew py38_conda_homebrew_libomp: PACKAGER: 'conda' - VERSION_PYTHON: '3.8' + PYTHON_VERSION: '3.8' BLAS: 'openblas' CC_OUTER_LOOP: 'clang' CC_INNER_LOOP: 'clang' @@ -164,7 +164,7 @@ stages: # MacOS env with OpenBLAS and OpenMP installed through conda-forge compilers py39_conda_forge_clang_openblas: PACKAGER: 'conda-forge' - VERSION_PYTHON: '*' + PYTHON_VERSION: '*' BLAS: 'openblas' CC_OUTER_LOOP: 'clang' CC_INNER_LOOP: 'clang' @@ -172,7 +172,7 @@ stages: # MacOS environment with OpenMP installed through conda-forge compilers pylatest_conda_forge_clang: PACKAGER: 'conda-forge' - VERSION_PYTHON: '*' + PYTHON_VERSION: '*' BLAS: 'mkl' CC_OUTER_LOOP: 'clang' CC_INNER_LOOP: 'clang' diff --git a/continuous_integration/install.cmd b/continuous_integration/install.cmd index 4da530ff..40731e05 100644 --- a/continuous_integration/install.cmd +++ b/continuous_integration/install.cmd @@ -9,8 +9,10 @@ set PIP_INSTALL=pip install -q @rem Deactivate any environment call deactivate @rem Clean up any left-over from a previous build and install version of python +conda update -n base conda conda-libmamba-solver -q -y +conda config --set solver libmamba conda remove --all -q -y -n %VIRTUALENV% -conda create -n %VIRTUALENV% -q -y python=%VERSION_PYTHON% +conda create -n %VIRTUALENV% -q -y python=%PYTHON_VERSION% call activate %VIRTUALENV% python -m pip install -U pip diff --git a/continuous_integration/install.sh b/continuous_integration/install.sh index 46d96b35..86d77fdb 100755 --- a/continuous_integration/install.sh +++ b/continuous_integration/install.sh @@ -17,7 +17,6 @@ make_conda() { if [[ "$UNAMESTR" == "Darwin" ]]; then if [[ "$INSTALL_LIBOMP" == "conda-forge" ]]; then # Install an OpenMP-enabled clang/llvm from conda-forge - # assumes conda-forge is set on priority channel TO_INSTALL="$TO_INSTALL compilers llvm-openmp" @@ -35,12 +34,14 @@ make_conda() { export LDFLAGS="$LDFLAGS -Wl,-rpath,/usr/local/opt/libomp/lib -L/usr/local/opt/libomp/lib -lomp" fi fi + conda update -n base conda conda-libmamba-solver -q --yes + conda config --set solver libmamba conda create -n $VIRTUALENV -q --yes $TO_INSTALL source activate $VIRTUALENV } if [[ "$PACKAGER" == "conda" ]]; then - TO_INSTALL="python=$VERSION_PYTHON pip" + TO_INSTALL="python=$PYTHON_VERSION pip" if [[ "$NO_NUMPY" != "true" ]]; then TO_INSTALL="$TO_INSTALL numpy scipy blas[build=$BLAS]" fi @@ -49,7 +50,7 @@ if [[ "$PACKAGER" == "conda" ]]; then elif [[ "$PACKAGER" == "conda-forge" ]]; then conda config --prepend channels conda-forge conda config --set channel_priority strict - TO_INSTALL="python=$VERSION_PYTHON numpy scipy blas[build=$BLAS]" + TO_INSTALL="python=$PYTHON_VERSION numpy scipy blas[build=$BLAS]" if [[ "$BLAS" == "openblas" && "$OPENBLAS_THREADING_LAYER" == "openmp" ]]; then TO_INSTALL="$TO_INSTALL libopenblas=*=*openmp*" fi @@ -58,7 +59,7 @@ elif [[ "$PACKAGER" == "conda-forge" ]]; then elif [[ "$PACKAGER" == "pip" ]]; then # Use conda to build an empty python env and then use pip to install # numpy and scipy - TO_INSTALL="python=$VERSION_PYTHON pip" + TO_INSTALL="python=$PYTHON_VERSION pip" make_conda $TO_INSTALL if [[ "$NO_NUMPY" != "true" ]]; then pip install numpy scipy diff --git a/continuous_integration/install_with_blis.sh b/continuous_integration/install_with_blis.sh index 89984f1e..cb880693 100755 --- a/continuous_integration/install_with_blis.sh +++ b/continuous_integration/install_with_blis.sh @@ -13,7 +13,9 @@ sudo ./llvm.sh 10 sudo apt-get install libomp-dev # create conda env -conda create -n $VIRTUALENV -q --yes -c conda-forge python=$VERSION_PYTHON pip cython +conda update -n base conda conda-libmamba-solver -q --yes +conda config --set solver libmamba +conda create -n $VIRTUALENV -q --yes -c conda-forge python=$PYTHON_VERSION pip cython source activate $VIRTUALENV if [[ "$BLIS_CC" == "gcc-8" ]]; then diff --git a/continuous_integration/test_script.cmd b/continuous_integration/test_script.cmd index c3c3da05..4da20228 100644 --- a/continuous_integration/test_script.cmd +++ b/continuous_integration/test_script.cmd @@ -1,5 +1,8 @@ call activate %VIRTUALENV% +# Display version information +python -m pip list + # Use the CLI to display the effective runtime environment prior to # launching the tests: python -m threadpoolctl -i numpy scipy.linalg tests._openmp_test_helper.openmp_helpers_inner diff --git a/continuous_integration/test_script.sh b/continuous_integration/test_script.sh index b1d3a64c..2864f584 100755 --- a/continuous_integration/test_script.sh +++ b/continuous_integration/test_script.sh @@ -4,11 +4,14 @@ set -e if [[ "$PACKAGER" == conda* ]]; then source activate $VIRTUALENV + conda list elif [[ "$PACKAGER" == "pip" ]]; then # we actually use conda to install the base environment: source activate $VIRTUALENV + pip list elif [[ "$PACKAGER" == "ubuntu" ]]; then source $VIRTUALENV/bin/activate + pip list fi set -x diff --git a/pyproject.toml b/pyproject.toml index e53f6814..5e9de5bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,10 +18,11 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Software Development :: Libraries :: Python Modules", ] [tool.black] line-length = 88 -target_version = ['py38', 'py39', 'py310', 'py311'] +target_version = ['py38', 'py39', 'py310', 'py311', 'py312'] preview = true diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index 0a14ddf8..e0d6786a 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -545,9 +545,20 @@ def test_libomp_libiomp_warning(recwarn): assert "multiple_openmp.md" in str(wm.message) -def test_command_line_empty(): +def test_command_line_empty_or_system_openmp(): + # When the command line is called without arguments, no library should be + # detected. The only exception is a system OpenMP library that can be + # linked to the Python interpreter, for instance via the libb2.so BLAKE2 + # library that can itself be linked to an OpenMP runtime on Gentoo. output = subprocess.check_output((sys.executable + " -m threadpoolctl").split()) - assert json.loads(output.decode("utf-8")) == [] + results = json.loads(output.decode("utf-8")) + conda_prefix = os.getenv("CONDA_PREFIX") + managed_by_conda = conda_prefix and sys.executable.startswith(conda_prefix) + if not managed_by_conda: # pragma: no cover + # When using a Python interpreter that does not come from a conda + # environment, we should ignore any system OpenMP library. + results = [r for r in results if r["user_api"] != "openmp"] + assert results == [] def test_command_line_command_flag(): From 1e35b97c0609fe200a98a3009c07d4e65da41aa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Tue, 19 Dec 2023 16:55:08 +0100 Subject: [PATCH 150/187] TST explicit noexcept in cython cdef because it's no longer the default (#158) --- tests/_openmp_test_helper/openmp_helpers_inner.pxd | 2 +- tests/_openmp_test_helper/openmp_helpers_inner.pyx | 2 +- threadpoolctl.py | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/_openmp_test_helper/openmp_helpers_inner.pxd b/tests/_openmp_test_helper/openmp_helpers_inner.pxd index 52522ad7..f4d0852c 100644 --- a/tests/_openmp_test_helper/openmp_helpers_inner.pxd +++ b/tests/_openmp_test_helper/openmp_helpers_inner.pxd @@ -1 +1 @@ -cdef int inner_openmp_loop(int) nogil +cdef int inner_openmp_loop(int) noexcept nogil diff --git a/tests/_openmp_test_helper/openmp_helpers_inner.pyx b/tests/_openmp_test_helper/openmp_helpers_inner.pyx index eca797b9..e7928d2f 100644 --- a/tests/_openmp_test_helper/openmp_helpers_inner.pyx +++ b/tests/_openmp_test_helper/openmp_helpers_inner.pyx @@ -15,7 +15,7 @@ def check_openmp_num_threads(int n): return num_threads -cdef int inner_openmp_loop(int n) nogil: +cdef int inner_openmp_loop(int n) noexcept nogil: """Run a short parallel section with OpenMP Return the number of threads that where effectively used by the diff --git a/threadpoolctl.py b/threadpoolctl.py index 2e231101..244ea6b6 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -4,6 +4,7 @@ thread pools (notably BLAS and OpenMP implementations) and dynamically set the maximal number of threads they can use. """ + # License: BSD 3-Clause # The code to introspect dynamically loaded libraries on POSIX systems is From dc260f1068bb521013ae6ad5ae6dc541871ce09f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Tue, 19 Dec 2023 17:01:20 +0100 Subject: [PATCH 151/187] TST Don't skip test_nested_prange_blas for BLIS (#157) --- continuous_integration/install_with_blis.sh | 6 +++++- tests/test_threadpoolctl.py | 13 ++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/continuous_integration/install_with_blis.sh b/continuous_integration/install_with_blis.sh index cb880693..dff5dc05 100755 --- a/continuous_integration/install_with_blis.sh +++ b/continuous_integration/install_with_blis.sh @@ -50,9 +50,13 @@ popd popd python -m pip install -q -r dev-requirements.txt -CFLAGS=-I$ABS_PATH/BLIS_install/include/blis LDFLAGS=-L$ABS_PATH/BLIS_install/lib \ +CFLAGS=-I$ABS_PATH/BLIS_install/include/blis \ + LDFLAGS="-L$ABS_PATH/BLIS_install/lib -Wl,-rpath,$ABS_PATH/BLIS_install/lib" \ bash ./continuous_integration/build_test_ext.sh +# Check dynamic linking +ldd tests/_openmp_test_helper/nested_prange_blas.cpython*.so + python --version python -c "import numpy; print(f'numpy {numpy.__version__}')" || echo "no numpy" python -c "import scipy; print(f'scipy {scipy.__version__}')" || echo "no scipy" diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index e0d6786a..87d65505 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -406,18 +406,17 @@ def test_multiple_shipped_openblas(): test_shipped_openblas() -@pytest.mark.skipif(scipy is None, reason="requires scipy") @pytest.mark.skipif( not cython_extensions_compiled, reason="Requires cython extensions to be compiled" ) @pytest.mark.parametrize("nthreads_outer", [None, 1, 2, 4]) def test_nested_prange_blas(nthreads_outer): - # Check that the BLAS linked to scipy effectively uses the number of - # threads requested by the context manager when nested in an outer OpenMP - # loop. - import numpy as np - import tests._openmp_test_helper.nested_prange_blas as prange_blas - + # Check that the BLAS uses the number of threads requested by the context manager + # when nested in an outer OpenMP loop. + # Remark: this test also works with sequential BLAS only because we limit the + # number of threads for the BLAS to 1. + np = pytest.importorskip("numpy") + prange_blas = pytest.importorskip("tests._openmp_test_helper.nested_prange_blas") check_nested_prange_blas = prange_blas.check_nested_prange_blas skip_if_openblas_openmp() From d1450c4ee0da896e2b9287eecc1cc3af938ea6fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Tue, 23 Jan 2024 17:23:01 +0100 Subject: [PATCH 152/187] MAINT Update BLIS install to use numpy's new build system (#160) --- continuous_integration/install_with_blis.sh | 25 +++++++++++++-------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/continuous_integration/install_with_blis.sh b/continuous_integration/install_with_blis.sh index dff5dc05..15ef3a5e 100755 --- a/continuous_integration/install_with_blis.sh +++ b/continuous_integration/install_with_blis.sh @@ -15,7 +15,8 @@ sudo apt-get install libomp-dev # create conda env conda update -n base conda conda-libmamba-solver -q --yes conda config --set solver libmamba -conda create -n $VIRTUALENV -q --yes -c conda-forge python=$PYTHON_VERSION pip cython +conda create -n $VIRTUALENV -q --yes -c conda-forge python=$PYTHON_VERSION \ + pip cython meson-python pkg-config source activate $VIRTUALENV if [[ "$BLIS_CC" == "gcc-8" ]]; then @@ -37,14 +38,20 @@ popd # build & install numpy git clone https://github.com/numpy/numpy.git pushd numpy -git checkout v1.26.0 # pin numpy < 2 for now git submodule update --init -echo "[blis] -libraries = blis -library_dirs = $ABS_PATH/BLIS_install/lib -include_dirs = $ABS_PATH/BLIS_install/include/blis -runtime_library_dirs = $ABS_PATH/BLIS_install/lib" > site.cfg -python setup.py develop + +echo "libdir=$ABS_PATH/BLIS_install/lib/ +includedir=$ABS_PATH/BLIS_install/include/blis/ +version=latest +extralib=-lm -lpthread -lgfortran +Name: blis +Description: BLIS +Version: \${version} +Libs: -L\${libdir} -lblis +Libs.private: \${extralib} +Cflags: -I\${includedir}" > blis.pc + +PKG_CONFIG_PATH=$ABS_PATH/numpy/ pip install . -v --no-build-isolation -Csetup-args=-Dblas=blis popd popd @@ -54,7 +61,7 @@ CFLAGS=-I$ABS_PATH/BLIS_install/include/blis \ LDFLAGS="-L$ABS_PATH/BLIS_install/lib -Wl,-rpath,$ABS_PATH/BLIS_install/lib" \ bash ./continuous_integration/build_test_ext.sh -# Check dynamic linking +# Check that BLIS is linked ldd tests/_openmp_test_helper/nested_prange_blas.cpython*.so python --version From 8faf5f5eec61bf297da8841924f15cf8ec463507 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Tue, 6 Feb 2024 09:44:09 +0100 Subject: [PATCH 153/187] FEA Add support for flexiblas (#156) Co-authored-by: Olivier Grisel --- .azure_pipeline.yml | 20 ++++- CHANGES.md | 3 + .../install_with_flexiblas.sh | 84 ++++++++++++++++++ continuous_integration/posix.yml | 10 ++- ...blis.pyx => nested_prange_blas_custom.pyx} | 0 .../setup_nested_prange_blas.py | 8 +- tests/test_threadpoolctl.py | 30 ++++++- tests/utils.py | 6 ++ threadpoolctl.py | 86 ++++++++++++++++++- 9 files changed, 232 insertions(+), 15 deletions(-) create mode 100755 continuous_integration/install_with_flexiblas.sh rename tests/_openmp_test_helper/{nested_prange_blas_blis.pyx => nested_prange_blas_custom.pyx} (100%) diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml index b88abe1c..99d0f841 100644 --- a/.azure_pipeline.yml +++ b/.azure_pipeline.yml @@ -53,7 +53,6 @@ stages: PYTHON_VERSION: '3.8' PACKAGER: 'pip' - - template: continuous_integration/posix.yml parameters: name: Linux @@ -123,7 +122,7 @@ stages: pylatest_blis_gcc_clang_openmp: PACKAGER: 'conda' PYTHON_VERSION: '*' - INSTALL_BLIS: 'true' + INSTALL_BLAS: 'blis' BLIS_NUM_THREADS: '4' CC_OUTER_LOOP: 'gcc' CC_INNER_LOOP: 'gcc' @@ -132,7 +131,7 @@ stages: pylatest_blis_clang_gcc_pthreads: PACKAGER: 'conda' PYTHON_VERSION: '*' - INSTALL_BLIS: 'true' + INSTALL_BLAS: 'blis' BLIS_NUM_THREADS: '4' CC_OUTER_LOOP: 'clang-10' CC_INNER_LOOP: 'clang-10' @@ -141,12 +140,19 @@ stages: pylatest_blis_no_threading: PACKAGER: 'conda' PYTHON_VERSION: '*' - INSTALL_BLIS: 'true' + INSTALL_BLAS: 'blis' BLIS_NUM_THREADS: '1' CC_OUTER_LOOP: 'gcc' CC_INNER_LOOP: 'gcc' BLIS_CC: 'gcc-8' BLIS_ENABLE_THREADING: 'no' + pylatest_flexiblas: + PACKAGER: 'conda' + PYTHON_VERSION: '*' + INSTALL_BLAS: 'flexiblas' + CC_OUTER_LOOP: 'gcc' + CC_INNER_LOOP: 'gcc' + - template: continuous_integration/posix.yml parameters: @@ -177,6 +183,12 @@ stages: CC_OUTER_LOOP: 'clang' CC_INNER_LOOP: 'clang' INSTALL_LIBOMP: 'conda-forge' + pylatest_flexiblas: + PACKAGER: 'conda' + PYTHON_VERSION: '*' + INSTALL_BLAS: 'flexiblas' + CC_OUTER_LOOP: 'clang' + CC_INNER_LOOP: 'clang' - stage: jobs: diff --git a/CHANGES.md b/CHANGES.md index 3dbb7e03..4df2962d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,9 @@ 3.2.1 (TBD) =========== +- Added support for FlexiBLAS + https://github.com/joblib/threadpoolctl/pull/156 + - Fixed a bug where an unsupported library would be detected because it shares a common prefix with one of the supported libraries. Now the symbols are also checked to identify the supported libraries. diff --git a/continuous_integration/install_with_flexiblas.sh b/continuous_integration/install_with_flexiblas.sh new file mode 100755 index 00000000..d9a16cf6 --- /dev/null +++ b/continuous_integration/install_with_flexiblas.sh @@ -0,0 +1,84 @@ +#!/bin/bash + +set -e + +pushd .. +ABS_PATH=$(pwd) +popd + +# create conda env +conda update -n base conda conda-libmamba-solver -q --yes +conda config --set solver libmamba +conda create -n $VIRTUALENV -q --yes -c conda-forge python=$PYTHON_VERSION \ + pip cython openblas meson-python pkg-config compilers +source activate $VIRTUALENV + +pushd .. + +# build & install FlexiBLAS +mkdir flexiblas_install +git clone https://github.com/mpimd-csc/flexiblas.git +pushd flexiblas + +mkdir build +pushd build + +EXTENSION=".so" +if [[ $(uname) == "Darwin" ]]; then + EXTENSION=".dylib" +fi + +# We intentionally restrict the list of backends to make it easier to +# write platform agnostic tests. In particular, we do not detect OS +# provided backends such as macOS' Apple/Accelerate/vecLib nor plaftorm +# specific BLAS implementations such as MKL that cannot be installed on +# arm64 hardware. +cmake ../ -DCMAKE_INSTALL_PREFIX=$ABS_PATH"/flexiblas_install" \ + -DBLAS_AUTO_DETECT="OFF" \ + -DEXTRA="OPENBLAS_CONDA" \ + -DFLEXIBLAS_DEFAULT="OPENBLAS_CONDA" \ + -DOPENBLAS_CONDA_LIBRARY=$CONDA_PREFIX"/lib/libopenblas"$EXTENSION \ +make +make install + +# Check that all 3 BLAS are listed in FlexiBLAS configuration +$ABS_PATH/flexiblas_install/bin/flexiblas list +popd +popd + +# build & install numpy +git clone https://github.com/numpy/numpy.git +pushd numpy +git submodule update --init + +echo "libdir=$ABS_PATH/flexiblas_install/lib/ +includedir=$ABS_PATH/flexiblas_install/include/flexiblas/ +version=3.3.1 +extralib=-lm -lpthread -lgfortran +Name: flexiblas +Description: FlexiBLAS - a BLAS wrapper +Version: \${version} +Libs: -L\${libdir} -lflexiblas +Libs.private: \${extralib} +Cflags: -I\${includedir}" > flexiblas.pc + +PKG_CONFIG_PATH=$ABS_PATH/numpy/ pip install . -v --no-build-isolation -Csetup-args=-Dblas=flexiblas -Csetup-args=-Dlapack=flexiblas +popd + +popd + +python -m pip install -q -r dev-requirements.txt +CFLAGS=-I$ABS_PATH/flexiblas_install/include/flexiblas \ + LDFLAGS="-L$ABS_PATH/flexiblas_install/lib -Wl,-rpath,$ABS_PATH/flexiblas_install/lib" \ + bash ./continuous_integration/build_test_ext.sh + +# Check that FlexiBLAS is linked +if [[ $(uname) != "Darwin" ]]; then + ldd tests/_openmp_test_helper/nested_prange_blas.cpython*.so +fi + +python --version +python -c "import numpy; print(f'numpy {numpy.__version__}')" || echo "no numpy" +python -c "import scipy; print(f'scipy {scipy.__version__}')" || echo "no scipy" + +python -m flit install --symlink diff --git a/continuous_integration/posix.yml b/continuous_integration/posix.yml index 5edb0ac7..5d4fe982 100644 --- a/continuous_integration/posix.yml +++ b/continuous_integration/posix.yml @@ -22,12 +22,16 @@ jobs: condition: eq('${{ parameters.name }}', 'macOS') - script: | continuous_integration/install.sh - displayName: 'Install without BLIS' - condition: ne(variables['INSTALL_BLIS'], 'true') + displayName: 'Install without custom BLAS' + condition: eq(variables['INSTALL_BLAS'], '') - script: | continuous_integration/install_with_blis.sh displayName: 'Install with BLIS' - condition: eq(variables['INSTALL_BLIS'], 'true') + condition: eq(variables['INSTALL_BLAS'], 'blis') + - script: | + continuous_integration/install_with_flexiblas.sh + displayName: 'Install with FlexiBLAS' + condition: eq(variables['INSTALL_BLAS'], 'flexiblas') - script: | continuous_integration/test_script.sh displayName: 'Test Library' diff --git a/tests/_openmp_test_helper/nested_prange_blas_blis.pyx b/tests/_openmp_test_helper/nested_prange_blas_custom.pyx similarity index 100% rename from tests/_openmp_test_helper/nested_prange_blas_blis.pyx rename to tests/_openmp_test_helper/nested_prange_blas_custom.pyx diff --git a/tests/_openmp_test_helper/setup_nested_prange_blas.py b/tests/_openmp_test_helper/setup_nested_prange_blas.py index 88b900b3..6a06d468 100644 --- a/tests/_openmp_test_helper/setup_nested_prange_blas.py +++ b/tests/_openmp_test_helper/setup_nested_prange_blas.py @@ -12,10 +12,10 @@ set_cc_variables("CC_OUTER_LOOP") openmp_flag = get_openmp_flag() - use_blis = os.getenv("INSTALL_BLIS", False) - libraries = ["blis"] if use_blis else [] - blis_suffix = "_blis" if use_blis else "" - filename = f"nested_prange_blas{blis_suffix}.pyx" + use_custom_blas = os.getenv("INSTALL_BLAS", False) + libraries = [use_custom_blas] if use_custom_blas else [] + custom_suffix = "_custom" if use_custom_blas else "" + filename = f"nested_prange_blas{custom_suffix}.pyx" ext_modules = [ Extension( diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index 87d65505..fb4d2580 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -10,6 +10,7 @@ from threadpoolctl import _ALL_PREFIXES, _ALL_USER_APIS from .utils import cython_extensions_compiled +from .utils import check_nested_prange_blas from .utils import libopenblas_paths from .utils import scipy from .utils import threadpool_info_from_subprocess @@ -409,15 +410,17 @@ def test_multiple_shipped_openblas(): @pytest.mark.skipif( not cython_extensions_compiled, reason="Requires cython extensions to be compiled" ) +@pytest.mark.skipif( + check_nested_prange_blas is None, + reason="Requires nested_prange_blas to be compiled", +) @pytest.mark.parametrize("nthreads_outer", [None, 1, 2, 4]) def test_nested_prange_blas(nthreads_outer): # Check that the BLAS uses the number of threads requested by the context manager # when nested in an outer OpenMP loop. # Remark: this test also works with sequential BLAS only because we limit the # number of threads for the BLAS to 1. - np = pytest.importorskip("numpy") - prange_blas = pytest.importorskip("tests._openmp_test_helper.nested_prange_blas") - check_nested_prange_blas = prange_blas.check_nested_prange_blas + import numpy as np skip_if_openblas_openmp() @@ -651,6 +654,27 @@ def test_openblas_threading_layer(): assert threading_layer in expected_openblas_threading_layers +def test_flexiblas(): + # Check that threadpool_info correctly recovers the FlexiBLAS backends + flexiblas_controller = ThreadpoolController().select(internal_api="flexiblas") + + if not flexiblas_controller: + pytest.skip("requires FlexiBLAS.") + flexiblas_controller = flexiblas_controller.lib_controllers[0] + + expected_backends = {"NETLIB", "OPENBLAS_CONDA"} + expected_backends_loaded = {"OPENBLAS_CONDA"} + expected_current_backend = "OPENBLAS_CONDA" # set as default at build time + + flexiblas_backends = flexiblas_controller.available_backends + flexiblas_backends_loaded = flexiblas_controller.loaded_backends + current_backend = flexiblas_controller.current_backend + + assert set(flexiblas_backends) == expected_backends + assert set(flexiblas_backends_loaded) == expected_backends_loaded + assert current_backend == expected_current_backend + + def test_threadpool_controller_as_decorator(): # Check that using the decorator can be nested and is restricted to the scope of # the decorated function. diff --git a/tests/utils.py b/tests/utils.py index e8d778ff..92ba5998 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -47,6 +47,12 @@ cython_extensions_compiled = False +try: + from tests._openmp_test_helper.nested_prange_blas import check_nested_prange_blas +except ImportError: + check_nested_prange_blas = None + + def threadpool_info_from_subprocess(module): """Utility to call threadpool_info in a subprocess diff --git a/threadpoolctl.py b/threadpoolctl.py index 244ea6b6..92efc94c 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -306,6 +306,84 @@ def _get_architecture(self): return bli_arch_string(bli_arch_query_id()).decode("utf-8") +class FlexiBLASController(LibController): + """Controller class for FlexiBLAS""" + + user_api = "blas" + internal_api = "flexiblas" + filename_prefixes = ("libflexiblas",) + check_symbols = ( + "flexiblas_get_num_threads", + "flexiblas_set_num_threads", + "flexiblas_get_version", + "flexiblas_list", + "flexiblas_list_loaded", + "flexiblas_current_backend", + ) + + def set_additional_attributes(self): + self.available_backends = self._get_backend_list(loaded=False) + self.loaded_backends = self._get_backend_list(loaded=True) + self.current_backend = self._get_current_backend() + + def get_num_threads(self): + get_func = getattr(self.dynlib, "flexiblas_get_num_threads", lambda: None) + num_threads = get_func() + # by default BLIS is single-threaded and get_num_threads + # returns -1. We map it to 1 for consistency with other libraries. + return 1 if num_threads == -1 else num_threads + + def set_num_threads(self, num_threads): + set_func = getattr( + self.dynlib, "flexiblas_set_num_threads", lambda num_threads: None + ) + return set_func(num_threads) + + def get_version(self): + get_version_ = getattr(self.dynlib, "flexiblas_get_version", None) + if get_version_ is None: + return None + + major = ctypes.c_int() + minor = ctypes.c_int() + patch = ctypes.c_int() + get_version_(ctypes.byref(major), ctypes.byref(minor), ctypes.byref(patch)) + return f"{major.value}.{minor.value}.{patch.value}" + + def _get_backend_list(self, loaded=False): + """Return the list of available backends for FlexiBLAS. + + If loaded is False, return the list of available backends from the FlexiBLAS + configuration. If loaded is True, return the list of actually loaded backends. + """ + func_name = f"flexiblas_list{'_loaded' if loaded else ''}" + get_backend_list_ = getattr(self.dynlib, func_name, None) + if get_backend_list_ is None: + return None + + n_backends = get_backend_list_(None, 0, 0) + + backends = [] + for i in range(n_backends): + backend_name = ctypes.create_string_buffer(1024) + get_backend_list_(backend_name, 1024, i) + if backend_name.value.decode("utf-8") != "__FALLBACK__": + # We don't know when to expect __FALLBACK__ but it is not a real + # backend and does not show up when running flexiblas list. + backends.append(backend_name.value.decode("utf-8")) + return backends + + def _get_current_backend(self): + """Return the backend of FlexiBLAS""" + get_backend_ = getattr(self.dynlib, "flexiblas_current_backend", None) + if get_backend_ is None: + return None + + backend = ctypes.create_string_buffer(1024) + get_backend_(backend, ctypes.sizeof(backend)) + return backend.value.decode("utf-8") + + class MKLController(LibController): """Controller class for MKL""" @@ -388,7 +466,13 @@ def get_version(self): # Controllers for the libraries that we'll look for in the loaded libraries. # Third party libraries can register their own controllers. -_ALL_CONTROLLERS = [OpenBLASController, BLISController, MKLController, OpenMPController] +_ALL_CONTROLLERS = [ + OpenBLASController, + BLISController, + MKLController, + OpenMPController, + FlexiBLASController, +] # Helpers for the doc and test names _ALL_USER_APIS = list(set(lib.user_api for lib in _ALL_CONTROLLERS)) From 1c71599cc8dc9893ea188c15995e5420a543807b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Wed, 7 Feb 2024 13:29:13 +0100 Subject: [PATCH 154/187] MAINT fix linting to comply to black latest version (#164) --- threadpoolctl.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/threadpoolctl.py b/threadpoolctl.py index 92efc94c..cfebcb5e 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -1116,7 +1116,8 @@ def _check_prefix(self, library_basename, filename_prefixes): def _warn_if_incompatible_openmp(self): """Raise a warning if llvm-OpenMP and intel-OpenMP are both loaded""" prefixes = [lib_controller.prefix for lib_controller in self.lib_controllers] - msg = textwrap.dedent(""" + msg = textwrap.dedent( + """ Found Intel OpenMP ('libiomp') and LLVM OpenMP ('libomp') loaded at the same time. Both libraries are known to be incompatible and this can cause random crashes or deadlocks on Linux when loaded in the @@ -1124,7 +1125,8 @@ def _warn_if_incompatible_openmp(self): Using threadpoolctl may cause crashes or deadlocks. For more information and possible workarounds, please see https://github.com/joblib/threadpoolctl/blob/master/multiple_openmp.md - """) + """ + ) if "libomp" in prefixes and "libiomp" in prefixes: warnings.warn(msg, RuntimeWarning) From c6cc9f08a52bdd1ea7d7aba6f1f356ad22d7337c Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Wed, 7 Feb 2024 13:28:09 +0000 Subject: [PATCH 155/187] BLD Limit version of flit_core specified to build with (#161) --- pyproject.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5e9de5bc..91327a84 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,9 @@ [build-system] -requires = ["flit_core"] +requires = ["flit_core >=2,<4"] build-backend = "flit_core.buildapi" +# TODO: replace the following section by the standard [project] section to be able +# to use flit v4. [tool.flit.metadata] module = "threadpoolctl" author = "Thomas Moreau" From a8d191740dedd7a20274703b5af958babb7a43e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Fri, 9 Feb 2024 14:45:56 +0100 Subject: [PATCH 156/187] make test case insensitive (#165) --- tests/test_threadpoolctl.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index fb4d2580..562ede87 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -613,12 +613,12 @@ def test_architecture(): expected_openblas_architectures = ( # XXX: add more as needed by CI or developer laptops "armv8", - "Haswell", - "Prescott", # see: https://github.com/xianyi/OpenBLAS/pull/3485 - "SkylakeX", - "Sandybridge", - "VORTEX", - "Zen", + "haswell", + "prescott", # see: https://github.com/xianyi/OpenBLAS/pull/3485 + "skylakex", + "sandybridge", + "vortex", + "zen", ) expected_blis_architectures = ( # XXX: add more as needed by CI or developer laptops @@ -627,9 +627,9 @@ def test_architecture(): ) for lib_info in threadpool_info(): if lib_info["internal_api"] == "openblas": - assert lib_info["architecture"] in expected_openblas_architectures + assert lib_info["architecture"].lower() in expected_openblas_architectures elif lib_info["internal_api"] == "blis": - assert lib_info["architecture"] in expected_blis_architectures + assert lib_info["architecture"].lower() in expected_blis_architectures else: # Not supported for other libraries assert "architecture" not in lib_info From 22098de05a7584d4f19b3eda0c5a9c6c82777065 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= Date: Wed, 14 Feb 2024 16:24:47 +0100 Subject: [PATCH 157/187] FEA Add support to switch FlexiBLAS backend (#163) Co-authored-by: Olivier Grisel --- .azure_pipeline.yml | 2 + CHANGES.md | 3 + README.md | 86 +++++++++++++++++++ .../install_with_flexiblas.sh | 2 +- tests/test_threadpoolctl.py | 56 +++++++++++- threadpoolctl.py | 73 ++++++++++++++-- 6 files changed, 211 insertions(+), 11 deletions(-) diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml index 99d0f841..701792a1 100644 --- a/.azure_pipeline.yml +++ b/.azure_pipeline.yml @@ -150,6 +150,7 @@ stages: PACKAGER: 'conda' PYTHON_VERSION: '*' INSTALL_BLAS: 'flexiblas' + PLATFORM_SPECIFIC_PACKAGES: 'mkl' CC_OUTER_LOOP: 'gcc' CC_INNER_LOOP: 'gcc' @@ -187,6 +188,7 @@ stages: PACKAGER: 'conda' PYTHON_VERSION: '*' INSTALL_BLAS: 'flexiblas' + PLATFORM_SPECIFIC_PACKAGES: 'mkl' CC_OUTER_LOOP: 'clang' CC_INNER_LOOP: 'clang' diff --git a/CHANGES.md b/CHANGES.md index 4df2962d..6a5f0748 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,9 @@ 3.2.1 (TBD) =========== +- Extended FlexiBLAS support to be able to switch backend at runtime. + https://github.com/joblib/threadpoolctl/pull/163 + - Added support for FlexiBLAS https://github.com/joblib/threadpoolctl/pull/156 diff --git a/README.md b/README.md index 3b22b45c..035f4dc9 100644 --- a/README.md +++ b/README.md @@ -192,6 +192,92 @@ decorators are accessible through their `wrap` method: ... ``` +### Switching the FlexiBLAS backend + +`FlexiBLAS` is a BLAS wrapper for which the BLAS backend can be switched at runtime. +`threadpoolctl` exposes python bindings for this feature. Here's an example but note +that this part of the API is experimental and subject to change without deprecation: + +```python +>>> from threadpoolctl import ThreadpoolController +>>> import numpy as np +>>> controller = ThreadpoolController() + +>>> controller.info() +[{'user_api': 'blas', + 'internal_api': 'flexiblas', + 'num_threads': 1, + 'prefix': 'libflexiblas', + 'filepath': '/usr/local/lib/libflexiblas.so.3.3', + 'version': '3.3.1', + 'available_backends': ['NETLIB', 'OPENBLASPTHREAD', 'ATLAS'], + 'loaded_backends': ['NETLIB'], + 'current_backend': 'NETLIB'}] + +# Retrieve the flexiblas controller +>>> flexiblas_ct = controller.select(internal_api="flexiblas").lib_controllers[0] + +# Switch the backend with one predefined at build time (listed in "available_backends") +>>> flexiblas_ct.switch_backend("OPENBLASPTHREAD") +>>> controller.info() +[{'user_api': 'blas', + 'internal_api': 'flexiblas', + 'num_threads': 4, + 'prefix': 'libflexiblas', + 'filepath': '/usr/local/lib/libflexiblas.so.3.3', + 'version': '3.3.1', + 'available_backends': ['NETLIB', 'OPENBLASPTHREAD', 'ATLAS'], + 'loaded_backends': ['NETLIB', 'OPENBLASPTHREAD'], + 'current_backend': 'OPENBLASPTHREAD'}, + {'user_api': 'blas', + 'internal_api': 'openblas', + 'num_threads': 4, + 'prefix': 'libopenblas', + 'filepath': '/usr/lib/x86_64-linux-gnu/openblas-pthread/libopenblasp-r0.3.8.so', + 'version': '0.3.8', + 'threading_layer': 'pthreads', + 'architecture': 'Haswell'}] + +# It's also possible to directly give the path to a shared library +>>> flexiblas_controller.switch_backend("/home/jeremie/miniforge/envs/flexiblas_threadpoolctl/lib/libmkl_rt.so") +>>> controller.info() +[{'user_api': 'blas', + 'internal_api': 'flexiblas', + 'num_threads': 2, + 'prefix': 'libflexiblas', + 'filepath': '/usr/local/lib/libflexiblas.so.3.3', + 'version': '3.3.1', + 'available_backends': ['NETLIB', 'OPENBLASPTHREAD', 'ATLAS'], + 'loaded_backends': ['NETLIB', + 'OPENBLASPTHREAD', + '/home/jeremie/miniforge/envs/flexiblas_threadpoolctl/lib/libmkl_rt.so'], + 'current_backend': '/home/jeremie/miniforge/envs/flexiblas_threadpoolctl/lib/libmkl_rt.so'}, + {'user_api': 'openmp', + 'internal_api': 'openmp', + 'num_threads': 4, + 'prefix': 'libomp', + 'filepath': '/home/jeremie/miniforge/envs/flexiblas_threadpoolctl/lib/libomp.so', + 'version': None}, + {'user_api': 'blas', + 'internal_api': 'openblas', + 'num_threads': 4, + 'prefix': 'libopenblas', + 'filepath': '/usr/lib/x86_64-linux-gnu/openblas-pthread/libopenblasp-r0.3.8.so', + 'version': '0.3.8', + 'threading_layer': 'pthreads', + 'architecture': 'Haswell'}, + {'user_api': 'blas', + 'internal_api': 'mkl', + 'num_threads': 2, + 'prefix': 'libmkl_rt', + 'filepath': '/home/jeremie/miniforge/envs/flexiblas_threadpoolctl/lib/libmkl_rt.so.2', + 'version': '2024.0-Product', + 'threading_layer': 'gnu'}] +``` + +You can observe that the previously linked OpenBLAS shared object stays loaded by +the Python program indefinitely, but FlexiBLAS itself no longer delegates BLAS calls +to OpenBLAS as indicated by the `current_backend` attribute. ### Writing a custom library controller Currently, `threadpoolctl` has support for `OpenMP` and the main `BLAS` libraries. diff --git a/continuous_integration/install_with_flexiblas.sh b/continuous_integration/install_with_flexiblas.sh index d9a16cf6..703f3ead 100755 --- a/continuous_integration/install_with_flexiblas.sh +++ b/continuous_integration/install_with_flexiblas.sh @@ -10,7 +10,7 @@ popd conda update -n base conda conda-libmamba-solver -q --yes conda config --set solver libmamba conda create -n $VIRTUALENV -q --yes -c conda-forge python=$PYTHON_VERSION \ - pip cython openblas meson-python pkg-config compilers + pip cython openblas $PLATFORM_SPECIFIC_PACKAGES meson-python pkg-config compilers source activate $VIRTUALENV pushd .. diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index 562ede87..d6e95363 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -654,8 +654,13 @@ def test_openblas_threading_layer(): assert threading_layer in expected_openblas_threading_layers +# skip test if not run in a azure pipelines job since it relies on a specific flexiblas +# installation. +@pytest.mark.skipif( + "TF_BUILD" not in os.environ, reason="not running in azure pipelines" +) def test_flexiblas(): - # Check that threadpool_info correctly recovers the FlexiBLAS backends + # Check that threadpool_info correctly recovers the FlexiBLAS backends. flexiblas_controller = ThreadpoolController().select(internal_api="flexiblas") if not flexiblas_controller: @@ -675,6 +680,55 @@ def test_flexiblas(): assert current_backend == expected_current_backend +def test_flexiblas_switch_error(): + # Check that an error is raised when trying to switch to an invalid backend. + flexiblas_controller = ThreadpoolController().select(internal_api="flexiblas") + + if not flexiblas_controller: + pytest.skip("requires FlexiBLAS.") + flexiblas_controller = flexiblas_controller.lib_controllers[0] + + with pytest.raises(RuntimeError, match="Failed to load backend"): + flexiblas_controller.switch_backend("INVALID_BACKEND") + + +# skip test if not run in a azure pipelines job since it relies on a specific flexiblas +# installation. +@pytest.mark.skipif( + "TF_BUILD" not in os.environ, reason="not running in azure pipelines" +) +def test_flexiblas_switch(): + # Check that the backend can be switched. + controller = ThreadpoolController() + fb_controller = controller.select(internal_api="flexiblas") + + if not fb_controller: + pytest.skip("requires FlexiBLAS.") + fb_controller = fb_controller.lib_controllers[0] + + # at first mkl is not loaded in the CI jobs where this test runs + assert len(controller.select(internal_api="mkl").lib_controllers) == 0 + + # at first, only "OPENBLAS_CONDA" is loaded + assert fb_controller.current_backend == "OPENBLAS_CONDA" + assert fb_controller.loaded_backends == ["OPENBLAS_CONDA"] + + fb_controller.switch_backend("NETLIB") + assert fb_controller.current_backend == "NETLIB" + assert fb_controller.loaded_backends == ["OPENBLAS_CONDA", "NETLIB"] + + ext = ".so" if sys.platform == "linux" else ".dylib" + mkl_path = f"{os.getenv('CONDA_PREFIX')}/lib/libmkl_rt{ext}" + fb_controller.switch_backend(mkl_path) + assert fb_controller.current_backend == mkl_path + assert fb_controller.loaded_backends == ["OPENBLAS_CONDA", "NETLIB", mkl_path] + # switching the backend triggered a new search for loaded shared libs + assert len(controller.select(internal_api="mkl").lib_controllers) == 1 + + # switch back to default to avoid side effects + fb_controller.switch_backend("OPENBLAS_CONDA") + + def test_threadpool_controller_as_decorator(): # Check that using the decorator can be nested and is restricted to the scope of # the decorated function. diff --git a/threadpoolctl.py b/threadpoolctl.py index cfebcb5e..4eb85d18 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -105,20 +105,17 @@ class LibController(ABC): """ @final - def __init__(self, *, filepath=None, prefix=None): + def __init__(self, *, filepath=None, prefix=None, parent=None): """This is not meant to be overriden by subclasses.""" + self.parent = parent self.prefix = prefix self.filepath = filepath self.dynlib = ctypes.CDLL(filepath, mode=_RTLD_NOLOAD) self.version = self.get_version() self.set_additional_attributes() - @final def info(self): - """Return relevant info wrapped in a dict - - This is not meant to be overriden by subclasses. - """ + """Return relevant info wrapped in a dict""" exposed_attrs = { "user_api": self.user_api, "internal_api": self.internal_api, @@ -126,6 +123,7 @@ def info(self): **vars(self), } exposed_attrs.pop("dynlib") + exposed_attrs.pop("parent") return exposed_attrs def set_additional_attributes(self): @@ -321,10 +319,26 @@ class FlexiBLASController(LibController): "flexiblas_current_backend", ) + @property + def loaded_backends(self): + return self._get_backend_list(loaded=True) + + @property + def current_backend(self): + return self._get_current_backend() + + def info(self): + """Return relevant info wrapped in a dict""" + # We override the info method because the loaded and current backends + # are dynamic properties + exposed_attrs = super().info() + exposed_attrs["loaded_backends"] = self.loaded_backends + exposed_attrs["current_backend"] = self.current_backend + + return exposed_attrs + def set_additional_attributes(self): self.available_backends = self._get_backend_list(loaded=False) - self.loaded_backends = self._get_backend_list(loaded=True) - self.current_backend = self._get_current_backend() def get_num_threads(self): get_func = getattr(self.dynlib, "flexiblas_get_num_threads", lambda: None) @@ -383,6 +397,40 @@ def _get_current_backend(self): get_backend_(backend, ctypes.sizeof(backend)) return backend.value.decode("utf-8") + def switch_backend(self, backend): + """Switch the backend of FlexiBLAS + + Parameters + ---------- + backend : str + The name or the path to the shared library of the backend to switch to. If + the backend is not already loaded, it will be loaded first. + """ + if backend not in self.loaded_backends: + if backend in self.available_backends: + load_func = getattr(self.dynlib, "flexiblas_load_backend", lambda _: -1) + else: # assume backend is a path to a shared library + load_func = getattr( + self.dynlib, "flexiblas_load_backend_library", lambda _: -1 + ) + res = load_func(str(backend).encode("utf-8")) + if res == -1: + raise RuntimeError( + f"Failed to load backend {backend!r}. It must either be the name of" + " a backend available in the FlexiBLAS configuration " + f"{self.available_backends} or the path to a valid shared library." + ) + + # Trigger a new search of loaded shared libraries since loading a new + # backend caused a dlopen. + self.parent._load_libraries() + + switch_func = getattr(self.dynlib, "flexiblas_switch", lambda _: -1) + idx = self.loaded_backends.index(backend) + res = switch_func(idx) + if res == -1: + raise RuntimeError(f"Failed to switch to backend {backend!r}.") + class MKLController(LibController): """Controller class for MKL""" @@ -1096,7 +1144,14 @@ def _make_controller_from_path(self, filepath): # expected library (e.g. a library having a common prefix with one of the # our supported libraries). Otherwise, create and store the library # controller. - lib_controller = controller_class(filepath=filepath, prefix=prefix) + lib_controller = controller_class( + filepath=filepath, prefix=prefix, parent=self + ) + + if filepath in (lib.filepath for lib in self.lib_controllers): + # We already have a controller for this library. + continue + if not hasattr(controller_class, "check_symbols") or any( hasattr(lib_controller.dynlib, func) for func in controller_class.check_symbols From 832f73284337b6a2f7860c9d24b79e8034057ac9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= Date: Wed, 14 Feb 2024 17:28:22 +0100 Subject: [PATCH 158/187] MAINT Release 3.3.0 (#168) --- CHANGES.md | 4 ++-- threadpoolctl.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6a5f0748..ae5642a3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,5 @@ -3.2.1 (TBD) -=========== +3.3.0 (2024-02-14) +================== - Extended FlexiBLAS support to be able to switch backend at runtime. https://github.com/joblib/threadpoolctl/pull/163 diff --git a/threadpoolctl.py b/threadpoolctl.py index 4eb85d18..371f1e80 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -23,7 +23,7 @@ from functools import lru_cache from contextlib import ContextDecorator -__version__ = "3.3.0.dev0" +__version__ = "3.3.0" __all__ = [ "threadpool_limits", "threadpool_info", From fef23038ee602e3c16db651de99fca4a85a0cb65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= Date: Sat, 17 Feb 2024 14:38:35 +0100 Subject: [PATCH 159/187] FEA Add support for Pyodide (#169) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jérémie du Boisberranger Co-authored-by: Olivier Grisel --- CHANGES.md | 6 ++++++ threadpoolctl.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index ae5642a3..fcd5c52c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,9 @@ +3.4.0 (TDB) +=========== + +- Added support for Pyodide + https://github.com/joblib/threadpoolctl/pull/169 + 3.3.0 (2024-02-14) ================== diff --git a/threadpoolctl.py b/threadpoolctl.py index 371f1e80..ed6fec4c 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -979,6 +979,8 @@ def _load_libraries(self): self._find_libraries_with_dyld() elif sys.platform == "win32": self._find_libraries_with_enum_process_module_ex() + elif "pyodide" in sys.modules: + self._find_libraries_pyodide() else: self._find_libraries_with_dl_iterate_phdr() @@ -1100,6 +1102,33 @@ def _find_libraries_with_enum_process_module_ex(self): finally: kernel_32.CloseHandle(h_process) + def _find_libraries_pyodide(self): + """Pyodide specific implementation for finding loaded libraries. + + Adapted from suggestion in https://github.com/joblib/threadpoolctl/pull/169#issuecomment-1946696449. + + One day, we may have a simpler solution. libc dl_iterate_phdr needs to + be implemented in Emscripten and exposed in Pyodide, see + https://github.com/emscripten-core/emscripten/issues/21354 for more + details. + """ + try: + from pyodide_js._module import LDSO + except ImportError: + warnings.warn( + "Unable to import LDSO from pyodide_js._module. This should never " + "happen." + ) + return + + for filepath in LDSO.loadedLibsByName.as_object_map(): + # Some libraries are duplicated by Pyodide and do not exist in the + # filesystem, so we first check for the existence of the file. For + # more details, see + # https://github.com/joblib/threadpoolctl/pull/169#issuecomment-1947946728 + if os.path.exists(filepath): + self._make_controller_from_path(filepath) + def _make_controller_from_path(self, filepath): """Store a library controller if it is supported and selected""" # Required to resolve symlinks From b6813a601fa7608919e1e606132e5f8b33c106f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= Date: Wed, 20 Mar 2024 08:58:28 +0100 Subject: [PATCH 160/187] FEA Support static and / or alternative libc (#171) --- CHANGES.md | 4 ++++ threadpoolctl.py | 25 +++++++++++++++---------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index fcd5c52c..7cf6c45e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,10 @@ 3.4.0 (TDB) =========== +- Added support for Python interpreters statically linked against libc or linked against + alternative implementations of libc like musl (on Alpine Linux for instance). + https://github.com/joblib/threadpoolctl/pull/171 + - Added support for Pyodide https://github.com/joblib/threadpoolctl/pull/169 diff --git a/threadpoolctl.py b/threadpoolctl.py index ed6fec4c..aa9c2d63 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -996,6 +996,10 @@ def _find_libraries_with_dl_iterate_phdr(self): """ libc = self._get_libc() if not hasattr(libc, "dl_iterate_phdr"): # pragma: no cover + warnings.warn( + "Could not find dl_iterate_phdr in the C standard library.", + RuntimeWarning, + ) return [] # Callback function for `dl_iterate_phdr` which is called for every @@ -1028,6 +1032,10 @@ def _find_libraries_with_dyld(self): """ libc = self._get_libc() if not hasattr(libc, "_dyld_image_count"): # pragma: no cover + warnings.warn( + "Could not find _dyld_image_count in the C standard library.", + RuntimeWarning, + ) return [] n_dyld = libc._dyld_image_count() @@ -1219,16 +1227,13 @@ def _get_libc(cls): """Load the lib-C for unix systems.""" libc = cls._system_libraries.get("libc") if libc is None: - libc_name = find_library("c") - if libc_name is None: # pragma: no cover - warnings.warn( - "libc not found. The ctypes module in Python" - f" {sys.version_info.major}.{sys.version_info.minor} is maybe" - " too old for this OS.", - RuntimeWarning, - ) - return None - libc = ctypes.CDLL(libc_name, mode=_RTLD_NOLOAD) + # Remark: If libc is statically linked or if Python is linked against an + # alternative implementation of libc like musl, find_library will return + # None and CDLL will load the main program itself which should contain the + # libc symbols. We still name it libc for convenience. + # If the main program does not contain the libc symbols, it's ok because + # we check their presence later anyway. + libc = ctypes.CDLL(find_library("c"), mode=_RTLD_NOLOAD) cls._system_libraries["libc"] = libc return libc From 81f70f01e830ecb56ed584263ed9d8e3bf86845e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= Date: Wed, 20 Mar 2024 14:35:52 +0100 Subject: [PATCH 161/187] Release 3.4.0 (#172) --- CHANGES.md | 4 ++-- threadpoolctl.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 7cf6c45e..1b13d7e5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,5 @@ -3.4.0 (TDB) -=========== +3.4.0 (2024-03-20) +================== - Added support for Python interpreters statically linked against libc or linked against alternative implementations of libc like musl (on Alpine Linux for instance). diff --git a/threadpoolctl.py b/threadpoolctl.py index aa9c2d63..36fec13d 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -23,7 +23,7 @@ from functools import lru_cache from contextlib import ContextDecorator -__version__ = "3.3.0" +__version__ = "3.4.0" __all__ = [ "threadpool_limits", "threadpool_info", From 78619695c730e7a406e26d0c7ff28293ef48b373 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= Date: Wed, 20 Mar 2024 15:53:19 +0100 Subject: [PATCH 162/187] MAINT bump version 3.5.0.dev0 (#173) --- threadpoolctl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/threadpoolctl.py b/threadpoolctl.py index 36fec13d..8cbb869e 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -23,7 +23,7 @@ from functools import lru_cache from contextlib import ContextDecorator -__version__ = "3.4.0" +__version__ = "3.5.0.dev0" __all__ = [ "threadpool_limits", "threadpool_info", From 5282c0b846454ea3d6cc5e9262704f8036f661d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= Date: Thu, 21 Mar 2024 10:38:20 +0100 Subject: [PATCH 163/187] update release instructions (#174) --- README.md | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 035f4dc9..6b2ec1d6 100644 --- a/README.md +++ b/README.md @@ -332,24 +332,40 @@ https://github.com/xianyi/OpenBLAS/issues/2985). To make a release: -Bump the version number (`__version__`) in `threadpoolctl.py`. +- Bump the version number (`__version__`) in `threadpoolctl.py` and update the + release date in `CHANGES.md`. -Build the distribution archives: +- Build the distribution archives: ```bash pip install flit flit build ``` -Check the contents of `dist/`. +and check the contents of `dist/`. -If everything is fine, make a commit for the release, tag it, push the -tag to github and then: +- If everything is fine, make a commit for the release, tag it and push the +tag to github: ```bash -flit publish +git tag -a X.Y.Z +git push git@github.com:joblib/threadpoolctl.git X.Y.Z ``` +- Upload the wheels and source distribution to PyPI using flit. Since PyPI doesn't + allow password authentication anymore, the username needs to be changed to the + generic name `__token__`: + +```bash +FLIT_USERNAME=__token__ flit publish +``` + + and a PyPI token has to be passed in place of the password. + +- Create a PR for the release on the [conda-forge feedstock](https://github.com/conda-forge/threadpoolctl-feedstock) (or wait for the bot to make it). + +- Publish the release on github. + ### Credits The initial dynamic library introspection code was written by @anton-malakhov From 8dd980b64073349a51e4b681338f9aee721ae79a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= Date: Mon, 29 Apr 2024 09:46:19 +0200 Subject: [PATCH 164/187] FEA Extend OpenBLAS controller to support scipy_openblas (#175) --- .azure_pipeline.yml | 6 ++ CHANGES.md | 8 ++ continuous_integration/install.sh | 9 ++ continuous_integration/posix.yml | 2 +- continuous_integration/test_script.sh | 2 +- threadpoolctl.py | 127 ++++++++++++-------------- 6 files changed, 84 insertions(+), 70 deletions(-) diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml index 701792a1..29267725 100644 --- a/.azure_pipeline.yml +++ b/.azure_pipeline.yml @@ -58,6 +58,12 @@ stages: name: Linux vmImage: ubuntu-20.04 matrix: + # Linux environment with development versions of numpy and scipy + pylatest_pip_dev: + PACKAGER: 'pip-dev' + PYTHON_VERSION: '*' + CC_OUTER_LOOP: 'gcc' + CC_INNER_LOOP: 'gcc' # Linux environment to test that packages that comes with Ubuntu 20.04 # are correctly handled. py38_ubuntu_atlas_gcc_gcc: diff --git a/CHANGES.md b/CHANGES.md index 1b13d7e5..e19c536d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,11 @@ +3.5.0 (TDB) +=========== + +- Added support for the Scientific Python version of OpenBLAS + (https://github.com/MacPython/openblas-libs), which exposes symbols with different + names than the ones of the original OpenBLAS library. + https://github.com/joblib/threadpoolctl/pull/175 + 3.4.0 (2024-03-20) ================== diff --git a/continuous_integration/install.sh b/continuous_integration/install.sh index 86d77fdb..d0e8402f 100755 --- a/continuous_integration/install.sh +++ b/continuous_integration/install.sh @@ -65,6 +65,15 @@ elif [[ "$PACKAGER" == "pip" ]]; then pip install numpy scipy fi +elif [[ "$PACKAGER" == "pip-dev" ]]; then + # Use conda to build an empty python env and then use pip to install + # numpy and scipy dev versions + TO_INSTALL="python=$PYTHON_VERSION pip" + make_conda $TO_INSTALL + + dev_anaconda_url=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple + pip install --pre --upgrade --timeout=60 --extra-index $dev_anaconda_url numpy scipy + elif [[ "$PACKAGER" == "ubuntu" ]]; then # Remove the ubuntu toolchain PPA that seems to be invalid: # https://github.com/scikit-learn/scikit-learn/pull/13934 diff --git a/continuous_integration/posix.yml b/continuous_integration/posix.yml index 5d4fe982..a1598d37 100644 --- a/continuous_integration/posix.yml +++ b/continuous_integration/posix.yml @@ -14,7 +14,7 @@ jobs: steps: - bash: echo "##vso[task.prependpath]$CONDA/bin" displayName: Add conda to PATH - condition: or(startsWith(variables['PACKAGER'], 'conda'), eq(variables['PACKAGER'], 'pip')) + condition: or(startsWith(variables['PACKAGER'], 'conda'), startsWith(variables['PACKAGER'], 'pip')) - bash: sudo chown -R $USER $CONDA # On Hosted macOS, the agent user doesn't have ownership of Miniconda's installation directory/ # We need to take ownership if we want to update conda or install packages globally diff --git a/continuous_integration/test_script.sh b/continuous_integration/test_script.sh index 2864f584..8eeaba19 100755 --- a/continuous_integration/test_script.sh +++ b/continuous_integration/test_script.sh @@ -5,7 +5,7 @@ set -e if [[ "$PACKAGER" == conda* ]]; then source activate $VIRTUALENV conda list -elif [[ "$PACKAGER" == "pip" ]]; then +elif [[ "$PACKAGER" == pip* ]]; then # we actually use conda to install the base environment: source activate $VIRTUALENV pip list diff --git a/threadpoolctl.py b/threadpoolctl.py index 8cbb869e..c53b33a7 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -15,6 +15,7 @@ import re import sys import ctypes +import itertools import textwrap from typing import final import warnings @@ -111,20 +112,19 @@ def __init__(self, *, filepath=None, prefix=None, parent=None): self.prefix = prefix self.filepath = filepath self.dynlib = ctypes.CDLL(filepath, mode=_RTLD_NOLOAD) + self._symbol_prefix, self._symbol_suffix = self._find_affixes() self.version = self.get_version() self.set_additional_attributes() def info(self): """Return relevant info wrapped in a dict""" - exposed_attrs = { + hidden_attrs = ("dynlib", "parent", "_symbol_prefix", "_symbol_suffix") + return { "user_api": self.user_api, "internal_api": self.internal_api, "num_threads": self.num_threads, - **vars(self), + **{k: v for k, v in vars(self).items() if k not in hidden_attrs}, } - exposed_attrs.pop("dynlib") - exposed_attrs.pop("parent") - return exposed_attrs def set_additional_attributes(self): """Set additional attributes meant to be exposed in the info dict""" @@ -149,96 +149,87 @@ def set_num_threads(self, num_threads): def get_version(self): """Return the version of the shared library""" + def _find_affixes(self): + """Return the affixes for the symbols of the shared library""" + return "", "" + + def _get_symbol(self, name): + """Return the symbol of the shared library accounding for the affixes""" + return getattr( + self.dynlib, f"{self._symbol_prefix}{name}{self._symbol_suffix}", None + ) + class OpenBLASController(LibController): """Controller class for OpenBLAS""" user_api = "blas" internal_api = "openblas" - filename_prefixes = ("libopenblas", "libblas") - check_symbols = ( - "openblas_get_num_threads", - "openblas_get_num_threads64_", - "openblas_set_num_threads", - "openblas_set_num_threads64_", - "openblas_get_config", - "openblas_get_config64_", - "openblas_get_parallel", - "openblas_get_parallel64_", - "openblas_get_corename", - "openblas_get_corename64_", + filename_prefixes = ("libopenblas", "libblas", "libscipy_openblas") + + _symbol_prefixes = ("", "scipy_") + _symbol_suffixes = ("", "64_", "_64") + + # All variations of "openblas_get_num_threads", accounting for the affixes + check_symbols = tuple( + f"{prefix}openblas_get_num_threads{suffix}" + for prefix, suffix in itertools.product(_symbol_prefixes, _symbol_suffixes) ) + def _find_affixes(self): + for prefix, suffix in itertools.product( + self._symbol_prefixes, self._symbol_suffixes + ): + if hasattr(self.dynlib, f"{prefix}openblas_get_num_threads{suffix}"): + return prefix, suffix + def set_additional_attributes(self): self.threading_layer = self._get_threading_layer() self.architecture = self._get_architecture() def get_num_threads(self): - get_func = getattr( - self.dynlib, - "openblas_get_num_threads", - # Symbols differ when built for 64bit integers in Fortran - getattr(self.dynlib, "openblas_get_num_threads64_", lambda: None), - ) - - return get_func() + get_num_threads_func = self._get_symbol("openblas_get_num_threads") + if get_num_threads_func is not None: + return get_num_threads_func() + return None def set_num_threads(self, num_threads): - set_func = getattr( - self.dynlib, - "openblas_set_num_threads", - # Symbols differ when built for 64bit integers in Fortran - getattr( - self.dynlib, "openblas_set_num_threads64_", lambda num_threads: None - ), - ) - return set_func(num_threads) + set_num_threads_func = self._get_symbol("openblas_set_num_threads") + if set_num_threads_func is not None: + return set_num_threads_func(num_threads) + return None def get_version(self): # None means OpenBLAS is not loaded or version < 0.3.4, since OpenBLAS # did not expose its version before that. - get_config = getattr( - self.dynlib, - "openblas_get_config", - getattr(self.dynlib, "openblas_get_config64_", None), - ) - if get_config is None: + get_version_func = self._get_symbol("openblas_get_config") + if get_version_func is not None: + get_version_func.restype = ctypes.c_char_p + config = get_version_func().split() + if config[0] == b"OpenBLAS": + return config[1].decode("utf-8") return None - - get_config.restype = ctypes.c_char_p - config = get_config().split() - if config[0] == b"OpenBLAS": - return config[1].decode("utf-8") return None def _get_threading_layer(self): """Return the threading layer of OpenBLAS""" - openblas_get_parallel = getattr( - self.dynlib, - "openblas_get_parallel", - getattr(self.dynlib, "openblas_get_parallel64_", None), - ) - if openblas_get_parallel is None: - return "unknown" - threading_layer = openblas_get_parallel() - if threading_layer == 2: - return "openmp" - elif threading_layer == 1: - return "pthreads" - return "disabled" + get_threading_layer_func = self._get_symbol("openblas_get_parallel") + if get_threading_layer_func is not None: + threading_layer = get_threading_layer_func() + if threading_layer == 2: + return "openmp" + elif threading_layer == 1: + return "pthreads" + return "disabled" + return "unknown" def _get_architecture(self): """Return the architecture detected by OpenBLAS""" - get_corename = getattr( - self.dynlib, - "openblas_get_corename", - getattr(self.dynlib, "openblas_get_corename64_", None), - ) - if get_corename is None: - return None - - get_corename.restype = ctypes.c_char_p - return get_corename().decode("utf-8") + get_architecture_func = self._get_symbol("openblas_get_corename") + if get_architecture_func is not None: + get_architecture_func.restype = ctypes.c_char_p + return get_architecture_func().decode("utf-8") + return None class BLISController(LibController): From 6b5590c962e9aab76e4bfa33eac83e13903ede6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= Date: Mon, 29 Apr 2024 15:40:33 +0200 Subject: [PATCH 165/187] Release 3.5.0 (#177) --- CHANGES.md | 4 ++-- threadpoolctl.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e19c536d..bb649f9c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,5 @@ -3.5.0 (TDB) -=========== +3.5.0 (2024-04-29) +================== - Added support for the Scientific Python version of OpenBLAS (https://github.com/MacPython/openblas-libs), which exposes symbols with different diff --git a/threadpoolctl.py b/threadpoolctl.py index c53b33a7..2a72d1b5 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -24,7 +24,7 @@ from functools import lru_cache from contextlib import ContextDecorator -__version__ = "3.5.0.dev0" +__version__ = "3.5.0" __all__ = [ "threadpool_limits", "threadpool_info", From 41df9c6968239972ef5106cb0e78923d7192cff1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= Date: Mon, 29 Apr 2024 18:34:04 +0200 Subject: [PATCH 166/187] MAINT 3.6.0 dev mode (#178) --- threadpoolctl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/threadpoolctl.py b/threadpoolctl.py index 2a72d1b5..218ff180 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -24,7 +24,7 @@ from functools import lru_cache from contextlib import ContextDecorator -__version__ = "3.5.0" +__version__ = "3.6.0.dev0" __all__ = [ "threadpool_limits", "threadpool_info", From c137dc8ed619cb90e42566e6fcd56b6bab1f7b51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= Date: Fri, 28 Jun 2024 18:17:29 +0200 Subject: [PATCH 167/187] CI Bump macos image to latest (#179) --- .azure_pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml index 29267725..e3824c7d 100644 --- a/.azure_pipeline.yml +++ b/.azure_pipeline.yml @@ -164,7 +164,7 @@ stages: - template: continuous_integration/posix.yml parameters: name: macOS - vmImage: macOS-11 + vmImage: macOS-latest matrix: # MacOS environment with OpenMP installed through homebrew py38_conda_homebrew_libomp: From 7eff7ebb6337e368bc58614933df500db6106bad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= Date: Fri, 28 Jun 2024 19:11:28 +0200 Subject: [PATCH 168/187] CI Fix MacOS Flexiblas build (#180) --- continuous_integration/install_with_flexiblas.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/continuous_integration/install_with_flexiblas.sh b/continuous_integration/install_with_flexiblas.sh index 703f3ead..59eec612 100755 --- a/continuous_integration/install_with_flexiblas.sh +++ b/continuous_integration/install_with_flexiblas.sh @@ -20,6 +20,9 @@ mkdir flexiblas_install git clone https://github.com/mpimd-csc/flexiblas.git pushd flexiblas +# Temporary ping Flexiblas commit to avoid openmp symbols not found at link time +git checkout v3.4.2 + mkdir build pushd build From b95c4b5d24e2dac30f1ebaf8112b01b361db115f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= Date: Thu, 6 Mar 2025 12:20:28 +0100 Subject: [PATCH 169/187] MNT Update black (#182) --- .azure_pipeline.yml | 2 +- continuous_integration/check_no_test_skipped.py | 1 - tests/_openmp_test_helper/setup_inner.py | 1 - tests/_openmp_test_helper/setup_nested_prange_blas.py | 1 - tests/_openmp_test_helper/setup_outer.py | 1 - tests/_pyMylib/__init__.py | 1 - tests/utils.py | 1 - 7 files changed, 1 insertion(+), 7 deletions(-) diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml index e3824c7d..44703825 100644 --- a/.azure_pipeline.yml +++ b/.azure_pipeline.yml @@ -27,7 +27,7 @@ stages: inputs: versionSpec: '3.12' - script: | - pip install black + pip install black==25.1.0 displayName: install black - script: | black --check --diff . diff --git a/continuous_integration/check_no_test_skipped.py b/continuous_integration/check_no_test_skipped.py index 7744aae3..f831d29a 100644 --- a/continuous_integration/check_no_test_skipped.py +++ b/continuous_integration/check_no_test_skipped.py @@ -6,7 +6,6 @@ import sys import xml.etree.ElementTree as ET - base_dir = sys.argv[1] # dict {test: result} where result is False if the test was skipped in every diff --git a/tests/_openmp_test_helper/setup_inner.py b/tests/_openmp_test_helper/setup_inner.py index e6103141..4e045c02 100644 --- a/tests/_openmp_test_helper/setup_inner.py +++ b/tests/_openmp_test_helper/setup_inner.py @@ -6,7 +6,6 @@ from build_utils import set_cc_variables from build_utils import get_openmp_flag - original_environ = os.environ.copy() try: # Make it possible to compile the 2 OpenMP enabled Cython extensions diff --git a/tests/_openmp_test_helper/setup_nested_prange_blas.py b/tests/_openmp_test_helper/setup_nested_prange_blas.py index 6a06d468..809f981c 100644 --- a/tests/_openmp_test_helper/setup_nested_prange_blas.py +++ b/tests/_openmp_test_helper/setup_nested_prange_blas.py @@ -6,7 +6,6 @@ from build_utils import set_cc_variables from build_utils import get_openmp_flag - original_environ = os.environ.copy() try: set_cc_variables("CC_OUTER_LOOP") diff --git a/tests/_openmp_test_helper/setup_outer.py b/tests/_openmp_test_helper/setup_outer.py index 8f875bdf..98a84d58 100644 --- a/tests/_openmp_test_helper/setup_outer.py +++ b/tests/_openmp_test_helper/setup_outer.py @@ -6,7 +6,6 @@ from build_utils import set_cc_variables from build_utils import get_openmp_flag - original_environ = os.environ.copy() try: # Make it possible to compile the 2 OpenMP enabled Cython extensions diff --git a/tests/_pyMylib/__init__.py b/tests/_pyMylib/__init__.py index af2867e5..92913fbc 100644 --- a/tests/_pyMylib/__init__.py +++ b/tests/_pyMylib/__init__.py @@ -3,7 +3,6 @@ from threadpoolctl import LibController, register - path = Path(__file__).parent / "my_threaded_lib.so" ctypes.CDLL(path) diff --git a/tests/utils.py b/tests/utils.py index 92ba5998..86fe1b0b 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -6,7 +6,6 @@ from os.path import dirname, normpath from subprocess import check_output - # Path to shipped openblas for libraries such as numpy or scipy libopenblas_patterns = [] From b6bbcc29566f45e69d48040f416f205752c3ded1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= Date: Thu, 6 Mar 2025 12:50:53 +0100 Subject: [PATCH 170/187] CI Add gh workflow for linting (#184) --- .github/workflows/test.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..bc9eb44c --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,29 @@ +name: CI +permissions: + contents: read + +on: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + + linting: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.12" + - name: Install black + run: | + pip install black==25.1.0 + - name: Run black + run: | + black --check --diff . From 2072b632648b89dc47ff63e87f22906cb4d73f80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= Date: Sun, 9 Mar 2025 15:25:49 +0100 Subject: [PATCH 171/187] CI Migrate CI to gh actions (#185) --- .azure_pipeline.yml | 214 --------------- .github/workflows/test.yml | 245 ++++++++++++++++++ continuous_integration/build_test_ext.sh | 2 +- .../check_no_test_skipped.py | 8 +- continuous_integration/install.cmd | 38 --- continuous_integration/install.sh | 86 ++++-- continuous_integration/install_blis.sh | 49 ++++ ...with_flexiblas.sh => install_flexiblas.sh} | 34 +-- continuous_integration/install_with_blis.sh | 71 ----- continuous_integration/posix.yml | 48 ---- .../{test_script.sh => run_tests.sh} | 14 +- continuous_integration/test_script.cmd | 10 - continuous_integration/upload_codecov.sh | 15 -- continuous_integration/windows.yml | 35 --- dev-requirements.txt | 1 + tests/_openmp_test_helper/setup_inner.py | 3 +- .../setup_nested_prange_blas.py | 3 +- tests/_openmp_test_helper/setup_outer.py | 3 +- tests/test_threadpoolctl.py | 19 +- 19 files changed, 385 insertions(+), 513 deletions(-) delete mode 100644 .azure_pipeline.yml delete mode 100644 continuous_integration/install.cmd mode change 100755 => 100644 continuous_integration/install.sh create mode 100644 continuous_integration/install_blis.sh rename continuous_integration/{install_with_flexiblas.sh => install_flexiblas.sh} (63%) mode change 100755 => 100644 delete mode 100755 continuous_integration/install_with_blis.sh delete mode 100644 continuous_integration/posix.yml rename continuous_integration/{test_script.sh => run_tests.sh} (60%) delete mode 100644 continuous_integration/test_script.cmd delete mode 100755 continuous_integration/upload_codecov.sh delete mode 100644 continuous_integration/windows.yml diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml deleted file mode 100644 index 44703825..00000000 --- a/.azure_pipeline.yml +++ /dev/null @@ -1,214 +0,0 @@ -# Adapted from https://github.com/pandas-dev/pandas/blob/master/azure-pipelines.yml - -# Global variables for all jobs -variables: - VIRTUALENV: 'testvenv' - JUNITXML: 'test-data.xml' - CODECOV_TOKEN: 'cee0e505-c12e-4139-aa43-621fb16a2347' - -schedules: -- cron: "0 1 * * *" # 1am UTC - displayName: Run nightly build - branches: - include: - - master - always: true - -stages: -- stage: - jobs: - - - job: 'linting' - displayName: Linting - pool: - vmImage: ubuntu-latest - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.12' - - script: | - pip install black==25.1.0 - displayName: install black - - script: | - black --check --diff . - displayName: Run black - - - template: continuous_integration/windows.yml - parameters: - name: Windows - vmImage: windows-latest - matrix: - pylatest_conda_forge_mkl: - PYTHON_VERSION: '*' - PACKAGER: 'conda-forge' - BLAS: 'mkl' - py310_conda_forge_openblas: - PYTHON_VERSION: '3.10' - PACKAGER: 'conda-forge' - BLAS: 'openblas' - py39_conda: - PYTHON_VERSION: '3.9' - PACKAGER: 'conda' - py38_pip: - PYTHON_VERSION: '3.8' - PACKAGER: 'pip' - - - template: continuous_integration/posix.yml - parameters: - name: Linux - vmImage: ubuntu-20.04 - matrix: - # Linux environment with development versions of numpy and scipy - pylatest_pip_dev: - PACKAGER: 'pip-dev' - PYTHON_VERSION: '*' - CC_OUTER_LOOP: 'gcc' - CC_INNER_LOOP: 'gcc' - # Linux environment to test that packages that comes with Ubuntu 20.04 - # are correctly handled. - py38_ubuntu_atlas_gcc_gcc: - PACKAGER: 'ubuntu' - APT_BLAS: 'libatlas3-base libatlas-base-dev' - PYTHON_VERSION: '3.8' - CC_OUTER_LOOP: 'gcc' - CC_INNER_LOOP: 'gcc' - py38_ubuntu_openblas_gcc_gcc: - PACKAGER: 'ubuntu' - APT_BLAS: 'libopenblas-base libopenblas-dev' - PYTHON_VERSION: '3.8' - CC_OUTER_LOOP: 'gcc' - CC_INNER_LOOP: 'gcc' - # Linux + Python 3.9 and homogeneous runtime nesting. - py39_conda_openblas_clang_clang: - PACKAGER: 'conda' - PYTHON_VERSION: '3.9' - BLAS: 'openblas' - CC_OUTER_LOOP: 'clang-10' - CC_INNER_LOOP: 'clang-10' - # Linux environment with MKL and Clang (known to be unsafe for - # threadpoolctl) to only test the warning from multiple OpenMP. - pylatest_conda_mkl_clang_gcc: - PACKAGER: 'conda' - PYTHON_VERSION: '*' - BLAS: 'mkl' - CC_OUTER_LOOP: 'clang-10' - CC_INNER_LOOP: 'gcc' - TESTS: 'libomp_libiomp_warning' - # Linux environment with MKL, safe for threadpoolctl. - pylatest_conda_mkl_gcc_gcc: - PACKAGER: 'conda' - PYTHON_VERSION: '*' - BLAS: 'mkl' - CC_OUTER_LOOP: 'gcc' - CC_INNER_LOOP: 'gcc' - MKL_THREADING_LAYER: 'INTEL' - # Linux + Python 3.8 with numpy / scipy installed with pip from PyPI - # and heterogeneous OpenMP runtimes. - py38_pip_openblas_gcc_clang: - PACKAGER: 'pip' - PYTHON_VERSION: '3.8' - CC_OUTER_LOOP: 'gcc' - CC_INNER_LOOP: 'clang-10' - # Linux environment with numpy from conda-forge channel and openblas-openmp - pylatest_conda_forge: - PACKAGER: 'conda-forge' - PYTHON_VERSION: '*' - BLAS: 'openblas' - OPENBLAS_THREADING_LAYER: 'openmp' - CC_OUTER_LOOP: 'gcc' - CC_INNER_LOOP: 'gcc' - # Linux environment with no numpy and heterogeneous OpenMP runtimes. - pylatest_conda_nonumpy_gcc_clang: - PACKAGER: 'conda' - NO_NUMPY: 'true' - PYTHON_VERSION: '*' - CC_OUTER_LOOP: 'gcc' - CC_INNER_LOOP: 'clang-10' - # Linux environment with numpy linked to BLIS - pylatest_blis_gcc_clang_openmp: - PACKAGER: 'conda' - PYTHON_VERSION: '*' - INSTALL_BLAS: 'blis' - BLIS_NUM_THREADS: '4' - CC_OUTER_LOOP: 'gcc' - CC_INNER_LOOP: 'gcc' - BLIS_CC: 'clang-10' - BLIS_ENABLE_THREADING: 'openmp' - pylatest_blis_clang_gcc_pthreads: - PACKAGER: 'conda' - PYTHON_VERSION: '*' - INSTALL_BLAS: 'blis' - BLIS_NUM_THREADS: '4' - CC_OUTER_LOOP: 'clang-10' - CC_INNER_LOOP: 'clang-10' - BLIS_CC: 'gcc-8' - BLIS_ENABLE_THREADING: 'pthreads' - pylatest_blis_no_threading: - PACKAGER: 'conda' - PYTHON_VERSION: '*' - INSTALL_BLAS: 'blis' - BLIS_NUM_THREADS: '1' - CC_OUTER_LOOP: 'gcc' - CC_INNER_LOOP: 'gcc' - BLIS_CC: 'gcc-8' - BLIS_ENABLE_THREADING: 'no' - pylatest_flexiblas: - PACKAGER: 'conda' - PYTHON_VERSION: '*' - INSTALL_BLAS: 'flexiblas' - PLATFORM_SPECIFIC_PACKAGES: 'mkl' - CC_OUTER_LOOP: 'gcc' - CC_INNER_LOOP: 'gcc' - - - - template: continuous_integration/posix.yml - parameters: - name: macOS - vmImage: macOS-latest - matrix: - # MacOS environment with OpenMP installed through homebrew - py38_conda_homebrew_libomp: - PACKAGER: 'conda' - PYTHON_VERSION: '3.8' - BLAS: 'openblas' - CC_OUTER_LOOP: 'clang' - CC_INNER_LOOP: 'clang' - INSTALL_LIBOMP: 'homebrew' - # MacOS env with OpenBLAS and OpenMP installed through conda-forge compilers - py39_conda_forge_clang_openblas: - PACKAGER: 'conda-forge' - PYTHON_VERSION: '*' - BLAS: 'openblas' - CC_OUTER_LOOP: 'clang' - CC_INNER_LOOP: 'clang' - INSTALL_LIBOMP: 'conda-forge' - # MacOS environment with OpenMP installed through conda-forge compilers - pylatest_conda_forge_clang: - PACKAGER: 'conda-forge' - PYTHON_VERSION: '*' - BLAS: 'mkl' - CC_OUTER_LOOP: 'clang' - CC_INNER_LOOP: 'clang' - INSTALL_LIBOMP: 'conda-forge' - pylatest_flexiblas: - PACKAGER: 'conda' - PYTHON_VERSION: '*' - INSTALL_BLAS: 'flexiblas' - PLATFORM_SPECIFIC_PACKAGES: 'mkl' - CC_OUTER_LOOP: 'clang' - CC_INNER_LOOP: 'clang' - -- stage: - jobs: - # Meta-test to ensure that at least of the above CI configurations had - # the necessary platform settings to execute each test without raising - # skipping. - - job: 'no_test_always_skipped' - displayName: 'No test always skipped' - pool: - vmImage: ubuntu-latest - steps: - - download: current - - script: | - python continuous_integration/check_no_test_skipped.py $(Pipeline.Workspace) - displayName: 'No test always skipped' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bc9eb44c..1520b10b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,6 +9,15 @@ on: pull_request: branches: - master + schedule: + # Daily build at 1:00 AM UTC + - cron: "0 1 * * *" + +# Cancel in-progress workflows when pushing +# a new commit on the same branch +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: @@ -27,3 +36,239 @@ jobs: - name: Run black run: | black --check --diff . + + testing: + needs: linting + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + + # Windows env with numpy, scipy,MKL installed through conda-forge + - name: pylatest_conda_forge_mkl + os: windows-latest + PYTHON_VERSION: "*" + PACKAGER: "conda-forge" + BLAS: "mkl" + # Windows env with numpy, scipy, OpenBLAS installed through conda-forge + - name: py310_conda_forge_openblas + os: windows-latest + PYTHON_VERSION: "3.10" + PACKAGER: "conda-forge" + BLAS: "openblas" + # Windows env with numpy, scipy installed through conda + - name: py39_conda + os: windows-latest + PYTHON_VERSION: "3.9" + PACKAGER: "conda" + # Windows env with numpy, scipy installed through pip + - name: py38_pip + os: windows-latest + PYTHON_VERSION: "3.8" + PACKAGER: "pip" + + # MacOS env with OpenMP installed through homebrew + - name: py38_conda_homebrew_libomp + os: macos-latest + PYTHON_VERSION: "3.8" + PACKAGER: "conda" + BLAS: "openblas" + CC_OUTER_LOOP: "clang" + CC_INNER_LOOP: "clang" + INSTALL_LIBOMP: "homebrew" + # MacOS env with OpenBLAS and OpenMP installed through conda-forge compilers + - name: pylatest_conda_forge_clang_openblas + os: macos-latest + PYTHON_VERSION: "*" + PACKAGER: "conda-forge" + BLAS: "openblas" + INSTALL_LIBOMP: "conda-forge" + # MacOS env with FlexiBLAS + - name: pylatest_flexiblas + os: macos-latest + PYTHON_VERSION: "*" + INSTALL_BLAS: "flexiblas" + PLATFORM_SPECIFIC_PACKAGES: "llvm-openmp" + + # Linux environments to test that packages that comes with Ubuntu 20.04 + # are correctly handled. + - name: py38_ubuntu_atlas_gcc_gcc + os: ubuntu-20.04 + PYTHON_VERSION: "3.8" + PACKAGER: "ubuntu" + APT_BLAS: "libatlas3-base libatlas-base-dev" + CC_OUTER_LOOP: "gcc" + CC_INNER_LOOP: "gcc" + - name: py38_ubuntu_openblas_gcc_gcc + os: ubuntu-20.04 + PYTHON_VERSION: "3.8" + PACKAGER: "ubuntu" + APT_BLAS: "libopenblas-base libopenblas-dev" + CC_OUTER_LOOP: "gcc" + CC_INNER_LOOP: "gcc" + + # Linux environment with development versions of numpy and scipy + - name: pylatest_pip_dev + os : ubuntu-latest + PACKAGER: "pip-dev" + PYTHON_VERSION: "*" + CC_OUTER_LOOP: "gcc" + CC_INNER_LOOP: "gcc" + # Linux + Python 3.9 and homogeneous runtime nesting. + - name: py39_conda_openblas_clang_clang + os: ubuntu-latest + PACKAGER: "conda" + PYTHON_VERSION: "3.9" + BLAS: "openblas" + CC_OUTER_LOOP: "clang-18" + CC_INNER_LOOP: "clang-18" + # Linux environment with MKL and Clang (known to be unsafe for + # threadpoolctl) to only test the warning from multiple OpenMP. + - name: pylatest_conda_mkl_clang_gcc + os: ubuntu-latest + PYTHON_VERSION: "*" + PACKAGER: "conda" + BLAS: "mkl" + CC_OUTER_LOOP: "clang-18" + CC_INNER_LOOP: "gcc" + TESTS: "libomp_libiomp_warning" + # Linux environment with MKL, safe for threadpoolctl. + - name: pylatest_conda_mkl_gcc_gcc + os: ubuntu-latest + PYTHON_VERSION: "*" + PACKAGER: "conda" + BLAS: "mkl" + CC_OUTER_LOOP: "gcc" + CC_INNER_LOOP: "gcc" + MKL_THREADING_LAYER: "INTEL" + # Linux + Python 3.8 with numpy / scipy installed with pip from PyPI + # and heterogeneous OpenMP runtimes. + - name: py38_pip_openblas_gcc_clang + os: ubuntu-latest + PACKAGER: "pip" + PYTHON_VERSION: "3.8" + CC_OUTER_LOOP: "gcc" + CC_INNER_LOOP: "clang-18" + # Linux environment with numpy from conda-forge channel and openblas-openmp + - name: pylatest_conda_forge + os: ubuntu-latest + PACKAGER: "conda-forge" + PYTHON_VERSION: "*" + BLAS: "openblas" + OPENBLAS_THREADING_LAYER: "openmp" + CC_OUTER_LOOP: "gcc" + CC_INNER_LOOP: "gcc" + # Linux environment with no numpy and heterogeneous OpenMP runtimes. + - name: pylatest_conda_nonumpy_gcc_clang + os: ubuntu-latest + PACKAGER: "conda" + PYTHON_VERSION: "*" + NO_NUMPY: "true" + CC_OUTER_LOOP: "gcc" + CC_INNER_LOOP: "clang-18" + + # Linux environments with numpy linked to BLIS + - name: pylatest_blis_gcc_clang_openmp + os: ubuntu-latest + PYTHON_VERSION: "*" + INSTALL_BLAS: "blis" + BLIS_NUM_THREAEDS: "4" + CC_OUTER_LOOP: "gcc" + CC_INNER_LOOP: "gcc" + BLIS_CC: "clang-18" + BLIS_ENABLE_THREADING: "openmp" + - name: pylatest_blis_clang_gcc_pthreads + os: ubuntu-latest + PYTHON_VERSION: "*" + INSTALL_BLAS: "blis" + BLIS_NUM_THREADS: "4" + CC_OUTER_LOOP: "clang-18" + CC_INNER_LOOP: "clang-18" + BLIS_CC: "gcc-12" + BLIS_ENABLE_THREADING: "pthreads" + - name: pylatest_blis_no_threading + os: ubuntu-latest + PYTHON_VERSION: "*" + INSTALL_BLAS: "blis" + BLIS_NUM_THREADS: "1" + CC_OUTER_LOOP: "gcc" + CC_INNER_LOOP: "gcc" + BLIS_CC: "gcc-12" + BLIS_ENABLE_THREADING: "no" + + # Linux env with FlexiBLAS + - name: pylatest_flexiblas + os: ubuntu-latest + PYTHON_VERSION: "*" + INSTALL_BLAS: "flexiblas" + PLATFORM_SPECIFIC_PACKAGES: "mkl" + CC_OUTER_LOOP: "gcc" + CC_INNER_LOOP: "gcc" + + env: ${{ matrix }} + + runs-on: ${{ matrix.os }} + + defaults: + run: + # Need to use this shell to get conda working properly. + # See https://github.com/marketplace/actions/setup-miniconda#important + shell: ${{ matrix.os == 'windows-latest' && 'cmd /C CALL {0}' || 'bash -el {0}' }} + + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Setup conda + uses: conda-incubator/setup-miniconda@v3 + with: + auto-activate-base: true + auto-update-conda: true + miniforge-version: latest + + - name: Install dependencies + run: | + bash -el continuous_integration/install.sh + + - name: Test library + run: | + bash -el continuous_integration/run_tests.sh + + - name: Upload test results + uses: actions/upload-artifact@v4 + with: + # Requires a unique name for each job in the matrix of this run + name: test_result_${{github.run_id}}_${{ matrix.os }}_${{ matrix.name }} + path: test_result.xml + retention-days: 1 + + - name: Upload to Codecov + uses: codecov/codecov-action@v5 + with: + files: coverage.xml + + # Meta-test to ensure that at least one of the above CI configurations had + # the necessary platform settings to execute each test without raising + # skipping. + meta_test: + needs: testing + runs-on: ubuntu-latest + steps: + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.12" + + - name: Checkout code + uses: actions/checkout@v3 + + - name: Download tests results + uses: actions/download-artifact@v4 + with: + path: test_results + + - name: Check no test always skipped + run: | + python continuous_integration/check_no_test_skipped.py test_results diff --git a/continuous_integration/build_test_ext.sh b/continuous_integration/build_test_ext.sh index bd0e1ae7..4bc27696 100755 --- a/continuous_integration/build_test_ext.sh +++ b/continuous_integration/build_test_ext.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -e +set -xe if [[ "$OSTYPE" == "linux-gnu"* ]]; then pushd tests/_pyMylib diff --git a/continuous_integration/check_no_test_skipped.py b/continuous_integration/check_no_test_skipped.py index f831d29a..18fd6bc2 100644 --- a/continuous_integration/check_no_test_skipped.py +++ b/continuous_integration/check_no_test_skipped.py @@ -13,11 +13,11 @@ always_skipped = {} for name in os.listdir(base_dir): - # all test result files are in /base_dir/jobs.*/ dirs - if name.startswith("stage1."): - print("> processing test result from job", name.replace("stage1", "")) + # all test result files are in base_dir/test_result_*/ dirs + if name.startswith("test_result_"): + print(f"> processing test result from job {name.replace('test_result_', '')}") print(" > tests skipped:") - result_file = os.path.join(base_dir, name, "test-data.xml") + result_file = os.path.join(base_dir, name, "test_result.xml") root = ET.parse(result_file).getroot() # All tests are identified by the xml tag testcase. diff --git a/continuous_integration/install.cmd b/continuous_integration/install.cmd deleted file mode 100644 index 40731e05..00000000 --- a/continuous_integration/install.cmd +++ /dev/null @@ -1,38 +0,0 @@ -@rem https://github.com/numba/numba/blob/master/buildscripts/incremental/setup_conda_environment.cmd -@rem The cmd /C hack circumvents a regression where conda installs a conda.bat -@rem script in non-root environments. -set CONDA_INSTALL=cmd /C conda install -q -y -set PIP_INSTALL=pip install -q - -@echo on - -@rem Deactivate any environment -call deactivate -@rem Clean up any left-over from a previous build and install version of python -conda update -n base conda conda-libmamba-solver -q -y -conda config --set solver libmamba -conda remove --all -q -y -n %VIRTUALENV% -conda create -n %VIRTUALENV% -q -y python=%PYTHON_VERSION% - -call activate %VIRTUALENV% -python -m pip install -U pip -python --version -pip --version - -@rem Install dependencies with either conda or pip. -set TO_INSTALL=numpy scipy cython pytest - -if "%PACKAGER%" == "conda" (%CONDA_INSTALL% %TO_INSTALL%) -if "%PACKAGER%" == "conda-forge" (%CONDA_INSTALL% -c conda-forge %TO_INSTALL% blas[build=%BLAS%]) -if "%PACKAGER%" == "pip" (%PIP_INSTALL% %TO_INSTALL%) - -@rem Install extra developer dependencies -pip install -q -r dev-requirements.txt - -@rem Install package -flit install --symlink - -@rem Build the cython test helper for openmp -bash ./continuous_integration/build_test_ext.sh - -if %errorlevel% neq 0 exit /b %errorlevel% diff --git a/continuous_integration/install.sh b/continuous_integration/install.sh old mode 100755 new mode 100644 index d0e8402f..7a958980 --- a/continuous_integration/install.sh +++ b/continuous_integration/install.sh @@ -1,28 +1,34 @@ #!/bin/bash -set -e +# License: BSD 3-Clause + +set -xe UNAMESTR=`uname` -if [[ "$CC_OUTER_LOOP" == "clang-10" || "$CC_INNER_LOOP" == "clang-10" ]]; then - # Assume Ubuntu: install a recent version of clang and libomp + +# Install a recent version of clang and libomp if needed +# Only applicable to linux jobs +if [[ "$CC_OUTER_LOOP" == "clang-18" ]] || \ + [[ "$CC_INNER_LOOP" == "clang-18" ]] || \ + [[ "$BLIS_CC" == "clang-18" ]] +then wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh - sudo ./llvm.sh 10 + sudo ./llvm.sh 18 sudo apt-get install libomp-dev fi + make_conda() { - TO_INSTALL="$@" + CHANNEL="$1" + TO_INSTALL="$2" if [[ "$UNAMESTR" == "Darwin" ]]; then if [[ "$INSTALL_LIBOMP" == "conda-forge" ]]; then # Install an OpenMP-enabled clang/llvm from conda-forge # assumes conda-forge is set on priority channel TO_INSTALL="$TO_INSTALL compilers llvm-openmp" - export CFLAGS="$CFLAGS -I$CONDA/envs/$VIRTUALENV/include" - export LDFLAGS="$LDFLAGS -Wl,-rpath,$CONDA/envs/$VIRTUALENV/lib -L$CONDA/envs/$VIRTUALENV/lib" - elif [[ "$INSTALL_LIBOMP" == "homebrew" ]]; then # Install a compiler with a working openmp HOMEBREW_NO_AUTO_UPDATE=1 brew install libomp @@ -30,37 +36,49 @@ make_conda() { # enable OpenMP support for Apple-clang export CC=/usr/bin/clang export CPPFLAGS="$CPPFLAGS -Xpreprocessor -fopenmp" - export CFLAGS="$CFLAGS -I/usr/local/opt/libomp/include" - export LDFLAGS="$LDFLAGS -Wl,-rpath,/usr/local/opt/libomp/lib -L/usr/local/opt/libomp/lib -lomp" + export CFLAGS="$CFLAGS -I/opt/homebrew/opt/libomp/include" + export LDFLAGS="$LDFLAGS -Wl,-rpath,/opt/homebrew/opt/libomp/lib -L/opt/homebrew/opt/libomp/lib -lomp" fi fi + + if [[ "$PYTHON_VERSION" == "*" ]]; then + # Avoid installing free-threaded python + TO_INSTALL="$TO_INSTALL python-gil" + fi + + # prevent mixing conda channels + conda config --set channel_priority strict + conda config --add channels $CHANNEL + conda update -n base conda conda-libmamba-solver -q --yes conda config --set solver libmamba - conda create -n $VIRTUALENV -q --yes $TO_INSTALL - source activate $VIRTUALENV + + conda create -n testenv -q --yes python=$PYTHON_VERSION $TO_INSTALL + conda activate testenv } + if [[ "$PACKAGER" == "conda" ]]; then - TO_INSTALL="python=$PYTHON_VERSION pip" + TO_INSTALL="" if [[ "$NO_NUMPY" != "true" ]]; then - TO_INSTALL="$TO_INSTALL numpy scipy blas[build=$BLAS]" + TO_INSTALL="$TO_INSTALL numpy scipy" + if [[ -n "$BLAS" ]]; then + TO_INSTALL="$TO_INSTALL blas=*=$BLAS" + fi fi - make_conda $TO_INSTALL + make_conda "defaults" "$TO_INSTALL" elif [[ "$PACKAGER" == "conda-forge" ]]; then - conda config --prepend channels conda-forge - conda config --set channel_priority strict - TO_INSTALL="python=$PYTHON_VERSION numpy scipy blas[build=$BLAS]" + TO_INSTALL="numpy scipy blas=*=$BLAS" if [[ "$BLAS" == "openblas" && "$OPENBLAS_THREADING_LAYER" == "openmp" ]]; then TO_INSTALL="$TO_INSTALL libopenblas=*=*openmp*" fi - make_conda $TO_INSTALL + make_conda "conda-forge" "$TO_INSTALL" elif [[ "$PACKAGER" == "pip" ]]; then # Use conda to build an empty python env and then use pip to install # numpy and scipy - TO_INSTALL="python=$PYTHON_VERSION pip" - make_conda $TO_INSTALL + make_conda "conda-forge" "" if [[ "$NO_NUMPY" != "true" ]]; then pip install numpy scipy fi @@ -68,8 +86,7 @@ elif [[ "$PACKAGER" == "pip" ]]; then elif [[ "$PACKAGER" == "pip-dev" ]]; then # Use conda to build an empty python env and then use pip to install # numpy and scipy dev versions - TO_INSTALL="python=$PYTHON_VERSION pip" - make_conda $TO_INSTALL + make_conda "conda-forge" "" dev_anaconda_url=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple pip install --pre --upgrade --timeout=60 --extra-index $dev_anaconda_url numpy scipy @@ -80,14 +97,29 @@ elif [[ "$PACKAGER" == "ubuntu" ]]; then sudo add-apt-repository --remove ppa:ubuntu-toolchain-r/test sudo apt-get update sudo apt-get install python3-scipy python3-virtualenv $APT_BLAS - python3 -m virtualenv --system-site-packages --python=python3 $VIRTUALENV - source $VIRTUALENV/bin/activate -fi + python3 -m virtualenv --system-site-packages --python=python3 testenv + source testenv/bin/activate + +elif [[ "$INSTALL_BLAS" == "blis" ]]; then + TO_INSTALL="cython meson-python pkg-config" + make_conda "conda-forge" "$TO_INSTALL" + source ./continuous_integration/install_blis.sh +elif [[ "$INSTALL_BLAS" == "flexiblas" ]]; then + TO_INSTALL="cython openblas $PLATFORM_SPECIFIC_PACKAGES meson-python pkg-config compilers" + make_conda "conda-forge" "$TO_INSTALL" + source ./continuous_integration/install_flexiblas.sh -python -m pip install -q -r dev-requirements.txt +fi + +python -m pip install -v -q -r dev-requirements.txt bash ./continuous_integration/build_test_ext.sh +# Check which BLAS is linked (only available on linux) +if [[ "$UNAMESTR" == "Linux" && "$NO_NUMPY" != "true" ]]; then + ldd tests/_openmp_test_helper/nested_prange_blas.cpython*.so +fi + python --version python -c "import numpy; print(f'numpy {numpy.__version__}')" || echo "no numpy" python -c "import scipy; print(f'scipy {scipy.__version__}')" || echo "no scipy" diff --git a/continuous_integration/install_blis.sh b/continuous_integration/install_blis.sh new file mode 100644 index 00000000..e8fa8074 --- /dev/null +++ b/continuous_integration/install_blis.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +set -xe + + +# Install gcc 12 to build BLIS +if [[ "$BLIS_CC" == "gcc-12" ]]; then + sudo apt install gcc-12 +fi + +# step outside of threadpoolctl directory +pushd .. +ABS_PATH=$(pwd) + +# build & install blis +mkdir BLIS_install +git clone https://github.com/flame/blis.git +pushd blis + +./configure --prefix=$ABS_PATH/BLIS_install --enable-cblas --enable-threading=$BLIS_ENABLE_THREADING CC=$BLIS_CC auto +make -j4 +make install +popd + +# build & install numpy +git clone https://github.com/numpy/numpy.git +pushd numpy +git submodule update --init + +echo "libdir=$ABS_PATH/BLIS_install/lib/ +includedir=$ABS_PATH/BLIS_install/include/blis/ +version=latest +extralib=-lm -lpthread -lgfortran +Name: blis +Description: BLIS +Version: \${version} +Libs: -L\${libdir} -lblis +Libs.private: \${extralib} +Cflags: -I\${includedir}" > blis.pc + +PKG_CONFIG_PATH=$ABS_PATH/numpy/ pip install . -v --no-build-isolation -Csetup-args=-Dblas=blis + +export CFLAGS=-I$ABS_PATH/BLIS_install/include/blis +export LDFLAGS="-L$ABS_PATH/BLIS_install/lib -Wl,-rpath,$ABS_PATH/BLIS_install/lib" + +popd + +# back to threadpoolctl directory +popd diff --git a/continuous_integration/install_with_flexiblas.sh b/continuous_integration/install_flexiblas.sh old mode 100755 new mode 100644 similarity index 63% rename from continuous_integration/install_with_flexiblas.sh rename to continuous_integration/install_flexiblas.sh index 59eec612..3d45ab54 --- a/continuous_integration/install_with_flexiblas.sh +++ b/continuous_integration/install_flexiblas.sh @@ -1,19 +1,10 @@ #!/bin/bash -set -e +set -xe +# step outside of threadpoolctl directory pushd .. ABS_PATH=$(pwd) -popd - -# create conda env -conda update -n base conda conda-libmamba-solver -q --yes -conda config --set solver libmamba -conda create -n $VIRTUALENV -q --yes -c conda-forge python=$PYTHON_VERSION \ - pip cython openblas $PLATFORM_SPECIFIC_PACKAGES meson-python pkg-config compilers -source activate $VIRTUALENV - -pushd .. # build & install FlexiBLAS mkdir flexiblas_install @@ -66,22 +57,11 @@ Libs.private: \${extralib} Cflags: -I\${includedir}" > flexiblas.pc PKG_CONFIG_PATH=$ABS_PATH/numpy/ pip install . -v --no-build-isolation -Csetup-args=-Dblas=flexiblas -Csetup-args=-Dlapack=flexiblas -popd -popd +export CFLAGS=-I$ABS_PATH/flexiblas_install/include/flexiblas \ +export LDFLAGS="-L$ABS_PATH/flexiblas_install/lib -Wl,-rpath,$ABS_PATH/flexiblas_install/lib" \ -python -m pip install -q -r dev-requirements.txt -CFLAGS=-I$ABS_PATH/flexiblas_install/include/flexiblas \ - LDFLAGS="-L$ABS_PATH/flexiblas_install/lib -Wl,-rpath,$ABS_PATH/flexiblas_install/lib" \ - bash ./continuous_integration/build_test_ext.sh - -# Check that FlexiBLAS is linked -if [[ $(uname) != "Darwin" ]]; then - ldd tests/_openmp_test_helper/nested_prange_blas.cpython*.so -fi - -python --version -python -c "import numpy; print(f'numpy {numpy.__version__}')" || echo "no numpy" -python -c "import scipy; print(f'scipy {scipy.__version__}')" || echo "no scipy" +popd -python -m flit install --symlink +# back to threadpoolctl directory +popd diff --git a/continuous_integration/install_with_blis.sh b/continuous_integration/install_with_blis.sh deleted file mode 100755 index 15ef3a5e..00000000 --- a/continuous_integration/install_with_blis.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash - -set -e - -pushd .. -ABS_PATH=$(pwd) -popd - -# Install a recent version of clang and libomp -wget https://apt.llvm.org/llvm.sh -chmod +x llvm.sh -sudo ./llvm.sh 10 -sudo apt-get install libomp-dev - -# create conda env -conda update -n base conda conda-libmamba-solver -q --yes -conda config --set solver libmamba -conda create -n $VIRTUALENV -q --yes -c conda-forge python=$PYTHON_VERSION \ - pip cython meson-python pkg-config -source activate $VIRTUALENV - -if [[ "$BLIS_CC" == "gcc-8" ]]; then - sudo apt install gcc-8 -fi - -pushd .. - -# build & install blis -mkdir BLIS_install -git clone https://github.com/flame/blis.git -pushd blis - -./configure --prefix=$ABS_PATH/BLIS_install --enable-cblas --enable-threading=$BLIS_ENABLE_THREADING CC=$BLIS_CC auto -make -j4 -make install -popd - -# build & install numpy -git clone https://github.com/numpy/numpy.git -pushd numpy -git submodule update --init - -echo "libdir=$ABS_PATH/BLIS_install/lib/ -includedir=$ABS_PATH/BLIS_install/include/blis/ -version=latest -extralib=-lm -lpthread -lgfortran -Name: blis -Description: BLIS -Version: \${version} -Libs: -L\${libdir} -lblis -Libs.private: \${extralib} -Cflags: -I\${includedir}" > blis.pc - -PKG_CONFIG_PATH=$ABS_PATH/numpy/ pip install . -v --no-build-isolation -Csetup-args=-Dblas=blis -popd - -popd - -python -m pip install -q -r dev-requirements.txt -CFLAGS=-I$ABS_PATH/BLIS_install/include/blis \ - LDFLAGS="-L$ABS_PATH/BLIS_install/lib -Wl,-rpath,$ABS_PATH/BLIS_install/lib" \ - bash ./continuous_integration/build_test_ext.sh - -# Check that BLIS is linked -ldd tests/_openmp_test_helper/nested_prange_blas.cpython*.so - -python --version -python -c "import numpy; print(f'numpy {numpy.__version__}')" || echo "no numpy" -python -c "import scipy; print(f'scipy {scipy.__version__}')" || echo "no scipy" - -python -m flit install --symlink diff --git a/continuous_integration/posix.yml b/continuous_integration/posix.yml deleted file mode 100644 index a1598d37..00000000 --- a/continuous_integration/posix.yml +++ /dev/null @@ -1,48 +0,0 @@ -parameters: - name: '' - vmImage: '' - matrix: [] - -jobs: -- job: ${{ parameters.name }} - pool: - vmImage: ${{ parameters.vmImage }} - strategy: - matrix: - ${{ insert }}: ${{ parameters.matrix }} - - steps: - - bash: echo "##vso[task.prependpath]$CONDA/bin" - displayName: Add conda to PATH - condition: or(startsWith(variables['PACKAGER'], 'conda'), startsWith(variables['PACKAGER'], 'pip')) - - bash: sudo chown -R $USER $CONDA - # On Hosted macOS, the agent user doesn't have ownership of Miniconda's installation directory/ - # We need to take ownership if we want to update conda or install packages globally - displayName: Take ownership of conda installation - condition: eq('${{ parameters.name }}', 'macOS') - - script: | - continuous_integration/install.sh - displayName: 'Install without custom BLAS' - condition: eq(variables['INSTALL_BLAS'], '') - - script: | - continuous_integration/install_with_blis.sh - displayName: 'Install with BLIS' - condition: eq(variables['INSTALL_BLAS'], 'blis') - - script: | - continuous_integration/install_with_flexiblas.sh - displayName: 'Install with FlexiBLAS' - condition: eq(variables['INSTALL_BLAS'], 'flexiblas') - - script: | - continuous_integration/test_script.sh - displayName: 'Test Library' - - task: PublishTestResults@2 - inputs: - testResultsFiles: '$(JUNITXML)' - testRunTitle: ${{ format('{0}-$(Agent.JobName)', parameters.name) }} - displayName: 'Publish Test Results' - condition: succeededOrFailed() - - publish: $(JUNITXML) - - script: | - bash continuous_integration/upload_codecov.sh - displayName: 'Upload to codecov' - condition: succeeded() diff --git a/continuous_integration/test_script.sh b/continuous_integration/run_tests.sh similarity index 60% rename from continuous_integration/test_script.sh rename to continuous_integration/run_tests.sh index 8eeaba19..f26f0fdc 100755 --- a/continuous_integration/test_script.sh +++ b/continuous_integration/run_tests.sh @@ -1,23 +1,21 @@ #!/bin/bash -set -e +set -xe -if [[ "$PACKAGER" == conda* ]]; then - source activate $VIRTUALENV +if [[ "$PACKAGER" == conda* ]] || [[ -z "$PACKAGER" ]]; then + conda activate testenv conda list elif [[ "$PACKAGER" == pip* ]]; then # we actually use conda to install the base environment: - source activate $VIRTUALENV + conda activate testenv pip list elif [[ "$PACKAGER" == "ubuntu" ]]; then - source $VIRTUALENV/bin/activate + source testenv/bin/activate pip list fi -set -x - # Use the CLI to display the effective runtime environment prior to # launching the tests: python -m threadpoolctl -i numpy scipy.linalg tests._openmp_test_helper.openmp_helpers_inner -pytest -vlrxXs -W error -k "$TESTS" --junitxml=$JUNITXML --cov=threadpoolctl +pytest -vlrxXs -W error -k "$TESTS" --junitxml=test_result.xml --cov=threadpoolctl --cov-report xml diff --git a/continuous_integration/test_script.cmd b/continuous_integration/test_script.cmd deleted file mode 100644 index 4da20228..00000000 --- a/continuous_integration/test_script.cmd +++ /dev/null @@ -1,10 +0,0 @@ -call activate %VIRTUALENV% - -# Display version information -python -m pip list - -# Use the CLI to display the effective runtime environment prior to -# launching the tests: -python -m threadpoolctl -i numpy scipy.linalg tests._openmp_test_helper.openmp_helpers_inner - -pytest -vlrxXs --junitxml=%JUNITXML% --cov=threadpoolctl diff --git a/continuous_integration/upload_codecov.sh b/continuous_integration/upload_codecov.sh deleted file mode 100755 index 1fb312c0..00000000 --- a/continuous_integration/upload_codecov.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -set -e - -if [[ "$PACKAGER" == "conda" ]]; then - source activate $VIRTUALENV -elif [[ "$PACKAGER" == "pip" ]]; then - source activate $VIRTUALENV -elif [[ "$PACKAGER" == "ubuntu" ]]; then - source $VIRTUALENV/bin/activate -fi - -pip install codecov - -codecov || echo "codecov upload failed" diff --git a/continuous_integration/windows.yml b/continuous_integration/windows.yml deleted file mode 100644 index 7d6e53da..00000000 --- a/continuous_integration/windows.yml +++ /dev/null @@ -1,35 +0,0 @@ - -parameters: - name: '' - vmImage: '' - matrix: [] - -jobs: -- job: ${{ parameters.name }} - pool: - vmImage: ${{ parameters.vmImage }} - - strategy: - matrix: - ${{ insert }}: ${{ parameters.matrix }} - - steps: - - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" - displayName: Add conda to PATH - - script: | - continuous_integration\\install.cmd - displayName: 'Install' - - script: | - continuous_integration\\test_script.cmd - displayName: 'Test threadpoolctl' - - task: PublishTestResults@2 - inputs: - testResultsFiles: '$(JUNITXML)' - testRunTitle: ${{ format('{0}-$(Agent.JobName)', parameters.name) }} - displayName: 'Publish Test Results' - condition: succeededOrFailed() - - publish: $(JUNITXML) - - script: | - bash continuous_integration\\upload_codecov.sh - displayName: 'Upload to codecov' - condition: succeeded() diff --git a/dev-requirements.txt b/dev-requirements.txt index d118f1e7..04901b66 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -3,3 +3,4 @@ coverage pytest pytest-cov cython +setuptools diff --git a/tests/_openmp_test_helper/setup_inner.py b/tests/_openmp_test_helper/setup_inner.py index 4e045c02..547e257a 100644 --- a/tests/_openmp_test_helper/setup_inner.py +++ b/tests/_openmp_test_helper/setup_inner.py @@ -1,7 +1,6 @@ import os -from distutils.core import setup +from setuptools import Extension, setup from Cython.Build import cythonize -from distutils.extension import Extension from build_utils import set_cc_variables from build_utils import get_openmp_flag diff --git a/tests/_openmp_test_helper/setup_nested_prange_blas.py b/tests/_openmp_test_helper/setup_nested_prange_blas.py index 809f981c..dcb39ca6 100644 --- a/tests/_openmp_test_helper/setup_nested_prange_blas.py +++ b/tests/_openmp_test_helper/setup_nested_prange_blas.py @@ -1,7 +1,6 @@ import os -from distutils.core import setup +from setuptools import Extension, setup from Cython.Build import cythonize -from distutils.extension import Extension from build_utils import set_cc_variables from build_utils import get_openmp_flag diff --git a/tests/_openmp_test_helper/setup_outer.py b/tests/_openmp_test_helper/setup_outer.py index 98a84d58..dee39b1a 100644 --- a/tests/_openmp_test_helper/setup_outer.py +++ b/tests/_openmp_test_helper/setup_outer.py @@ -1,7 +1,6 @@ import os -from distutils.core import setup +from setuptools import Extension, setup from Cython.Build import cythonize -from distutils.extension import Extension from build_utils import set_cc_variables from build_utils import get_openmp_flag diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index d6e95363..7f6e090e 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -624,6 +624,7 @@ def test_architecture(): # XXX: add more as needed by CI or developer laptops "skx", "haswell", + "zen3", ) for lib_info in threadpool_info(): if lib_info["internal_api"] == "openblas": @@ -657,7 +658,7 @@ def test_openblas_threading_layer(): # skip test if not run in a azure pipelines job since it relies on a specific flexiblas # installation. @pytest.mark.skipif( - "TF_BUILD" not in os.environ, reason="not running in azure pipelines" + "GITHUB_ACTIONS" not in os.environ, reason="not running in azure pipelines" ) def test_flexiblas(): # Check that threadpool_info correctly recovers the FlexiBLAS backends. @@ -695,7 +696,7 @@ def test_flexiblas_switch_error(): # skip test if not run in a azure pipelines job since it relies on a specific flexiblas # installation. @pytest.mark.skipif( - "TF_BUILD" not in os.environ, reason="not running in azure pipelines" + "GITHUB_ACTIONS" not in os.environ, reason="not running in azure pipelines" ) def test_flexiblas_switch(): # Check that the backend can be switched. @@ -717,13 +718,13 @@ def test_flexiblas_switch(): assert fb_controller.current_backend == "NETLIB" assert fb_controller.loaded_backends == ["OPENBLAS_CONDA", "NETLIB"] - ext = ".so" if sys.platform == "linux" else ".dylib" - mkl_path = f"{os.getenv('CONDA_PREFIX')}/lib/libmkl_rt{ext}" - fb_controller.switch_backend(mkl_path) - assert fb_controller.current_backend == mkl_path - assert fb_controller.loaded_backends == ["OPENBLAS_CONDA", "NETLIB", mkl_path] - # switching the backend triggered a new search for loaded shared libs - assert len(controller.select(internal_api="mkl").lib_controllers) == 1 + if sys.platform == "linux": + mkl_path = f"{os.getenv('CONDA_PREFIX')}/lib/libmkl_rt.so" + fb_controller.switch_backend(mkl_path) + assert fb_controller.current_backend == mkl_path + assert fb_controller.loaded_backends == ["OPENBLAS_CONDA", "NETLIB", mkl_path] + # switching the backend triggered a new search for loaded shared libs + assert len(controller.select(internal_api="mkl").lib_controllers) == 1 # switch back to default to avoid side effects fb_controller.switch_backend("OPENBLAS_CONDA") From ca27594336e7b2125db9cd65d1ac4a2fea79f563 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= Date: Sun, 9 Mar 2025 16:25:17 +0100 Subject: [PATCH 172/187] MNT Drop Python 3.8 support (#186) --- .github/workflows/test.yml | 30 +++++++++++++++--------------- CHANGES.md | 6 ++++++ 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1520b10b..43560981 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -52,26 +52,26 @@ jobs: PACKAGER: "conda-forge" BLAS: "mkl" # Windows env with numpy, scipy, OpenBLAS installed through conda-forge - - name: py310_conda_forge_openblas + - name: py311_conda_forge_openblas os: windows-latest - PYTHON_VERSION: "3.10" + PYTHON_VERSION: "3.11" PACKAGER: "conda-forge" BLAS: "openblas" # Windows env with numpy, scipy installed through conda - - name: py39_conda + - name: py310_conda os: windows-latest - PYTHON_VERSION: "3.9" + PYTHON_VERSION: "3.10" PACKAGER: "conda" # Windows env with numpy, scipy installed through pip - - name: py38_pip + - name: py39_pip os: windows-latest - PYTHON_VERSION: "3.8" + PYTHON_VERSION: "3.9" PACKAGER: "pip" # MacOS env with OpenMP installed through homebrew - - name: py38_conda_homebrew_libomp + - name: py39_conda_homebrew_libomp os: macos-latest - PYTHON_VERSION: "3.8" + PYTHON_VERSION: "3.9" PACKAGER: "conda" BLAS: "openblas" CC_OUTER_LOOP: "clang" @@ -93,16 +93,16 @@ jobs: # Linux environments to test that packages that comes with Ubuntu 20.04 # are correctly handled. - - name: py38_ubuntu_atlas_gcc_gcc + - name: py39_ubuntu_atlas_gcc_gcc os: ubuntu-20.04 - PYTHON_VERSION: "3.8" + PYTHON_VERSION: "3.9" PACKAGER: "ubuntu" APT_BLAS: "libatlas3-base libatlas-base-dev" CC_OUTER_LOOP: "gcc" CC_INNER_LOOP: "gcc" - - name: py38_ubuntu_openblas_gcc_gcc + - name: py39_ubuntu_openblas_gcc_gcc os: ubuntu-20.04 - PYTHON_VERSION: "3.8" + PYTHON_VERSION: "3.9" PACKAGER: "ubuntu" APT_BLAS: "libopenblas-base libopenblas-dev" CC_OUTER_LOOP: "gcc" @@ -142,12 +142,12 @@ jobs: CC_OUTER_LOOP: "gcc" CC_INNER_LOOP: "gcc" MKL_THREADING_LAYER: "INTEL" - # Linux + Python 3.8 with numpy / scipy installed with pip from PyPI + # Linux + Python 3.11 with numpy / scipy installed with pip from PyPI # and heterogeneous OpenMP runtimes. - - name: py38_pip_openblas_gcc_clang + - name: py311_pip_openblas_gcc_clang os: ubuntu-latest PACKAGER: "pip" - PYTHON_VERSION: "3.8" + PYTHON_VERSION: "3.11" CC_OUTER_LOOP: "gcc" CC_INNER_LOOP: "clang-18" # Linux environment with numpy from conda-forge channel and openblas-openmp diff --git a/CHANGES.md b/CHANGES.md index bb649f9c..56a6b51d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,9 @@ +3.6.0 (TBD) +=========== + +- Dropped official support for Python 3.8. + https://github.com/joblib/threadpoolctl/pull/186 + 3.5.0 (2024-04-29) ================== From 86f3f0b622ed3edd2b7adb9166fe7874b239862a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= Date: Sun, 9 Mar 2025 16:46:01 +0100 Subject: [PATCH 173/187] DOC Replace azure pipelines badge with github actions badge (#187) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6b2ec1d6..fc22c84e 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Thread-pool Controls [![Build Status](https://dev.azure.com/joblib/threadpoolctl/_apis/build/status/joblib.threadpoolctl?branchName=master)](https://dev.azure.com/joblib/threadpoolctl/_build/latest?definitionId=1&branchName=master) [![codecov](https://codecov.io/gh/joblib/threadpoolctl/branch/master/graph/badge.svg)](https://codecov.io/gh/joblib/threadpoolctl) +# Thread-pool Controls [![Build Status](https://github.com/joblib/threadpoolctl/actions/workflows/test.yml/badge.svg?branch=master)](https://github.com/joblib/threadpoolctl/actions?query=branch%3Amaster) [![codecov](https://codecov.io/gh/joblib/threadpoolctl/branch/master/graph/badge.svg)](https://codecov.io/gh/joblib/threadpoolctl) Python helpers to limit the number of threads used in the threadpool-backed of common native libraries used for scientific From ad953f536f52aff324e14a4ae6764c8e13e97a84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= Date: Sun, 9 Mar 2025 16:46:38 +0100 Subject: [PATCH 174/187] CI Bump ubuntu image to 22.04 for system deps based jobs (#188) --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 43560981..39fc19e2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -91,17 +91,17 @@ jobs: INSTALL_BLAS: "flexiblas" PLATFORM_SPECIFIC_PACKAGES: "llvm-openmp" - # Linux environments to test that packages that comes with Ubuntu 20.04 + # Linux environments to test that packages that comes with Ubuntu 22.04 # are correctly handled. - name: py39_ubuntu_atlas_gcc_gcc - os: ubuntu-20.04 + os: ubuntu-22.04 PYTHON_VERSION: "3.9" PACKAGER: "ubuntu" APT_BLAS: "libatlas3-base libatlas-base-dev" CC_OUTER_LOOP: "gcc" CC_INNER_LOOP: "gcc" - name: py39_ubuntu_openblas_gcc_gcc - os: ubuntu-20.04 + os: ubuntu-22.04 PYTHON_VERSION: "3.9" PACKAGER: "ubuntu" APT_BLAS: "libopenblas-base libopenblas-dev" From 931a7641c66a4171e05236f7083e3179d226a064 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= Date: Tue, 11 Mar 2025 16:03:29 +0100 Subject: [PATCH 175/187] ENH Support path length > MAX_PATH on Windows (#189) --- CHANGES.md | 4 ++++ threadpoolctl.py | 18 +++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 56a6b51d..e868628d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,10 @@ 3.6.0 (TBD) =========== +- Added support for libraries with a path longer than 260 on Windows. The supported path + length is now 10 times higher but not unlimited for security reasons. + https://github.com/joblib/threadpoolctl/pull/189 + - Dropped official support for Python 3.8. https://github.com/joblib/threadpoolctl/pull/186 diff --git a/threadpoolctl.py b/threadpoolctl.py index 218ff180..2efc30cc 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -1086,7 +1086,10 @@ def _find_libraries_with_enum_process_module_ex(self): h_modules = map(HMODULE, buf[:count]) # Loop through all the module headers and get the library path - buf = ctypes.create_unicode_buffer(MAX_PATH) + # Allocate a buffer for the path 10 times the size of MAX_PATH to take + # into account long path names. + max_path = 10 * MAX_PATH + buf = ctypes.create_unicode_buffer(max_path) n_size = DWORD() for h_module in h_modules: # Get the path of the current module @@ -1096,8 +1099,17 @@ def _find_libraries_with_enum_process_module_ex(self): raise OSError("GetModuleFileNameEx failed") filepath = buf.value - # Store the library controller if it is supported and selected - self._make_controller_from_path(filepath) + if len(filepath) == max_path: # pragma: no cover + warnings.warn( + "Could not get the full path of a dynamic library (path too " + "long). This library will be ignored and threadpoolctl might " + "not be able to control or display information about all " + f"loaded libraries. Here's the truncated path: {filepath!r}", + RuntimeWarning, + ) + else: + # Store the library controller if it is supported and selected + self._make_controller_from_path(filepath) finally: kernel_32.CloseHandle(h_process) From 4d1e0c1fc46e7064de41ebe81dcf494f153ddbfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= Date: Thu, 13 Mar 2025 12:27:08 +0100 Subject: [PATCH 176/187] MNT Drop Python 3.8 support follow-up (#191) --- CHANGES.md | 1 + pyproject.toml | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e868628d..19c6f830 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,7 @@ - Dropped official support for Python 3.8. https://github.com/joblib/threadpoolctl/pull/186 + https://github.com/joblib/threadpoolctl/pull/191 3.5.0 (2024-04-29) ================== diff --git a/pyproject.toml b/pyproject.toml index 91327a84..c0ea1696 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,21 +10,21 @@ author = "Thomas Moreau" author-email = "thomas.moreau.2010@gmail.com" home-page = "https://github.com/joblib/threadpoolctl" description-file = "README.md" -requires-python = ">=3.8" +requires-python = ">=3.9" license = "BSD-3-Clause" classifiers = [ "Intended Audience :: Developers", "License :: OSI Approved :: BSD License", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Topic :: Software Development :: Libraries :: Python Modules", ] [tool.black] line-length = 88 -target_version = ['py38', 'py39', 'py310', 'py311', 'py312'] +target_version = ['py39', 'py310', 'py311', 'py312', 'py313'] preview = true From 527ef5f1ef020dc5c7876b1a357508c6f672ebb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= Date: Thu, 13 Mar 2025 13:26:19 +0100 Subject: [PATCH 177/187] REL Automate release process with github actions (#190) --- .github/workflows/publish-to-pypi.yml | 128 ++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 .github/workflows/publish-to-pypi.yml diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml new file mode 100644 index 00000000..4e5bf7a6 --- /dev/null +++ b/.github/workflows/publish-to-pypi.yml @@ -0,0 +1,128 @@ +name: Publish threadpoolctl 🎮 distribution 📦 to PyPI and TestPyPI +# Taken from: +# https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/ + +on: + push: + branches: master + +jobs: + build: + name: Build distribution 📦 + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Install pypa/build + run: python -m pip install build --user + + - name: Build a binary wheel and a source tarball + run: python -m build + + - name: Store the distribution packages + uses: actions/upload-artifact@v4 + with: + name: python-package-distributions + path: dist/ + retention-days: 1 + + publish-to-pypi: + name: >- + Publish threadpoolctl 🎮 distribution 📦 to PyPI + if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes + needs: + - build + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/project/threadpoolctl/ + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution 📦 to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + github-release: + name: >- + Sign the threadpoolctl 🎮 distribution 📦 with Sigstore + and upload them to GitHub Release + needs: + - publish-to-pypi + runs-on: ubuntu-latest + + permissions: + contents: write # IMPORTANT: mandatory for making GitHub Releases + id-token: write # IMPORTANT: mandatory for sigstore + + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Sign the dists with Sigstore + uses: sigstore/gh-action-sigstore-python@v3.0.0 + with: + inputs: >- + ./dist/*.tar.gz + ./dist/*.whl + - name: Create GitHub Release + env: + GITHUB_TOKEN: ${{ github.token }} + run: >- + gh release create + "$GITHUB_REF_NAME" + --repo "$GITHUB_REPOSITORY" + --notes "" + - name: Upload artifact signatures to GitHub Release + env: + GITHUB_TOKEN: ${{ github.token }} + # Upload to GitHub Release using the `gh` CLI. + # `dist/` contains the built packages, and the + # sigstore-produced signatures and certificates. + run: >- + gh release upload + "$GITHUB_REF_NAME" dist/** + --repo "$GITHUB_REPOSITORY" + + publish-to-testpypi: + name: Publish threadpoolctl 🎮 distribution 📦 to TestPyPI + needs: + - build + runs-on: ubuntu-latest + + environment: + name: testpypi + url: https://test.pypi.org/project/threadpoolctl/ + + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution 📦 to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + verbose: true + # skip-existing is required for .dev0 versions with fixed names but + # different contents. See also: https://github.com/pypa/flit/issues/257 + skip-existing: true + repository-url: https://test.pypi.org/legacy/ From d5bf10bcf90d9a8a315fe9496fb1bc97007d2fea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= Date: Thu, 13 Mar 2025 14:28:09 +0100 Subject: [PATCH 178/187] Release 3.6.0 (#192) --- CHANGES.md | 4 ++-- threadpoolctl.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 19c6f830..7040d695 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,5 @@ -3.6.0 (TBD) -=========== +3.6.0 (2025-03-13) +================== - Added support for libraries with a path longer than 260 on Windows. The supported path length is now 10 times higher but not unlimited for security reasons. diff --git a/threadpoolctl.py b/threadpoolctl.py index 2efc30cc..aea33509 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -24,7 +24,7 @@ from functools import lru_cache from contextlib import ContextDecorator -__version__ = "3.6.0.dev0" +__version__ = "3.6.0" __all__ = [ "threadpool_limits", "threadpool_info", From dbcc74b0a4d64556f55d263f8a3dcdb84366b5dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= Date: Thu, 13 Mar 2025 14:41:33 +0100 Subject: [PATCH 179/187] CI fix publish workflow (#193) --- .github/workflows/publish-to-pypi.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml index 4e5bf7a6..c127bd52 100644 --- a/.github/workflows/publish-to-pypi.yml +++ b/.github/workflows/publish-to-pypi.yml @@ -2,9 +2,7 @@ name: Publish threadpoolctl 🎮 distribution 📦 to PyPI and TestPyPI # Taken from: # https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/ -on: - push: - branches: master +on: push jobs: build: From a8d725810a5a77a48f843b1000c3e07497590a15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= Date: Thu, 13 Mar 2025 14:48:43 +0100 Subject: [PATCH 180/187] CI Temporary disable tag condition to publish to PyPI (#194) --- .github/workflows/publish-to-pypi.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml index c127bd52..b2160858 100644 --- a/.github/workflows/publish-to-pypi.yml +++ b/.github/workflows/publish-to-pypi.yml @@ -35,7 +35,8 @@ jobs: publish-to-pypi: name: >- Publish threadpoolctl 🎮 distribution 📦 to PyPI - if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes + # temp: always publish to PyPI because the tag is already pushed + # if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes needs: - build runs-on: ubuntu-latest From f8e068cdaf82e621a29311004f911d293279988f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= Date: Thu, 13 Mar 2025 15:26:36 +0100 Subject: [PATCH 181/187] MNT Set version to 3.7.0.dev0 (#195) --- .github/workflows/publish-to-pypi.yml | 3 +-- threadpoolctl.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml index b2160858..c127bd52 100644 --- a/.github/workflows/publish-to-pypi.yml +++ b/.github/workflows/publish-to-pypi.yml @@ -35,8 +35,7 @@ jobs: publish-to-pypi: name: >- Publish threadpoolctl 🎮 distribution 📦 to PyPI - # temp: always publish to PyPI because the tag is already pushed - # if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes + if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes needs: - build runs-on: ubuntu-latest diff --git a/threadpoolctl.py b/threadpoolctl.py index aea33509..e6ac58d8 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -24,7 +24,7 @@ from functools import lru_cache from contextlib import ContextDecorator -__version__ = "3.6.0" +__version__ = "3.7.0.dev0" __all__ = [ "threadpool_limits", "threadpool_info", From 29a0734e245a60ab3a35dab022b080b646585e69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= Date: Thu, 13 Mar 2025 16:23:42 +0100 Subject: [PATCH 182/187] DOC Update maintainers doc (#196) --- README.md | 50 +++++++++++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index fc22c84e..648143ad 100644 --- a/README.md +++ b/README.md @@ -332,40 +332,48 @@ https://github.com/xianyi/OpenBLAS/issues/2985). To make a release: -- Bump the version number (`__version__`) in `threadpoolctl.py` and update the - release date in `CHANGES.md`. +- Create a PR to bump the version number (`__version__`) in `threadpoolctl.py` and + update the release date in `CHANGES.md`. -- Build the distribution archives: +- Merge the PR and check that the `Publish threadpoolctl distribution to TestPyPI` job + of the `publish-to-pypi.yml` workflow successfully uploaded the wheel and source + distribution to Test PyPI. -```bash -pip install flit -flit build -``` +- If everything is fine create a tag for the release and push it to github: -and check the contents of `dist/`. + ```bash + git tag -a X.Y.Z + git push git@github.com:joblib/threadpoolctl.git X.Y.Z + ``` -- If everything is fine, make a commit for the release, tag it and push the -tag to github: +- Check that the `Publish threadpoolctl distribution to PyPI` job of the + `publish-to-pypi.yml` workflow successfully uploaded the wheel and source distribution + to PyPI this time. -```bash -git tag -a X.Y.Z -git push git@github.com:joblib/threadpoolctl.git X.Y.Z -``` +- Create a PR for the release on the [conda-forge feedstock](https://github.com/conda-forge/threadpoolctl-feedstock) (or wait for the bot to make it). + +- Publish the release on github. + +If for some reason the steps above can't be achieved and a munual upload of the wheel +and source distribution is needed: + +- Build the distribution archives: + + ```bash + pip install flit + flit build + ``` - Upload the wheels and source distribution to PyPI using flit. Since PyPI doesn't allow password authentication anymore, the username needs to be changed to the generic name `__token__`: -```bash -FLIT_USERNAME=__token__ flit publish -``` + ```bash + FLIT_USERNAME=__token__ flit publish + ``` and a PyPI token has to be passed in place of the password. -- Create a PR for the release on the [conda-forge feedstock](https://github.com/conda-forge/threadpoolctl-feedstock) (or wait for the bot to make it). - -- Publish the release on github. - ### Credits The initial dynamic library introspection code was written by @anton-malakhov From 495eae96b53189a6b4c1d36934b4b012d9db191f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= Date: Thu, 13 Mar 2025 23:03:17 +0100 Subject: [PATCH 183/187] CI skip workflow on forks (#197) --- .github/workflows/publish-to-pypi.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml index c127bd52..b38ed7d7 100644 --- a/.github/workflows/publish-to-pypi.yml +++ b/.github/workflows/publish-to-pypi.yml @@ -7,6 +7,8 @@ on: push jobs: build: name: Build distribution 📦 + # Don't run on forked repositories + if: github.event.repository.fork != true runs-on: ubuntu-latest steps: From 1b35392f49e6486567165808e3a4d72d14b07940 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= Date: Thu, 8 May 2025 14:02:54 +0200 Subject: [PATCH 184/187] TST Add missing openblas arch (#198) --- tests/test_threadpoolctl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index 7f6e090e..ea27cc2b 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -614,6 +614,7 @@ def test_architecture(): # XXX: add more as needed by CI or developer laptops "armv8", "haswell", + "neoversen1", "prescott", # see: https://github.com/xianyi/OpenBLAS/pull/3485 "skylakex", "sandybridge", From cf38a18fde026097b3a8fecc4730460868115a95 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sat, 8 Nov 2025 05:15:01 +0100 Subject: [PATCH 185/187] Use `as_py_json` in place of deprecated `as_object_map` for Pyodide>=0.29 (#201) --- threadpoolctl.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/threadpoolctl.py b/threadpoolctl.py index e6ac58d8..ceed5b88 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -1132,7 +1132,12 @@ def _find_libraries_pyodide(self): ) return - for filepath in LDSO.loadedLibsByName.as_object_map(): + if hasattr(LDSO.loadedLibsByName, "as_py_json"): # Pyodide >= 0.29 + libs_iter = LDSO.loadedLibsByName.as_py_json() + else: + libs_iter = LDSO.loadedLibsByName.as_object_map() # Pyodide < 0.29 + + for filepath in libs_iter: # Some libraries are duplicated by Pyodide and do not exist in the # filesystem, so we first check for the existence of the file. For # more details, see From 757e83b83f0ce00ff9a24524c92958eeeb8d38fd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 18 Jun 2026 16:15:46 -0400 Subject: [PATCH 186/187] DOC discuss inconsistencies in openMP's limitation thread locality (#209) --- README.md | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 648143ad..65fba02c 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,19 @@ Fine control of the underlying thread-pool size can be useful in workloads that involve nested parallelism so as to mitigate oversubscription issues. +> **Important:** In its current state, `threadpoolctl` is only designed for +> situations where BLAS and OpenMP are only called from the main Python thread. +> Or, to be more accurate, `threadpoolctl` and BLAS/OpenMP APIs should only ever +> called from the same, single Python thread. For example: +> +> * When you're using it to configure a worker in a process pool, which then calls BLAS or OpenMP APIs directly in the main thread. +> * A Jupyter notebook, where the BLAS or OpenMP APIs are being called from code running in the cell's main thread. +> +> However, once you start calling BLAS or OpenMP APIs and `threadpoolctl` from +> multiple different Python threads, the impact of the `threadpoolctl` limiting +> APIs will be very inconsistent. For more details and a plan to fix this, see +> https://github.com/joblib/threadpoolctl/issues/208 + ## Installation - For users, install the last published version from PyPI: @@ -322,11 +335,16 @@ https://github.com/xianyi/OpenBLAS/issues/2985). and workarounds: https://github.com/joblib/threadpoolctl/blob/master/multiple_openmp.md -- Setting the maximum number of threads of the OpenMP and BLAS libraries has a global - effect and impacts the whole Python process. There is no thread level isolation as - these libraries do not offer thread-local APIs to configure the number of threads to - use in nested parallel calls. +- Setting the maximum number of threads of the OpenMP and BLAS libraries has + inconsistent scope and semantics (thread-local vs process-wide) depending on + the underlying library. For more details see + https://github.com/joblib/threadpoolctl/issues/208 + For example, if you're using OpenMP with libgomp (gcc) or libomp (clang), the + setting is thread-local and sets how many OpenMP threads will be started in + the current thread. On the other hand, with OpenBLAS with pthreads backend or + on Windows, the setting is process-wide and impacts the size of a process-wide + thread pool shared across all threads in the process. ## Maintainers From 432d8ba9b89e495d0f0ff29f150827afe8477f47 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 10 Jul 2026 08:42:20 +0000 Subject: [PATCH 187/187] CI fix macOS FlexiBLAS build with newer CMake Newer CMake versions reject the minimum policy version bundled with FlexiBLAS v3.4.2. Set CMAKE_POLICY_VERSION_MINIMUM=3.5 so the existing FlexiBLAS pin keeps working without upgrading the library. Co-authored-by: Olivier Grisel --- continuous_integration/install_flexiblas.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/continuous_integration/install_flexiblas.sh b/continuous_integration/install_flexiblas.sh index 3d45ab54..1f50801b 100644 --- a/continuous_integration/install_flexiblas.sh +++ b/continuous_integration/install_flexiblas.sh @@ -28,6 +28,7 @@ fi # specific BLAS implementations such as MKL that cannot be installed on # arm64 hardware. cmake ../ -DCMAKE_INSTALL_PREFIX=$ABS_PATH"/flexiblas_install" \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ -DBLAS_AUTO_DETECT="OFF" \ -DEXTRA="OPENBLAS_CONDA" \ -DFLEXIBLAS_DEFAULT="OPENBLAS_CONDA" \