diff --git a/docs/changelog.md b/docs/changelog.md index 4bc9680..f8f311b 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,9 +2,12 @@ ### 1.11.0 - dropped support for python < 3.9 and fixed - pytest 8.4.0 (in progress) -- Dropped support for python <3.9 and added a proper github pages workflow (explicit). Removed `six` from dependencies. - Fixed [#79](https://github.com/smarie/python-pytest-harvest/issues/79) - +- Dropped support for python <3.9 and added a proper github pages workflow (explicit). Removed `six` from + dependencies. Fixed [#79](https://github.com/smarie/python-pytest-harvest/issues/79) +- Fixed EOFError during pytest session finish in some conditions, with `pytest-xdist plugin` and `-n` option + activated. This requires `cloudpickle` to be installed (it is not done by default). Fixed + [#72](https://github.com/smarie/python-pytest-harvest/issues/72). + ### 1.10.6 - bugfixes and maintenance chores - Refactored layout and CI. Fixed [#56](https://github.com/smarie/python-pytest-harvest/issues/56). diff --git a/setup.cfg b/setup.cfg index 87f5603..6092401 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,7 +35,7 @@ install_requires = packaging decopatch makefun>=1.5 - + ; cloudpickle is needed for pytest-xdist tests_require = pytest numpy diff --git a/src/pytest_harvest/plugin.py b/src/pytest_harvest/plugin.py index e08793e..2c80b3d 100644 --- a/src/pytest_harvest/plugin.py +++ b/src/pytest_harvest/plugin.py @@ -1,4 +1,3 @@ -import pickle from collections import OrderedDict from logging import warning from shutil import rmtree @@ -391,6 +390,14 @@ def __init__(self, config): @pytest.hookimpl(trylast=True) def pytest_harvest_xdist_init(self): + try: + import cloudpickle + except ImportError as e: + print("ERROR : `cloudpickle` must be installed for `pytest-harvest` to collect distributed results from " + "`pytest-xdist` parallel workers. If you do not wish to install cloudpickle, please remove the '-n' " + "option from the commandline to disable xdist parallelization.\n") + raise e + # reset the recipient folder if self.results_path.exists(): rmtree(str(self.results_path)) @@ -399,20 +406,24 @@ def pytest_harvest_xdist_init(self): @pytest.hookimpl(trylast=True) def pytest_harvest_xdist_worker_dump(self, worker_id, session_items, fixture_store): + import cloudpickle + with open(str(self.results_path / ('%s.pkl' % worker_id)), 'wb') as f: try: - pickle.dump((session_items, fixture_store), f) + cloudpickle.dump((session_items, fixture_store), f) except Exception as e: warning("Error while pickling worker %s's harvested results: [%s] %s", (worker_id, e.__class__, e)) return True @pytest.hookimpl(trylast=True) def pytest_harvest_xdist_load(self): + import cloudpickle + workers_saved_material = dict() for pkl_file in self.results_path.glob('*.pkl'): wid = pkl_file.stem with pkl_file.open('rb') as f: - workers_saved_material[wid] = pickle.load(f) + workers_saved_material[wid] = cloudpickle.load(f) return workers_saved_material @pytest.hookimpl(trylast=True)