From 45d41a0a4e74293e6cfe5779477e7606c34982c1 Mon Sep 17 00:00:00 2001 From: Sylvain MARIE Date: Mon, 30 Sep 2024 12:41:34 +0200 Subject: [PATCH 1/4] Fixed EOFError during pytest session finish in some conditions, with `pytest-xdist plugin` and `-n` option activated. Fixed #72 --- setup.cfg | 1 + src/pytest_harvest/plugin.py | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 93354cd..9cb6ef7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -48,6 +48,7 @@ install_requires = six pathlib2;python_version<'3.2' tests_require = +; cloudpickle pytest numpy pandas diff --git a/src/pytest_harvest/plugin.py b/src/pytest_harvest/plugin.py index c289d67..a72be54 100644 --- a/src/pytest_harvest/plugin.py +++ b/src/pytest_harvest/plugin.py @@ -1,4 +1,3 @@ -import pickle from collections import OrderedDict from logging import warning from shutil import rmtree @@ -399,6 +398,14 @@ def __init__(self, config): @pytest.hookimpl(trylast=True) def pytest_harvest_xdist_init(self): + try: + import cloudpickle + except ImportError as e: + print("ERROR : `cloudpickle` must be installed for `pytest-harvest` to collect distributed results from " + "`pytest-xdist` parallel workers. If you do not wish to install cloudpickle, please remove the '-n' " + "option from the commandline to disable xdist parallelization.\n") + raise e + # reset the recipient folder if self.results_path.exists(): rmtree(str(self.results_path)) @@ -407,20 +414,24 @@ def pytest_harvest_xdist_init(self): @pytest.hookimpl(trylast=True) def pytest_harvest_xdist_worker_dump(self, worker_id, session_items, fixture_store): + import cloudpickle + with open(str(self.results_path / ('%s.pkl' % worker_id)), 'wb') as f: try: - pickle.dump((session_items, fixture_store), f) + cloudpickle.dump((session_items, fixture_store), f) except Exception as e: warning("Error while pickling worker %s's harvested results: [%s] %s", (worker_id, e.__class__, e)) return True @pytest.hookimpl(trylast=True) def pytest_harvest_xdist_load(self): + import cloudpickle + workers_saved_material = dict() for pkl_file in self.results_path.glob('*.pkl'): wid = pkl_file.stem with pkl_file.open('rb') as f: - workers_saved_material[wid] = pickle.load(f) + workers_saved_material[wid] = cloudpickle.load(f) return workers_saved_material @pytest.hookimpl(trylast=True) From 50eb9c6dd1203203b50dee4efa48c72a59d2d566 Mon Sep 17 00:00:00 2001 From: Sylvain MARIE Date: Mon, 30 Sep 2024 12:42:19 +0200 Subject: [PATCH 2/4] Updated changelog --- docs/changelog.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/changelog.md b/docs/changelog.md index a9a1aa1..6b62d9b 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,7 +1,9 @@ # Changelog -### 1.10.6 - bugfixes and maintenance chores +### 1.10.6 - bugfix and maintenance + - Fixed EOFError during pytest session finish in some conditions, with `pytest-xdist plugin` and `-n` option + activated. Fixed [#72](https://github.com/smarie/python-pytest-harvest/issues/72). - Refactored layout and CI. Fixed [#56](https://github.com/smarie/python-pytest-harvest/issues/56). ### 1.10.5 - pytest 8.1 compat From 455aab04f02f67e31b9f2f1c10c6add74d27a987 Mon Sep 17 00:00:00 2001 From: Sylvain MARIE Date: Thu, 13 Nov 2025 18:06:43 +0100 Subject: [PATCH 3/4] Minor fix for setup.cfg --- setup.cfg | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 20c442e..6092401 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,9 +35,8 @@ install_requires = packaging decopatch makefun>=1.5 - + ; cloudpickle is needed for pytest-xdist tests_require = -; cloudpickle pytest numpy pandas From d8569124ad2295c1c6128680044741b6cb79f23f Mon Sep 17 00:00:00 2001 From: Sylvain MARIE Date: Thu, 13 Nov 2025 18:10:36 +0100 Subject: [PATCH 4/4] Changelog update --- docs/changelog.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/changelog.md b/docs/changelog.md index a76e052..f8f311b 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -5,7 +5,8 @@ - Dropped support for python <3.9 and added a proper github pages workflow (explicit). Removed `six` from dependencies. Fixed [#79](https://github.com/smarie/python-pytest-harvest/issues/79) - Fixed EOFError during pytest session finish in some conditions, with `pytest-xdist plugin` and `-n` option - activated. Fixed [#72](https://github.com/smarie/python-pytest-harvest/issues/72). + activated. This requires `cloudpickle` to be installed (it is not done by default). Fixed + [#72](https://github.com/smarie/python-pytest-harvest/issues/72). ### 1.10.6 - bugfixes and maintenance chores