Python 3.14 support

Author: crusaderky

.github/workflows/tests.yaml

@@ -26,23 +26,13 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        environment: [mindeps, "3.10", "3.11", "3.12", "3.13"]
+        environment: [mindeps, "3.10", "3.11", "3.12", "3.13", "3.14"]
         label: [default]
         extra_packages: [null]
         # Cherry-pick test modules to split the overall runtime roughly in half
         partition: [ci1, not ci1]
 
         exclude:
-          # MacOS CI does not have any hosts available; run it on 3.12 only
-          - os: macos-latest
-            environment: mindeps
-          - os: macos-latest
-            environment: "3.10"
-          - os: macos-latest
-            environment: "3.11"
-          - os: macos-latest
-            environment: "3.13"
-
           - os: windows-latest
             environment: mindeps
 

continuous_integration/environment-3.14.yaml

@@ -0,0 +1,55 @@
+name: dask-distributed-313
+channels:
+  - conda-forge
+dependencies:
+  - python=3.14
+  - packaging
+  - pip
+  - asyncssh
+  - bokeh>3
+  - click
+  - cloudpickle
+  - coverage
+  - dask  # overridden by git tip below
+  - fsspec  # overridden by git tip below
+  # - gilknocker  # conda-forge package not yet available for Python 3.14
+  - h5py
+  - ipykernel
+  - ipywidgets
+  - jinja2
+  - jupyter_events<0.11
+  - jupyter-server-proxy
+  - jupyterlab
+  - locket
+  - msgpack-python
+  - netcdf4
+  - paramiko
+  - pre-commit
+  - prometheus_client
+  - psutil
+  - pyarrow
+  - pytest<8.4  # Pin due to https://github.com/pytest-dev/pytest-cov/issues/693
+  - pytest-cov
+  - pytest-faulthandler
+  - pytest-repeat
+  - pytest-rerunfailures
+  - pytest-timeout
+  - requests
+  - scikit-learn
+  - scipy
+  - sortedcollections
+  - tblib !=3.2.0,!=3.2.1
+  - toolz
+  - tornado
+  - zict  # overridden by git tip below
+  - zstandard
+  # Temporary fix for https://github.com/pypa/setuptools/issues/4496
+  - setuptools < 71
+  # Temporary fix for https://github.com/jupyterlab/jupyterlab/issues/17012
+  - httpx<0.28.0
+  - pip:
+      - git+https://github.com/dask/dask
+      - git+https://github.com/dask/zict
+      - git+https://github.com/fsspec/filesystem_spec
+      - keras
+      - gilknocker  # conda-forge package not yet available for Python 3.14 

distributed/client.py

@@ -3250,8 +3250,20 @@ def _get_computation_code(
                     "|".join([f"(?:{mod})" for mod in ignore_modules])
                 )
             if ignore_files:
+                # Given ignore-files = [foo], match:
+                #   /path/to/foo
+                #   /path/to/foo.py[c]
+                #   /path/to/foo/bar.py[c]
+                #   \path\to\foo
+                #   \path\to\foo.py[c]
+                #   \path\to\foo\bar.py[c]
+                #   <frozen foo>
+                # Do not match files that have 'foo' as a substring,
+                # unless the user explicitly states '.*foo.*'.
+                ignore_files_or = "|".join(mod for mod in ignore_files)
                 fname_pattern = re.compile(
-                    r".*[\\/](" + "|".join(mod for mod in ignore_files) + r")([\\/]|$)"
+                    rf".*[\\/]({ignore_files_or})([\\/]|\.pyc?$|$)"
+                    rf"|<frozen ({ignore_files_or})>$"
                 )
         else:
             # stacklevel 0 or less - shows dask internals which likely isn't helpful

distributed/cluster_dump.py

@@ -304,7 +304,8 @@ def to_yamls(
         import yaml
 
         root_dir = Path(root_dir) if root_dir else Path.cwd()
-        dumper = yaml.CSafeDumper
+        # libyaml C bindings may be missing
+        dumper = getattr(yaml, "CSafeDumper", yaml.SafeDumper)
         scheduler_expand_keys = set(scheduler_expand_keys)
         worker_expand_keys = set(worker_expand_keys)
 

distributed/distributed.yaml

@@ -289,14 +289,15 @@ distributed:
         - __channelexec__  # more xdist
         - execnet  # more xdist
       ignore-files:
-        - runpy\.py  # `python -m pytest` (or other module) shell command
-        - pytest  # `pytest` shell command
-        - py\.test  # `py.test` shell command
-        - pytest-script\.py  # `pytest` shell command in Windows
-        - _pytest  # pytest implementation
+        # `python -m pytest` (or other module)
+        # runpy.py on Python <=3.13; <frozen runpy> on >=3.14
+        - runpy
+        # Many variations of pytest:
+        # pytest, py.test, pytest-script (on Windows),
+        # _pytest (implementation), vscode_pytest
+        - .*py\.?test.*
         - pycharm  # Run pytest from PyCharm GUI
-        - vscode_pytest
-        - get_output_via_markers\.py
+        - get_output_via_markers
     erred-tasks:
       max-history: 100
 

distributed/protocol/tests/test_protocol.py

@@ -162,16 +162,15 @@ def test_sizeof_serialize(Wrapper, Wrapped):
 @pytest.mark.skipif(WINDOWS, reason="On windows this is triggering a stackoverflow")
 def test_deeply_nested_structures():
     # These kind of deeply nested structures are generated in our profiling code
-    def gen_deeply_nested(depth, msg=None):
-        d = msg or {}
-        while depth:
-            depth -= 1
+    def gen_deeply_nested(depth):
+        d = {}
+        for _ in range(depth):
             d = {"children": d}
         return d
 
-    msg = {}
-    for _ in range(10):
-        msg = gen_deeply_nested(sys.getrecursionlimit() // 2, msg=msg)
+    # Note: Python <=3.13 already fails with 2x the recursion limit;
+    # 3.14 keeps working until 14x for some reason
+    msg = gen_deeply_nested(sys.getrecursionlimit() * 14)
 
     with pytest.raises(RecursionError):
         copy.deepcopy(msg)

distributed/shuffle/tests/test_shuffle_plugins.py

@@ -1,11 +1,11 @@
 from __future__ import annotations
 
-from asyncio import iscoroutinefunction
 
 import pytest
 
 from distributed.shuffle._scheduler_plugin import ShuffleSchedulerPlugin
 from distributed.shuffle._worker_plugin import ShuffleWorkerPlugin
+from distributed.utils import iscoroutinefunction
 from distributed.utils_test import gen_cluster
 
 pd = pytest.importorskip("pandas")

pyproject.toml

@@ -20,6 +20,7 @@ classifiers = [
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
     "Topic :: Scientific/Engineering",
     "Topic :: System :: Distributed Computing",
 ]