Merge pull request #30 from funkelab/zarr3

bumps version up to 0.7 since it adds quite a few changes, some of which are not backwards compatible.

Breaking change:
- now depends on python 3.11 or above. This is necessary because we rely on iohub for zarr3 compatibility which supports only python 3.11 and above.

Other changes:
- bulk write api. Added `bulk_write_nodes`, `bulk_write_edges` and `bulk_write_graph`. These use more efficient forms of insert queries into the relevant databases to better handle large amounts of data. Some features of the existing write api may be unavailable (`fail_if_exists` for example)
- bulk write context manager. Graph providers now have a bulk write context manager whose primary purpose is to remove indexes on enter and rebuild on exit. Note that you do not want to call this on workers. There is a flag to call on workers which can be helpful and will do things like turn of synchronous commits.

minor improvements and bug fixes:
- removed duplicate conditions on queries with attribute filters
- Better respect half open properties of ROI requests. Sometimes nodes would get through before if exactly on the upper bound.
- Better ROI queries for edges. Instead of fetching all nodes in a ROI and then serializing the list of nodes into a string to put in the SQL query, we now send a single request with a JOIN statement to both read nodes in roi and match on endpoints to get edges. This results in faster reads and won't run into the memory issues occasionally encountered before. Note if using a node attribute filter it will still fetch nodes first matching that criteria, then serialize to string and query edges with u contained in list of nodes.

Tests:
- greatly expanded tests for new features and bug fixes

Author: pattonw

.github/workflows/tests.yaml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10", "3.11", "3.12"]
+        python-version: ["3.11", "3.12", "3.13"]
         resolution: ["highest", "lowest-direct"]
 
     services:

.github/workflows/mypy.yaml .github/workflows/ty.yaml.github/workflows/mypy.yaml renamed to .github/workflows/ty.yaml

@@ -1,15 +1,15 @@
-name: mypy
+name: ty
 
 on: [push, pull_request]
 
 jobs:
   static-analysis:
-    name: Python mypy
+    name: Python ty
     runs-on: ubuntu-latest
     steps:
       - name: Setup checkout
         uses: actions/checkout@master
       - name: Install uv
         uses: astral-sh/setup-uv@main
-      - name: mypy
-        run: uv run --extra dev mypy funlib/persistence tests
+      - name: ty
+        run: uv run --extra dev ty check src tests

.gitignore

@@ -130,3 +130,5 @@ dmypy.json
 .vscode/
 
 *.sw[pmno]
+daisy_logs
+uv.lock

README.md

@@ -1,6 +1,6 @@
 [![tests](https://github.com/funkelab/funlib.persistence/actions/workflows/tests.yaml/badge.svg)](https://github.com/funkelab/funlib.persistence/actions/workflows/tests.yaml)
 [![ruff](https://github.com/funkelab/funlib.persistence/actions/workflows/ruff.yaml/badge.svg)](https://github.com/funkelab/funlib.persistence/actions/workflows/ruff.yaml)
-[![mypy](https://github.com/funkelab/funlib.persistence/actions/workflows/mypy.yaml/badge.svg)](https://github.com/funkelab/funlib.persistence/actions/workflows/mypy.yaml)
+[![ty](https://github.com/funkelab/funlib.persistence/actions/workflows/ty.yaml/badge.svg)](https://github.com/funkelab/funlib.persistence/actions/workflows/ty.yaml)
 [![pypi](https://github.com/funkelab/funlib.persistence/actions/workflows/publish.yaml/badge.svg)](https://pypi.org/project/funlib.persistence/)
 
 # funlib.persistence

funlib/persistence/arrays/lazy_ops.py

@@ -13,25 +13,21 @@ authors = [
 ]
 dynamic = ['version']
 
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 classifiers = ["Programming Language :: Python :: 3"]
 keywords = []
 
 dependencies = [
-    "zarr>=2,<3",
-    # ImportError: cannot import name 'cbuffer_sizes' from 'numcodecs.blosc'
-    # We can pin zarr to >2.18.7 but then we have to drop python 3.10
-    # pin numcodecs to avoid breaking change
-    "numcodecs>0.13,<0.16.0",
-    "iohub>=0.2.0b0",
+    "zarr>=3,<4",
+    "iohub>=0.3.0a5",
     "funlib.geometry>=0.3.0",
     "networkx>=3.0.0",
     "pymongo>=4.0.0",
     "numpy>=2.0.0",
     "pydantic>=2.0.0",
     "dask>=2024.0.0",
     "toml>=0.10.0",
-    "psycopg2-binary>=2.9.5",
+    "psycopg2-binary>=2.9.11",
 ]
 
 [tool.setuptools.dynamic]
@@ -40,10 +36,10 @@ version = { attr = "funlib.persistence.__version__" }
 [project.optional-dependencies]
 dev = [
     "coverage>=7.7.1",
-    "mypy>=1.15.0",
     "pytest>=8.3.5",
     "pytest-mock>=3.14.0",
     "ruff>=0.11.2",
+    "ty>=0.0.16",
     "types-networkx",
     "types-psycopg2",
     "types-toml",
@@ -56,10 +52,8 @@ lint.select = ["F", "W", "I001"]
 [tool.setuptools.package-data]
 "funlib.persistence" = ["py.typed"]
 
-[tool.mypy]
-explicit_package_bases = true
-
 # # module specific overrides
 [[tool.mypy.overrides]]
 module = ["zarr.*", "iohub.*"]
 ignore_missing_imports = true
+

funlib/persistence/graphs/pgsql_graph_database.py

@@ -1,4 +1,4 @@
 from .arrays import Array, open_ds, prepare_ds, open_ome_ds, prepare_ome_ds  # noqa
 
-__version__ = "0.6.1"
+__version__ = "0.7.0"
 __version_info__ = tuple(int(i) for i in __version__.split("."))

pyproject.toml

@@ -6,9 +6,8 @@
 import dask.array as da
 import numpy as np
 from dask.array.optimization import fuse_slice
-from zarr import Array as ZarrArray
-
 from funlib.geometry import Coordinate, Roi
+from zarr import Array as ZarrArray
 
 from .freezable import Freezable
 from .lazy_ops import LazyOp
@@ -135,7 +134,7 @@ def attrs(self) -> dict:
 
     @property
     def chunk_shape(self) -> Coordinate:
-        return Coordinate(self.data.chunksize)
+        return Coordinate(self.data.chunksize)  # ty: ignore[unresolved-attribute]
 
     def uncollapsed_dims(self, physical: bool = False) -> list[bool]:
         """
@@ -344,18 +343,21 @@ def __getitem__(self, key) -> np.ndarray:
         else:
             return self.data[key].compute()
 
-    def __setitem__(self, key, value: np.ndarray):
-        """Set the data of this array within the given ROI.
+    def __setitem__(self, key: Roi | slice | tuple, value: np.ndarray | float | int):
+        """Set the data of this array.
 
         Args:
 
-            key (`class:Roi`):
+            key (`class:Roi` or any numpy compatible key):
 
-                The ROI to write to.
+                The region to write to. Can be a `Roi` for world-unit indexing,
+                or any numpy-compatible key (e.g. ``np.s_[:]``, a slice, a tuple
+                of slices).
 
-            value (``ndarray``):
+            value (``ndarray`` or scalar):
 
-                The value to write.
+                The value to write. Can be a numpy array or a scalar that will
+                be broadcast.
         """
 
         if self.is_writeable:
@@ -474,7 +476,7 @@ def _is_slice(self, lazy_op: LazyOp, writeable: bool = False) -> bool:
         elif isinstance(lazy_op, list) and all([isinstance(a, int) for a in lazy_op]):
             return True
         elif isinstance(lazy_op, tuple) and all(
-            [self._is_slice(a, writeable) for a in lazy_op]
+            [self._is_slice(a, writeable) for a in lazy_op]  # type: ignore[arg-type]  # ty can't narrow parameterized tuple iteration
         ):
             return True
         elif (
@@ -508,7 +510,7 @@ def validate(self, strict: bool = False):
             )
 
     def to_pixel_space(
-        self, world_loc: Roi | Coordinate | Sequence[int | float]
+        self, world_loc: Roi | Coordinate | Sequence[int | float] | np.ndarray
     ) -> Roi | Coordinate | np.ndarray:
         """Convert a point or roi in world space into the pixel space of this array.
         Works on sequences of floats by returning a numpy array that is not guaranteed
@@ -539,7 +541,7 @@ def to_pixel_space(
             )
 
     def to_world_space(
-        self, pixel_loc: Roi | Coordinate | Sequence[int | float]
+        self, pixel_loc: Roi | Coordinate | Sequence[int | float] | np.ndarray
     ) -> Roi | Coordinate:
         """Convert a point or roi from pixel space in this array to the world
         coordinate system defined by this array's roi and voxel size.