Skip to content

Add benchmarks for sharded + local store indexing #3663

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
d-v-b merged 6 commits into from
Jan 25, 2026
+17 −5
Merged

Add benchmarks for sharded + local store indexing #3663

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
c15bd60
Add benchmarks for sharded + local store indexing
aldenks Jan 20, 2026
0542993
Merge branch 'main' into shard-slice-indexing-benchmark
d-v-b Jan 20, 2026
f62eb68
Merge branch 'main' into shard-slice-indexing-benchmark
d-v-b Jan 24, 2026
11e0f5b
use latencystore with 10ms get latency instead of localstore
d-v-b Jan 25, 2026
520f915
use .01s of latency
d-v-b Jan 25, 2026
6271050
set get latency to .0001s
d-v-b Jan 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 12 additions & 3 deletions tests/benchmarks/test_indexing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,18 +22,27 @@
(slice(None), slice(0, 3, 2), slice(0, 10)),
)

shards = (
None,
(50,) * 3,
)


@pytest.mark.parametrize("store", ["memory"], indirect=["store"])
@pytest.mark.parametrize("store", ["memory", "memory_get_latency"], indirect=["store"])
@pytest.mark.parametrize("indexer", indexers, ids=str)
@pytest.mark.parametrize("shards", shards, ids=str)
def test_slice_indexing(
store: Store, indexer: tuple[int | slice], benchmark: BenchmarkFixture
store: Store,
indexer: tuple[int | slice],
shards: tuple[int, ...] | None,
benchmark: BenchmarkFixture,
) -> None:
data = create_array(
store=store,
shape=(105,) * 3,
dtype="uint8",
chunks=(10,) * 3,
shards=None,
shards=shards,
compressors=None,
filters=None,
fill_value=0,
Expand Down
7 changes: 5 additions & 2 deletions tests/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
from zarr.core.metadata.v3 import ArrayV3Metadata
from zarr.core.sync import sync
from zarr.storage import FsspecStore, LocalStore, MemoryStore, StorePath, ZipStore
from zarr.testing.store import LatencyStore

if TYPE_CHECKING:
from collections.abc import Generator
Expand All @@ -58,8 +59,8 @@


async def parse_store(
store: Literal["local", "memory", "fsspec", "zip"], path: str
) -> LocalStore | MemoryStore | FsspecStore | ZipStore:
store: Literal["local", "memory", "fsspec", "zip", "memory_get_latency"], path: str
) -> LocalStore | MemoryStore | FsspecStore | ZipStore | LatencyStore:
if store == "local":
return await LocalStore.open(path)
if store == "memory":
Expand All @@ -68,6 +69,8 @@ async def parse_store(
return await FsspecStore.open(url=path)
if store == "zip":
return await ZipStore.open(path + "/zarr.zip", mode="w")
if store == "memory_get_latency":
return LatencyStore(MemoryStore(), get_latency=0.0001, set_latency=0)
raise AssertionError


Expand Down
Loading