fix: skip object store prefix markers in list_dir

changes/4032.bugfix.md

@@ -0,0 +1 @@
+Fixed `ObjectStore.list_dir` for object-store listings that include a directory-marker object matching the requested non-root prefix.

src/zarr/core/array.py

@@ -4018,7 +4018,11 @@ async def _shards_initialized(
         x async for x in array.store_path.store.list_prefix(prefix=array.store_path.path)
     ]
     store_contents_relative = [
-        _relativize_path(path=key, prefix=array.store_path.path) for key in store_contents
+        _relativize_path(path=key, prefix=array.store_path.path)
+        for key in store_contents
+        # obstore can include a directory marker whose key matches the listed prefix;
+        # it is not an initialized shard and must be excluded before relativizing.
+        if array.store_path.path == "" or key != array.store_path.path
     ]
     return tuple(
         chunk_key for chunk_key in array._iter_shard_keys() if chunk_key in store_contents_relative

src/zarr/storage/_memory.py

@@ -219,7 +219,7 @@ async def list_dir(self, prefix: str) -> AsyncIterator[str]:
             keys_unique = {
                 key.removeprefix(f"{prefix}/").split("/")[0]
                 for key in self._store_dict
-                if key.startswith(f"{prefix}/") and key != prefix
+                if key.startswith(f"{prefix}/") and key not in {prefix, f"{prefix}/"}
             }
 
         for key in keys_unique:

src/zarr/storage/_obstore.py

@@ -267,7 +267,11 @@ async def _transform_list_dir(
     for path in chain(
         list_result["common_prefixes"], map(itemgetter("path"), list_result["objects"])
     ):
-        yield _relativize_path(path=path, prefix=prefix)
+        if prefix != "" and path == prefix:
+            continue
+        relpath = _relativize_path(path=path, prefix=prefix)
+        if relpath:
+            yield relpath
 
 
 class _BoundedRequest(TypedDict):

tests/test_array.py

@@ -45,7 +45,7 @@
     default_serializer_v3,
 )
 from zarr.core.array_spec import ArrayConfig, ArrayConfigParams
-from zarr.core.buffer import NDArrayLike, NDArrayLikeOrScalar, default_buffer_prototype
+from zarr.core.buffer import NDArrayLike, NDArrayLikeOrScalar, cpu, default_buffer_prototype
 from zarr.core.chunk_grids import (
     SHARDED_INNER_CHUNK_MAX_BYTES,
     guess_chunks,
@@ -447,6 +447,8 @@ async def test_chunks_initialized(
     arr = zarr.create_array(
         store, name=path, shape=shape, shards=shard_shape, chunks=chunk_shape, dtype="i1"
     )
+    if path:
+        await store.set(path, cpu.Buffer.from_bytes(b""))
 
     chunks_accumulated = tuple(
         accumulate(tuple(tuple(v.split(" ")) for v in arr._iter_shard_keys()))

tests/test_store/test_object.py

@@ -1,4 +1,5 @@
 # ruff: noqa: E402
+import re
 from pathlib import Path
 from typing import TypedDict
 
@@ -9,9 +10,10 @@
 from hypothesis.stateful import (
     run_state_machine_as_test,
 )
-from obstore.store import LocalStore, MemoryStore
+from obstore.store import LocalStore, MemoryStore, S3Store
 
 from zarr.core.buffer import Buffer, cpu
+from zarr.core.sync import _collect_aiterator
 from zarr.storage import ObjectStore
 from zarr.testing.stateful import ZarrHierarchyStateMachine
 from zarr.testing.store import StoreTests
@@ -97,6 +99,39 @@ async def test_store_getsize_prefix(self, store: ObjectStore[LocalStore]) -> Non
         assert total_size == len(buf) * 2
 
 
+@pytest.mark.filterwarnings(
+    re.escape("ignore:datetime.datetime.utcnow() is deprecated:DeprecationWarning")
+)
+async def test_list_dir_ignores_s3_prefix_marker(moto_server: str) -> None:
+    """Ensure obstore's exact-prefix S3 directory marker is not listed as a child."""
+    boto3 = pytest.importorskip("boto3")
+    bucket = "object-store-prefix-marker"
+    client = boto3.client(
+        "s3",
+        endpoint_url=moto_server,
+        region_name="us-east-1",
+        aws_access_key_id="x",
+        aws_secret_access_key="x",
+    )
+    client.create_bucket(Bucket=bucket)
+    client.put_object(Bucket=bucket, Key="g/", Body=b"")
+
+    store = ObjectStore(
+        S3Store(
+            bucket=bucket,
+            endpoint=moto_server,
+            region="us-east-1",
+            access_key_id="x",
+            secret_access_key="x",
+            client_options={"allow_http": True},
+            virtual_hosted_style_request=False,
+        )
+    )
+
+    assert await _collect_aiterator(store.list_dir("g")) == ()
+    assert await _collect_aiterator(store.list_dir("g/")) == ()
+
+
 @pytest.mark.slow_hypothesis
 def test_zarr_hierarchy() -> None:
     sync_store = ObjectStore(MemoryStore())