Add cross-property BM25 And operator

.github/workflows/main.yaml

@@ -29,7 +29,7 @@ env:
   WEAVIATE_135: 1.35.18
   WEAVIATE_136: 1.36.12
   WEAVIATE_137: 1.37.5-e0fe0d5.amd64
-  WEAVIATE_139: 1.39.0-rc.0-b41225e.amd64
+  WEAVIATE_139: 1.39.0-rc.1-89299a5.amd64
 
 jobs:
   lint-and-format:

docs/changelog.rst

@@ -1,6 +1,12 @@
 Changelog
 =========
 
+Version 4.23.0
+--------------
+This minor version includes:
+    - Support for new 1.39 features:
+        - Add support for the new cross-property ``And`` BM25 search operator through ``BM25Operator.and_cross()``, available on the ``bm25`` and ``hybrid`` query, generative-query, and aggregation methods. Unlike ``BM25Operator.and_()``, a query token may be matched by any of the searched properties instead of having to occur within a single one. All searched properties must share the same tokenization and analyzer settings. Requires Weaviate ``1.37.15``, ``1.38.8``, ``1.39.0`` or higher
+
 Version 4.22.0
 --------------
 This minor version includes:

integration/test_collection.py

@@ -36,12 +36,17 @@
 )
 from weaviate.collections.classes.internal import Object, ReferenceToMulti, _CrossReference
 from weaviate.collections.classes.types import PhoneNumber, WeaviateProperties, _PhoneNumber
+from weaviate.collections.grpc.shared import (
+    _BM25_AND_CROSS_MIN_VERSIONS,
+    _BM25_AND_CROSS_MIN_VERSIONS_STR,
+)
 from weaviate.exceptions import (
     UnexpectedStatusCodeError,
     WeaviateInsertInvalidPropertyError,
     WeaviateInsertManyAllFailedError,
     WeaviateInvalidInputError,
     WeaviateQueryError,
+    WeaviateUnsupportedFeatureError,
 )
 from weaviate.types import UUID, UUIDS
 
@@ -1756,3 +1761,76 @@ def test_bm25_operators(collection_factory: CollectionFactory) -> None:
     assert len(objs.objects) == 4
     assert objs.objects[0].uuid == uuid2
     assert sorted(obj.uuid for obj in objs.objects[1:]) == sorted([uuid1, uuid3, uuid4])
+
+
+def test_bm25_operator_and_cross(collection_factory: CollectionFactory) -> None:
+    collection = collection_factory(
+        properties=[
+            Property(name="title", data_type=DataType.TEXT),
+            Property(name="body", data_type=DataType.TEXT),
+        ],
+        vectorizer_config=Configure.Vectorizer.none(),
+    )
+
+    if not collection._connection._weaviate_version.is_at_least_any(*_BM25_AND_CROSS_MIN_VERSIONS):
+        pytest.skip(f"bm25 cross-property AND requires {_BM25_AND_CROSS_MIN_VERSIONS_STR}")
+
+    # Only `split_across` needs cross-property matching: neither of its properties holds both tokens.
+    split_across = collection.data.insert({"title": "banana", "body": "split"})
+    single_property = collection.data.insert({"title": "banana split", "body": "dessert"})
+    partial = collection.data.insert({"title": "banana", "body": "bread"})
+
+    and_cross = collection.query.bm25("banana split", operator=wvc.query.BM25Operator.and_cross())
+    assert sorted(obj.uuid for obj in and_cross.objects) == sorted([split_across, single_property])
+
+    or_ = collection.query.bm25(
+        "banana split", operator=wvc.query.BM25Operator.or_(minimum_match=1)
+    )
+    assert sorted(obj.uuid for obj in or_.objects) == sorted(
+        [split_across, single_property, partial]
+    )
+
+    # Per-property AND is at most as permissive as cross-property AND. Which of the two it is
+    # depends on whether the server took the BlockMax or the legacy WAND path, so only the
+    # subset relation is asserted.
+    and_ = collection.query.bm25("banana split", operator=wvc.query.BM25Operator.and_())
+    assert {obj.uuid for obj in and_.objects} <= {obj.uuid for obj in and_cross.objects}
+
+
+def test_bm25_operator_and_cross_mismatched_tokenization(
+    collection_factory: CollectionFactory,
+) -> None:
+    collection = collection_factory(
+        properties=[
+            Property(name="title", data_type=DataType.TEXT, tokenization=Tokenization.WORD),
+            Property(name="body", data_type=DataType.TEXT, tokenization=Tokenization.FIELD),
+        ],
+        vectorizer_config=Configure.Vectorizer.none(),
+    )
+
+    if not collection._connection._weaviate_version.is_at_least_any(*_BM25_AND_CROSS_MIN_VERSIONS):
+        pytest.skip(f"bm25 cross-property AND requires {_BM25_AND_CROSS_MIN_VERSIONS_STR}")
+
+    collection.data.insert({"title": "banana", "body": "split"})
+
+    with pytest.raises(WeaviateQueryError):
+        collection.query.bm25(
+            "banana split",
+            query_properties=["title", "body"],
+            operator=wvc.query.BM25Operator.and_cross(),
+        )
+
+
+def test_bm25_operator_and_cross_unsupported_version(
+    collection_factory: CollectionFactory,
+) -> None:
+    collection = collection_factory(
+        properties=[Property(name="name", data_type=DataType.TEXT)],
+        vectorizer_config=Configure.Vectorizer.none(),
+    )
+
+    if collection._connection._weaviate_version.is_at_least_any(*_BM25_AND_CROSS_MIN_VERSIONS):
+        pytest.skip("server supports bm25 cross-property AND")
+
+    with pytest.raises(WeaviateUnsupportedFeatureError):
+        collection.query.bm25("banana split", operator=wvc.query.BM25Operator.and_cross())

integration/test_collection_hybrid.py

@@ -21,6 +21,10 @@
     _HybridNearVector,
 )
 from weaviate.collections.classes.internal import Object
+from weaviate.collections.grpc.shared import (
+    _BM25_AND_CROSS_MIN_VERSIONS,
+    _BM25_AND_CROSS_MIN_VERSIONS_STR,
+)
 from weaviate.exceptions import (
     WeaviateUnsupportedFeatureError,
 )
@@ -528,3 +532,51 @@ def test_hybrid_bm25_operators(collection_factory: CollectionFactory) -> None:
     assert len(objs.objects) == 4
     assert objs.objects[0].uuid == uuid2
     assert sorted(obj.uuid for obj in objs.objects[1:]) == sorted([uuid1, uuid3, uuid4])
+
+
+def test_hybrid_bm25_operator_and_cross(collection_factory: CollectionFactory) -> None:
+    collection = collection_factory(
+        properties=[
+            Property(name="title", data_type=DataType.TEXT),
+            Property(name="body", data_type=DataType.TEXT),
+        ],
+        vectorizer_config=Configure.Vectorizer.none(),
+    )
+
+    if not collection._connection._weaviate_version.is_at_least_any(*_BM25_AND_CROSS_MIN_VERSIONS):
+        pytest.skip(f"bm25 cross-property AND requires {_BM25_AND_CROSS_MIN_VERSIONS_STR}")
+
+    # Only `split_across` needs cross-property matching: neither of its properties holds both tokens.
+    split_across = collection.data.insert({"title": "banana", "body": "split"}, vector=[1, 0, 0, 0])
+    single_property = collection.data.insert(
+        {"title": "banana split", "body": "dessert"}, vector=[0, 1, 0, 0]
+    )
+    collection.data.insert({"title": "banana", "body": "bread"}, vector=[0, 0, 1, 0])
+
+    objs = collection.query.hybrid(
+        "banana split",
+        vector=None,
+        alpha=0.0,
+        bm25_operator=wvc.query.BM25Operator.and_cross(),
+    )
+    assert sorted(obj.uuid for obj in objs.objects) == sorted([split_across, single_property])
+
+
+def test_hybrid_bm25_operator_and_cross_unsupported_version(
+    collection_factory: CollectionFactory,
+) -> None:
+    collection = collection_factory(
+        properties=[Property(name="name", data_type=DataType.TEXT)],
+        vectorizer_config=Configure.Vectorizer.none(),
+    )
+
+    if collection._connection._weaviate_version.is_at_least_any(*_BM25_AND_CROSS_MIN_VERSIONS):
+        pytest.skip("server supports bm25 cross-property AND")
+
+    with pytest.raises(WeaviateUnsupportedFeatureError):
+        collection.query.hybrid(
+            "banana split",
+            vector=None,
+            alpha=0.0,
+            bm25_operator=wvc.query.BM25Operator.and_cross(),
+        )

test/collection/test_bm25_operator.py

@@ -0,0 +1,155 @@
+"""Unit tests: BM25 search operators are wired into the gRPC request and version-gated.
+
+``BM25Operator.and_cross()`` maps to the ``OPERATOR_AND_CROSS`` enum added in Weaviate 1.39.0 and
+backported to 1.38.8 and 1.37.15, so the client must reject it on servers outside those ranges
+rather than send an enum value the server will not understand.
+"""
+
+from typing import Optional
+
+import pytest
+
+from weaviate.classes.query import BM25Operator
+from weaviate.collections.classes.grpc import BM25OperatorOptions
+from weaviate.collections.grpc.aggregate import _AggregateGRPC
+from weaviate.collections.grpc.query import _QueryGRPC
+from weaviate.exceptions import WeaviateUnsupportedFeatureError
+from weaviate.proto.v1 import base_search_pb2
+from weaviate.util import _ServerVersion
+
+_SUPPORTED = ["1.37.15", "1.38.8", "1.39.0", "1.40.0"]
+_UNSUPPORTED = ["1.36.9", "1.37.14", "1.38.0", "1.38.7"]
+
+
+def _query(version: str = "1.39.0") -> _QueryGRPC:
+    return _QueryGRPC(
+        weaviate_version=_ServerVersion.from_string(version),
+        name="Dummy",
+        tenant=None,
+        consistency_level=None,
+        validate_arguments=True,
+        uses_125_api=True,
+        uses_127_api=True,
+    )
+
+
+def _aggregate(version: str = "1.39.0") -> _AggregateGRPC:
+    return _AggregateGRPC(
+        weaviate_version=_ServerVersion.from_string(version),
+        name="Dummy",
+        tenant=None,
+        consistency_level=None,
+        validate_arguments=True,
+    )
+
+
+def _aggregate_hybrid(
+    builder: _AggregateGRPC, operator: Optional[BM25OperatorOptions]
+) -> base_search_pb2.Hybrid:
+    return builder.hybrid(
+        query="banana two",
+        alpha=0.0,
+        vector=None,
+        properties=None,
+        target_vector=None,
+        bm25_operator=operator,
+        aggregations=[],
+        filters=None,
+        group_by=None,
+        limit=None,
+        object_limit=None,
+        objects_count=False,
+    ).hybrid
+
+
+@pytest.mark.parametrize(
+    "operator,expected,minimum_or_tokens_match",
+    [
+        (BM25Operator.and_(), base_search_pb2.SearchOperatorOptions.OPERATOR_AND, 0),
+        (BM25Operator.or_(minimum_match=2), base_search_pb2.SearchOperatorOptions.OPERATOR_OR, 2),
+        (
+            BM25Operator.and_cross(),
+            base_search_pb2.SearchOperatorOptions.OPERATOR_AND_CROSS,
+            0,
+        ),
+    ],
+)
+def test_bm25_operator_wired_into_request(
+    operator: BM25OperatorOptions,
+    expected: "base_search_pb2.SearchOperatorOptions.Operator",
+    minimum_or_tokens_match: int,
+) -> None:
+    search_operator = (
+        _query().bm25(query="banana two", operator=operator).bm25_search.search_operator
+    )
+    assert search_operator.operator == expected
+    assert search_operator.minimum_or_tokens_match == minimum_or_tokens_match
+
+
+@pytest.mark.parametrize(
+    "operator,expected",
+    [
+        (BM25Operator.and_(), base_search_pb2.SearchOperatorOptions.OPERATOR_AND),
+        (BM25Operator.or_(minimum_match=2), base_search_pb2.SearchOperatorOptions.OPERATOR_OR),
+        (BM25Operator.and_cross(), base_search_pb2.SearchOperatorOptions.OPERATOR_AND_CROSS),
+    ],
+)
+def test_hybrid_bm25_operator_wired_into_request(
+    operator: BM25OperatorOptions, expected: "base_search_pb2.SearchOperatorOptions.Operator"
+) -> None:
+    req = _query().hybrid(query="banana two", alpha=0.0, bm25_operator=operator)
+    assert req.hybrid_search.bm25_search_operator.operator == expected
+
+
+def test_no_operator_leaves_search_operator_unset() -> None:
+    assert not _query().bm25(query="banana two").bm25_search.HasField("search_operator")
+    req = _query().hybrid(query="banana two", alpha=0.0)
+    assert not req.hybrid_search.HasField("bm25_search_operator")
+
+
+@pytest.mark.parametrize("version", _SUPPORTED)
+def test_and_cross_allowed_on_supported_versions(version: str) -> None:
+    req = _query(version).bm25(query="banana two", operator=BM25Operator.and_cross())
+    assert (
+        req.bm25_search.search_operator.operator
+        == base_search_pb2.SearchOperatorOptions.OPERATOR_AND_CROSS
+    )
+
+
+@pytest.mark.parametrize("version", _UNSUPPORTED)
+def test_and_cross_rejected_on_unsupported_versions(version: str) -> None:
+    with pytest.raises(WeaviateUnsupportedFeatureError) as e:
+        _query(version).bm25(query="banana two", operator=BM25Operator.and_cross())
+    assert "1.37.15 or 1.38.8 or 1.39.0" in str(e.value)
+    assert version in str(e.value)
+
+
+@pytest.mark.parametrize("version", _UNSUPPORTED)
+def test_hybrid_and_cross_rejected_on_unsupported_versions(version: str) -> None:
+    with pytest.raises(WeaviateUnsupportedFeatureError):
+        _query(version).hybrid(
+            query="banana two", alpha=0.0, bm25_operator=BM25Operator.and_cross()
+        )
+
+
+@pytest.mark.parametrize("version", _UNSUPPORTED)
+def test_aggregate_hybrid_and_cross_rejected_on_unsupported_versions(version: str) -> None:
+    with pytest.raises(WeaviateUnsupportedFeatureError):
+        _aggregate_hybrid(_aggregate(version), BM25Operator.and_cross())
+
+
+def test_aggregate_hybrid_and_cross_allowed_on_supported_version() -> None:
+    hybrid = _aggregate_hybrid(_aggregate("1.39.0"), BM25Operator.and_cross())
+    assert (
+        hybrid.bm25_search_operator.operator
+        == base_search_pb2.SearchOperatorOptions.OPERATOR_AND_CROSS
+    )
+
+
+@pytest.mark.parametrize("version", _UNSUPPORTED)
+@pytest.mark.parametrize("operator", [BM25Operator.and_(), BM25Operator.or_(minimum_match=1), None])
+def test_other_operators_unaffected_by_gate(
+    version: str, operator: Optional[BM25OperatorOptions]
+) -> None:
+    _query(version).bm25(query="banana two", operator=operator)
+    _query(version).hybrid(query="banana two", alpha=0.0, bm25_operator=operator)

test/test_server_version.py

@@ -60,6 +60,34 @@ def test_server_version_is_at_least(is_valid: bool) -> None:
     assert is_valid
 
 
+@pytest.mark.parametrize(
+    "version,expected",
+    [
+        ("1.36.9", False),
+        ("1.37.14", False),
+        ("1.37.15", True),
+        ("1.37.99", True),
+        ("1.38.0", False),
+        ("1.38.7", False),
+        ("1.38.8", True),
+        ("1.38.20", True),
+        ("1.39.0", True),
+        ("1.40.0", True),
+        ("2.0.0", True),
+    ],
+)
+def test_server_version_is_at_least_any(version: str, expected: bool) -> None:
+    assert (
+        _ServerVersion.from_string(version).is_at_least_any((1, 37, 15), (1, 38, 8), (1, 39, 0))
+        is expected
+    )
+
+
+def test_server_version_is_at_least_any_single_minimum() -> None:
+    assert _ServerVersion(1, 39, 0).is_at_least_any((1, 39, 0))
+    assert not _ServerVersion(1, 38, 99).is_at_least_any((1, 39, 0))
+
+
 def test_server_version_magic_methods() -> None:
     # Test __eq__
     assert _ServerVersion(1, 2, 3) == _ServerVersion(1, 2, 3)