add low book confidence detector

machine/quality_estimation/__init__.py

@@ -1,3 +1,4 @@
+from .book_confidence import LOW_BOOK_CONFIDENCE_THRESHOLD, is_book_confidence_unusually_low
 from .chrf3_quality_estimator import ChrF3QualityEstimator
 from .scripture_book_usability import ScriptureBookUsability
 from .scripture_chapter_usability import ScriptureChapterUsability
@@ -11,6 +12,8 @@
 
 __all__ = [
     "ChrF3QualityEstimator",
+    "is_book_confidence_unusually_low",
+    "LOW_BOOK_CONFIDENCE_THRESHOLD",
     "ScriptureBookUsability",
     "ScriptureChapterUsability",
     "ScriptureSegmentUsability",

machine/quality_estimation/book_confidence.py

@@ -0,0 +1,15 @@
+from typing import Optional
+
+LOW_BOOK_CONFIDENCE_THRESHOLD = 0.42
+
+
+def is_book_confidence_unusually_low(
+    confidence: float, book_id: Optional[str] = None, model: Optional[str] = None
+) -> bool:
+    if not 0 <= confidence <= 1:
+        raise ValueError(
+            f"The book confidence {confidence} is invalid. "
+            f"It is calculated as the geometric mean of the segment confidences, "
+            f"and it must be between 0 and 1, inclusive."
+        )
+    return confidence < LOW_BOOK_CONFIDENCE_THRESHOLD

tests/quality_estimation/test_book_confidence.py

@@ -0,0 +1,35 @@
+from pytest import raises
+
+from machine.quality_estimation import LOW_BOOK_CONFIDENCE_THRESHOLD, is_book_confidence_unusually_low
+
+
+def test_is_book_confidence_unusually_low_at_threshold() -> None:
+    assert not is_book_confidence_unusually_low(LOW_BOOK_CONFIDENCE_THRESHOLD)
+
+
+def test_is_book_confidence_unusually_low_min_confidence() -> None:
+    assert is_book_confidence_unusually_low(0.0)
+
+
+def test_is_book_confidence_unusually_low_max_confidence() -> None:
+    assert not is_book_confidence_unusually_low(1.0)
+
+
+def test_is_book_confidence_unusually_low_with_book_id_and_model() -> None:
+    assert is_book_confidence_unusually_low(0.3, "MAT", "facebook/nllb-200-distilled-1.3B")
+    assert not is_book_confidence_unusually_low(0.9, "MAT", "facebook/nllb-200-distilled-1.3B")
+
+
+def test_is_book_confidence_unusually_low_negative() -> None:
+    with raises(ValueError):
+        is_book_confidence_unusually_low(-0.5)
+
+
+def test_is_book_confidence_unusually_low_greater_than_one() -> None:
+    with raises(ValueError):
+        is_book_confidence_unusually_low(1.5)
+
+
+def test_is_book_confidence_unusually_low_nan() -> None:
+    with raises(ValueError):
+        is_book_confidence_unusually_low(float("nan"))