From d6ccce134a567fcad446ddf0fa0b61a3766a27ae Mon Sep 17 00:00:00 2001 From: mshannon-sil Date: Mon, 17 Aug 2026 15:34:16 -0400 Subject: [PATCH 1/2] add low book confidence detector --- machine/quality_estimation/__init__.py | 3 ++ machine/quality_estimation/book_confidence.py | 11 +++++++ .../test_book_confidence.py | 30 +++++++++++++++++++ 3 files changed, 44 insertions(+) create mode 100644 machine/quality_estimation/book_confidence.py create mode 100644 tests/quality_estimation/test_book_confidence.py diff --git a/machine/quality_estimation/__init__.py b/machine/quality_estimation/__init__.py index 9b90e8f3..357bec91 100644 --- a/machine/quality_estimation/__init__.py +++ b/machine/quality_estimation/__init__.py @@ -1,3 +1,4 @@ +from .book_confidence import LOW_BOOK_CONFIDENCE_THRESHOLD, is_book_confidence_unusually_low from .chrf3_quality_estimator import ChrF3QualityEstimator from .scripture_book_usability import ScriptureBookUsability from .scripture_chapter_usability import ScriptureChapterUsability @@ -11,6 +12,8 @@ __all__ = [ "ChrF3QualityEstimator", + "is_book_confidence_unusually_low", + "LOW_BOOK_CONFIDENCE_THRESHOLD", "ScriptureBookUsability", "ScriptureChapterUsability", "ScriptureSegmentUsability", diff --git a/machine/quality_estimation/book_confidence.py b/machine/quality_estimation/book_confidence.py new file mode 100644 index 00000000..39df8b0f --- /dev/null +++ b/machine/quality_estimation/book_confidence.py @@ -0,0 +1,11 @@ +LOW_BOOK_CONFIDENCE_THRESHOLD = 0.42 + + +def is_book_confidence_unusually_low(confidence: float) -> bool: + if not 0 <= confidence <= 1: + raise ValueError( + f"The book confidence {confidence} is invalid. " + f"It is calculated as the geometric mean of the segment confidences, " + f"and it must be between 0 and 1, inclusive." + ) + return confidence < LOW_BOOK_CONFIDENCE_THRESHOLD diff --git a/tests/quality_estimation/test_book_confidence.py b/tests/quality_estimation/test_book_confidence.py new file mode 100644 index 00000000..fced19a9 --- /dev/null +++ b/tests/quality_estimation/test_book_confidence.py @@ -0,0 +1,30 @@ +from pytest import raises + +from machine.quality_estimation import LOW_BOOK_CONFIDENCE_THRESHOLD, is_book_confidence_unusually_low + + +def test_is_book_confidence_unusually_low_at_threshold() -> None: + assert not is_book_confidence_unusually_low(LOW_BOOK_CONFIDENCE_THRESHOLD) + + +def test_is_book_confidence_unusually_low_min_confidence() -> None: + assert is_book_confidence_unusually_low(0.0) + + +def test_is_book_confidence_unusually_low_max_confidence() -> None: + assert not is_book_confidence_unusually_low(1.0) + + +def test_is_book_confidence_unusually_low_negative() -> None: + with raises(ValueError): + is_book_confidence_unusually_low(-0.5) + + +def test_is_book_confidence_unusually_low_greater_than_one() -> None: + with raises(ValueError): + is_book_confidence_unusually_low(1.5) + + +def test_is_book_confidence_unusually_low_nan() -> None: + with raises(ValueError): + is_book_confidence_unusually_low(float("nan")) From 098f83bfed1a562d1bc9bd9c0154f93688e9520d Mon Sep 17 00:00:00 2001 From: mshannon-sil Date: Mon, 17 Aug 2026 18:26:56 -0400 Subject: [PATCH 2/2] match serval function signature --- machine/quality_estimation/book_confidence.py | 6 +++++- tests/quality_estimation/test_book_confidence.py | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/machine/quality_estimation/book_confidence.py b/machine/quality_estimation/book_confidence.py index 39df8b0f..42f5fd30 100644 --- a/machine/quality_estimation/book_confidence.py +++ b/machine/quality_estimation/book_confidence.py @@ -1,7 +1,11 @@ +from typing import Optional + LOW_BOOK_CONFIDENCE_THRESHOLD = 0.42 -def is_book_confidence_unusually_low(confidence: float) -> bool: +def is_book_confidence_unusually_low( + confidence: float, book_id: Optional[str] = None, model: Optional[str] = None +) -> bool: if not 0 <= confidence <= 1: raise ValueError( f"The book confidence {confidence} is invalid. " diff --git a/tests/quality_estimation/test_book_confidence.py b/tests/quality_estimation/test_book_confidence.py index fced19a9..cc5e8e98 100644 --- a/tests/quality_estimation/test_book_confidence.py +++ b/tests/quality_estimation/test_book_confidence.py @@ -15,6 +15,11 @@ def test_is_book_confidence_unusually_low_max_confidence() -> None: assert not is_book_confidence_unusually_low(1.0) +def test_is_book_confidence_unusually_low_with_book_id_and_model() -> None: + assert is_book_confidence_unusually_low(0.3, "MAT", "facebook/nllb-200-distilled-1.3B") + assert not is_book_confidence_unusually_low(0.9, "MAT", "facebook/nllb-200-distilled-1.3B") + + def test_is_book_confidence_unusually_low_negative() -> None: with raises(ValueError): is_book_confidence_unusually_low(-0.5)