diff --git a/machine/quality_estimation/__init__.py b/machine/quality_estimation/__init__.py index 9b90e8f3..357bec91 100644 --- a/machine/quality_estimation/__init__.py +++ b/machine/quality_estimation/__init__.py @@ -1,3 +1,4 @@ +from .book_confidence import LOW_BOOK_CONFIDENCE_THRESHOLD, is_book_confidence_unusually_low from .chrf3_quality_estimator import ChrF3QualityEstimator from .scripture_book_usability import ScriptureBookUsability from .scripture_chapter_usability import ScriptureChapterUsability @@ -11,6 +12,8 @@ __all__ = [ "ChrF3QualityEstimator", + "is_book_confidence_unusually_low", + "LOW_BOOK_CONFIDENCE_THRESHOLD", "ScriptureBookUsability", "ScriptureChapterUsability", "ScriptureSegmentUsability", diff --git a/machine/quality_estimation/book_confidence.py b/machine/quality_estimation/book_confidence.py new file mode 100644 index 00000000..42f5fd30 --- /dev/null +++ b/machine/quality_estimation/book_confidence.py @@ -0,0 +1,15 @@ +from typing import Optional + +LOW_BOOK_CONFIDENCE_THRESHOLD = 0.42 + + +def is_book_confidence_unusually_low( + confidence: float, book_id: Optional[str] = None, model: Optional[str] = None +) -> bool: + if not 0 <= confidence <= 1: + raise ValueError( + f"The book confidence {confidence} is invalid. " + f"It is calculated as the geometric mean of the segment confidences, " + f"and it must be between 0 and 1, inclusive." + ) + return confidence < LOW_BOOK_CONFIDENCE_THRESHOLD diff --git a/tests/quality_estimation/test_book_confidence.py b/tests/quality_estimation/test_book_confidence.py new file mode 100644 index 00000000..cc5e8e98 --- /dev/null +++ b/tests/quality_estimation/test_book_confidence.py @@ -0,0 +1,35 @@ +from pytest import raises + +from machine.quality_estimation import LOW_BOOK_CONFIDENCE_THRESHOLD, is_book_confidence_unusually_low + + +def test_is_book_confidence_unusually_low_at_threshold() -> None: + assert not is_book_confidence_unusually_low(LOW_BOOK_CONFIDENCE_THRESHOLD) + + +def test_is_book_confidence_unusually_low_min_confidence() -> None: + assert is_book_confidence_unusually_low(0.0) + + +def test_is_book_confidence_unusually_low_max_confidence() -> None: + assert not is_book_confidence_unusually_low(1.0) + + +def test_is_book_confidence_unusually_low_with_book_id_and_model() -> None: + assert is_book_confidence_unusually_low(0.3, "MAT", "facebook/nllb-200-distilled-1.3B") + assert not is_book_confidence_unusually_low(0.9, "MAT", "facebook/nllb-200-distilled-1.3B") + + +def test_is_book_confidence_unusually_low_negative() -> None: + with raises(ValueError): + is_book_confidence_unusually_low(-0.5) + + +def test_is_book_confidence_unusually_low_greater_than_one() -> None: + with raises(ValueError): + is_book_confidence_unusually_low(1.5) + + +def test_is_book_confidence_unusually_low_nan() -> None: + with raises(ValueError): + is_book_confidence_unusually_low(float("nan"))