diff --git a/src/SIL.Machine/QualityEstimation/BookConfidence.cs b/src/SIL.Machine/QualityEstimation/BookConfidence.cs new file mode 100644 index 00000000..c9567fef --- /dev/null +++ b/src/SIL.Machine/QualityEstimation/BookConfidence.cs @@ -0,0 +1,40 @@ +using System; + +namespace SIL.Machine.QualityEstimation +{ + public static class BookConfidence + { + public static readonly double LowBookConfidenceThreshold = 0.42; + + /// + /// Determines whether a book confidence is unusually low, i.e. below + /// . + /// + /// + /// The book confidence, calculated as the geometric mean of the segment confidences. Must be between 0 and 1, + /// inclusive. + /// + /// The book id. Reserved for future book-specific logic; not currently used. + /// The model name. Reserved for future model-specific logic; not currently used. + /// + /// true if is below ; otherwise, + /// false. + /// + /// + /// is not between 0 and 1, inclusive. + /// + public static bool IsBookConfidenceUnusuallyLow(double confidence, string bookId = null, string model = null) + { + if (!(confidence >= 0 && confidence <= 1)) + { + throw new ArgumentOutOfRangeException( + nameof(confidence), + confidence, + "The book confidence must be between 0 and 1, inclusive." + ); + } + + return confidence < LowBookConfidenceThreshold; + } + } +} diff --git a/tests/SIL.Machine.Tests/QualityEstimation/BookConfidenceTests.cs b/tests/SIL.Machine.Tests/QualityEstimation/BookConfidenceTests.cs new file mode 100644 index 00000000..145758fc --- /dev/null +++ b/tests/SIL.Machine.Tests/QualityEstimation/BookConfidenceTests.cs @@ -0,0 +1,59 @@ +using NUnit.Framework; + +namespace SIL.Machine.QualityEstimation; + +[TestFixture] +public class BookConfidenceTests +{ + [Test] + public void IsBookConfidenceUnusuallyLow_AtThreshold() + { + Assert.That(BookConfidence.IsBookConfidenceUnusuallyLow(BookConfidence.LowBookConfidenceThreshold), Is.False); + } + + [Test] + public void IsBookConfidenceUnusuallyLow_MinConfidence() + { + Assert.That(BookConfidence.IsBookConfidenceUnusuallyLow(0.0), Is.True); + } + + [Test] + public void IsBookConfidenceUnusuallyLow_MaxConfidence() + { + Assert.That(BookConfidence.IsBookConfidenceUnusuallyLow(1.0), Is.False); + } + + [Test] + public void IsBookConfidenceUnusuallyLow_WithBookIdAndModel() + { + using (Assert.EnterMultipleScope()) + { + Assert.That( + BookConfidence.IsBookConfidenceUnusuallyLow(0.3, "MAT", "facebook/nllb-200-distilled-1.3B"), + Is.True + ); + Assert.That( + BookConfidence.IsBookConfidenceUnusuallyLow(0.9, "MAT", "facebook/nllb-200-distilled-1.3B"), + Is.False + ); + } + } + + [Test] + public void IsBookConfidenceUnusuallyLow_Negative() + { + Assert.Throws(() => BookConfidence.IsBookConfidenceUnusuallyLow(-0.5)); + } + + [Test] + public void IsBookConfidenceUnusuallyLow_GreaterThanOne() + { + Assert.Throws(() => BookConfidence.IsBookConfidenceUnusuallyLow(1.5)); + } + + [Test] + public void IsBookConfidenceUnusuallyLow_Nan() + { + Assert.Throws(() => BookConfidence.IsBookConfidenceUnusuallyLow(double.NaN)); + } +}