Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 40 additions & 0 deletions src/SIL.Machine/QualityEstimation/BookConfidence.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
using System;

namespace SIL.Machine.QualityEstimation
{
public static class BookConfidence
{
public static readonly double LowBookConfidenceThreshold = 0.42;

/// <summary>
/// Determines whether a book confidence is unusually low, i.e. below
/// <see cref="LowBookConfidenceThreshold"/>.
/// </summary>
/// <param name="confidence">
/// The book confidence, calculated as the geometric mean of the segment confidences. Must be between 0 and 1,
/// inclusive.
/// </param>
/// <param name="bookId">The book id. Reserved for future book-specific logic; not currently used.</param>
/// <param name="model">The model name. Reserved for future model-specific logic; not currently used.</param>
/// <returns>
/// <c>true</c> if <paramref name="confidence"/> is below <see cref="LowBookConfidenceThreshold"/>; otherwise,
/// <c>false</c>.
/// </returns>
/// <exception cref="ArgumentOutOfRangeException">
/// <paramref name="confidence"/> is not between 0 and 1, inclusive.
/// </exception>
public static bool IsBookConfidenceUnusuallyLow(double confidence, string bookId = null, string model = null)
{
if (!(confidence >= 0 && confidence <= 1))
{
throw new ArgumentOutOfRangeException(
nameof(confidence),
confidence,
"The book confidence must be between 0 and 1, inclusive."
);
}

return confidence < LowBookConfidenceThreshold;
}
}
}
59 changes: 59 additions & 0 deletions tests/SIL.Machine.Tests/QualityEstimation/BookConfidenceTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
using NUnit.Framework;

namespace SIL.Machine.QualityEstimation;

[TestFixture]
public class BookConfidenceTests
{
[Test]
public void IsBookConfidenceUnusuallyLow_AtThreshold()
{
Assert.That(BookConfidence.IsBookConfidenceUnusuallyLow(BookConfidence.LowBookConfidenceThreshold), Is.False);
}

[Test]
public void IsBookConfidenceUnusuallyLow_MinConfidence()
{
Assert.That(BookConfidence.IsBookConfidenceUnusuallyLow(0.0), Is.True);
}

[Test]
public void IsBookConfidenceUnusuallyLow_MaxConfidence()
{
Assert.That(BookConfidence.IsBookConfidenceUnusuallyLow(1.0), Is.False);
}

[Test]
public void IsBookConfidenceUnusuallyLow_WithBookIdAndModel()
{
using (Assert.EnterMultipleScope())
{
Assert.That(
BookConfidence.IsBookConfidenceUnusuallyLow(0.3, "MAT", "facebook/nllb-200-distilled-1.3B"),
Is.True
);
Assert.That(
BookConfidence.IsBookConfidenceUnusuallyLow(0.9, "MAT", "facebook/nllb-200-distilled-1.3B"),
Is.False
);
}
}

[Test]
public void IsBookConfidenceUnusuallyLow_Negative()
{
Assert.Throws<ArgumentOutOfRangeException>(() => BookConfidence.IsBookConfidenceUnusuallyLow(-0.5));
}

[Test]
public void IsBookConfidenceUnusuallyLow_GreaterThanOne()
{
Assert.Throws<ArgumentOutOfRangeException>(() => BookConfidence.IsBookConfidenceUnusuallyLow(1.5));
}

[Test]
public void IsBookConfidenceUnusuallyLow_Nan()
{
Assert.Throws<ArgumentOutOfRangeException>(() => BookConfidence.IsBookConfidenceUnusuallyLow(double.NaN));
}
}
Loading