From 4b4bba06dfbb9bf866356956d03abcdbce84f20f Mon Sep 17 00:00:00 2001
From: Enkidu93 <lowryec17@gcc.edu>
Date: Tue, 21 Jul 2026 16:25:42 -0400
Subject: [PATCH 01/24] Use native eflomal implementation

---
 machine/corpora/aligned_word_pair.py     |   2 +-
 machine/jobs/eflomal_aligner.py          | 167 -----------------------
 machine/jobs/nmt_engine_build_job.py     |  92 ++++++++++---
 machine/jobs/settings.yaml               |   3 +-
 machine/jobs/translation_file_service.py |   2 +
 machine/jobs/word_alignment_build_job.py |   3 +-
 6 files changed, 77 insertions(+), 192 deletions(-)
 delete mode 100644 machine/jobs/eflomal_aligner.py

diff --git a/machine/corpora/aligned_word_pair.py b/machine/corpora/aligned_word_pair.py
index a18251fd..326f84a0 100644
--- a/machine/corpora/aligned_word_pair.py
+++ b/machine/corpora/aligned_word_pair.py
@@ -71,7 +71,7 @@ def format_score(score: float) -> str:
         source_index = "NULL" if self.source_index < 0 else str(self.source_index)
         target_index = "NULL" if self.target_index < 0 else str(self.target_index)
         repr = f"{source_index}-{target_index}"
-        if include_scores and self.translation_score >= 0:
+        if include_scores and (self.translation_score >= 0 or self.alignment_score >= 0):
             repr += f":{format_score(self.translation_score)}"
             if self.alignment_score >= 0:
                 repr += f":{format_score(self.alignment_score)}"
diff --git a/machine/jobs/eflomal_aligner.py b/machine/jobs/eflomal_aligner.py
deleted file mode 100644
index eea6b2f4..00000000
--- a/machine/jobs/eflomal_aligner.py
+++ /dev/null
@@ -1,167 +0,0 @@
-# NOTE: this is a temporary solution to be able to use the eflomal aligner inside of machine.py.
-# The vast majority of this code is taken from the silnlp repository.
-
-import os
-import subprocess
-from contextlib import ExitStack
-from importlib.util import find_spec
-from math import sqrt
-from pathlib import Path
-from tempfile import TemporaryDirectory
-from typing import IO, Iterable, List, Sequence, Tuple
-
-from ..corpora import AlignedWordPair
-from ..corpora.token_processors import escape_spaces, lowercase, normalize
-from ..tokenization import LatinWordTokenizer
-from ..translation import SymmetrizationHeuristic, WordAlignmentMatrix
-
-
-# From silnlp.common.package_utils
-def is_eflomal_available() -> bool:
-    return find_spec("eflomal") is not None
-
-
-if is_eflomal_available():
-    from eflomal import read_text, write_text  # type: ignore
-
-EFLOMAL_PATH = Path(os.getenv("EFLOMAL_PATH", "."), "eflomal")
-TOKENIZER = LatinWordTokenizer()
-
-
-# From silnlp.alignment.tools
-def execute_eflomal(
-    source_path: Path,
-    target_path: Path,
-    forward_links_path: Path,
-    reverse_links_path: Path,
-    n_iterations: Tuple[int, int, int],
-) -> None:
-    if not is_eflomal_available():
-        raise RuntimeError("eflomal is not installed.")
-
-    args = [
-        str(EFLOMAL_PATH),
-        "-s",
-        str(source_path),
-        "-t",
-        str(target_path),
-        "-f",
-        str(forward_links_path),
-        "-r",
-        str(reverse_links_path),
-        # "-q",
-        "-m",
-        "3",
-        "-n",
-        "3",
-        "-N",
-        "0.2",
-        "-1",
-        str(n_iterations[0]),
-        "-2",
-        str(n_iterations[1]),
-        "-3",
-        str(n_iterations[2]),
-    ]
-    subprocess.run(args, stderr=subprocess.DEVNULL)
-
-
-# From silnlp.alignment.eflomal
-def to_word_alignment_matrix(alignment_str: str) -> WordAlignmentMatrix:
-    word_pairs = AlignedWordPair.from_string(alignment_str)
-    row_count = 0
-    column_count = 0
-    for pair in word_pairs:
-        if pair.source_index + 1 > row_count:
-            row_count = pair.source_index + 1
-        if pair.target_index + 1 > column_count:
-            column_count = pair.target_index + 1
-    return WordAlignmentMatrix.from_word_pairs(row_count, column_count, word_pairs)
-
-
-# From silnlp.alignment.eflomal
-def to_eflomal_text_file(input: Iterable[str], output_file: IO[bytes], prefix_len: int = 0, suffix_len: int = 0) -> int:
-    sents, index = read_text(input, True, prefix_len, suffix_len)
-    n_sents = len(sents)
-    voc_size = len(index)
-    write_text(output_file, tuple(sents), voc_size)
-    return n_sents
-
-
-# From silnlp.alignment.eflomal
-def prepare_files(
-    src_input: Iterable[str], src_output_file: IO[bytes], trg_input: Iterable[str], trg_output_file: IO[bytes]
-) -> int:
-    n_src_sents = to_eflomal_text_file(src_input, src_output_file)
-    n_trg_sents = to_eflomal_text_file(trg_input, trg_output_file)
-    if n_src_sents != n_trg_sents:
-        raise ValueError("Mismatched file sizes")
-    return n_src_sents
-
-
-def tokenize(sent: str) -> Sequence[str]:
-    return list(TOKENIZER.tokenize(sent))
-
-
-def normalize_for_alignment(sent: Sequence[str]) -> str:
-    return " ".join(lowercase(normalize("NFC", escape_spaces(sent))))
-
-
-# From silnlp.alignment.eflomal
-class EflomalAligner:
-    def __init__(self, model_dir: Path) -> None:
-        self._model_dir = model_dir
-
-    def train(self, src_toks: Sequence[Sequence[str]], trg_toks: Sequence[Sequence[str]]) -> None:
-        self._model_dir.mkdir(exist_ok=True)
-        with TemporaryDirectory() as temp_dir:
-            src_eflomal_path = Path(temp_dir, "source")
-            trg_eflomal_path = Path(temp_dir, "target")
-            with ExitStack() as stack:
-                src_output_file = stack.enter_context(src_eflomal_path.open("wb"))
-                trg_output_file = stack.enter_context(trg_eflomal_path.open("wb"))
-                # Write input files for the eflomal binary
-                n_sentences = prepare_files(
-                    [normalize_for_alignment(s) for s in src_toks],
-                    src_output_file,
-                    [normalize_for_alignment(s) for s in trg_toks],
-                    trg_output_file,
-                )
-
-            iters = max(2, int(round(1.0 * 5000 / sqrt(n_sentences))) if n_sentences > 0 else 0)
-            iters4 = max(1, iters // 4)
-            n_iterations = (max(2, iters4), iters4, iters)
-
-            # Run wrapper for the eflomal binary
-            execute_eflomal(
-                src_eflomal_path,
-                trg_eflomal_path,
-                self._model_dir / "forward-align.txt",
-                self._model_dir / "reverse-align.txt",
-                n_iterations,
-            )
-
-    def align(self, sym_heuristic: str = "grow-diag-final-and") -> List[str]:
-        forward_align_path = self._model_dir / "forward-align.txt"
-        reverse_align_path = self._model_dir / "reverse-align.txt"
-
-        alignments = []
-        heuristic = SymmetrizationHeuristic[sym_heuristic.upper().replace("-", "_")]
-        with ExitStack() as stack:
-            forward_file = stack.enter_context(forward_align_path.open("r", encoding="utf-8-sig"))
-            reverse_file = stack.enter_context(reverse_align_path.open("r", encoding="utf-8-sig"))
-
-            for forward_line, reverse_line in zip(forward_file, reverse_file):
-                forward_matrix = to_word_alignment_matrix(forward_line.strip())
-                reverse_matrix = to_word_alignment_matrix(reverse_line.strip())
-                src_len = max(forward_matrix.row_count, reverse_matrix.row_count)
-                trg_len = max(forward_matrix.column_count, reverse_matrix.column_count)
-
-                forward_matrix.resize(src_len, trg_len)
-                reverse_matrix.resize(src_len, trg_len)
-
-                forward_matrix.symmetrize_with(reverse_matrix, heuristic)
-
-                alignments.append(str(forward_matrix))
-
-        return alignments
diff --git a/machine/jobs/nmt_engine_build_job.py b/machine/jobs/nmt_engine_build_job.py
index cf4020d6..ab39825f 100644
--- a/machine/jobs/nmt_engine_build_job.py
+++ b/machine/jobs/nmt_engine_build_job.py
@@ -1,15 +1,25 @@
 import logging
 from contextlib import ExitStack
-from pathlib import Path
-from tempfile import TemporaryDirectory
 from typing import Any, Callable, Optional, Sequence, Tuple
 
+from ..corpora.aligned_word_pair import AlignedWordPair
+
+from ..corpora.text_row import TextRow
+
+from ..corpora.memory_text import MemoryText
+
+from ..corpora.dictionary_text_corpus import DictionaryTextCorpus
+from itertools import chain
+
+from .config import SETTINGS
+from .thot.thot_word_alignment_model_factory import ThotWordAlignmentModelFactory
+from ..tokenization.tokenizer_factory import create_tokenizer
+
 from ..corpora.corpora_utils import batch
 from ..corpora.parallel_text_corpus import ParallelTextCorpus
 from ..corpora.text_corpus import TextCorpus
 from ..utils.phased_progress_reporter import Phase, PhasedProgressReporter
 from ..utils.progress_status import ProgressStatus
-from .eflomal_aligner import EflomalAligner, is_eflomal_available, tokenize
 from .nmt_model_factory import NmtModelFactory
 from .translation_engine_build_job import TranslationEngineBuildJob
 from .translation_file_service import PretranslationInfo, TranslationFileService
@@ -94,6 +104,7 @@ def _train_model(
 
     def _batch_inference(
         self,
+        parallel_training_corpus: ParallelTextCorpus,
         progress_reporter: PhasedProgressReporter,
         check_canceled: Optional[Callable[[], None]],
     ) -> None:
@@ -119,9 +130,11 @@ def _batch_inference(
                 current_inference_step += len(seg_batch)
                 phase_progress(ProgressStatus.from_step(current_inference_step, inference_step_count))
 
-            if self._config.align_pretranslations and is_eflomal_available():
+            if self._config.align_pretranslations:
                 logger.info("Aligning source to pretranslations")
-                pretranslations = self._align(src_segments, pretranslations, progress_reporter, check_canceled)
+                pretranslations = self._align(
+                    src_segments, pretranslations, parallel_training_corpus, progress_reporter, check_canceled
+                )
 
             writer = stack.enter_context(self._translation_file_service.open_target_pretranslation_writer())
             for pretranslation in pretranslations:
@@ -129,8 +142,9 @@ def _batch_inference(
 
     def _align(
         self,
-        src_segments: Sequence[str],
+        src_pretranslate_segments: Sequence[str],
         pretranslations: Sequence[PretranslationInfo],
+        parallel_training_corpus: ParallelTextCorpus,
         progress_reporter: PhasedProgressReporter,
         check_canceled: Optional[Callable[[], None]],
     ) -> Sequence[PretranslationInfo]:
@@ -138,29 +152,67 @@ def _align(
             check_canceled()
 
         logger.info("Aligning source to pretranslations")
-        progress_reporter.start_next_phase()
 
-        src_tokenized = [tokenize(s) for s in src_segments]
-        trg_tokenized = [tokenize(pt_info["translation"]) for pt_info in pretranslations]
+        alignment_model_factory = ThotWordAlignmentModelFactory(SETTINGS)
 
-        with TemporaryDirectory() as td:
-            aligner = EflomalAligner(Path(td))
-            logger.info("Training aligner")
-            aligner.train(src_tokenized, trg_tokenized)
+        tokenizer = create_tokenizer(SETTINGS.thot_align.tokenizer)
 
-            if check_canceled is not None:
-                check_canceled()
+        source_inference_corpus = DictionaryTextCorpus(
+            MemoryText(
+                "pretranslations",
+                [
+                    TextRow("pretranslations", i, list(tokenizer.tokenize(segment)))
+                    for i, segment in enumerate(src_pretranslate_segments)
+                ],
+            )
+        )
+        target_inference_corpus = DictionaryTextCorpus(
+            MemoryText(
+                "pretranslations",
+                [
+                    TextRow("pretranslations", i, list(tokenizer.tokenize(pretranslation["translation"])))
+                    for i, pretranslation in enumerate(pretranslations)
+                ],
+            )
+        )
 
-            logger.info("Aligning pretranslations")
-            alignments = aligner.align()
+        parallel_pretranslation_rows = list(source_inference_corpus.align_rows(target_inference_corpus))
+
+        alignment_parallel_corpus = ParallelTextCorpus.from_parallel_rows(
+            chain(
+                parallel_pretranslation_rows,
+                parallel_training_corpus.get_rows(),
+            )
+        )
+
+        logger.info("Training aligner")
+        with (
+            progress_reporter.start_next_phase() as phase_progress,
+            alignment_model_factory.create_model_trainer(tokenizer, alignment_parallel_corpus) as trainer,
+        ):
+            trainer.train(progress=phase_progress, check_canceled=check_canceled)
+            trainer.save()
+
+        logger.info("Aligning pretranslations")
+        alignment_model = alignment_model_factory.create_alignment_model()
+
+        alignments = alignment_model.align_batch(parallel_pretranslation_rows)
+
+        all_word_pairs = []
+        for parallel_text_row, alignment in zip(parallel_pretranslation_rows, alignments, strict=True):
+            word_pairs = alignment.to_aligned_word_pairs(include_null=True)
+            alignment_model.compute_aligned_word_pair_scores(
+                parallel_text_row.source_segment, parallel_text_row.target_segment, word_pairs
+            )
+            all_word_pairs.append(word_pairs)
 
         if check_canceled is not None:
             check_canceled()
 
         for i in range(len(pretranslations)):
-            pretranslations[i]["sourceTokens"] = list(src_tokenized[i])
-            pretranslations[i]["translationTokens"] = list(trg_tokenized[i])
-            pretranslations[i]["alignment"] = alignments[i]
+            pretranslations[i]["sourceTokens"] = list(parallel_pretranslation_rows[i].source_segment)
+            pretranslations[i]["translationTokens"] = list(parallel_pretranslation_rows[i].target_segment)
+            pretranslations[i]["alignment"] = AlignedWordPair.to_string(all_word_pairs[i], include_scores=True)
 
         return pretranslations
 
diff --git a/machine/jobs/settings.yaml b/machine/jobs/settings.yaml
index b1ee5402..47b7548f 100644
--- a/machine/jobs/settings.yaml
+++ b/machine/jobs/settings.yaml
@@ -2,7 +2,6 @@ default:
   data_dir: ~/machine
   shared_file_uri: s3:/silnlp/
   shared_file_folder: production
-  inference_batch_size: 1024
   align_pretranslations: true
   huggingface:
     parent_model_name: facebook/nllb-200-distilled-1.3B
@@ -35,7 +34,7 @@ default:
     tokenizer: latin
   thot_align:
     word_alignment_heuristic: grow-diag-final-and
-    model_type: hmm
+    model_type: eflomal
     tokenizer: latin
 development:
   shared_file_folder: dev
diff --git a/machine/jobs/translation_file_service.py b/machine/jobs/translation_file_service.py
index 3472aa80..90c7363c 100644
--- a/machine/jobs/translation_file_service.py
+++ b/machine/jobs/translation_file_service.py
@@ -22,6 +22,7 @@ class PretranslationInfo(TypedDict):
     translationTokens: List[str]  # noqa: N815
     alignment: str
     sequenceConfidence: float  # noqa: N815
+    alignmentScore: float
 
 
 class TranslationFileService:
@@ -102,6 +103,7 @@ def generator() -> Generator[PretranslationInfo, None, None]:
                         translationTokens=list(),
                         alignment="",
                         sequenceConfidence=0,
+                        alignmentScore=0,
                     )
 
         return ContextManagedGenerator(generator())
diff --git a/machine/jobs/word_alignment_build_job.py b/machine/jobs/word_alignment_build_job.py
index 8838ddb3..842151a8 100644
--- a/machine/jobs/word_alignment_build_job.py
+++ b/machine/jobs/word_alignment_build_job.py
@@ -99,7 +99,6 @@ def _batch_inference(
             writer = stack.enter_context(self._word_alignment_file_service.open_alignment_output_writer())
             current_inference_step = 0
             phase_progress(ProgressStatus.from_step(current_inference_step, inference_step_count))
-            batch_size = self._config["inference_batch_size"]
 
             parallel_corpus = ParallelTextCorpus.from_parallel_rows(
                 [
@@ -114,7 +113,7 @@ def _batch_inference(
                 ]
             ).lowercase()
 
-            segment_batch = list(parallel_corpus.take(batch_size))
+            segment_batch = list(parallel_corpus)
             if check_canceled is not None:
                 check_canceled()
             alignments = alignment_model.align_batch(segment_batch)

From 718b06777fd94f8e317ac67e165d2c6f99ab0c75 Mon Sep 17 00:00:00 2001
From: Enkidu93 <lowryec17@gcc.edu>
Date: Tue, 21 Jul 2026 16:28:05 -0400
Subject: [PATCH 02/24] Remove redundant alignment score

---
 machine/jobs/translation_file_service.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/machine/jobs/translation_file_service.py b/machine/jobs/translation_file_service.py
index 90c7363c..3472aa80 100644
--- a/machine/jobs/translation_file_service.py
+++ b/machine/jobs/translation_file_service.py
@@ -22,7 +22,6 @@ class PretranslationInfo(TypedDict):
     translationTokens: List[str]  # noqa: N815
     alignment: str
     sequenceConfidence: float  # noqa: N815
-    alignmentScore: float
 
 
 class TranslationFileService:
@@ -103,7 +102,6 @@ def generator() -> Generator[PretranslationInfo, None, None]:
                         translationTokens=list(),
                         alignment="",
                         sequenceConfidence=0,
-                        alignmentScore=0,
                     )
 
         return ContextManagedGenerator(generator())

From a65e9462f55fae7e00cf5331ec2f035b25e10a28 Mon Sep 17 00:00:00 2001
From: Enkidu93 <lowryec17@gcc.edu>
Date: Wed, 22 Jul 2026 14:22:22 -0400
Subject: [PATCH 03/24] Fix api usage; adjust test

---
 machine/jobs/nmt_engine_build_job.py         | 28 ++++++-------
 machine/jobs/smt_engine_build_job.py         |  1 +
 machine/jobs/translation_engine_build_job.py |  3 +-
 tests/jobs/test_nmt_engine_build_job.py      | 41 ++++++++++----------
 4 files changed, 36 insertions(+), 37 deletions(-)

diff --git a/machine/jobs/nmt_engine_build_job.py b/machine/jobs/nmt_engine_build_job.py
index ab39825f..6c18ecfe 100644
--- a/machine/jobs/nmt_engine_build_job.py
+++ b/machine/jobs/nmt_engine_build_job.py
@@ -1,26 +1,21 @@
 import logging
 from contextlib import ExitStack
+from itertools import chain
 from typing import Any, Callable, Optional, Sequence, Tuple
 
 from ..corpora.aligned_word_pair import AlignedWordPair
-
-from ..corpora.text_row import TextRow
-
-from ..corpora.memory_text import MemoryText
-
-from ..corpora.dictionary_text_corpus import DictionaryTextCorpus
-from itertools import chain
-
-from .config import SETTINGS
-from .thot.thot_word_alignment_model_factory import ThotWordAlignmentModelFactory
-from ..tokenization.tokenizer_factory import create_tokenizer
-
 from ..corpora.corpora_utils import batch
+from ..corpora.dictionary_text_corpus import DictionaryTextCorpus
+from ..corpora.memory_text import MemoryText
 from ..corpora.parallel_text_corpus import ParallelTextCorpus
 from ..corpora.text_corpus import TextCorpus
+from ..corpora.text_row import TextRow
+from ..tokenization.tokenizer_factory import create_tokenizer
 from ..utils.phased_progress_reporter import Phase, PhasedProgressReporter
 from ..utils.progress_status import ProgressStatus
+from .config import SETTINGS
 from .nmt_model_factory import NmtModelFactory
+from .thot.thot_word_alignment_model_factory import ThotWordAlignmentModelFactory
 from .translation_engine_build_job import TranslationEngineBuildJob
 from .translation_file_service import PretranslationInfo, TranslationFileService
 
@@ -153,9 +148,9 @@ def _align(
 
         logger.info("Aligning source to pretranslations")
 
-        alignment_model_factory = ThotWordAlignmentModelFactory(SETTINGS)
+        alignment_model_factory = ThotWordAlignmentModelFactory(self._config)
 
-        tokenizer = create_tokenizer(SETTINGS.thot_align.tokenizer)
+        tokenizer = create_tokenizer(self._config.thot_align.tokenizer)
 
         source_inference_corpus = DictionaryTextCorpus(
             MemoryText(
@@ -176,11 +171,11 @@ def _align(
             )
         )
 
-        parallel_pretranslation_rows = list(source_inference_corpus.align_rows(target_inference_corpus))
+        parallel_pretranslation_rows = source_inference_corpus.align_rows(target_inference_corpus)
 
         alignment_parallel_corpus = ParallelTextCorpus.from_parallel_rows(
             chain(
-                parallel_pretranslation_rows,
+                parallel_pretranslation_rows.get_rows(),
                 parallel_training_corpus.get_rows(),
             )
         )
@@ -196,6 +191,7 @@ def _align(
         logger.info("Aligning pretranslations")
         alignment_model = alignment_model_factory.create_alignment_model()
 
+        parallel_pretranslation_rows = list(parallel_pretranslation_rows)
         alignments = alignment_model.align_batch(parallel_pretranslation_rows)
 
         all_word_pairs = []
diff --git a/machine/jobs/smt_engine_build_job.py b/machine/jobs/smt_engine_build_job.py
index 286e5425..370a7180 100644
--- a/machine/jobs/smt_engine_build_job.py
+++ b/machine/jobs/smt_engine_build_job.py
@@ -71,6 +71,7 @@ def _train_model(
 
     def _batch_inference(
         self,
+        parallel_training_corpus: ParallelTextCorpus,
         progress_reporter: PhasedProgressReporter,
         check_canceled: Optional[Callable[[], None]],
     ) -> None:
diff --git a/machine/jobs/translation_engine_build_job.py b/machine/jobs/translation_engine_build_job.py
index 7effa62f..cf796d87 100644
--- a/machine/jobs/translation_engine_build_job.py
+++ b/machine/jobs/translation_engine_build_job.py
@@ -42,7 +42,7 @@ def run(
             check_canceled()
 
         logger.info("Pretranslating segments")
-        self._batch_inference(progress_reporter, check_canceled)
+        self._batch_inference(parallel_corpus, progress_reporter, check_canceled)
 
         self._save_model()
         return train_corpus_size, confidence
@@ -70,6 +70,7 @@ def _train_model(
     @abstractmethod
     def _batch_inference(
         self,
+        parallel_training_corpus: ParallelTextCorpus,
         progress_reporter: PhasedProgressReporter,
         check_canceled: Optional[Callable[[], None]],
     ) -> None: ...
diff --git a/tests/jobs/test_nmt_engine_build_job.py b/tests/jobs/test_nmt_engine_build_job.py
index e3524f8b..0c48ee18 100644
--- a/tests/jobs/test_nmt_engine_build_job.py
+++ b/tests/jobs/test_nmt_engine_build_job.py
@@ -17,7 +17,6 @@
     PretranslationInfo,
     TranslationFileService,
 )
-from machine.jobs.eflomal_aligner import is_eflomal_available
 from machine.translation import (
     Phrase,
     Trainer,
@@ -37,25 +36,19 @@ def test_run(decoy: Decoy) -> None:
     pretranslations = json.loads(env.target_pretranslations)
     assert len(pretranslations) == 1
     assert pretranslations[0]["translation"] == "Please, I have booked a room."
-    if is_eflomal_available():
-        assert pretranslations[0]["sourceTokens"] == [
-            "Por",
-            "favor",
-            ",",
-            "tengo",
-            "reservada",
-            "una",
-            "habitación",
-            ".",
-        ]
-        assert pretranslations[0]["translationTokens"] == ["Please", ",", "I", "have", "booked", "a", "room", "."]
-        assert len(pretranslations[0]["alignment"]) > 0
-        assert pretranslations[0]["sequenceConfidence"] == 0.5
-    else:
-        assert pretranslations[0]["sourceTokens"] == []
-        assert pretranslations[0]["translationTokens"] == []
-        assert len(pretranslations[0]["alignment"]) == 0
-        assert pretranslations[0]["sequenceConfidence"] == 0.5
+    assert pretranslations[0]["sourceTokens"] == [
+        "Por",
+        "favor",
+        ",",
+        "tengo",
+        "reservada",
+        "una",
+        "habitación",
+        ".",
+    ]
+    assert pretranslations[0]["translationTokens"] == ["Please", ",", "I", "have", "booked", "a", "room", "."]
+    assert len(pretranslations[0]["alignment"]) > 0
+    assert pretranslations[0]["sequenceConfidence"] == 0.5
     decoy.verify(env.translation_file_service.save_model(Path("model.tar.gz"), "models/save-model.tar.gz"), times=1)
 
 
@@ -168,6 +161,14 @@ def open_target_pretranslation_writer(env: _TestEnvironment) -> Iterator[DictToJ
                     "save_model": "save-model",
                     "inference_batch_size": 100,
                     "align_pretranslations": True,
+                    "build_id": "my_build",
+                    "thot_align": {
+                        "tokenizer": "latin",
+                        "model_type": "eflomal",
+                        "word_alignment_heuristic": "grow-diag-final-and",
+                    },
+                    "data_dir": "~/machine",
+                    "shared_file_folder": "dev",
                 }
             ),
             self.nmt_model_factory,

From a87817c7e03556a96b562ada7330af3779ae0cf9 Mon Sep 17 00:00:00 2001
From: Enkidu93 <lowryec17@gcc.edu>
Date: Wed, 22 Jul 2026 14:27:23 -0400
Subject: [PATCH 04/24] Remove unused import

---
 machine/jobs/nmt_engine_build_job.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/machine/jobs/nmt_engine_build_job.py b/machine/jobs/nmt_engine_build_job.py
index 6c18ecfe..b00be351 100644
--- a/machine/jobs/nmt_engine_build_job.py
+++ b/machine/jobs/nmt_engine_build_job.py
@@ -13,7 +13,6 @@
 from ..tokenization.tokenizer_factory import create_tokenizer
 from ..utils.phased_progress_reporter import Phase, PhasedProgressReporter
 from ..utils.progress_status import ProgressStatus
-from .config import SETTINGS
 from .nmt_model_factory import NmtModelFactory
 from .thot.thot_word_alignment_model_factory import ThotWordAlignmentModelFactory
 from .translation_engine_build_job import TranslationEngineBuildJob

From c211c6e280dae107cf4317fe37171a78d10f5a24 Mon Sep 17 00:00:00 2001
From: Enkidu93 <lowryec17@gcc.edu>
Date: Wed, 22 Jul 2026 16:13:31 -0400
Subject: [PATCH 05/24] Properly mock word alignment; revert settings

---
 machine/jobs/build_nmt_engine.py            |  5 +++-
 machine/jobs/nmt_engine_build_job.py        | 24 ++++++++++--------
 machine/jobs/settings.yaml                  |  1 +
 tests/jobs/test_nmt_engine_build_job.py     | 27 +++++++++++++++++++++
 tests/jobs/test_word_alignment_build_job.py |  2 --
 5 files changed, 46 insertions(+), 13 deletions(-)

diff --git a/machine/jobs/build_nmt_engine.py b/machine/jobs/build_nmt_engine.py
index 09522975..e3e57f27 100644
--- a/machine/jobs/build_nmt_engine.py
+++ b/machine/jobs/build_nmt_engine.py
@@ -12,6 +12,7 @@
 from .nmt_engine_build_job import NmtEngineBuildJob
 from .nmt_model_factory import NmtModelFactory
 from .shared_file_service_factory import SharedFileServiceType
+from .thot.thot_word_alignment_model_factory import ThotWordAlignmentModelFactory
 from .translation_file_service import TranslationFileService
 
 # Setup logging
@@ -57,7 +58,9 @@ def clearml_progress(status: ProgressStatus) -> None:
         else:
             raise RuntimeError("The model type is invalid.")
 
-        job = NmtEngineBuildJob(SETTINGS, nmt_model_factory, translation_file_service)
+        job = NmtEngineBuildJob(
+            SETTINGS, nmt_model_factory, translation_file_service, ThotWordAlignmentModelFactory(SETTINGS)
+        )
         train_corpus_size, _ = job.run(progress, check_canceled)
         if task is not None:
             task.get_logger().report_single_value(name="train_corpus_size", value=train_corpus_size)
diff --git a/machine/jobs/nmt_engine_build_job.py b/machine/jobs/nmt_engine_build_job.py
index b00be351..ddf861ee 100644
--- a/machine/jobs/nmt_engine_build_job.py
+++ b/machine/jobs/nmt_engine_build_job.py
@@ -14,19 +14,25 @@
 from ..utils.phased_progress_reporter import Phase, PhasedProgressReporter
 from ..utils.progress_status import ProgressStatus
 from .nmt_model_factory import NmtModelFactory
-from .thot.thot_word_alignment_model_factory import ThotWordAlignmentModelFactory
 from .translation_engine_build_job import TranslationEngineBuildJob
 from .translation_file_service import PretranslationInfo, TranslationFileService
+from .word_alignment_model_factory import WordAlignmentModelFactory
 
 logger = logging.getLogger(__name__)
 
 
 class NmtEngineBuildJob(TranslationEngineBuildJob):
     def __init__(
-        self, config: Any, nmt_model_factory: NmtModelFactory, translation_file_service: TranslationFileService
+        self,
+        config: Any,
+        nmt_model_factory: NmtModelFactory,
+        translation_file_service: TranslationFileService,
+        alignment_model_factory: WordAlignmentModelFactory,
     ) -> None:
         self._nmt_model_factory = nmt_model_factory
         self._nmt_model_factory.init()
+        self._alignment_model_factory = alignment_model_factory
+        self._alignment_model_factory.init()
         super().__init__(config, translation_file_service)
 
     def _get_progress_reporter(
@@ -147,8 +153,6 @@ def _align(
 
         logger.info("Aligning source to pretranslations")
 
-        alignment_model_factory = ThotWordAlignmentModelFactory(self._config)
-
         tokenizer = create_tokenizer(self._config.thot_align.tokenizer)
 
         source_inference_corpus = DictionaryTextCorpus(
@@ -170,27 +174,27 @@ def _align(
             )
         )
 
-        parallel_pretranslation_rows = source_inference_corpus.align_rows(target_inference_corpus)
+        parallel_pretranslation_corpus = source_inference_corpus.align_rows(target_inference_corpus)
 
         alignment_parallel_corpus = ParallelTextCorpus.from_parallel_rows(
             chain(
-                parallel_pretranslation_rows.get_rows(),
+                parallel_pretranslation_corpus.get_rows(),
                 parallel_training_corpus.get_rows(),
             )
-        )
+        )  # TODO .lowercase()?
 
         logger.info("Training aligner")
         with (
             progress_reporter.start_next_phase() as phase_progress,
-            alignment_model_factory.create_model_trainer(tokenizer, alignment_parallel_corpus) as trainer,
+            self._alignment_model_factory.create_model_trainer(tokenizer, alignment_parallel_corpus) as trainer,
         ):
             trainer.train(progress=phase_progress, check_canceled=check_canceled)
             trainer.save()
 
         logger.info("Aligning pretranslations")
-        alignment_model = alignment_model_factory.create_alignment_model()
+        alignment_model = self._alignment_model_factory.create_alignment_model()
 
-        parallel_pretranslation_rows = list(parallel_pretranslation_rows)
+        parallel_pretranslation_rows = list(parallel_pretranslation_corpus)  # TODO .lowercase()?
         alignments = alignment_model.align_batch(parallel_pretranslation_rows)
 
         all_word_pairs = []
diff --git a/machine/jobs/settings.yaml b/machine/jobs/settings.yaml
index 47b7548f..ae28a7b0 100644
--- a/machine/jobs/settings.yaml
+++ b/machine/jobs/settings.yaml
@@ -3,6 +3,7 @@ default:
   shared_file_uri: s3:/silnlp/
   shared_file_folder: production
   align_pretranslations: true
+  inference_batch_size: 1024
   huggingface:
     parent_model_name: facebook/nllb-200-distilled-1.3B
     train_params:
diff --git a/tests/jobs/test_nmt_engine_build_job.py b/tests/jobs/test_nmt_engine_build_job.py
index 0c48ee18..dd18fc62 100644
--- a/tests/jobs/test_nmt_engine_build_job.py
+++ b/tests/jobs/test_nmt_engine_build_job.py
@@ -16,6 +16,7 @@
     NmtModelFactory,
     PretranslationInfo,
     TranslationFileService,
+    WordAlignmentModelFactory,
 )
 from machine.translation import (
     Phrase,
@@ -25,6 +26,7 @@
     TranslationResult,
     TranslationSources,
     WordAlignmentMatrix,
+    WordAlignmentModel,
 )
 from machine.utils import CanceledError, ContextManagedGenerator
 
@@ -153,6 +155,30 @@ def open_target_pretranslation_writer(env: _TestEnvironment) -> Iterator[DictToJ
             lambda: open_target_pretranslation_writer(self)
         )
 
+        self.alignment_model_trainer = decoy.mock(cls=Trainer)
+        decoy.when(self.alignment_model_trainer.__enter__()).then_return(self.alignment_model_trainer)
+        stats = TrainStats()
+        decoy.when(self.alignment_model_trainer.stats).then_return(stats)
+
+        self.model = decoy.mock(cls=WordAlignmentModel)
+        decoy.when(self.model.__enter__()).then_return(self.model)
+        decoy.when(self.model.align_batch(matchers.Anything())).then_return(
+            [
+                WordAlignmentMatrix.from_word_pairs(
+                    row_count=8,
+                    column_count=8,
+                    set_values=[(0, 0), (1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6), (7, 7)],
+                ),
+            ]
+        )
+
+        self.word_alignment_model_factory = decoy.mock(cls=WordAlignmentModelFactory)
+        decoy.when(
+            self.word_alignment_model_factory.create_model_trainer(matchers.Anything(), matchers.Anything())
+        ).then_return(self.alignment_model_trainer)
+        decoy.when(self.word_alignment_model_factory.create_alignment_model()).then_return(self.model)
+        decoy.when(self.word_alignment_model_factory.save_model()).then_return(Path("model.zip"))
+
         self.job = NmtEngineBuildJob(
             MockSettings(
                 {
@@ -173,6 +199,7 @@ def open_target_pretranslation_writer(env: _TestEnvironment) -> Iterator[DictToJ
             ),
             self.nmt_model_factory,
             self.translation_file_service,
+            self.word_alignment_model_factory,
         )
 
 
diff --git a/tests/jobs/test_word_alignment_build_job.py b/tests/jobs/test_word_alignment_build_job.py
index 63b6b816..03387808 100644
--- a/tests/jobs/test_word_alignment_build_job.py
+++ b/tests/jobs/test_word_alignment_build_job.py
@@ -43,8 +43,6 @@ def __init__(self, decoy: Decoy) -> None:
         self.model_trainer = decoy.mock(cls=Trainer)
         decoy.when(self.model_trainer.__enter__()).then_return(self.model_trainer)
         stats = TrainStats()
-        stats.train_corpus_size = 3
-        stats.metrics["bleu"] = 30.0
         decoy.when(self.model_trainer.stats).then_return(stats)
 
         self.model = decoy.mock(cls=WordAlignmentModel)

From 883f37d7c36056a79041d84d4cde1ed757daf98f Mon Sep 17 00:00:00 2001
From: Enkidu93 <lowryec17@gcc.edu>
Date: Thu, 23 Jul 2026 09:55:09 -0400
Subject: [PATCH 06/24] Fix logging; do not include NULL alignments

---
 machine/jobs/nmt_engine_build_job.py     | 4 +---
 machine/jobs/word_alignment_build_job.py | 2 +-
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/machine/jobs/nmt_engine_build_job.py b/machine/jobs/nmt_engine_build_job.py
index ddf861ee..299e4d90 100644
--- a/machine/jobs/nmt_engine_build_job.py
+++ b/machine/jobs/nmt_engine_build_job.py
@@ -151,8 +151,6 @@ def _align(
         if check_canceled is not None:
             check_canceled()
 
-        logger.info("Aligning source to pretranslations")
-
         tokenizer = create_tokenizer(self._config.thot_align.tokenizer)
 
         source_inference_corpus = DictionaryTextCorpus(
@@ -199,7 +197,7 @@ def _align(
 
         all_word_pairs = []
         for parallel_text_row, alignment in zip(parallel_pretranslation_rows, alignments, strict=True):
-            word_pairs = alignment.to_aligned_word_pairs(include_null=True)
+            word_pairs = alignment.to_aligned_word_pairs(include_null=False)
             alignment_model.compute_aligned_word_pair_scores(
                 parallel_text_row.source_segment, parallel_text_row.target_segment, word_pairs
             )
diff --git a/machine/jobs/word_alignment_build_job.py b/machine/jobs/word_alignment_build_job.py
index 842151a8..f47cd20d 100644
--- a/machine/jobs/word_alignment_build_job.py
+++ b/machine/jobs/word_alignment_build_job.py
@@ -111,7 +111,7 @@ def _batch_inference(
                     )
                     for ii in inference_inputs
                 ]
-            ).lowercase()
+            ).lowercase()  # TODO we're calling lowercase() here; is that correct?
 
             segment_batch = list(parallel_corpus)
             if check_canceled is not None:

From d1cd61dc73ad28500d70b134125993078b6dad60 Mon Sep 17 00:00:00 2001
From: Enkidu93 <lowryec17@gcc.edu>
Date: Thu, 23 Jul 2026 11:24:53 -0400
Subject: [PATCH 07/24] Make naming more consistent

---
 machine/jobs/nmt_engine_build_job.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/machine/jobs/nmt_engine_build_job.py b/machine/jobs/nmt_engine_build_job.py
index 299e4d90..a62070a2 100644
--- a/machine/jobs/nmt_engine_build_job.py
+++ b/machine/jobs/nmt_engine_build_job.py
@@ -27,11 +27,11 @@ def __init__(
         config: Any,
         nmt_model_factory: NmtModelFactory,
         translation_file_service: TranslationFileService,
-        alignment_model_factory: WordAlignmentModelFactory,
+        word_alignment_model_factory: WordAlignmentModelFactory,
     ) -> None:
         self._nmt_model_factory = nmt_model_factory
         self._nmt_model_factory.init()
-        self._alignment_model_factory = alignment_model_factory
+        self._alignment_model_factory = word_alignment_model_factory
         self._alignment_model_factory.init()
         super().__init__(config, translation_file_service)
 

From 7e96370718c2d28eaf7ce054825aa73b63ed256a Mon Sep 17 00:00:00 2001
From: Enkidu93 <lowryec17@gcc.edu>
Date: Thu, 23 Jul 2026 11:35:08 -0400
Subject: [PATCH 08/24] Revert settings.yml change

---
 machine/jobs/settings.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/machine/jobs/settings.yaml b/machine/jobs/settings.yaml
index ae28a7b0..cdddcf3d 100644
--- a/machine/jobs/settings.yaml
+++ b/machine/jobs/settings.yaml
@@ -2,8 +2,8 @@ default:
   data_dir: ~/machine
   shared_file_uri: s3:/silnlp/
   shared_file_folder: production
-  align_pretranslations: true
   inference_batch_size: 1024
+  align_pretranslations: true
   huggingface:
     parent_model_name: facebook/nllb-200-distilled-1.3B
     train_params:

From dc5fba1bb9e54b2ddcaadc7a3dfb8069bc3725b4 Mon Sep 17 00:00:00 2001
From: Damien Daspit <damien_daspit@sil.org>
Date: Fri, 24 Jul 2026 17:48:58 -0400
Subject: [PATCH 09/24] Use transductive API for getting alignments

- full streaming pipeline for inferencing
- fall back to inductive API
- fix text id handling in "flatten"
---
 machine/corpora/corpora_utils.py              |   1 +
 machine/corpora/flatten.py                    |  15 +-
 machine/jobs/nmt_engine_build_job.py          | 198 +++++++++++-------
 .../thot/thot_word_alignment_model_factory.py |  14 +-
 tests/corpora/test_flatten.py                 |  50 +++++
 tests/jobs/test_nmt_engine_build_job.py       | 148 +++++++++----
 6 files changed, 289 insertions(+), 137 deletions(-)
 create mode 100644 tests/corpora/test_flatten.py

diff --git a/machine/corpora/corpora_utils.py b/machine/corpora/corpora_utils.py
index ddd8e3c5..bc9e43e0 100644
--- a/machine/corpora/corpora_utils.py
+++ b/machine/corpora/corpora_utils.py
@@ -22,6 +22,7 @@ def alignment_exception(refs: Iterable[str]) -> RuntimeError:
 def batch(iterable: Iterable[T], batch_size: int) -> Iterable[Sequence[T]]:
     if isinstance(iterable, Sequence) and len(iterable) <= batch_size:
         yield iterable
+        return
 
     batch: List[T] = []
     for item in iterable:
diff --git a/machine/corpora/flatten.py b/machine/corpora/flatten.py
index 7f0e56d9..0e460f37 100644
--- a/machine/corpora/flatten.py
+++ b/machine/corpora/flatten.py
@@ -33,14 +33,13 @@ def flatten(corpora: Iterable[Corpus]) -> Corpus:
     if len(corpus_list) == 1:
         return corpus_list[0]
 
-    if any(type(corpus_list[0]) != type(corpus) for corpus in corpus_list[1:]):  # noqa: E721
-        raise TypeError("All corpora must be of the same type.")
-
-    if isinstance(corpus_list[0], TextCorpus):
+    if all(isinstance(corpus, TextCorpus) for corpus in corpus_list):
         return _FlattenTextCorpus(cast(List[TextCorpus], corpus_list))
-    if isinstance(corpus_list[0], AlignmentCorpus):
+    if all(isinstance(corpus, AlignmentCorpus) for corpus in corpus_list):
         return _FlattenAlignmentCorpus(cast(List[AlignmentCorpus], corpus_list))
-    return _FlattenParallelTextCorpus(cast(List[ParallelTextCorpus], corpus_list))
+    if all(isinstance(corpus, ParallelTextCorpus) for corpus in corpus_list):
+        return _FlattenParallelTextCorpus(cast(List[ParallelTextCorpus], corpus_list))
+    raise TypeError("All corpora must be of the same type.")
 
 
 class _FlattenTextCorpus(TextCorpus):
@@ -100,7 +99,7 @@ def is_target_tokenized(self) -> bool:
     def count(self, include_empty: bool = True, text_ids: Optional[Iterable[str]] = None) -> int:
         return sum(c.count(include_empty, text_ids) for c in self._corpora)
 
-    def _get_rows(self) -> Generator[ParallelTextRow, None, None]:
+    def _get_rows(self, text_ids: Optional[Iterable[str]] = None) -> Generator[ParallelTextRow, None, None]:
         for corpus in self._corpora:
-            with corpus.get_rows() as rows:
+            with corpus.get_rows(text_ids) as rows:
                 yield from rows
diff --git a/machine/jobs/nmt_engine_build_job.py b/machine/jobs/nmt_engine_build_job.py
index a62070a2..2f15ca1b 100644
--- a/machine/jobs/nmt_engine_build_job.py
+++ b/machine/jobs/nmt_engine_build_job.py
@@ -1,16 +1,21 @@
+import json
 import logging
 from contextlib import ExitStack
-from itertools import chain
-from typing import Any, Callable, Optional, Sequence, Tuple
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Any, Callable, Generator, Iterable, Optional, Sequence, Tuple
 
 from ..corpora.aligned_word_pair import AlignedWordPair
 from ..corpora.corpora_utils import batch
-from ..corpora.dictionary_text_corpus import DictionaryTextCorpus
-from ..corpora.memory_text import MemoryText
+from ..corpora.flatten import flatten
 from ..corpora.parallel_text_corpus import ParallelTextCorpus
 from ..corpora.text_corpus import TextCorpus
-from ..corpora.text_row import TextRow
+from ..corpora.text_file_text_corpus import TextFileTextCorpus
+from ..corpora.token_processors import lowercase
 from ..tokenization.tokenizer_factory import create_tokenizer
+from ..translation.transductive_word_alignment_model import TransductiveWordAlignmentModel
+from ..translation.translation_engine import TranslationEngine
+from ..translation.word_alignment_matrix import WordAlignmentMatrix
 from ..utils.phased_progress_reporter import Phase, PhasedProgressReporter
 from ..utils.progress_status import ProgressStatus
 from .nmt_model_factory import NmtModelFactory
@@ -114,104 +119,129 @@ def _batch_inference(
         with ExitStack() as stack:
             phase_progress = stack.enter_context(progress_reporter.start_next_phase())
             engine = stack.enter_context(self._nmt_model_factory.create_engine())
-            pretranslations = [
-                pt_info for pt_info in stack.enter_context(self._translation_file_service.get_source_pretranslations())
-            ]
-            src_segments = [pt_info["translation"] for pt_info in pretranslations]
-            current_inference_step = 0
-            phase_progress(ProgressStatus.from_step(current_inference_step, inference_step_count))
-            batch_size = self._config["inference_batch_size"]
-            for seg_batch in batch(iter(src_segments), batch_size):
-                if check_canceled is not None:
-                    check_canceled()
-                for i, result in enumerate(engine.translate_batch(seg_batch)):
-                    pretranslations[current_inference_step + i]["translation"] = result.translation
-                    pretranslations[current_inference_step + i]["sequenceConfidence"] = result.sequence_confidence
-                current_inference_step += len(seg_batch)
-                phase_progress(ProgressStatus.from_step(current_inference_step, inference_step_count))
-
+            src_pretranslations = stack.enter_context(self._translation_file_service.get_source_pretranslations())
+            pretranslations = self._translate(
+                engine, src_pretranslations, inference_step_count, phase_progress, check_canceled
+            )
             if self._config.align_pretranslations:
-                logger.info("Aligning source to pretranslations")
-                pretranslations = self._align(
-                    src_segments, pretranslations, parallel_training_corpus, progress_reporter, check_canceled
+                results: Iterable[PretranslationInfo] = self._align(
+                    pretranslations, parallel_training_corpus, progress_reporter, check_canceled
                 )
+            else:
+                results = (pt_info for _, pt_info in pretranslations)
 
             writer = stack.enter_context(self._translation_file_service.open_target_pretranslation_writer())
-            for pretranslation in pretranslations:
-                writer.write(pretranslation)
+            for pt_info in results:
+                writer.write(pt_info)
+
+    def _translate(
+        self,
+        engine: TranslationEngine,
+        src_pretranslations: Iterable[PretranslationInfo],
+        inference_step_count: int,
+        phase_progress: Callable[[ProgressStatus], None],
+        check_canceled: Optional[Callable[[], None]],
+    ) -> Generator[Tuple[str, PretranslationInfo], None, None]:
+        current_inference_step = 0
+        phase_progress(ProgressStatus.from_step(current_inference_step, inference_step_count))
+        batch_size: int = self._config["inference_batch_size"]
+        for pt_batch in batch(src_pretranslations, batch_size):
+            if check_canceled is not None:
+                check_canceled()
+            source_segments = [pt_info["translation"] for pt_info in pt_batch]
+            for pt_info, result in zip(pt_batch, engine.translate_batch(source_segments), strict=True):
+                pt_info["translation"] = result.translation
+                pt_info["sequenceConfidence"] = result.sequence_confidence
+            current_inference_step += len(pt_batch)
+            phase_progress(ProgressStatus.from_step(current_inference_step, inference_step_count))
+            yield from zip(source_segments, pt_batch)
 
     def _align(
         self,
-        src_pretranslate_segments: Sequence[str],
-        pretranslations: Sequence[PretranslationInfo],
+        pretranslations: Iterable[Tuple[str, PretranslationInfo]],
         parallel_training_corpus: ParallelTextCorpus,
         progress_reporter: PhasedProgressReporter,
         check_canceled: Optional[Callable[[], None]],
-    ) -> Sequence[PretranslationInfo]:
+    ) -> Generator[PretranslationInfo, None, None]:
         if check_canceled is not None:
             check_canceled()
 
         tokenizer = create_tokenizer(self._config.thot_align.tokenizer)
 
-        source_inference_corpus = DictionaryTextCorpus(
-            MemoryText(
-                "pretranslations",
-                [
-                    TextRow("pretranslations", i, list(tokenizer.tokenize(segment)))
-                    for i, segment in enumerate(src_pretranslate_segments)
-                ],
-            )
-        )
-        target_inference_corpus = DictionaryTextCorpus(
-            MemoryText(
-                "pretranslations",
-                [
-                    TextRow("pretranslations", i, list(tokenizer.tokenize(pretranslation["translation"])))
-                    for i, pretranslation in enumerate(pretranslations)
-                ],
-            )
-        )
+        with TemporaryDirectory() as temp_dir:
+            # Spool the translated pretranslations to disk, so that the aligner can make multiple
+            # passes over them without keeping the entire corpus in memory.
+            logger.info("Aligning source to pretranslations")
+            source_path = Path(temp_dir) / "pretranslations.src.txt"
+            translation_path = Path(temp_dir) / "pretranslations.trg.txt"
+            pretranslations_path = Path(temp_dir) / "pretranslations.json"
+            with (
+                source_path.open("w", encoding="utf-8", newline="\n") as source_file,
+                translation_path.open("w", encoding="utf-8", newline="\n") as translation_file,
+                pretranslations_path.open("w", encoding="utf-8", newline="\n") as pretranslations_file,
+            ):
+                for source_segment, pt_info in pretranslations:
+                    source_file.write(source_segment + "\n")
+                    translation_file.write(pt_info["translation"] + "\n")
+                    pretranslations_file.write(json.dumps(pt_info, ensure_ascii=False) + "\n")
 
-        parallel_pretranslation_corpus = source_inference_corpus.align_rows(target_inference_corpus)
+            if check_canceled is not None:
+                check_canceled()
 
-        alignment_parallel_corpus = ParallelTextCorpus.from_parallel_rows(
-            chain(
-                parallel_pretranslation_corpus.get_rows(),
-                parallel_training_corpus.get_rows(),
+            parallel_pretranslation_corpus = TextFileTextCorpus(source_path).align_rows(
+                TextFileTextCorpus(translation_path)
             )
-        )  # TODO .lowercase()?
 
-        logger.info("Training aligner")
-        with (
-            progress_reporter.start_next_phase() as phase_progress,
-            self._alignment_model_factory.create_model_trainer(tokenizer, alignment_parallel_corpus) as trainer,
-        ):
-            trainer.train(progress=phase_progress, check_canceled=check_canceled)
-            trainer.save()
-
-        logger.info("Aligning pretranslations")
-        alignment_model = self._alignment_model_factory.create_alignment_model()
+            # The pretranslations are placed at the beginning of the training corpus, so that a
+            # transductive model's training alignments can be matched back to them by index.
+            alignment_parallel_corpus = flatten([parallel_pretranslation_corpus, parallel_training_corpus])
 
-        parallel_pretranslation_rows = list(parallel_pretranslation_corpus)  # TODO .lowercase()?
-        alignments = alignment_model.align_batch(parallel_pretranslation_rows)
+            logger.info("Training aligner")
+            with (
+                progress_reporter.start_next_phase() as phase_progress,
+                self._alignment_model_factory.create_model_trainer(tokenizer, alignment_parallel_corpus) as trainer,
+            ):
+                trainer.train(progress=phase_progress, check_canceled=check_canceled)
+                trainer.save()
 
-        all_word_pairs = []
-        for parallel_text_row, alignment in zip(parallel_pretranslation_rows, alignments, strict=True):
-            word_pairs = alignment.to_aligned_word_pairs(include_null=False)
-            alignment_model.compute_aligned_word_pair_scores(
-                parallel_text_row.source_segment, parallel_text_row.target_segment, word_pairs
-            )
-            all_word_pairs.append(word_pairs)
-
-        if check_canceled is not None:
-            check_canceled()
-
-        for i in range(len(pretranslations)):
-            pretranslations[i]["sourceTokens"] = list(parallel_pretranslation_rows[i].source_segment)
-            pretranslations[i]["translationTokens"] = list(parallel_pretranslation_rows[i].target_segment)
-            pretranslations[i]["alignment"] = AlignedWordPair.to_string(all_word_pairs[i], include_scores=True)
+            if check_canceled is not None:
+                check_canceled()
 
-        return pretranslations
+            logger.info("Aligning pretranslations")
+            batch_size: int = self._config["inference_batch_size"]
+            with (
+                self._alignment_model_factory.create_alignment_model() as alignment_model,
+                parallel_pretranslation_corpus.tokenize(tokenizer).get_rows() as rows,
+            ):
+                transductive_model: Optional[TransductiveWordAlignmentModel] = None
+                if isinstance(alignment_model, TransductiveWordAlignmentModel):
+                    transductive_model = alignment_model
+                index = 0
+                for pt_batch in batch(zip(_read_pretranslations(pretranslations_path), rows, strict=True), batch_size):
+                    if check_canceled is not None:
+                        check_canceled()
+                    # The aligner is trained on lowercased tokens, so it must also be given
+                    # lowercased tokens when aligning and scoring; the original-cased tokens are
+                    # written to the pretranslations.
+                    segments = [(lowercase(row.source_segment), lowercase(row.target_segment)) for _, row in pt_batch]
+                    if transductive_model is not None:
+                        # The pretranslations are the first rows of the training corpus, so their
+                        # alignments were already computed during training.
+                        alignments: Sequence[WordAlignmentMatrix] = [
+                            transductive_model.get_training_alignment(index + i) for i in range(len(pt_batch))
+                        ]
+                    else:
+                        alignments = alignment_model.align_batch(segments)
+                    for (pt_info, row), (source_segment, target_segment), alignment in zip(
+                        pt_batch, segments, alignments, strict=True
+                    ):
+                        word_pairs = alignment.to_aligned_word_pairs(include_null=False)
+                        alignment_model.compute_aligned_word_pair_scores(source_segment, target_segment, word_pairs)
+                        pt_info["sourceTokens"] = list(row.source_segment)
+                        pt_info["translationTokens"] = list(row.target_segment)
+                        pt_info["alignment"] = AlignedWordPair.to_string(word_pairs, include_scores=True)
+                        yield pt_info
+                    index += len(pt_batch)
 
     def _save_model(self) -> None:
         if "save_model" in self._config and self._config.save_model is not None:
@@ -220,3 +250,9 @@ def _save_model(self) -> None:
             self._translation_file_service.save_model(
                 model_path, f"models/{self._config.save_model + ''.join(model_path.suffixes)}"
             )
+
+
+def _read_pretranslations(path: Path) -> Generator[PretranslationInfo, None, None]:
+    with path.open("r", encoding="utf-8") as file:
+        for line in file:
+            yield json.loads(line)
diff --git a/machine/jobs/thot/thot_word_alignment_model_factory.py b/machine/jobs/thot/thot_word_alignment_model_factory.py
index 7e9d5fc8..aa2d0a06 100644
--- a/machine/jobs/thot/thot_word_alignment_model_factory.py
+++ b/machine/jobs/thot/thot_word_alignment_model_factory.py
@@ -15,19 +15,21 @@ class ThotWordAlignmentModelFactory(WordAlignmentModelFactory):
 
     def create_model_trainer(self, tokenizer: Tokenizer[str, int, str], corpus: ParallelTextCorpus) -> Trainer:
         self._model_dir.mkdir(parents=True, exist_ok=True)
+        corpus = corpus.tokenize(tokenizer).lowercase()
+        # Retain the alignments computed during training, so that the model created by
+        # create_alignment_model can align the training corpus transductively without a separate
+        # inference pass. The alignments are saved with the model and restored when it is loaded.
         direct_trainer = ThotWordAlignmentModelTrainer(
             self._config.thot_align.model_type,
-            corpus.lowercase(),
+            corpus,
             prefix_filename=self._direct_model_path,
-            source_tokenizer=tokenizer,
-            target_tokenizer=tokenizer,
+            emit_training_alignments=True,
         )
         inverse_trainer = ThotWordAlignmentModelTrainer(
             self._config.thot_align.model_type,
-            corpus.invert().lowercase(),
+            corpus.invert(),
             prefix_filename=self._inverse_model_path,
-            source_tokenizer=tokenizer,
-            target_tokenizer=tokenizer,
+            emit_training_alignments=True,
         )
         return SymmetrizedWordAlignmentModelTrainer(direct_trainer, inverse_trainer)
 
diff --git a/tests/corpora/test_flatten.py b/tests/corpora/test_flatten.py
new file mode 100644
index 00000000..2b9de799
--- /dev/null
+++ b/tests/corpora/test_flatten.py
@@ -0,0 +1,50 @@
+from machine.corpora import DictionaryTextCorpus, MemoryText, TextRow, flatten
+
+
+def test_flatten_text_corpus() -> None:
+    corpus = flatten(
+        [_create_text_corpus("text1", ["source 1", "source 2"]), _create_text_corpus("text2", ["source 3"])]
+    )
+
+    rows = list(corpus.get_rows())
+    assert len(rows) == 3
+    assert [row.text for row in rows] == ["source 1", "source 2", "source 3"]
+    # the corpus is re-iterable
+    assert len(list(corpus.get_rows())) == 3
+
+
+def test_flatten_parallel_text_corpus() -> None:
+    corpus1 = _create_text_corpus("text1", ["source 1"]).align_rows(_create_text_corpus("text1", ["target 1"]))
+    corpus2 = _create_text_corpus("text2", ["source 2"]).align_rows(_create_text_corpus("text2", ["target 2"]))
+    corpus = flatten([corpus1, corpus2])
+
+    rows = list(corpus.get_rows())
+    assert len(rows) == 2
+    assert [row.source_text for row in rows] == ["source 1", "source 2"]
+    assert [row.target_text for row in rows] == ["target 1", "target 2"]
+    # the corpus is re-iterable
+    assert len(list(corpus.get_rows())) == 2
+
+
+def test_flatten_parallel_text_corpus_different_classes() -> None:
+    corpus1 = _create_text_corpus("text1", ["Source 1"]).align_rows(_create_text_corpus("text1", ["Target 1"]))
+    corpus2 = _create_text_corpus("text2", ["Source 2"]).align_rows(_create_text_corpus("text2", ["Target 2"]))
+    corpus = flatten([corpus1, corpus2.lowercase()])
+
+    rows = list(corpus.get_rows())
+    assert [row.source_text for row in rows] == ["Source 1", "source 2"]
+
+
+def test_flatten_parallel_text_corpus_text_ids() -> None:
+    corpus1 = _create_text_corpus("text1", ["source 1"]).align_rows(_create_text_corpus("text1", ["target 1"]))
+    corpus2 = _create_text_corpus("text2", ["source 2"]).align_rows(_create_text_corpus("text2", ["target 2"]))
+    corpus = flatten([corpus1, corpus2])
+
+    rows = list(corpus.get_rows(["text2"]))
+    assert [row.source_text for row in rows] == ["source 2"]
+
+
+def _create_text_corpus(text_id: str, sentences: list) -> DictionaryTextCorpus:
+    return DictionaryTextCorpus(
+        MemoryText(text_id, [TextRow(text_id, i, [sentence]) for i, sentence in enumerate(sentences)])
+    )
diff --git a/tests/jobs/test_nmt_engine_build_job.py b/tests/jobs/test_nmt_engine_build_job.py
index dd18fc62..57905ce9 100644
--- a/tests/jobs/test_nmt_engine_build_job.py
+++ b/tests/jobs/test_nmt_engine_build_job.py
@@ -2,7 +2,7 @@
 from contextlib import contextmanager
 from io import StringIO
 from pathlib import Path
-from typing import Iterator
+from typing import Iterator, List
 
 from decoy import Decoy, matchers
 from pytest import raises
@@ -22,6 +22,7 @@
     Phrase,
     Trainer,
     TrainStats,
+    TransductiveWordAlignmentModel,
     TranslationEngine,
     TranslationResult,
     TranslationSources,
@@ -32,7 +33,7 @@
 
 
 def test_run(decoy: Decoy) -> None:
-    env = _TestEnvironment(decoy)
+    env = _TestEnvironment(decoy, transductive=True)
     env.job.run()
 
     pretranslations = json.loads(env.target_pretranslations)
@@ -51,9 +52,55 @@ def test_run(decoy: Decoy) -> None:
     assert pretranslations[0]["translationTokens"] == ["Please", ",", "I", "have", "booked", "a", "room", "."]
     assert len(pretranslations[0]["alignment"]) > 0
     assert pretranslations[0]["sequenceConfidence"] == 0.5
+    decoy.verify(env.model.align_batch(matchers.Anything()), times=0)
     decoy.verify(env.translation_file_service.save_model(Path("model.tar.gz"), "models/save-model.tar.gz"), times=1)
 
 
+def test_run_inductive_fallback(decoy: Decoy) -> None:
+    env = _TestEnvironment(decoy, transductive=False)
+    env.job.run()
+
+    pretranslations = json.loads(env.target_pretranslations)
+    assert len(pretranslations) == 1
+    assert pretranslations[0]["sourceTokens"] == [
+        "Por",
+        "favor",
+        ",",
+        "tengo",
+        "reservada",
+        "una",
+        "habitación",
+        ".",
+    ]
+    assert pretranslations[0]["translationTokens"] == ["Please", ",", "I", "have", "booked", "a", "room", "."]
+    assert len(pretranslations[0]["alignment"]) > 0
+
+
+def test_run_batched_transductive(decoy: Decoy) -> None:
+    env = _TestEnvironment(decoy, transductive=True, num_pretranslations=3, batch_size=2)
+    env.job.run()
+
+    pretranslations = json.loads(env.target_pretranslations)
+    assert len(pretranslations) == 3
+    assert [pt["textId"] for pt in pretranslations] == ["text1", "text2", "text3"]
+    for pretranslation in pretranslations:
+        assert pretranslation["translation"] == "Please, I have booked a room."
+        assert len(pretranslation["alignment"]) > 0
+    assert env.translate_batch_sizes == [2, 1]
+    assert env.training_alignment_requests == [0, 1, 2]
+
+
+def test_run_batched_inductive(decoy: Decoy) -> None:
+    env = _TestEnvironment(decoy, transductive=False, num_pretranslations=3, batch_size=2)
+    env.job.run()
+
+    pretranslations = json.loads(env.target_pretranslations)
+    assert len(pretranslations) == 3
+    for pretranslation in pretranslations:
+        assert len(pretranslation["alignment"]) > 0
+    assert env.align_batch_sizes == [2, 1]
+
+
 def test_cancel(decoy: Decoy) -> None:
     env = _TestEnvironment(decoy)
     checker = _CancellationChecker(3)
@@ -63,8 +110,29 @@ def test_cancel(decoy: Decoy) -> None:
     assert env.target_pretranslations == ""
 
 
+class _TransductiveModel(WordAlignmentModel, TransductiveWordAlignmentModel):
+    pass
+
+
+def _create_translation_result() -> TranslationResult:
+    return TranslationResult(
+        translation="Please, I have booked a room.",
+        source_tokens="Por favor , tengo reservada una habitación .".split(),
+        target_tokens="Please , I have booked a room .".split(),
+        confidences=[0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5],
+        sequence_confidence=0.5,
+        sources=[TranslationSources.NMT] * 8,
+        alignment=WordAlignmentMatrix.from_word_pairs(
+            8, 8, {(0, 0), (1, 0), (2, 1), (3, 2), (3, 3), (4, 4), (5, 5), (6, 6), (7, 7)}
+        ),
+        phrases=[Phrase(Range.create(0, 8), 8)],
+    )
+
+
 class _TestEnvironment:
-    def __init__(self, decoy: Decoy) -> None:
+    def __init__(
+        self, decoy: Decoy, transductive: bool = False, num_pretranslations: int = 1, batch_size: int = 100
+    ) -> None:
         self.source_tokenizer_trainer = decoy.mock(cls=Trainer)
         self.target_tokenizer_trainer = decoy.mock(cls=Trainer)
 
@@ -74,33 +142,15 @@ def __init__(self, decoy: Decoy) -> None:
         stats.metrics["bleu"] = 30.0
         decoy.when(self.model_trainer.stats).then_return(stats)
 
+        self.translate_batch_sizes: List[int] = []
+
+        def _translate_batch(segments: List[str]) -> List[TranslationResult]:
+            self.translate_batch_sizes.append(len(segments))
+            return [_create_translation_result() for _ in segments]
+
         self.engine = decoy.mock(cls=TranslationEngine)
         decoy.when(self.engine.__enter__()).then_return(self.engine)
-        decoy.when(self.engine.translate_batch(matchers.Anything())).then_return(
-            [
-                TranslationResult(
-                    translation="Please, I have booked a room.",
-                    source_tokens="Por favor , tengo reservada una habitación .".split(),
-                    target_tokens="Please , I have booked a room .".split(),
-                    confidences=[0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5],
-                    sequence_confidence=0.5,
-                    sources=[
-                        TranslationSources.NMT,
-                        TranslationSources.NMT,
-                        TranslationSources.NMT,
-                        TranslationSources.NMT,
-                        TranslationSources.NMT,
-                        TranslationSources.NMT,
-                        TranslationSources.NMT,
-                        TranslationSources.NMT,
-                    ],
-                    alignment=WordAlignmentMatrix.from_word_pairs(
-                        8, 8, {(0, 0), (1, 0), (2, 1), (3, 2), (3, 3), (4, 4), (5, 5), (6, 6), (7, 7)}
-                    ),
-                    phrases=[Phrase(Range.create(0, 8), 8)],
-                )
-            ]
-        )
+        decoy.when(self.engine.translate_batch(matchers.Anything())).then_do(_translate_batch)
 
         self.nmt_model_factory = decoy.mock(cls=NmtModelFactory)
         decoy.when(self.nmt_model_factory.train_tokenizer).then_return(True)
@@ -127,15 +177,16 @@ def __init__(self, decoy: Decoy) -> None:
                     for pi in [
                         PretranslationInfo(
                             corpusId="corpus1",
-                            textId="text1",
-                            sourceRefs=["ref1"],
-                            targetRefs=["ref1"],
+                            textId=f"text{i + 1}",
+                            sourceRefs=[f"ref{i + 1}"],
+                            targetRefs=[f"ref{i + 1}"],
                             translation="Por favor, tengo reservada una habitación.",
                             sourceTokens=[],
                             translationTokens=[],
                             alignment="",
                             sequenceConfidence=0.5,
                         )
+                        for i in range(num_pretranslations)
                     ]
                 )
             )
@@ -160,17 +211,30 @@ def open_target_pretranslation_writer(env: _TestEnvironment) -> Iterator[DictToJ
         stats = TrainStats()
         decoy.when(self.alignment_model_trainer.stats).then_return(stats)
 
-        self.model = decoy.mock(cls=WordAlignmentModel)
-        decoy.when(self.model.__enter__()).then_return(self.model)
-        decoy.when(self.model.align_batch(matchers.Anything())).then_return(
-            [
-                WordAlignmentMatrix.from_word_pairs(
-                    row_count=8,
-                    column_count=8,
-                    set_values=[(0, 0), (1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6), (7, 7)],
-                ),
-            ]
+        alignment = WordAlignmentMatrix.from_word_pairs(
+            row_count=8,
+            column_count=8,
+            set_values=[(0, 0), (1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6), (7, 7)],
         )
+        self.training_alignment_requests: List[int] = []
+        self.align_batch_sizes: List[int] = []
+
+        def _get_training_alignment(n: int) -> WordAlignmentMatrix:
+            self.training_alignment_requests.append(n)
+            return alignment
+
+        def _align_batch(segments: List[object]) -> List[WordAlignmentMatrix]:
+            self.align_batch_sizes.append(len(segments))
+            return [alignment for _ in segments]
+
+        if transductive:
+            self.model = decoy.mock(cls=_TransductiveModel)
+            decoy.when(self.model.__enter__()).then_return(self.model)
+            decoy.when(self.model.get_training_alignment(matchers.Anything())).then_do(_get_training_alignment)
+        else:
+            self.model = decoy.mock(cls=WordAlignmentModel)
+            decoy.when(self.model.__enter__()).then_return(self.model)
+            decoy.when(self.model.align_batch(matchers.Anything())).then_do(_align_batch)
 
         self.word_alignment_model_factory = decoy.mock(cls=WordAlignmentModelFactory)
         decoy.when(
@@ -185,7 +249,7 @@ def open_target_pretranslation_writer(env: _TestEnvironment) -> Iterator[DictToJ
                     "src_lang": "es",
                     "trg_lang": "en",
                     "save_model": "save-model",
-                    "inference_batch_size": 100,
+                    "inference_batch_size": batch_size,
                     "align_pretranslations": True,
                     "build_id": "my_build",
                     "thot_align": {

From 586dedaf2c1849eb2eec11a5fd96514eb240348a Mon Sep 17 00:00:00 2001
From: Enkidu93 <lowryec17@gcc.edu>
Date: Wed, 29 Jul 2026 13:55:00 -0400
Subject: [PATCH 10/24] Update word alignment build job to properly use
 alignment API and batch

---
 machine/jobs/nmt_engine_build_job.py        |  4 +-
 machine/jobs/word_alignment_build_job.py    | 97 ++++++++++++---------
 tests/jobs/test_word_alignment_build_job.py | 16 +++-
 3 files changed, 73 insertions(+), 44 deletions(-)

diff --git a/machine/jobs/nmt_engine_build_job.py b/machine/jobs/nmt_engine_build_job.py
index 2f15ca1b..ed5cf821 100644
--- a/machine/jobs/nmt_engine_build_job.py
+++ b/machine/jobs/nmt_engine_build_job.py
@@ -169,7 +169,7 @@ def _align(
         tokenizer = create_tokenizer(self._config.thot_align.tokenizer)
 
         with TemporaryDirectory() as temp_dir:
-            # Spool the translated pretranslations to disk, so that the aligner can make multiple
+            # Spool the translated pretranslations to disk so that the aligner can make multiple
             # passes over them without keeping the entire corpus in memory.
             logger.info("Aligning source to pretranslations")
             source_path = Path(temp_dir) / "pretranslations.src.txt"
@@ -192,7 +192,7 @@ def _align(
                 TextFileTextCorpus(translation_path)
             )
 
-            # The pretranslations are placed at the beginning of the training corpus, so that a
+            # The pretranslations are placed at the beginning of the training corpus so that a
             # transductive model's training alignments can be matched back to them by index.
             alignment_parallel_corpus = flatten([parallel_pretranslation_corpus, parallel_training_corpus])
 
diff --git a/machine/jobs/word_alignment_build_job.py b/machine/jobs/word_alignment_build_job.py
index f47cd20d..b8b04948 100644
--- a/machine/jobs/word_alignment_build_job.py
+++ b/machine/jobs/word_alignment_build_job.py
@@ -1,14 +1,19 @@
+import json
 import logging
 from contextlib import ExitStack
-from typing import Any, Callable, Optional
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Any, Callable, Generator, Optional
 
 from ..corpora.aligned_word_pair import AlignedWordPair
+from ..corpora.corpora_utils import batch
 from ..corpora.parallel_text_corpus import ParallelTextCorpus
-from ..corpora.parallel_text_row import ParallelTextRow
+from ..corpora.text_file_text_corpus import TextFileTextCorpus
+from ..corpora.token_processors import lowercase
 from ..tokenization.tokenizer_factory import create_tokenizer
 from ..utils.phased_progress_reporter import Phase, PhasedProgressReporter
 from ..utils.progress_status import ProgressStatus
-from .word_alignment_file_service import WordAlignmentFileService
+from .word_alignment_file_service import WordAlignmentFileService, WordAlignmentInput
 from .word_alignment_model_factory import WordAlignmentModelFactory
 
 logger = logging.getLogger(__name__)
@@ -95,49 +100,57 @@ def _batch_inference(
 
         with ExitStack() as stack:
             phase_progress = stack.enter_context(progress_reporter.start_next_phase())
-            alignment_model = stack.enter_context(self._word_alignment_model_factory.create_alignment_model())
             writer = stack.enter_context(self._word_alignment_file_service.open_alignment_output_writer())
             current_inference_step = 0
             phase_progress(ProgressStatus.from_step(current_inference_step, inference_step_count))
 
-            parallel_corpus = ParallelTextCorpus.from_parallel_rows(
-                [
-                    ParallelTextRow(
-                        ii["textId"],
-                        ii["sourceRefs"],
-                        ii["targetRefs"],
-                        list(self._tokenizer.tokenize(ii["source"])),
-                        list(self._tokenizer.tokenize(ii["target"])),
-                    )
-                    for ii in inference_inputs
-                ]
-            ).lowercase()  # TODO we're calling lowercase() here; is that correct?
-
-            segment_batch = list(parallel_corpus)
-            if check_canceled is not None:
-                check_canceled()
-            alignments = alignment_model.align_batch(segment_batch)
+            temp_dir = stack.enter_context(TemporaryDirectory())
+            # Spool the parallel data to disk so that the aligner can make multiple
+            # passes over them without keeping the entire corpus in memory.
+            source_path = Path(temp_dir) / "word_align.src.txt"
+            target_path = Path(temp_dir) / "word_align.trg.txt"
+            word_alignments_path = Path(temp_dir) / "word_align.json"
+            with (
+                source_path.open("w", encoding="utf-8", newline="\n") as source_file,
+                target_path.open("w", encoding="utf-8", newline="\n") as target_file,
+                word_alignments_path.open("w", encoding="utf-8", newline="\n") as word_alignments_file,
+            ):
+                for wa_input in inference_inputs:
+                    source_file.write(wa_input["source"] + "\n")
+                    target_file.write(wa_input["target"] + "\n")
+                    word_alignments_file.write(json.dumps(wa_input, ensure_ascii=False) + "\n")
+
             if check_canceled is not None:
                 check_canceled()
 
-            for parallel_text_row, inference_input, alignment in zip(
-                parallel_corpus.get_rows(), inference_inputs, alignments
-            ):
-                word_pairs = alignment.to_aligned_word_pairs(include_null=True)
-                alignment_model.compute_aligned_word_pair_scores(
-                    parallel_text_row.source_segment, parallel_text_row.target_segment, word_pairs
-                )
-
-                word_alignment_info = {
-                    "corpusId": inference_input["corpusId"],
-                    "textId": inference_input["textId"],
-                    "sourceRefs": [str(ref) for ref in inference_input["sourceRefs"]],
-                    "targetRefs": [str(ref) for ref in inference_input["targetRefs"]],
-                    "sourceTokens": parallel_text_row.source_segment,
-                    "targetTokens": parallel_text_row.target_segment,
-                    "alignment": AlignedWordPair.to_string(word_pairs),
-                }
-                writer.write(word_alignment_info)
+            parallel_corpus = TextFileTextCorpus(source_path).align_rows(TextFileTextCorpus(target_path))
+
+            batch_size: int = self._config["inference_batch_size"]
+            alignment_model = stack.enter_context(self._word_alignment_model_factory.create_alignment_model())
+            rows = stack.enter_context(parallel_corpus.tokenize(self._tokenizer).get_rows())
+            for wa_batch in batch(zip(_read_word_alignments(word_alignments_path), rows, strict=True), batch_size):
+                if check_canceled is not None:
+                    check_canceled()
+                segments = [(lowercase(row.source_segment), lowercase(row.target_segment)) for _, row in wa_batch]
+                alignments = alignment_model.align_batch(segments)
+                if check_canceled is not None:
+                    check_canceled()
+                for (wa_input, row), (source_segment, target_segment), alignment in zip(
+                    wa_batch, segments, alignments, strict=True
+                ):
+                    word_pairs = alignment.to_aligned_word_pairs(include_null=False)
+                    alignment_model.compute_aligned_word_pair_scores(source_segment, target_segment, word_pairs)
+
+                    word_alignment_info = {
+                        "corpusId": wa_input["corpusId"],
+                        "textId": wa_input["textId"],
+                        "sourceRefs": [str(ref) for ref in wa_input["sourceRefs"]],
+                        "targetRefs": [str(ref) for ref in wa_input["targetRefs"]],
+                        "sourceTokens": row.source_segment,
+                        "targetTokens": row.target_segment,
+                        "alignment": AlignedWordPair.to_string(word_pairs),
+                    }
+                    writer.write(word_alignment_info)
 
     def _save_model(self) -> None:
         logger.info("Saving model")
@@ -145,3 +158,9 @@ def _save_model(self) -> None:
         self._word_alignment_file_service.save_model(
             model_path, f"builds/{self._config['build_id']}/model{''.join(model_path.suffixes)}"
         )
+
+
+def _read_word_alignments(path: Path) -> Generator[WordAlignmentInput, None, None]:
+    with path.open("r", encoding="utf-8") as file:
+        for line in file:
+            yield json.loads(line)
diff --git a/tests/jobs/test_word_alignment_build_job.py b/tests/jobs/test_word_alignment_build_job.py
index 03387808..dad39bb0 100644
--- a/tests/jobs/test_word_alignment_build_job.py
+++ b/tests/jobs/test_word_alignment_build_job.py
@@ -21,8 +21,8 @@ def test_run(decoy: Decoy) -> None:
     env.job.run()
 
     alignments = json.loads(env.alignment_json)
-    assert len(alignments) == 1
-    assert alignments[0]["alignment"] == "0-0 1-1 2-2"
+    assert len(alignments) == 3
+    assert alignments[0]["alignment"] == "2-2 3-3 4-5 6-8 9-7 11-6 12-10"
     decoy.verify(
         env.word_alignment_file_service.save_model(matchers.Anything(), f"builds/{env.job._config.build_id}/model.zip"),
         times=1,
@@ -49,7 +49,17 @@ def __init__(self, decoy: Decoy) -> None:
         decoy.when(self.model.__enter__()).then_return(self.model)
         decoy.when(self.model.align_batch(matchers.Anything())).then_return(
             [
-                WordAlignmentMatrix.from_word_pairs(row_count=3, column_count=3, set_values=[(0, 0), (1, 1), (2, 2)]),
+                WordAlignmentMatrix.from_word_pairs(
+                    row_count=13,
+                    column_count=11,
+                    set_values=[(2, 2), (3, 3), (4, 5), (6, 8), (9, 7), (11, 6), (12, 10)],
+                ),
+                WordAlignmentMatrix.from_word_pairs(
+                    row_count=10, column_count=10, set_values=[(2, 2), (3, 3), (4, 5), (5, 6), (6, 8), (8, 7), (9, 9)]
+                ),
+                WordAlignmentMatrix.from_word_pairs(
+                    row_count=7, column_count=7, set_values=[(0, 0), (1, 1), (3, 3), (4, 4), (5, 5), (6, 6)]
+                ),
             ]
         )
 

From 8257dc4c01c3c21c3f64e407734a0dcc8d71d588 Mon Sep 17 00:00:00 2001
From: Enkidu93 <lowryec17@gcc.edu>
Date: Mon, 10 Aug 2026 12:06:17 -0400
Subject: [PATCH 11/24] Address reviewer comments

---
 machine/jobs/word_alignment_build_job.py    | 12 +++++---
 machine/jobs/word_alignment_file_service.py | 34 +++++++++++----------
 2 files changed, 25 insertions(+), 21 deletions(-)

diff --git a/machine/jobs/word_alignment_build_job.py b/machine/jobs/word_alignment_build_job.py
index b8b04948..c7a97de6 100644
--- a/machine/jobs/word_alignment_build_job.py
+++ b/machine/jobs/word_alignment_build_job.py
@@ -94,15 +94,13 @@ def _batch_inference(
         check_canceled: Optional[Callable[[], None]],
     ) -> None:
 
-        inference_inputs = self._word_alignment_file_service.get_word_alignment_inputs()
-
-        inference_step_count = len(inference_inputs)
+        with self._word_alignment_file_service.get_word_alignment_inputs() as inference_inputs:
+            inference_step_count = sum(1 for _ in inference_inputs)
 
         with ExitStack() as stack:
             phase_progress = stack.enter_context(progress_reporter.start_next_phase())
             writer = stack.enter_context(self._word_alignment_file_service.open_alignment_output_writer())
-            current_inference_step = 0
-            phase_progress(ProgressStatus.from_step(current_inference_step, inference_step_count))
+            inference_inputs = stack.enter_context(self._word_alignment_file_service.get_word_alignment_inputs())
 
             temp_dir = stack.enter_context(TemporaryDirectory())
             # Spool the parallel data to disk so that the aligner can make multiple
@@ -125,6 +123,8 @@ def _batch_inference(
 
             parallel_corpus = TextFileTextCorpus(source_path).align_rows(TextFileTextCorpus(target_path))
 
+            current_inference_step = 0
+            phase_progress(ProgressStatus.from_step(current_inference_step, inference_step_count))
             batch_size: int = self._config["inference_batch_size"]
             alignment_model = stack.enter_context(self._word_alignment_model_factory.create_alignment_model())
             rows = stack.enter_context(parallel_corpus.tokenize(self._tokenizer).get_rows())
@@ -151,6 +151,8 @@ def _batch_inference(
                         "alignment": AlignedWordPair.to_string(word_pairs),
                     }
                     writer.write(word_alignment_info)
+                    current_inference_step += len(wa_batch)
+                    phase_progress(ProgressStatus.from_step(current_inference_step, inference_step_count))
 
     def _save_model(self) -> None:
         logger.info("Saving model")
diff --git a/machine/jobs/word_alignment_file_service.py b/machine/jobs/word_alignment_file_service.py
index 0c02d9cf..6f191afd 100644
--- a/machine/jobs/word_alignment_file_service.py
+++ b/machine/jobs/word_alignment_file_service.py
@@ -1,11 +1,12 @@
 from contextlib import contextmanager
 from pathlib import Path
-from typing import Any, Iterator, List, Optional, TypedDict, Union
+from typing import Any, Generator, Iterator, List, Optional, TypedDict, Union
 
 import json_stream
 
 from ..corpora.text_corpus import TextCorpus
 from ..corpora.text_file_text_corpus import TextFileTextCorpus
+from ..utils.context_managed_generator import ContextManagedGenerator
 from .shared_file_service_base import DictToJsonWriter, SharedFileServiceBase
 from .shared_file_service_factory import SharedFileServiceType, get_shared_file_service
 
@@ -54,23 +55,24 @@ def create_target_corpus(self) -> TextCorpus:
             for target_filename in self._target_filenames
         )
 
-    def get_word_alignment_inputs(self) -> List[WordAlignmentInput]:
-        src_pretranslate_path = self.shared_file_service.download_file(
+    def get_word_alignment_inputs(self) -> ContextManagedGenerator[WordAlignmentInput, None, None]:
+        src_word_alignment_path = self.shared_file_service.download_file(
             f"{self.shared_file_service.build_path}/{self._word_alignment_input_filename}"
         )
-        with src_pretranslate_path.open("r", encoding="utf-8-sig") as file:
-            wa_inputs = [
-                WordAlignmentInput(
-                    corpusId=pi["corpusId"],
-                    textId=pi["textId"],
-                    sourceRefs=list(pi["sourceRefs"]),
-                    targetRefs=list(pi["targetRefs"]),
-                    source=pi["source"],
-                    target=pi["target"],
-                )
-                for pi in json_stream.load(file)
-            ]
-        return wa_inputs
+
+        def generator() -> Generator[WordAlignmentInput, None, None]:
+            with src_word_alignment_path.open("r", encoding="utf-8-sig") as file:
+                for wi in json_stream.load(file):
+                    yield WordAlignmentInput(
+                        corpusId=wi["corpusId"],
+                        textId=wi["textId"],
+                        sourceRefs=list(wi["sourceRefs"]),
+                        targetRefs=list(wi["targetRefs"]),
+                        source=wi["source"],
+                        target=wi["target"],
+                    )
+
+        return ContextManagedGenerator(generator())
 
     def exists_source_corpus(self) -> bool:
         return all(

From f512b491886129c38ce6a4c393222ec67f0909cd Mon Sep 17 00:00:00 2001
From: Enkidu93 <lowryec17@gcc.edu>
Date: Mon, 10 Aug 2026 12:14:55 -0400
Subject: [PATCH 12/24] Update word alignment job test mock to return correct
 type

---
 tests/jobs/test_word_alignment_build_job.py | 61 +++++++++++----------
 1 file changed, 33 insertions(+), 28 deletions(-)

diff --git a/tests/jobs/test_word_alignment_build_job.py b/tests/jobs/test_word_alignment_build_job.py
index dad39bb0..77251021 100644
--- a/tests/jobs/test_word_alignment_build_job.py
+++ b/tests/jobs/test_word_alignment_build_job.py
@@ -13,7 +13,7 @@
 from machine.jobs.word_alignment_file_service import WordAlignmentFileService, WordAlignmentInput
 from machine.translation import Trainer, TrainStats, WordAlignmentMatrix
 from machine.translation.word_alignment_model import WordAlignmentModel
-from machine.utils import CanceledError
+from machine.utils import CanceledError, ContextManagedGenerator
 
 
 def test_run(decoy: Decoy) -> None:
@@ -98,33 +98,38 @@ def __init__(self, decoy: Decoy) -> None:
         decoy.when(self.word_alignment_file_service.exists_source_corpus()).then_return(True)
         decoy.when(self.word_alignment_file_service.exists_target_corpus()).then_return(True)
 
-        decoy.when(self.word_alignment_file_service.get_word_alignment_inputs()).then_return(
-            [
-                WordAlignmentInput(
-                    corpusId="corpus1",
-                    textId="text1",
-                    sourceRefs=["1"],
-                    targetRefs=["1"],
-                    source="¿Le importaría darnos las llaves de la habitación, por favor?",
-                    target="Would you mind giving us the room keys, please?",
-                ),
-                WordAlignmentInput(
-                    corpusId="corpus1",
-                    textId="text1",
-                    sourceRefs=["2"],
-                    targetRefs=["2"],
-                    source="¿Le importaría cambiarme a otra habitación más tranquila?",
-                    target="Would you mind moving me to another quieter room?",
-                ),
-                WordAlignmentInput(
-                    corpusId="corpus1",
-                    textId="text1",
-                    sourceRefs=["3"],
-                    targetRefs=["3"],
-                    source="Me parece que existe un problema.",
-                    target="I think there is a problem.",
-                ),
-            ]
+        decoy.when(self.word_alignment_file_service.get_word_alignment_inputs()).then_do(
+            lambda: ContextManagedGenerator(
+                (
+                    wa
+                    for wa in [
+                        WordAlignmentInput(
+                            corpusId="corpus1",
+                            textId="text1",
+                            sourceRefs=["1"],
+                            targetRefs=["1"],
+                            source="¿Le importaría darnos las llaves de la habitación, por favor?",
+                            target="Would you mind giving us the room keys, please?",
+                        ),
+                        WordAlignmentInput(
+                            corpusId="corpus1",
+                            textId="text1",
+                            sourceRefs=["2"],
+                            targetRefs=["2"],
+                            source="¿Le importaría cambiarme a otra habitación más tranquila?",
+                            target="Would you mind moving me to another quieter room?",
+                        ),
+                        WordAlignmentInput(
+                            corpusId="corpus1",
+                            textId="text1",
+                            sourceRefs=["3"],
+                            targetRefs=["3"],
+                            source="Me parece que existe un problema.",
+                            target="I think there is a problem.",
+                        ),
+                    ]
+                )
+            )
         )
 
         self.alignment_json = ""

From 9ccbf07e06d0ae6ea49ba8bb7716e65ac831e814 Mon Sep 17 00:00:00 2001
From: Enkidu93 <lowryec17@gcc.edu>
Date: Wed, 12 Aug 2026 13:41:41 -0400
Subject: [PATCH 13/24] Add logging for debugging

---
 machine/jobs/nmt_engine_build_job.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/machine/jobs/nmt_engine_build_job.py b/machine/jobs/nmt_engine_build_job.py
index ed5cf821..ae2867c7 100644
--- a/machine/jobs/nmt_engine_build_job.py
+++ b/machine/jobs/nmt_engine_build_job.py
@@ -227,9 +227,11 @@ def _align(
                     if transductive_model is not None:
                         # The pretranslations are the first rows of the training corpus, so their
                         # alignments were already computed during training.
-                        alignments: Sequence[WordAlignmentMatrix] = [
-                            transductive_model.get_training_alignment(index + i) for i in range(len(pt_batch))
-                        ]
+                        alignments: Sequence[WordAlignmentMatrix] = []
+                        for i in range(len(pt_batch)):
+                            logger.info(f"Index: {index}, i: {i}, pt_info: {pt_batch[i]}")
+                            alignments.append(transductive_model.get_training_alignment(index + i))
+
                     else:
                         alignments = alignment_model.align_batch(segments)
                     for (pt_info, row), (source_segment, target_segment), alignment in zip(

From 4f7ef98755548ecbff2b2e5ec83237a22adceaf6 Mon Sep 17 00:00:00 2001
From: Enkidu93 <lowryec17@gcc.edu>
Date: Wed, 12 Aug 2026 13:45:22 -0400
Subject: [PATCH 14/24] Revert debugging changes

---
 machine/jobs/nmt_engine_build_job.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/machine/jobs/nmt_engine_build_job.py b/machine/jobs/nmt_engine_build_job.py
index ae2867c7..ed5cf821 100644
--- a/machine/jobs/nmt_engine_build_job.py
+++ b/machine/jobs/nmt_engine_build_job.py
@@ -227,11 +227,9 @@ def _align(
                     if transductive_model is not None:
                         # The pretranslations are the first rows of the training corpus, so their
                         # alignments were already computed during training.
-                        alignments: Sequence[WordAlignmentMatrix] = []
-                        for i in range(len(pt_batch)):
-                            logger.info(f"Index: {index}, i: {i}, pt_info: {pt_batch[i]}")
-                            alignments.append(transductive_model.get_training_alignment(index + i))
-
+                        alignments: Sequence[WordAlignmentMatrix] = [
+                            transductive_model.get_training_alignment(index + i) for i in range(len(pt_batch))
+                        ]
                     else:
                         alignments = alignment_model.align_batch(segments)
                     for (pt_info, row), (source_segment, target_segment), alignment in zip(

From b247d13c461edaaaa6b0b56529cb1ca1ed64bf99 Mon Sep 17 00:00:00 2001
From: Enkidu93 <lowryec17@gcc.edu>
Date: Wed, 19 Aug 2026 15:59:18 -0400
Subject: [PATCH 15/24] Debug commit

---
 machine/jobs/nmt_engine_build_job.py | 136 +++++++++++++--------------
 1 file changed, 65 insertions(+), 71 deletions(-)

diff --git a/machine/jobs/nmt_engine_build_job.py b/machine/jobs/nmt_engine_build_job.py
index ed5cf821..9ebbd921 100644
--- a/machine/jobs/nmt_engine_build_job.py
+++ b/machine/jobs/nmt_engine_build_job.py
@@ -7,10 +7,13 @@
 
 from ..corpora.aligned_word_pair import AlignedWordPair
 from ..corpora.corpora_utils import batch
+from ..corpora.dictionary_text_corpus import DictionaryTextCorpus
 from ..corpora.flatten import flatten
+from ..corpora.memory_text import MemoryText
 from ..corpora.parallel_text_corpus import ParallelTextCorpus
 from ..corpora.text_corpus import TextCorpus
 from ..corpora.text_file_text_corpus import TextFileTextCorpus
+from ..corpora.text_row import TextRow
 from ..corpora.token_processors import lowercase
 from ..tokenization.tokenizer_factory import create_tokenizer
 from ..translation.transductive_word_alignment_model import TransductiveWordAlignmentModel
@@ -168,80 +171,71 @@ def _align(
 
         tokenizer = create_tokenizer(self._config.thot_align.tokenizer)
 
-        with TemporaryDirectory() as temp_dir:
-            # Spool the translated pretranslations to disk so that the aligner can make multiple
-            # passes over them without keeping the entire corpus in memory.
-            logger.info("Aligning source to pretranslations")
-            source_path = Path(temp_dir) / "pretranslations.src.txt"
-            translation_path = Path(temp_dir) / "pretranslations.trg.txt"
-            pretranslations_path = Path(temp_dir) / "pretranslations.json"
-            with (
-                source_path.open("w", encoding="utf-8", newline="\n") as source_file,
-                translation_path.open("w", encoding="utf-8", newline="\n") as translation_file,
-                pretranslations_path.open("w", encoding="utf-8", newline="\n") as pretranslations_file,
-            ):
-                for source_segment, pt_info in pretranslations:
-                    source_file.write(source_segment + "\n")
-                    translation_file.write(pt_info["translation"] + "\n")
-                    pretranslations_file.write(json.dumps(pt_info, ensure_ascii=False) + "\n")
+        if check_canceled is not None:
+            check_canceled()
 
-            if check_canceled is not None:
-                check_canceled()
+        pretranslations = list(pretranslations)
 
-            parallel_pretranslation_corpus = TextFileTextCorpus(source_path).align_rows(
-                TextFileTextCorpus(translation_path)
+        parallel_pretranslation_corpus = DictionaryTextCorpus(
+            MemoryText("pt", [TextRow("pt", i, pi[0].split()) for i, pi in enumerate(pretranslations)])
+        ).align_rows(
+            DictionaryTextCorpus(
+                MemoryText(
+                    "pt", [TextRow("pt", i, pi[1]["translation"].split()) for i, pi in enumerate(pretranslations)]
+                )
             )
+        )
 
-            # The pretranslations are placed at the beginning of the training corpus so that a
-            # transductive model's training alignments can be matched back to them by index.
-            alignment_parallel_corpus = flatten([parallel_pretranslation_corpus, parallel_training_corpus])
+        # The pretranslations are placed at the beginning of the training corpus so that a
+        # transductive model's training alignments can be matched back to them by index.
+        alignment_parallel_corpus = flatten([parallel_pretranslation_corpus, parallel_training_corpus])
 
-            logger.info("Training aligner")
-            with (
-                progress_reporter.start_next_phase() as phase_progress,
-                self._alignment_model_factory.create_model_trainer(tokenizer, alignment_parallel_corpus) as trainer,
-            ):
-                trainer.train(progress=phase_progress, check_canceled=check_canceled)
-                trainer.save()
+        logger.info("Training aligner")
+        with (
+            progress_reporter.start_next_phase() as phase_progress,
+            self._alignment_model_factory.create_model_trainer(tokenizer, alignment_parallel_corpus) as trainer,
+        ):
+            trainer.train(progress=phase_progress, check_canceled=check_canceled)
+            trainer.save()
 
-            if check_canceled is not None:
-                check_canceled()
+        if check_canceled is not None:
+            check_canceled()
 
-            logger.info("Aligning pretranslations")
-            batch_size: int = self._config["inference_batch_size"]
-            with (
-                self._alignment_model_factory.create_alignment_model() as alignment_model,
-                parallel_pretranslation_corpus.tokenize(tokenizer).get_rows() as rows,
-            ):
-                transductive_model: Optional[TransductiveWordAlignmentModel] = None
-                if isinstance(alignment_model, TransductiveWordAlignmentModel):
-                    transductive_model = alignment_model
-                index = 0
-                for pt_batch in batch(zip(_read_pretranslations(pretranslations_path), rows, strict=True), batch_size):
-                    if check_canceled is not None:
-                        check_canceled()
-                    # The aligner is trained on lowercased tokens, so it must also be given
-                    # lowercased tokens when aligning and scoring; the original-cased tokens are
-                    # written to the pretranslations.
-                    segments = [(lowercase(row.source_segment), lowercase(row.target_segment)) for _, row in pt_batch]
-                    if transductive_model is not None:
-                        # The pretranslations are the first rows of the training corpus, so their
-                        # alignments were already computed during training.
-                        alignments: Sequence[WordAlignmentMatrix] = [
-                            transductive_model.get_training_alignment(index + i) for i in range(len(pt_batch))
-                        ]
-                    else:
-                        alignments = alignment_model.align_batch(segments)
-                    for (pt_info, row), (source_segment, target_segment), alignment in zip(
-                        pt_batch, segments, alignments, strict=True
-                    ):
-                        word_pairs = alignment.to_aligned_word_pairs(include_null=False)
-                        alignment_model.compute_aligned_word_pair_scores(source_segment, target_segment, word_pairs)
-                        pt_info["sourceTokens"] = list(row.source_segment)
-                        pt_info["translationTokens"] = list(row.target_segment)
-                        pt_info["alignment"] = AlignedWordPair.to_string(word_pairs, include_scores=True)
-                        yield pt_info
-                    index += len(pt_batch)
+        logger.info("Aligning pretranslations")
+        batch_size: int = self._config["inference_batch_size"]
+        with (
+            self._alignment_model_factory.create_alignment_model() as alignment_model,
+            parallel_pretranslation_corpus.tokenize(tokenizer).get_rows() as rows,
+        ):
+            transductive_model: Optional[TransductiveWordAlignmentModel] = None
+            if isinstance(alignment_model, TransductiveWordAlignmentModel):
+                transductive_model = alignment_model
+            index = 0
+            for pt_batch in batch(zip((pi[1] for pi in pretranslations), rows, strict=True), batch_size):
+                if check_canceled is not None:
+                    check_canceled()
+                # The aligner is trained on lowercased tokens, so it must also be given
+                # lowercased tokens when aligning and scoring; the original-cased tokens are
+                # written to the pretranslations.
+                segments = [(lowercase(row.source_segment), lowercase(row.target_segment)) for _, row in pt_batch]
+                if transductive_model is not None:
+                    # The pretranslations are the first rows of the training corpus, so their
+                    # alignments were already computed during training.
+                    alignments: Sequence[WordAlignmentMatrix] = [
+                        transductive_model.get_training_alignment(index + i) for i in range(len(pt_batch))
+                    ]
+                else:
+                    alignments = alignment_model.align_batch(segments)
+                for (pt_info, row), (source_segment, target_segment), alignment in zip(
+                    pt_batch, segments, alignments, strict=True
+                ):
+                    word_pairs = alignment.to_aligned_word_pairs(include_null=False)
+                    alignment_model.compute_aligned_word_pair_scores(source_segment, target_segment, word_pairs)
+                    pt_info["sourceTokens"] = list(row.source_segment)
+                    pt_info["translationTokens"] = list(row.target_segment)
+                    pt_info["alignment"] = AlignedWordPair.to_string(word_pairs, include_scores=True)
+                    yield pt_info
+                index += len(pt_batch)
 
     def _save_model(self) -> None:
         if "save_model" in self._config and self._config.save_model is not None:
@@ -252,7 +246,7 @@ def _save_model(self) -> None:
             )
 
 
-def _read_pretranslations(path: Path) -> Generator[PretranslationInfo, None, None]:
-    with path.open("r", encoding="utf-8") as file:
-        for line in file:
-            yield json.loads(line)
+# def _read_pretranslations(path: Path) -> Generator[PretranslationInfo, None, None]:
+#     with path.open("r", encoding="utf-8") as file:
+#         for line in file:
+#             yield json.loads(line)

From 6599e832105533f5f29bb8bcfe34f4289d23c048 Mon Sep 17 00:00:00 2001
From: Enkidu93 <lowryec17@gcc.edu>
Date: Thu, 20 Aug 2026 10:33:13 -0400
Subject: [PATCH 16/24] Update thot; report steps (debug)

---
 machine/jobs/nmt_engine_build_job.py | 140 ++++++++++++++-------------
 poetry.lock                          |  66 ++++++-------
 pyproject.toml                       |   2 +-
 3 files changed, 107 insertions(+), 101 deletions(-)

diff --git a/machine/jobs/nmt_engine_build_job.py b/machine/jobs/nmt_engine_build_job.py
index 9ebbd921..f68536a7 100644
--- a/machine/jobs/nmt_engine_build_job.py
+++ b/machine/jobs/nmt_engine_build_job.py
@@ -7,13 +7,10 @@
 
 from ..corpora.aligned_word_pair import AlignedWordPair
 from ..corpora.corpora_utils import batch
-from ..corpora.dictionary_text_corpus import DictionaryTextCorpus
 from ..corpora.flatten import flatten
-from ..corpora.memory_text import MemoryText
 from ..corpora.parallel_text_corpus import ParallelTextCorpus
 from ..corpora.text_corpus import TextCorpus
 from ..corpora.text_file_text_corpus import TextFileTextCorpus
-from ..corpora.text_row import TextRow
 from ..corpora.token_processors import lowercase
 from ..tokenization.tokenizer_factory import create_tokenizer
 from ..translation.transductive_word_alignment_model import TransductiveWordAlignmentModel
@@ -51,7 +48,7 @@ def _get_progress_reporter(
                 phases = [
                     Phase(message="Training NMT model", percentage=0.8, stage="train"),
                     Phase(message="Pretranslating segments", percentage=0.1, stage="inference"),
-                    Phase(message="Aligning segments", percentage=0.1, report_steps=False),
+                    Phase(message="Aligning segments", percentage=0.1, report_steps=True),
                 ]
             else:
                 phases = [
@@ -62,7 +59,7 @@ def _get_progress_reporter(
             if self._config.align_pretranslations:
                 phases = [
                     Phase(message="Pretranslating segments", percentage=0.9, stage="inference"),
-                    Phase(message="Aligning segments", percentage=0.1, report_steps=False),
+                    Phase(message="Aligning segments", percentage=0.1, report_steps=True),
                 ]
             else:
                 phases = [Phase(message="Pretranslating segments", percentage=1.0, stage="inference")]
@@ -171,71 +168,80 @@ def _align(
 
         tokenizer = create_tokenizer(self._config.thot_align.tokenizer)
 
-        if check_canceled is not None:
-            check_canceled()
+        with TemporaryDirectory() as temp_dir:
+            # Spool the translated pretranslations to disk so that the aligner can make multiple
+            # passes over them without keeping the entire corpus in memory.
+            logger.info("Aligning source to pretranslations")
+            source_path = Path(temp_dir) / "pretranslations.src.txt"
+            translation_path = Path(temp_dir) / "pretranslations.trg.txt"
+            pretranslations_path = Path(temp_dir) / "pretranslations.json"
+            with (
+                source_path.open("w", encoding="utf-8", newline="\n") as source_file,
+                translation_path.open("w", encoding="utf-8", newline="\n") as translation_file,
+                pretranslations_path.open("w", encoding="utf-8", newline="\n") as pretranslations_file,
+            ):
+                for source_segment, pt_info in pretranslations:
+                    source_file.write(source_segment + "\n")
+                    translation_file.write(pt_info["translation"] + "\n")
+                    pretranslations_file.write(json.dumps(pt_info, ensure_ascii=False) + "\n")
 
-        pretranslations = list(pretranslations)
+            if check_canceled is not None:
+                check_canceled()
 
-        parallel_pretranslation_corpus = DictionaryTextCorpus(
-            MemoryText("pt", [TextRow("pt", i, pi[0].split()) for i, pi in enumerate(pretranslations)])
-        ).align_rows(
-            DictionaryTextCorpus(
-                MemoryText(
-                    "pt", [TextRow("pt", i, pi[1]["translation"].split()) for i, pi in enumerate(pretranslations)]
-                )
+            parallel_pretranslation_corpus = TextFileTextCorpus(source_path).align_rows(
+                TextFileTextCorpus(translation_path)
             )
-        )
 
-        # The pretranslations are placed at the beginning of the training corpus so that a
-        # transductive model's training alignments can be matched back to them by index.
-        alignment_parallel_corpus = flatten([parallel_pretranslation_corpus, parallel_training_corpus])
+            # The pretranslations are placed at the beginning of the training corpus so that a
+            # transductive model's training alignments can be matched back to them by index.
+            alignment_parallel_corpus = flatten([parallel_pretranslation_corpus, parallel_training_corpus])
 
-        logger.info("Training aligner")
-        with (
-            progress_reporter.start_next_phase() as phase_progress,
-            self._alignment_model_factory.create_model_trainer(tokenizer, alignment_parallel_corpus) as trainer,
-        ):
-            trainer.train(progress=phase_progress, check_canceled=check_canceled)
-            trainer.save()
+            logger.info("Training aligner")
+            with (
+                progress_reporter.start_next_phase() as phase_progress,
+                self._alignment_model_factory.create_model_trainer(tokenizer, alignment_parallel_corpus) as trainer,
+            ):
+                trainer.train(progress=phase_progress, check_canceled=check_canceled)
+                trainer.save()
 
-        if check_canceled is not None:
-            check_canceled()
+            if check_canceled is not None:
+                check_canceled()
 
-        logger.info("Aligning pretranslations")
-        batch_size: int = self._config["inference_batch_size"]
-        with (
-            self._alignment_model_factory.create_alignment_model() as alignment_model,
-            parallel_pretranslation_corpus.tokenize(tokenizer).get_rows() as rows,
-        ):
-            transductive_model: Optional[TransductiveWordAlignmentModel] = None
-            if isinstance(alignment_model, TransductiveWordAlignmentModel):
-                transductive_model = alignment_model
-            index = 0
-            for pt_batch in batch(zip((pi[1] for pi in pretranslations), rows, strict=True), batch_size):
-                if check_canceled is not None:
-                    check_canceled()
-                # The aligner is trained on lowercased tokens, so it must also be given
-                # lowercased tokens when aligning and scoring; the original-cased tokens are
-                # written to the pretranslations.
-                segments = [(lowercase(row.source_segment), lowercase(row.target_segment)) for _, row in pt_batch]
-                if transductive_model is not None:
-                    # The pretranslations are the first rows of the training corpus, so their
-                    # alignments were already computed during training.
-                    alignments: Sequence[WordAlignmentMatrix] = [
-                        transductive_model.get_training_alignment(index + i) for i in range(len(pt_batch))
-                    ]
-                else:
-                    alignments = alignment_model.align_batch(segments)
-                for (pt_info, row), (source_segment, target_segment), alignment in zip(
-                    pt_batch, segments, alignments, strict=True
-                ):
-                    word_pairs = alignment.to_aligned_word_pairs(include_null=False)
-                    alignment_model.compute_aligned_word_pair_scores(source_segment, target_segment, word_pairs)
-                    pt_info["sourceTokens"] = list(row.source_segment)
-                    pt_info["translationTokens"] = list(row.target_segment)
-                    pt_info["alignment"] = AlignedWordPair.to_string(word_pairs, include_scores=True)
-                    yield pt_info
-                index += len(pt_batch)
+            logger.info("Aligning pretranslations")
+            batch_size: int = self._config["inference_batch_size"]
+            with (
+                self._alignment_model_factory.create_alignment_model() as alignment_model,
+                parallel_pretranslation_corpus.tokenize(tokenizer).get_rows() as rows,
+            ):
+                transductive_model: Optional[TransductiveWordAlignmentModel] = None
+                if isinstance(alignment_model, TransductiveWordAlignmentModel):
+                    transductive_model = alignment_model
+                index = 0
+                for pt_batch in batch(zip(_read_pretranslations(pretranslations_path), rows, strict=True), batch_size):
+                    if check_canceled is not None:
+                        check_canceled()
+                    # The aligner is trained on lowercased tokens, so it must also be given
+                    # lowercased tokens when aligning and scoring; the original-cased tokens are
+                    # written to the pretranslations.
+                    segments = [(lowercase(row.source_segment), lowercase(row.target_segment)) for _, row in pt_batch]
+                    if transductive_model is not None:
+                        # The pretranslations are the first rows of the training corpus, so their
+                        # alignments were already computed during training.
+                        alignments: Sequence[WordAlignmentMatrix] = [
+                            transductive_model.get_training_alignment(index + i) for i in range(len(pt_batch))
+                        ]
+                    else:
+                        alignments = alignment_model.align_batch(segments)
+                    for (pt_info, row), (source_segment, target_segment), alignment in zip(
+                        pt_batch, segments, alignments, strict=True
+                    ):
+                        word_pairs = alignment.to_aligned_word_pairs(include_null=False)
+                        alignment_model.compute_aligned_word_pair_scores(source_segment, target_segment, word_pairs)
+                        pt_info["sourceTokens"] = list(row.source_segment)
+                        pt_info["translationTokens"] = list(row.target_segment)
+                        pt_info["alignment"] = AlignedWordPair.to_string(word_pairs, include_scores=True)
+                        yield pt_info
+                    index += len(pt_batch)
 
     def _save_model(self) -> None:
         if "save_model" in self._config and self._config.save_model is not None:
@@ -246,7 +252,7 @@ def _save_model(self) -> None:
             )
 
 
-# def _read_pretranslations(path: Path) -> Generator[PretranslationInfo, None, None]:
-#     with path.open("r", encoding="utf-8") as file:
-#         for line in file:
-#             yield json.loads(line)
+def _read_pretranslations(path: Path) -> Generator[PretranslationInfo, None, None]:
+    with path.open("r", encoding="utf-8") as file:
+        for line in file:
+            yield json.loads(line)
diff --git a/poetry.lock b/poetry.lock
index 05631520..9638dbaa 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -4808,43 +4808,43 @@ type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.deve
 
 [[package]]
 name = "sil-thot"
-version = "3.5.1"
+version = "3.5.2"
 description = "A toolkit for statistical word alignment and machine translation"
 optional = false
 python-versions = "<4.0,>=3.10"
 groups = ["main"]
 files = [
-    {file = "sil_thot-3.5.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:896c9d285cad8febdb053110e2cb69098cdbf749f58cb8fc401ecaef4ce3b31c"},
-    {file = "sil_thot-3.5.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e16737d6c39d7596864e8b61df688ba69ea3855dfae2df9db269cd459ad10246"},
-    {file = "sil_thot-3.5.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b1c701109371a1b0bcc48b86cccd84edadb30ae872496fb559ec93352463f6c"},
-    {file = "sil_thot-3.5.1-cp310-cp310-win32.whl", hash = "sha256:d58040685f1cdb6a8500944c49b6f7369382059f62d6a76e818262458076af00"},
-    {file = "sil_thot-3.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:b46e971ba975db88f4f186c89f2f1225b4e6555141184eae647613af12a8ec38"},
-    {file = "sil_thot-3.5.1-cp310-cp310-win_arm64.whl", hash = "sha256:1acbbb874a284684fc717184af6fa613257e7689cd459e28ceaadc196d264409"},
-    {file = "sil_thot-3.5.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8e81fb6f8344a8346fac772b20b773bdaadaa83be89b8d841b07c9cd7eb99abd"},
-    {file = "sil_thot-3.5.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b7e870101a22e0fee84f323c57f596e8466bbbdf98c062b2ec55854c855a54dd"},
-    {file = "sil_thot-3.5.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e72e9c625301f4f7ea87918fd33ca4915873a937f4efe934fb144bdfe05f0281"},
-    {file = "sil_thot-3.5.1-cp311-cp311-win32.whl", hash = "sha256:5365d1d5a6bb21d10659ef26dbf7d1d74aae3de12b25e7f1207c8ee1873a95bb"},
-    {file = "sil_thot-3.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:5318cbbd08c1f567d6d1d248e24aba00eb6cd2110ca92c3019db3fbe251aec44"},
-    {file = "sil_thot-3.5.1-cp311-cp311-win_arm64.whl", hash = "sha256:806936112ae26d907c6391047a5547f7aaf458c7c565a520aafa1823878ae31a"},
-    {file = "sil_thot-3.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:21f1e7e22b23b1292f6e2347f093967426d9dfa523b4bb09ea49382fc40cee28"},
-    {file = "sil_thot-3.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5e8b8200c9d8f958eb1b425101252997c072225bbd21973c51656cd853bef9dc"},
-    {file = "sil_thot-3.5.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ab04c62ae204cec86c766c7c0d9a74dc0c96f45774c5b210c3921a8f4d000004"},
-    {file = "sil_thot-3.5.1-cp312-cp312-win32.whl", hash = "sha256:ac9cab93314d54b0328ff9a78b84182563bb92d350d5ebecf595ae35a4c23781"},
-    {file = "sil_thot-3.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:d9cf20fe13f662070a801e611f9cacbc6c9e547ff42989080dd2a4ab350cbaae"},
-    {file = "sil_thot-3.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:9bf45b58d7335f6693fe074f450f7219f3211916da42a7ae4cc5243932e0ea43"},
-    {file = "sil_thot-3.5.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:3ec0159e492e6eadedef52f67c6b5f397c0359e42b65309a6607edd6d0e7b69d"},
-    {file = "sil_thot-3.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:65a699fc9ff6e7917c4b065a99cb3c7f6df5fe41d1e0cf599b5d7e7a33f2b112"},
-    {file = "sil_thot-3.5.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:865d65c7354083d50c5e0b29acf0c50fcaa68256e0856a9c6d01a749adc4bc8b"},
-    {file = "sil_thot-3.5.1-cp313-cp313-win32.whl", hash = "sha256:311df2a158cf6a0b9febee144d822da964e287b3c3ae996cbb6ec810eca7aba3"},
-    {file = "sil_thot-3.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:0fb8f015d5f1486651c2136f64e00fcc6038990a43d3fce9516612a6c7ed4af0"},
-    {file = "sil_thot-3.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:2eb9131ec0be2ffb7fb3ddd3f3ac9801c6fddb8cda6fd73ca19e66ed250af836"},
-    {file = "sil_thot-3.5.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:34d1663f402382a72f0feb97b4961e35d8b87859c1de00bc890147a05bee1120"},
-    {file = "sil_thot-3.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:afa926a1c7e261c33008a0c0ee9d0a923b0837c277fc0da07c57d28c8c6cdf9d"},
-    {file = "sil_thot-3.5.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:24f24a9a19f95b7d5f3954e2bb4eff7ee9798a44fd7cae1c48fdd6601f01af31"},
-    {file = "sil_thot-3.5.1-cp314-cp314-win32.whl", hash = "sha256:19a4d070cb3b86135f2837742daef87eb8d9f5101215699605cc4a4133fe1a41"},
-    {file = "sil_thot-3.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:3c40d3fc149a7b8d1eb4aed0f8e7d63e533f76e1e5f61284e756234d830468b9"},
-    {file = "sil_thot-3.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:dccb748942ca9e1a3bddf68f20096cbac8217457389f8b674cdad0fb409f8868"},
-    {file = "sil_thot-3.5.1.tar.gz", hash = "sha256:29defec1d82b1017e6241b74d9ae8278f2283fefe6ab2581cf4b68365f659fed"},
+    {file = "sil_thot-3.5.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ccab3398dd075ee7a479afba911c630b4bb33e9b5ec0956de03b45fcb6621435"},
+    {file = "sil_thot-3.5.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f35b16aaa91259f6f9fb2203eeb7170ba82c99359a4a192e850cefc3b5cb0d5c"},
+    {file = "sil_thot-3.5.2-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f4d9c8c6cc5906be1c12eca26b099989e4de64db96e6aff68864858ec9df5c3d"},
+    {file = "sil_thot-3.5.2-cp310-cp310-win32.whl", hash = "sha256:4f3cb853ce621d4d282a2423392f29ff054e461f43c42056c45aaaa713efd16c"},
+    {file = "sil_thot-3.5.2-cp310-cp310-win_amd64.whl", hash = "sha256:696a16bef2333c4dff08b4088641dd701726ddc444655e03dcef69fad58dc03c"},
+    {file = "sil_thot-3.5.2-cp310-cp310-win_arm64.whl", hash = "sha256:7a86628b4231ad97a1e982b7f99c9246dec767323eb06bb065b7133dc259d163"},
+    {file = "sil_thot-3.5.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8e4250e8b89da25c16d0e1af321ff696b7637984ee1132fb0a8580babe34d16c"},
+    {file = "sil_thot-3.5.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9bf535897a0cc48c1cc9b1e6ae7907c05971575a2f49d547081e1cfa2b1eb521"},
+    {file = "sil_thot-3.5.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:297158d8062dad0e1c294c0f15eb1f17035e7fcc4b0ee90b6792950eaa5dc4e0"},
+    {file = "sil_thot-3.5.2-cp311-cp311-win32.whl", hash = "sha256:2fdbd1ebee192bb477363026bcf3a1541518cf46b25123964df7e3474103d821"},
+    {file = "sil_thot-3.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:c86e1a68e0c2423571bf837985a65ab34f308917f9db443834b8f68270ed9965"},
+    {file = "sil_thot-3.5.2-cp311-cp311-win_arm64.whl", hash = "sha256:0225b2ebeacaf91e18ff7db27c6b3f21a1de8a2f4416e567cf26c007cdde09a9"},
+    {file = "sil_thot-3.5.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:266ae6761c71ba2c2e76c721258f914483f56646c45391547b4aa5b1f64ce8cd"},
+    {file = "sil_thot-3.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2fba0ca4cf151951e1b648d424ec1ac0dd5a62bad45c1a9dfc94af6deec1585b"},
+    {file = "sil_thot-3.5.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4a69597cf1ca333e515cf298e936ecfd76ff1266894d1f42c1504149ebeacd7e"},
+    {file = "sil_thot-3.5.2-cp312-cp312-win32.whl", hash = "sha256:1c6485952ea992e53cf0a10499de1d3922468fcb152325b1aecbc40dd4514f3a"},
+    {file = "sil_thot-3.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:92f48451ae6e1ee540d3294ea88c3e33d217beef661f283c11c1acd65b9c32e5"},
+    {file = "sil_thot-3.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:d6b192d02a5d4d6fb7bcfc9845389b2601da8bff179fa61f375a1febcac9859f"},
+    {file = "sil_thot-3.5.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:be868f6835c1ec0e5aa762d441a07405304cba59355241e0b89a3b0ad563ad95"},
+    {file = "sil_thot-3.5.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:268b04067ace8b0927baa87583d4f88a74f260c0f7a529c5d37ba6433791b165"},
+    {file = "sil_thot-3.5.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b9c68b22b6dc3556a12b87b79627bcaa7cbf9c78eeac11943b6781101c5eb7ca"},
+    {file = "sil_thot-3.5.2-cp313-cp313-win32.whl", hash = "sha256:1cd6603fb927996bd899091ff0fd409f29a749000aff826c67b21f03549684ba"},
+    {file = "sil_thot-3.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:6fdde44c8450593f8e64fe819f20f58d71908d3e477132f705fb08f1ce64aa7d"},
+    {file = "sil_thot-3.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:6dbe2848bc950baaf32455626d240da8122c3fc26baef70b3fc4dc4aa4c49f9c"},
+    {file = "sil_thot-3.5.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:b4c32a322698002586701e97e030636a943020d78f559e3fb87eb352d463ceb0"},
+    {file = "sil_thot-3.5.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:777ba0b0bbcc236e74121412e198a15bef54157a8a104f5a98da72db823458fb"},
+    {file = "sil_thot-3.5.2-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:896d1530e6a22bd48bc8ec57520b17b6e19580cefdaa4363e3a792db4af60863"},
+    {file = "sil_thot-3.5.2-cp314-cp314-win32.whl", hash = "sha256:f3a5acb021326d13cfac0b1da2da4faed27bc797f508ba0d00d46205e9e2e002"},
+    {file = "sil_thot-3.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:9965ec127e50a3f98c01629c074afa1a7c62e0bd5e373bc69a8e200958a10c51"},
+    {file = "sil_thot-3.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:0d56bfe498e4da7a038cabd65ada67d8da8c2390aed6d62a9f558c8a6bb5885e"},
+    {file = "sil_thot-3.5.2.tar.gz", hash = "sha256:41ce88201cba7bd5290075dba4b1881390be98cc5f7838878a5da5afb02ce970"},
 ]
 
 [package.extras]
@@ -5729,4 +5729,4 @@ thot = ["sil-thot"]
 [metadata]
 lock-version = "2.1"
 python-versions = ">=3.10,<3.15"
-content-hash = "59d6f3ef4b829f4810ea3344fe3e27884556d3fd28f0bbcbe7ba3b03c19da1c7"
+content-hash = "bc6517c12bd05032a6086806b3bb6dd5233c270c896ee52dd4cf6a9840e19cb4"
diff --git a/pyproject.toml b/pyproject.toml
index 0ce5935e..040e5005 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -47,7 +47,7 @@ dependencies = [
   "charset-normalizer>=2.1.1,<3.0",
   "urllib3<2.0",
   "sentencepiece~=0.2",
-  "sil-thot~=3.5.1",
+  "sil-thot~=3.5.2",
   "transformers==4.47.1",
   "datasets~=4.1",
   "sacremoses>=0.0.53,<1",

From 963ee7c6681025b4d054ce59bbdd4cdc13dc8d17 Mon Sep 17 00:00:00 2001
From: Enkidu93 <lowryec17@gcc.edu>
Date: Thu, 20 Aug 2026 11:23:43 -0400
Subject: [PATCH 17/24] Debug steps and reporting

---
 .../translation/thot/thot_word_alignment_model_trainer.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/machine/translation/thot/thot_word_alignment_model_trainer.py b/machine/translation/thot/thot_word_alignment_model_trainer.py
index e9de1524..008783d9 100644
--- a/machine/translation/thot/thot_word_alignment_model_trainer.py
+++ b/machine/translation/thot/thot_word_alignment_model_trainer.py
@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import logging
 import sys
 from pathlib import Path
 from typing import Callable, List, Optional, Tuple, Union, cast, overload
@@ -17,6 +18,8 @@
 from .thot_word_alignment_model_type import ThotWordAlignmentModelType
 from .thot_word_alignment_parameters import ThotWordAlignmentParameters
 
+logger = logging.getLogger(__name__)
+
 
 class ThotWordAlignmentModelTrainer(Trainer):
     @overload
@@ -192,12 +195,15 @@ def train(
             else None
         )
         cur_step = 0
+        update_frequency = (num_steps // 100) + 1 if num_steps else 1
+        logger.info("Num steps:", num_steps, "| Update Frequency:", update_frequency)
 
         def report() -> None:
-            if progress is not None:
+            if progress is not None and (cur_step % update_frequency) == 0:
                 progress(
                     ProgressStatus.from_step(cur_step, num_steps) if num_steps is not None else ProgressStatus(cur_step)
                 )
+                logger.info("Step:", cur_step, "| Num Steps:", num_steps)
 
         report()
 

From 01ddc0e30662795ce4b065ea473c2f5d543e820f Mon Sep 17 00:00:00 2001
From: Enkidu93 <lowryec17@gcc.edu>
Date: Thu, 20 Aug 2026 11:53:54 -0400
Subject: [PATCH 18/24] Remove debug logging; add proper align phase

---
 machine/jobs/nmt_engine_build_job.py                         | 4 ++--
 .../translation/thot/thot_word_alignment_model_trainer.py    | 5 -----
 2 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/machine/jobs/nmt_engine_build_job.py b/machine/jobs/nmt_engine_build_job.py
index f68536a7..08ab3012 100644
--- a/machine/jobs/nmt_engine_build_job.py
+++ b/machine/jobs/nmt_engine_build_job.py
@@ -48,7 +48,7 @@ def _get_progress_reporter(
                 phases = [
                     Phase(message="Training NMT model", percentage=0.8, stage="train"),
                     Phase(message="Pretranslating segments", percentage=0.1, stage="inference"),
-                    Phase(message="Aligning segments", percentage=0.1, report_steps=True),
+                    Phase(message="Aligning segments", percentage=0.1, stage="align"),
                 ]
             else:
                 phases = [
@@ -59,7 +59,7 @@ def _get_progress_reporter(
             if self._config.align_pretranslations:
                 phases = [
                     Phase(message="Pretranslating segments", percentage=0.9, stage="inference"),
-                    Phase(message="Aligning segments", percentage=0.1, report_steps=True),
+                    Phase(message="Aligning segments", percentage=0.1, stage="align"),
                 ]
             else:
                 phases = [Phase(message="Pretranslating segments", percentage=1.0, stage="inference")]
diff --git a/machine/translation/thot/thot_word_alignment_model_trainer.py b/machine/translation/thot/thot_word_alignment_model_trainer.py
index 008783d9..cd90dabc 100644
--- a/machine/translation/thot/thot_word_alignment_model_trainer.py
+++ b/machine/translation/thot/thot_word_alignment_model_trainer.py
@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import logging
 import sys
 from pathlib import Path
 from typing import Callable, List, Optional, Tuple, Union, cast, overload
@@ -18,8 +17,6 @@
 from .thot_word_alignment_model_type import ThotWordAlignmentModelType
 from .thot_word_alignment_parameters import ThotWordAlignmentParameters
 
-logger = logging.getLogger(__name__)
-
 
 class ThotWordAlignmentModelTrainer(Trainer):
     @overload
@@ -196,14 +193,12 @@ def train(
         )
         cur_step = 0
         update_frequency = (num_steps // 100) + 1 if num_steps else 1
-        logger.info("Num steps:", num_steps, "| Update Frequency:", update_frequency)
 
         def report() -> None:
             if progress is not None and (cur_step % update_frequency) == 0:
                 progress(
                     ProgressStatus.from_step(cur_step, num_steps) if num_steps is not None else ProgressStatus(cur_step)
                 )
-                logger.info("Step:", cur_step, "| Num Steps:", num_steps)
 
         report()
 

From 41a5f2b27a3e1cb4d0e0ee7eebdec6f55f8587e4 Mon Sep 17 00:00:00 2001
From: Enkidu93 <lowryec17@gcc.edu>
Date: Thu, 20 Aug 2026 12:38:34 -0400
Subject: [PATCH 19/24] Debug logging

---
 .../translation/thot/thot_word_alignment_model_trainer.py  | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/machine/translation/thot/thot_word_alignment_model_trainer.py b/machine/translation/thot/thot_word_alignment_model_trainer.py
index cd90dabc..4de176da 100644
--- a/machine/translation/thot/thot_word_alignment_model_trainer.py
+++ b/machine/translation/thot/thot_word_alignment_model_trainer.py
@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import logging
 import sys
 from pathlib import Path
 from typing import Callable, List, Optional, Tuple, Union, cast, overload
@@ -17,6 +18,8 @@
 from .thot_word_alignment_model_type import ThotWordAlignmentModelType
 from .thot_word_alignment_parameters import ThotWordAlignmentParameters
 
+logger = logging.getLogger(__name__)
+
 
 class ThotWordAlignmentModelTrainer(Trainer):
     @overload
@@ -192,13 +195,15 @@ def train(
             else None
         )
         cur_step = 0
-        update_frequency = (num_steps // 100) + 1 if num_steps else 1
+        update_frequency = num_steps // 10 if num_steps and num_steps > 10 else 1
+        logger.info("step frequency %d", update_frequency)
 
         def report() -> None:
             if progress is not None and (cur_step % update_frequency) == 0:
                 progress(
                     ProgressStatus.from_step(cur_step, num_steps) if num_steps is not None else ProgressStatus(cur_step)
                 )
+                logger.info("step %d of total %d", cur_step, num_steps)
 
         report()
 

From 7946322e79b786fe57c747a82139d62ebddef6cd Mon Sep 17 00:00:00 2001
From: Enkidu93 <lowryec17@gcc.edu>
Date: Thu, 20 Aug 2026 13:37:15 -0400
Subject: [PATCH 20/24] Fix update frequency setting

---
 .../translation/thot/thot_word_alignment_model_trainer.py  | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/machine/translation/thot/thot_word_alignment_model_trainer.py b/machine/translation/thot/thot_word_alignment_model_trainer.py
index 4de176da..d2d3db5e 100644
--- a/machine/translation/thot/thot_word_alignment_model_trainer.py
+++ b/machine/translation/thot/thot_word_alignment_model_trainer.py
@@ -195,11 +195,14 @@ def train(
             else None
         )
         cur_step = 0
-        update_frequency = num_steps // 10 if num_steps and num_steps > 10 else 1
+        update_frequency = num_steps // 100 if num_steps and num_steps > 100 else None
         logger.info("step frequency %d", update_frequency)
 
         def report() -> None:
-            if progress is not None and (cur_step % update_frequency) == 0:
+            nonlocal update_frequency
+            if update_frequency is None and num_steps is not None:
+                update_frequency = num_steps // 100 if num_steps > 100 else 1
+            if progress is not None and (update_frequency is None or (cur_step % update_frequency) == 0):
                 progress(
                     ProgressStatus.from_step(cur_step, num_steps) if num_steps is not None else ProgressStatus(cur_step)
                 )

From ab22a8437cf981189cececfc23e236ed6769e05f Mon Sep 17 00:00:00 2001
From: Enkidu93 <lowryec17@gcc.edu>
Date: Thu, 20 Aug 2026 14:12:23 -0400
Subject: [PATCH 21/24] Remove debug logging; make update frequency dynamic

---
 .../thot/thot_word_alignment_model_trainer.py            | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/machine/translation/thot/thot_word_alignment_model_trainer.py b/machine/translation/thot/thot_word_alignment_model_trainer.py
index d2d3db5e..96ce2c07 100644
--- a/machine/translation/thot/thot_word_alignment_model_trainer.py
+++ b/machine/translation/thot/thot_word_alignment_model_trainer.py
@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import logging
 import sys
 from pathlib import Path
 from typing import Callable, List, Optional, Tuple, Union, cast, overload
@@ -18,8 +17,6 @@
 from .thot_word_alignment_model_type import ThotWordAlignmentModelType
 from .thot_word_alignment_parameters import ThotWordAlignmentParameters
 
-logger = logging.getLogger(__name__)
-
 
 class ThotWordAlignmentModelTrainer(Trainer):
     @overload
@@ -195,18 +192,16 @@ def train(
             else None
         )
         cur_step = 0
-        update_frequency = num_steps // 100 if num_steps and num_steps > 100 else None
-        logger.info("step frequency %d", update_frequency)
+        update_frequency = num_steps // 10 if num_steps and num_steps > 10 else None
 
         def report() -> None:
             nonlocal update_frequency
             if update_frequency is None and num_steps is not None:
-                update_frequency = num_steps // 100 if num_steps > 100 else 1
+                update_frequency = num_steps // 10 if num_steps > 10 else 1
             if progress is not None and (update_frequency is None or (cur_step % update_frequency) == 0):
                 progress(
                     ProgressStatus.from_step(cur_step, num_steps) if num_steps is not None else ProgressStatus(cur_step)
                 )
-                logger.info("step %d of total %d", cur_step, num_steps)
 
         report()
 

From 0a98932c3b6c26ce33fe1e02e653fd2789cd6ca1 Mon Sep 17 00:00:00 2001
From: Enkidu93 <lowryec17@gcc.edu>
Date: Thu, 20 Aug 2026 15:55:49 -0400
Subject: [PATCH 22/24] Remove reporting (debug commit)

---
 .../thot/thot_word_alignment_model_trainer.py | 26 +++++++++----------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/machine/translation/thot/thot_word_alignment_model_trainer.py b/machine/translation/thot/thot_word_alignment_model_trainer.py
index 96ce2c07..f31ea0a5 100644
--- a/machine/translation/thot/thot_word_alignment_model_trainer.py
+++ b/machine/translation/thot/thot_word_alignment_model_trainer.py
@@ -186,22 +186,22 @@ def train(
         # so the total step count is not known up front; progress is reported as indeterminate until
         # it is resolved below.
         iteration_count_known = not self._is_eflomal or self._models[0][1] > 0
-        num_steps: Optional[int] = (
-            sum(iterations + 1 for _, iterations in self._models if iterations > 0) + 1
-            if iteration_count_known
-            else None
-        )
+        # num_steps: Optional[int] = (
+        #     sum(iterations + 1 for _, iterations in self._models if iterations > 0) + 1
+        #     if iteration_count_known
+        #     else None
+        # )
         cur_step = 0
-        update_frequency = num_steps // 10 if num_steps and num_steps > 10 else None
 
         def report() -> None:
-            nonlocal update_frequency
-            if update_frequency is None and num_steps is not None:
-                update_frequency = num_steps // 10 if num_steps > 10 else 1
-            if progress is not None and (update_frequency is None or (cur_step % update_frequency) == 0):
-                progress(
-                    ProgressStatus.from_step(cur_step, num_steps) if num_steps is not None else ProgressStatus(cur_step)
-                )
+            pass
+            # nonlocal update_frequency
+            # if update_frequency is None and num_steps is not None:
+            #     update_frequency = num_steps // 10 if num_steps > 10 else 1
+            # if progress is not None and (update_frequency is None or (cur_step % update_frequency) == 0):
+            #     progress(
+            #         ProgressStatus.from_step(cur_step, num_steps) if num_steps is not None else ProgressStatus(cur_step)
+            #     )
 
         report()
 

From 33c584443b4d79adf887c8a2d1a4eba5aee39351 Mon Sep 17 00:00:00 2001
From: Enkidu93 <lowryec17@gcc.edu>
Date: Thu, 20 Aug 2026 15:59:00 -0400
Subject: [PATCH 23/24] Fix typo (debug)

---
 machine/translation/thot/thot_word_alignment_model_trainer.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/machine/translation/thot/thot_word_alignment_model_trainer.py b/machine/translation/thot/thot_word_alignment_model_trainer.py
index f31ea0a5..cb3c779c 100644
--- a/machine/translation/thot/thot_word_alignment_model_trainer.py
+++ b/machine/translation/thot/thot_word_alignment_model_trainer.py
@@ -185,7 +185,7 @@ def train(
         # iteration count is derived from the corpus during start_training (stored as 0 until then),
         # so the total step count is not known up front; progress is reported as indeterminate until
         # it is resolved below.
-        iteration_count_known = not self._is_eflomal or self._models[0][1] > 0
+        # iteration_count_known = not self._is_eflomal or self._models[0][1] > 0
         # num_steps: Optional[int] = (
         #     sum(iterations + 1 for _, iterations in self._models if iterations > 0) + 1
         #     if iteration_count_known
@@ -242,7 +242,7 @@ def report() -> None:
                 # The automatic schedule is resolved during start_training; ask the model how many
                 # sweeps to run and finalize the (until now indeterminate) total step count.
                 iteration_count = cast(ta.EflomalAlignmentModel, model).scheduled_iterations
-                num_steps = cur_step + iteration_count + 1
+                # num_steps = cur_step + iteration_count + 1
 
             cur_step += 1
             report()

From a4f1cf230223905e16304b7cc89d4776f5d85e2f Mon Sep 17 00:00:00 2001
From: Enkidu93 <lowryec17@gcc.edu>
Date: Thu, 20 Aug 2026 16:37:12 -0400
Subject: [PATCH 24/24] Remove debug changes; add comment

---
 .../thot/thot_word_alignment_model_trainer.py | 31 ++++++++++---------
 1 file changed, 16 insertions(+), 15 deletions(-)

diff --git a/machine/translation/thot/thot_word_alignment_model_trainer.py b/machine/translation/thot/thot_word_alignment_model_trainer.py
index cb3c779c..e57de6fe 100644
--- a/machine/translation/thot/thot_word_alignment_model_trainer.py
+++ b/machine/translation/thot/thot_word_alignment_model_trainer.py
@@ -185,23 +185,24 @@ def train(
         # iteration count is derived from the corpus during start_training (stored as 0 until then),
         # so the total step count is not known up front; progress is reported as indeterminate until
         # it is resolved below.
-        # iteration_count_known = not self._is_eflomal or self._models[0][1] > 0
-        # num_steps: Optional[int] = (
-        #     sum(iterations + 1 for _, iterations in self._models if iterations > 0) + 1
-        #     if iteration_count_known
-        #     else None
-        # )
+        iteration_count_known = not self._is_eflomal or self._models[0][1] > 0
+        num_steps: Optional[int] = (
+            sum(iterations + 1 for _, iterations in self._models if iterations > 0) + 1
+            if iteration_count_known
+            else None
+        )
         cur_step = 0
+        # Throttle the progress reporting
+        update_frequency = num_steps // 10 if num_steps is not None and num_steps > 10 else 1
 
         def report() -> None:
-            pass
-            # nonlocal update_frequency
-            # if update_frequency is None and num_steps is not None:
-            #     update_frequency = num_steps // 10 if num_steps > 10 else 1
-            # if progress is not None and (update_frequency is None or (cur_step % update_frequency) == 0):
-            #     progress(
-            #         ProgressStatus.from_step(cur_step, num_steps) if num_steps is not None else ProgressStatus(cur_step)
-            #     )
+            nonlocal update_frequency
+            if update_frequency is None and num_steps is not None:
+                update_frequency = num_steps // 10 if num_steps > 10 else 1
+            if progress is not None and (update_frequency is None or (cur_step % update_frequency) == 0):
+                progress(
+                    ProgressStatus.from_step(cur_step, num_steps) if num_steps is not None else ProgressStatus(cur_step)
+                )
 
         report()
 
@@ -242,7 +243,7 @@ def report() -> None:
                 # The automatic schedule is resolved during start_training; ask the model how many
                 # sweeps to run and finalize the (until now indeterminate) total step count.
                 iteration_count = cast(ta.EflomalAlignmentModel, model).scheduled_iterations
-                # num_steps = cur_step + iteration_count + 1
+                num_steps = cur_step + iteration_count + 1
 
             cur_step += 1
             report()