From f0a167496af21fb78e1da883b5c9d50da1a7cae7 Mon Sep 17 00:00:00 2001 From: HugoFara Date: Thu, 2 Jul 2026 14:17:59 +0200 Subject: [PATCH 1/3] fix(db): use session TEMPORARY tables for parse/import scratch (#247) temp_word_occurrences, temp_words and tempexprs were persistent, globally-shared InnoDB tables that every parse/import TRUNCATEd and refilled. That caused two problems: - TRUNCATE on file-per-table InnoDB drops and recreates the .ibd. When that file went missing (notably on Windows) the table was left with a missing tablespace and every import crashed with InnoDB error 194 "Tablespace is missing for a table" (issue #247). - Because the tables were shared, concurrent parses/imports read and truncated each other's rows, silently corrupting sentences and word occurrences. Convert all three to per-connection CREATE TEMPORARY TABLE: - Add ScratchTables helper owning the DDL (moved out of baseline.sql). - TextParsing / TextParsingPersistence: ensure()+DELETE instead of TRUNCATE. The two self-referencing UPDATEs (TiSeID and tempexprs realignment) are rewritten via helper temp tables temp_seid_map / temp_sent_map, because MySQL/MariaDB cannot open a TEMPORARY table twice in the same statement. - CompleteImportService: create temp_words in initTempTables; cleanup uses DROP TEMPORARY TABLE so it does not implicitly commit the import transaction. - ParsingCoordinator (unused duplicate path): route through the helper and ensure tempexprs exists before reading it. - Migration 20260702_120000 drops the old persistent tables, which also repairs installs whose tablespace was already orphaned. Temporary InnoDB tables live in the session temp tablespace (no per-table .ibd to orphan) and are private to the connection, so error 194 cannot recur and concurrent parses can no longer collide. SqlValidator::ALLOWED_TABLES entries are kept so old backups still restore; backup uses its own BACKUP_TABLES list that never included these tables. Claude-Session: https://claude.ai/code/session_01X5PujuwBEMtMJxK1BA97fi --- ..._120000_drop_persistent_scratch_tables.sql | 21 +++ db/schema/baseline.sql | 23 +--- .../Services/ParsingCoordinator.php | 15 ++- .../Services/CompleteImportService.php | 25 ++-- .../Infrastructure/Database/ScratchTables.php | 121 ++++++++++++++++++ .../Infrastructure/Database/TextParsing.php | 12 +- .../Database/TextParsingPersistence.php | 56 ++++---- 7 files changed, 214 insertions(+), 59 deletions(-) create mode 100644 db/migrations/20260702_120000_drop_persistent_scratch_tables.sql create mode 100644 src/Shared/Infrastructure/Database/ScratchTables.php diff --git a/db/migrations/20260702_120000_drop_persistent_scratch_tables.sql b/db/migrations/20260702_120000_drop_persistent_scratch_tables.sql new file mode 100644 index 000000000..9ab819da2 --- /dev/null +++ b/db/migrations/20260702_120000_drop_persistent_scratch_tables.sql @@ -0,0 +1,21 @@ +-- Migration: Drop the persistent parse/import scratch tables. +-- +-- temp_word_occurrences, temp_words and tempexprs used to be persistent, +-- globally-shared InnoDB tables that every parse/import TRUNCATEd and refilled. +-- That caused two problems: +-- * "Tablespace is missing for a table" (InnoDB error 194) crashes on +-- TRUNCATE when the table's .ibd file went missing (notably on Windows) — +-- every text import then failed. See issue #247. +-- * Concurrent parses/imports corrupted each other's rows (no isolation). +-- +-- They are now created per database connection as CREATE TEMPORARY TABLE at +-- runtime (see src/Shared/Infrastructure/Database/ScratchTables.php), so the +-- persistent versions are no longer needed. Dropping them here also repairs +-- installs whose tablespace was already orphaned (the reported crash). +-- +-- DROP TABLE IF EXISTS is idempotent and succeeds even when the tablespace is +-- missing, so it clears the orphaned dictionary entry as well. + +DROP TABLE IF EXISTS temp_word_occurrences; +DROP TABLE IF EXISTS temp_words; +DROP TABLE IF EXISTS tempexprs; diff --git a/db/schema/baseline.sql b/db/schema/baseline.sql index e21f3c2f1..dc438f327 100644 --- a/db/schema/baseline.sql +++ b/db/schema/baseline.sql @@ -112,23 +112,12 @@ CREATE TABLE IF NOT EXISTS word_occurrences ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8; -CREATE TABLE IF NOT EXISTS temp_word_occurrences ( - TiCount smallint(5) unsigned NOT NULL, - TiSeID mediumint(8) unsigned NOT NULL, - TiOrder smallint(5) unsigned NOT NULL, - TiWordCount tinyint(3) unsigned NOT NULL, - TiText varchar(250) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; - -CREATE TABLE IF NOT EXISTS temp_words ( - WoText varchar(250) DEFAULT NULL, - WoTextLC varchar(250) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, - WoTranslation varchar(500) NOT NULL DEFAULT '*', - WoRomanization varchar(100) DEFAULT NULL, - WoSentence varchar(1000) DEFAULT NULL, - WoTaglist varchar(255) DEFAULT NULL, - PRIMARY KEY(WoTextLC) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +-- The parse/import scratch tables (temp_word_occurrences, temp_words, tempexprs) +-- are no longer persistent shared tables. They are created per database +-- connection as CREATE TEMPORARY TABLE at runtime (see +-- src/Shared/Infrastructure/Database/ScratchTables.php). This avoids InnoDB +-- "Tablespace is missing" (error 194) crashes on TRUNCATE and isolates +-- concurrent parses/imports from each other. See issue #247. CREATE TABLE IF NOT EXISTS texts ( TxID smallint(5) unsigned NOT NULL AUTO_INCREMENT, diff --git a/src/Modules/Language/Application/Services/ParsingCoordinator.php b/src/Modules/Language/Application/Services/ParsingCoordinator.php index 3b4be44d1..2a3f72895 100644 --- a/src/Modules/Language/Application/Services/ParsingCoordinator.php +++ b/src/Modules/Language/Application/Services/ParsingCoordinator.php @@ -25,6 +25,7 @@ use Lwt\Shared\Infrastructure\Database\Connection; use Lwt\Shared\Infrastructure\Database\Escaping; use Lwt\Shared\Infrastructure\Database\QueryBuilder; +use Lwt\Shared\Infrastructure\Database\ScratchTables; use Lwt\Shared\Infrastructure\Database\UserScopedQuery; /** @@ -152,8 +153,13 @@ protected function saveToDatabase( ): void { $lid = $language->id()->toInt(); - // Clear temporary table - QueryBuilder::table('temp_word_occurrences')->truncate(); + // Create (per connection) and clear the scratch table. See ScratchTables. + // NOTE: this class duplicates the parsing pipeline in TextParsing / + // TextParsingPersistence and is not currently wired to any route; unlike + // that path it never populates `tempexprs`, so its multi-word branch reads + // an empty table. Unifying the two implementations is Phase 2. + ScratchTables::ensureWordOccurrences(); + QueryBuilder::table('temp_word_occurrences')->delete(); // Get next sentence ID $dbname = Globals::getDatabaseName(); @@ -290,6 +296,11 @@ protected function registerSentencesTextItems( // Insert text items if ($hasmultiword) { + // Ensure the scratch table exists before reading it. This class does + // not populate tempexprs (see the note in saveToDatabase), and it is + // now a per-connection TEMPORARY table, so without this the SELECT + // below would fail with "table doesn't exist". + ScratchTables::ensureExpressions(); // Build SQL and bindings in lockstep so that the two user-scope // injections inside the UNION land in left-to-right placeholder // order. Pre-bundling all six values up-front and appending the diff --git a/src/Modules/Vocabulary/Application/Services/CompleteImportService.php b/src/Modules/Vocabulary/Application/Services/CompleteImportService.php index 728be73ad..89a811800 100644 --- a/src/Modules/Vocabulary/Application/Services/CompleteImportService.php +++ b/src/Modules/Vocabulary/Application/Services/CompleteImportService.php @@ -20,6 +20,7 @@ use Lwt\Shared\Infrastructure\Database\Connection; use Lwt\Shared\Infrastructure\Database\DB; use Lwt\Shared\Infrastructure\Database\QueryBuilder; +use Lwt\Shared\Infrastructure\Database\ScratchTables; use Lwt\Shared\Infrastructure\Database\Settings; use Lwt\Shared\Infrastructure\Database\UserScopedQuery; use Lwt\Modules\Tags\Application\TagsFacade; @@ -131,6 +132,10 @@ public function importComplete( */ private function initTempTables(): void { + // Per-connection scratch table for the import rows (was a persistent + // shared table; see ScratchTables for why it is now TEMPORARY). + ScratchTables::ensureWords(); + Connection::execute("DELETE FROM temp_words"); Connection::execute( "CREATE TEMPORARY TABLE IF NOT EXISTS numbers( n tinyint(3) unsigned NOT NULL @@ -646,8 +651,11 @@ private function handleTagsImport(int $langId): void */ private function cleanupTempTables(): void { - Connection::execute("DROP TABLE IF EXISTS numbers"); - QueryBuilder::table('temp_words')->truncate(); + // DROP TEMPORARY (not DROP TABLE / TRUNCATE): the former does not + // implicitly commit, so it is safe to call inside the import transaction. + // Both tables are recreated by initTempTables() on the next import. + Connection::execute("DROP TEMPORARY TABLE IF EXISTS numbers"); + Connection::execute("DROP TEMPORARY TABLE IF EXISTS temp_words"); } /** @@ -662,6 +670,10 @@ private function cleanupTempTables(): void */ public function importTagsOnly(array $fields, string $tabType, string $fileName, bool $ignoreFirst): void { + // Create the per-connection scratch tables (temp_words + numbers) before + // loading the tag rows into temp_words below. + $this->initTempTables(); + $columns = ''; $tlField = $fields["tl"]; for ($j = 1; $j <= $tlField; $j++) { @@ -736,15 +748,6 @@ public function importTagsOnly(array $fields, string $tabType, string $fileName, } } - // Create numbers table and insert tags - Connection::execute( - "CREATE TEMPORARY TABLE IF NOT EXISTS numbers( - n tinyint(3) unsigned NOT NULL - )" - ); - Connection::execute("INSERT IGNORE INTO numbers(n) VALUES ('1'),('2'),('3'), - ('4'),('5'),('6'),('7'),('8'),('9')"); - // Stamp TgUsID so the imported tags are owned by the caller — // see handleTagsImport() for context. Without this the rows // land with TgUsID NULL and never appear on the user's tag diff --git a/src/Shared/Infrastructure/Database/ScratchTables.php b/src/Shared/Infrastructure/Database/ScratchTables.php new file mode 100644 index 000000000..9019dbaf5 --- /dev/null +++ b/src/Shared/Infrastructure/Database/ScratchTables.php @@ -0,0 +1,121 @@ + + * @license Unlicense + * @link https://hugofara.github.io/lwt/developer/api + * @since 3.2.2 + */ + +declare(strict_types=1); + +namespace Lwt\Shared\Infrastructure\Database; + +/** + * Creates the per-connection scratch tables used while parsing texts and + * importing vocabulary. + * + * These tables (`temp_word_occurrences`, `temp_words`, `tempexprs`) used to be + * persistent, globally-shared InnoDB tables that every parse/import `TRUNCATE`d + * and refilled. That design caused two problems: + * + * - **Tablespace crashes.** On file-per-table InnoDB, `TRUNCATE` physically + * drops and recreates the `.ibd`. When that file went missing (notably on + * Windows) the table was left with a missing tablespace and every subsequent + * import failed with InnoDB error 194 "Tablespace is missing for a table" + * (see issue #247). + * - **Concurrency corruption.** Because the tables were shared, two parses or + * imports running at once (multiple tabs, a feed refresh overlapping a manual + * import, or any two users in multi-user mode) read and truncated each + * other's rows, silently producing scrambled sentences and word occurrences. + * + * They are now created per database connection with `CREATE TEMPORARY TABLE`, so + * each request gets its own private copy. Temporary InnoDB tables live in the + * shared session temporary tablespace (there is no per-table `.ibd` to orphan), + * and they are dropped automatically when the connection closes. + * + * Callers must `ensure*()` the table exists before use, then clear rows with + * `DELETE FROM` — never `TRUNCATE`, which is DDL that recreates the tablespace + * and implicitly commits any open transaction. + * + * @since 3.2.2 + */ +final class ScratchTables +{ + /** + * Ensure the word-occurrence staging table exists for this connection. + * + * Holds the tokenised word occurrences of a single text while it is being + * parsed into `sentences` / `word_occurrences`. + * + * @return void + */ + public static function ensureWordOccurrences(): void + { + Connection::query( + <<<'SQL' + CREATE TEMPORARY TABLE IF NOT EXISTS temp_word_occurrences ( + TiCount smallint(5) unsigned NOT NULL, + TiSeID mediumint(8) unsigned NOT NULL, + TiOrder smallint(5) unsigned NOT NULL, + TiWordCount tinyint(3) unsigned NOT NULL, + TiText varchar(250) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci + SQL + ); + } + + /** + * Ensure the word-import staging table exists for this connection. + * + * Holds the rows of a vocabulary import file before they are merged into + * the `words` table. + * + * @return void + */ + public static function ensureWords(): void + { + Connection::query( + <<<'SQL' + CREATE TEMPORARY TABLE IF NOT EXISTS temp_words ( + WoText varchar(250) DEFAULT NULL, + WoTextLC varchar(250) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + WoTranslation varchar(500) NOT NULL DEFAULT '*', + WoRomanization varchar(100) DEFAULT NULL, + WoSentence varchar(1000) DEFAULT NULL, + WoTaglist varchar(255) DEFAULT NULL, + PRIMARY KEY(WoTextLC) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci + SQL + ); + } + + /** + * Ensure the multi-word-expression staging table exists for this connection. + * + * Populated by the multi-word detection pass while parsing a text. + * + * @return void + */ + public static function ensureExpressions(): void + { + Connection::query( + <<<'SQL' + CREATE TEMPORARY TABLE IF NOT EXISTS tempexprs ( + sent mediumint unsigned, + word varchar(250), + lword varchar(250), + TiOrder smallint unsigned, + n tinyint(3) unsigned NOT NULL + ) + SQL + ); + } +} diff --git a/src/Shared/Infrastructure/Database/TextParsing.php b/src/Shared/Infrastructure/Database/TextParsing.php index 1accbe363..a64ee5650 100644 --- a/src/Shared/Infrastructure/Database/TextParsing.php +++ b/src/Shared/Infrastructure/Database/TextParsing.php @@ -89,7 +89,7 @@ public static function parseAndDisplayPreview(string $text, int $lid): void TextParsingPersistence::displayStatistics($lid, $rtlScript, !empty($wl)); // Clean up - QueryBuilder::table('temp_word_occurrences')->truncate(); + QueryBuilder::table('temp_word_occurrences')->delete(); } /** @@ -138,7 +138,7 @@ public static function parseAndSave(string $text, int $lid, int $textId): void TextParsingPersistence::registerSentencesTextItems($textId, $lid, !empty($wl)); // Clean up - QueryBuilder::table('temp_word_occurrences')->truncate(); + QueryBuilder::table('temp_word_occurrences')->delete(); } /** @@ -218,7 +218,7 @@ public static function checkText(string $text, int $lid): array : 100.0; // Clean up temp_word_occurrences - QueryBuilder::table('temp_word_occurrences')->truncate(); + QueryBuilder::table('temp_word_occurrences')->delete(); return [ 'sentences' => $sentenceCount, @@ -255,7 +255,11 @@ private static function prepare(string $text, int $id, int $lid): ?array $termchar = (string)$record['LgRegexpWordCharacters']; $replace = explode("|", (string) $record['LgCharacterSubstitutions']); $text = Escaping::prepareTextdata($text); - QueryBuilder::table('temp_word_occurrences')->truncate(); + // Create the per-connection scratch table (if needed) and clear it. + // DELETE (not TRUNCATE): TRUNCATE recreates the tablespace, which crashed + // on Windows with InnoDB error 194 and implicitly commits transactions. + ScratchTables::ensureWordOccurrences(); + QueryBuilder::table('temp_word_occurrences')->delete(); // because of sentence special characters $text = str_replace(array('}', '{'), array(']', '['), $text); diff --git a/src/Shared/Infrastructure/Database/TextParsingPersistence.php b/src/Shared/Infrastructure/Database/TextParsingPersistence.php index 9d6fe0156..5f9ec0587 100644 --- a/src/Shared/Infrastructure/Database/TextParsingPersistence.php +++ b/src/Shared/Infrastructure/Database/TextParsingPersistence.php @@ -261,36 +261,50 @@ public static function registerSentencesTextItems(int $tid, int $lid, bool $hasm // The pre-computed TiSeID may not match actual AUTO_INCREMENT values, // so we update temp_word_occurrences to use the actual SeID from inserted sentences. // ROW_NUMBER() maps TiSeID rank to SeOrder, which we JOIN to get SeID. + // Materialise the TiSeID->rank mapping in its own temporary table first. + // temp_word_occurrences is now a TEMPORARY table, and MySQL/MariaDB cannot + // open a temporary table twice in the same statement ("Can't reopen table"), + // so the mapping cannot be an inline self-subquery of the UPDATE. + Connection::execute("DROP TEMPORARY TABLE IF EXISTS temp_seid_map"); + Connection::execute( + "CREATE TEMPORARY TABLE temp_seid_map AS + SELECT TiSeID AS old_seid, + ROW_NUMBER() OVER (ORDER BY TiSeID) AS rn + FROM temp_word_occurrences + GROUP BY TiSeID" + ); Connection::preparedExecute( "UPDATE temp_word_occurrences t - JOIN ( - SELECT TiSeID AS old_seid, - ROW_NUMBER() OVER (ORDER BY TiSeID) AS rn - FROM temp_word_occurrences - GROUP BY TiSeID - ) mapping ON t.TiSeID = mapping.old_seid + JOIN temp_seid_map mapping ON t.TiSeID = mapping.old_seid JOIN sentences s ON s.SeOrder = mapping.rn AND s.SeTxID = ? SET t.TiSeID = s.SeID", [$tid] ); + Connection::execute("DROP TEMPORARY TABLE IF EXISTS temp_seid_map"); // STEP 1.6: Also update tempexprs.sent if multiword expressions exist. // tempexprs was populated before sentences were inserted, so its `sent` // field also needs to be aligned with actual SeID values. if ($hasmultiword) { + // Same "can't reopen a temporary table" constraint as STEP 1.5: + // materialise the sent->rank mapping before the self-referencing UPDATE. + Connection::execute("DROP TEMPORARY TABLE IF EXISTS temp_sent_map"); + Connection::execute( + "CREATE TEMPORARY TABLE temp_sent_map AS + SELECT sent AS old_sent, + ROW_NUMBER() OVER (ORDER BY sent) AS rn + FROM tempexprs + WHERE sent IS NOT NULL + GROUP BY sent" + ); Connection::preparedExecute( "UPDATE tempexprs e - JOIN ( - SELECT sent AS old_sent, - ROW_NUMBER() OVER (ORDER BY sent) AS rn - FROM tempexprs - WHERE sent IS NOT NULL - GROUP BY sent - ) mapping ON e.sent = mapping.old_sent + JOIN temp_sent_map mapping ON e.sent = mapping.old_sent JOIN sentences s ON s.SeOrder = mapping.rn AND s.SeTxID = ? SET e.sent = s.SeID", [$tid] ); + Connection::execute("DROP TEMPORARY TABLE IF EXISTS temp_sent_map"); } // STEP 2: Insert text items. TiSeID and tempexprs.sent now equal actual SeID. @@ -444,23 +458,15 @@ public static function checkExpressions(array $wl): void n tinyint(3) unsigned NOT NULL );" ); - Connection::execute("TRUNCATE TABLE numbers"); + Connection::execute("DELETE FROM numbers"); Connection::query( "INSERT IGNORE INTO numbers(n) VALUES (" . implode('),(', $wl) . ');' ); - // Store garbage - Connection::query( - "CREATE TABLE IF NOT EXISTS tempexprs ( - sent mediumint unsigned, - word varchar(250), - lword varchar(250), - TiOrder smallint unsigned, - n tinyint(3) unsigned NOT NULL - )" - ); - Connection::execute("TRUNCATE TABLE tempexprs"); + // Store garbage (per-connection temporary table) + ScratchTables::ensureExpressions(); + Connection::execute("DELETE FROM tempexprs"); Connection::query( "INSERT IGNORE INTO tempexprs (sent, word, lword, TiOrder, n) From 132722b5dad222b0189eb891fe9761e79e2179c6 Mon Sep 17 00:00:00 2001 From: HugoFara Date: Thu, 2 Jul 2026 15:27:03 +0200 Subject: [PATCH 2/3] refactor(parsing): parse texts fully in PHP, drop scratch tables (#247) Phase 2 of the #247 scratch-table work. Text parsing no longer uses any scratch table: tokenization, multi-word detection, and the sentence / word_occurrence inserts all happen in PHP. Removed: - temp_word_occurrences and tempexprs from the parse path entirely. - The LOAD DATA LOCAL INFILE path and its two divergent fallbacks (saveWithSql / saveWithSqlFallback and the inline INSERT branch). - The stateful @-variable multi-word-detection SQL (checkExpressions). - TextParsingPersistence and the dead, never-wired ParsingCoordinator. Added: - ParsedToken value object (was a temp_word_occurrences row). - TokenPersistence: inserts sentences (reads back real SeIDs), detects multi-word expressions with a per-sentence windowed hash lookup, and batch-inserts word_occurrences. - StandardTextParser::tokenize()/splitSentences() and JapaneseTextParser::tokenize()/buildTokensFromMecab() return ParsedToken[]; the tokenizers themselves are unchanged. This fixes two pre-existing bugs affecting installs without LOAD DATA LOCAL INFILE (the same managed-DB / Windows population as #247): - saveWithSqlFallback trimmed the "\r" sentence marker, so every text parsed as a single sentence with no spacing. - The @-variable detector only found multi-word expressions in the first sentence of space-less (MeCab/CJK) languages. Verified with a differential harness against the canonical LOAD DATA output over 14 cases (Latin/CJK/MeCab, 2- and 3-word expressions, overlaps, paragraphs, quotes, numbers, abbreviations): 13 byte-identical, the 14th differing only by correctly detecting a multi-word expression the old SQL missed. checkText() is now pure (it previously echoed preview HTML despite its docblock). The Japanese path was tested end-to-end against a real MeCab install, and buildTokensFromMecab() has a unit test using captured MeCab output (no binary needed). Claude-Session: https://claude.ai/code/session_01X5PujuwBEMtMJxK1BA97fi --- .../Services/ParsingCoordinator.php | 359 ------------ .../Language/LanguageServiceProvider.php | 7 - .../Database/JapaneseTextParser.php | 226 +++----- .../Infrastructure/Database/ParsedToken.php | 64 +++ .../Infrastructure/Database/ScratchTables.php | 70 +-- .../Database/StandardTextParser.php | 208 ++++--- .../Infrastructure/Database/TextParsing.php | 188 ++----- .../Database/TextParsingPersistence.php | 524 ------------------ .../Database/TokenPersistence.php | 495 +++++++++++++++++ .../Database/JapaneseTextParserTest.php | 71 +-- .../Database/StandardTextParserTest.php | 16 - .../Database/TextParsingPersistenceTest.php | 379 ------------- 12 files changed, 837 insertions(+), 1770 deletions(-) delete mode 100644 src/Modules/Language/Application/Services/ParsingCoordinator.php create mode 100644 src/Shared/Infrastructure/Database/ParsedToken.php delete mode 100644 src/Shared/Infrastructure/Database/TextParsingPersistence.php create mode 100644 src/Shared/Infrastructure/Database/TokenPersistence.php delete mode 100644 tests/backend/Shared/Infrastructure/Database/TextParsingPersistenceTest.php diff --git a/src/Modules/Language/Application/Services/ParsingCoordinator.php b/src/Modules/Language/Application/Services/ParsingCoordinator.php deleted file mode 100644 index 2a3f72895..000000000 --- a/src/Modules/Language/Application/Services/ParsingCoordinator.php +++ /dev/null @@ -1,359 +0,0 @@ - - * @license Unlicense - * @link https://hugofara.github.io/lwt/developer/api - * @since 3.0.0 - */ - -declare(strict_types=1); - -namespace Lwt\Modules\Language\Application\Services; - -use Lwt\Modules\Language\Domain\Language; -use Lwt\Modules\Language\Domain\Parser\ParserConfig; -use Lwt\Modules\Language\Domain\Parser\ParserResult; -use Lwt\Modules\Language\Infrastructure\Parser\ParserRegistry; -use Lwt\Shared\Infrastructure\Globals; -use Lwt\Shared\Infrastructure\Database\Connection; -use Lwt\Shared\Infrastructure\Database\Escaping; -use Lwt\Shared\Infrastructure\Database\QueryBuilder; -use Lwt\Shared\Infrastructure\Database\ScratchTables; -use Lwt\Shared\Infrastructure\Database\UserScopedQuery; - -/** - * Coordinates parsing operations, providing a facade for the parser system. - * - * This class handles parser selection, preprocessing, and database persistence. - * It serves as the main entry point for text parsing operations. - * - * @since 3.0.0 - */ -class ParsingCoordinator -{ - public function __construct( - private ParserRegistry $registry - ) { - } - - /** - * Split text into sentences without database operations. - * - * @param string $text Text to parse - * @param Language $language Language entity - * - * @return string[] Array of sentences - */ - public function splitIntoSentences(string $text, Language $language): array - { - $text = $this->preprocess($text, $language); - $config = ParserConfig::fromLanguage($language); - $parser = $this->registry->getParserForLanguage($language); - - $result = $parser->parse($text, $config); - return $result->getSentences(); - } - - /** - * Parse text and return the result without database operations. - * - * @param string $text Text to parse - * @param Language $language Language entity - * - * @return ParserResult Parsing result - */ - public function parseForPreview(string $text, Language $language): ParserResult - { - $text = $this->preprocess($text, $language); - $config = ParserConfig::fromLanguage($language); - $parser = $this->registry->getParserForLanguage($language); - - return $parser->parse($text, $config); - } - - /** - * Parse text and save to database. - * - * @param string $text Text to parse - * @param Language $language Language entity - * @param int $textId Text ID (must be positive) - * - * @return void - * - * @throws \InvalidArgumentException If textId is not positive - */ - public function parseAndSave(string $text, Language $language, int $textId): void - { - if ($textId <= 0) { - throw new \InvalidArgumentException( - "Text ID must be positive, got: $textId" - ); - } - - $text = $this->preprocess($text, $language); - $config = ParserConfig::fromLanguage($language); - $parser = $this->registry->getParserForLanguage($language); - - $result = $parser->parse($text, $config); - - $this->saveToDatabase($result, $language, $textId); - } - - /** - * Preprocess text before parsing. - * - * Applies character substitutions and other text cleanup. - * - * @param string $text Raw text - * @param Language $language Language entity - * - * @return string Preprocessed text - */ - protected function preprocess(string $text, Language $language): string - { - $text = Escaping::prepareTextdata($text); - - // Replace special characters that interfere with sentence parsing - $text = str_replace(['}', '{'], [']', '['], $text); - - // Apply character substitutions - $substitutions = $language->characterSubstitutions(); - if ($substitutions !== '') { - foreach (explode('|', $substitutions) as $value) { - $fromto = explode('=', trim($value)); - if (count($fromto) >= 2) { - $text = str_replace(trim($fromto[0]), trim($fromto[1]), $text); - } - } - } - - return $text; - } - - /** - * Save parsing result to database. - * - * @param ParserResult $result Parsing result - * @param Language $language Language entity - * @param int $textId Text ID - * - * @return void - */ - protected function saveToDatabase( - ParserResult $result, - Language $language, - int $textId - ): void { - $lid = $language->id()->toInt(); - - // Create (per connection) and clear the scratch table. See ScratchTables. - // NOTE: this class duplicates the parsing pipeline in TextParsing / - // TextParsingPersistence and is not currently wired to any route; unlike - // that path it never populates `tempexprs`, so its multi-word branch reads - // an empty table. Unifying the two implementations is Phase 2. - ScratchTables::ensureWordOccurrences(); - QueryBuilder::table('temp_word_occurrences')->delete(); - - // Get next sentence ID - $dbname = Globals::getDatabaseName(); - $sentencesTable = Globals::table('sentences'); - $nextSeID = (int) Connection::preparedFetchValue( - "SELECT AUTO_INCREMENT as value FROM information_schema.TABLES - WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?", - [$dbname, $sentencesTable] - ); - if ($nextSeID <= 0) { - $bindings = []; - $nextSeID = (int) Connection::preparedFetchValue( - "SELECT IFNULL(MAX(`SeID`)+1,1) as value FROM sentences WHERE 1=1" - . UserScopedQuery::forTablePrepared('sentences', $bindings), - $bindings - ); - } - - // Insert tokens into temp_word_occurrences - $this->insertTokensToTemp($result, $nextSeID); - - // Check for multi-word expressions - $hasMultiword = $this->checkExpressions($lid); - - // Register sentences and text items - $this->registerSentencesTextItems($textId, $lid, $hasMultiword); - } - - /** - * Insert tokens into temp_word_occurrences table. - * - * @param ParserResult $result Parsing result - * @param int $startSeID Starting sentence ID - * - * @return void - */ - protected function insertTokensToTemp(ParserResult $result, int $startSeID): void - { - $tokens = $result->getTokens(); - if (empty($tokens)) { - return; - } - - $rows = []; - $currentSeID = $startSeID; - $lastSentenceIndex = -1; - $count = 0; - - foreach ($tokens as $token) { - // Track sentence changes - if ($token->getSentenceIndex() !== $lastSentenceIndex) { - if ($lastSentenceIndex >= 0) { - $currentSeID++; - $count = 0; - } - $lastSentenceIndex = $token->getSentenceIndex(); - } - - $tiSeID = $currentSeID; - $tiCount = $count + 1; - $count += mb_strlen($token->getText()); - $tiOrder = $token->getOrder() + 1; // 1-based in DB - $tiWordCount = $token->isWord() ? 1 : 0; - - $rows[] = [$tiSeID, $tiCount, $tiOrder, $token->getText(), $tiWordCount]; - } - - $placeholders = []; - $flatParams = []; - foreach ($rows as $row) { - $placeholders[] = "(?, ?, ?, ?, ?)"; - $flatParams = array_merge($flatParams, $row); - } - - Connection::preparedExecute( - "INSERT INTO temp_word_occurrences ( - TiSeID, TiCount, TiOrder, TiText, TiWordCount - ) VALUES " . implode(',', $placeholders), - $flatParams - ); - } - - /** - * Check for multi-word expressions and populate tempexprs. - * - * @param int $lid Language ID - * - * @return bool True if multi-word expressions were found - */ - protected function checkExpressions(int $lid): bool - { - // This is a simplified version - the full implementation is in TextParsing - $bindings = [$lid, $lid]; - $result = Connection::preparedFetchAll( - "SELECT WoWordCount FROM words WHERE WoWordCount > 1 AND WoLgID = ?" - . UserScopedQuery::forTablePrepared('words', $bindings, '') - . " GROUP BY WoWordCount", - $bindings - ); - - return !empty($result); - } - - /** - * Register sentences and text items in the database. - * - * @param int $tid Text ID - * @param int $lid Language ID - * @param bool $hasmultiword Whether to process multi-word expressions - * - * @return void - */ - protected function registerSentencesTextItems( - int $tid, - int $lid, - bool $hasmultiword - ): void { - // Insert sentences - Connection::query('SET @i=0;'); - Connection::preparedExecute( - "INSERT INTO sentences ( - SeLgID, SeTxID, SeOrder, SeFirstPos, SeText - ) SELECT - ?, - ?, - @i:=@i+1, - MIN(IF(TiWordCount=0, TiOrder+1, TiOrder)), - GROUP_CONCAT(TiText ORDER BY TiOrder SEPARATOR \"\") - FROM temp_word_occurrences - GROUP BY TiSeID - ORDER BY TiSeID", - [$lid, $tid] - ); - - // Insert text items - if ($hasmultiword) { - // Ensure the scratch table exists before reading it. This class does - // not populate tempexprs (see the note in saveToDatabase), and it is - // now a per-connection TEMPORARY table, so without this the SELECT - // below would fail with "table doesn't exist". - ScratchTables::ensureExpressions(); - // Build SQL and bindings in lockstep so that the two user-scope - // injections inside the UNION land in left-to-right placeholder - // order. Pre-bundling all six values up-front and appending the - // two $userIds at the end mismatches positions 3-7 and swaps - // Ti2LgID/Ti2TxID, triggering an FK violation on word_occurrences. - $bindings = [$lid, $tid]; - $sql = "INSERT INTO word_occurrences ( - Ti2WoID, Ti2LgID, Ti2TxID, Ti2SeID, Ti2Order, Ti2WordCount, Ti2Text - ) SELECT WoID, ?, ?, sent, TiOrder - (2*(n-1)) TiOrder, - n TiWordCount, word - FROM tempexprs - JOIN words - ON WoTextLC = lword AND WoWordCount = n" - . UserScopedQuery::forTablePrepared('words', $bindings, '') - . " WHERE lword IS NOT NULL AND WoLgID = ?"; - $bindings[] = $lid; - $bindings[] = $lid; - $bindings[] = $tid; - $sql .= " UNION ALL - SELECT WoID, ?, ?, TiSeID, TiOrder, TiWordCount, TiText - FROM temp_word_occurrences - LEFT JOIN words - ON LOWER(TiText) = WoTextLC AND TiWordCount=1 AND WoLgID = ?"; - $bindings[] = $lid; - $sql .= UserScopedQuery::forTablePrepared('words', $bindings, '') - . " ORDER BY TiOrder, TiWordCount"; - - $stmt = Connection::prepare($sql); - $stmt->bindValues($bindings); - $stmt->execute(); - } else { - $bindings = [$lid, $tid, $lid]; - Connection::preparedExecute( - "INSERT INTO word_occurrences ( - Ti2WoID, Ti2LgID, Ti2TxID, Ti2SeID, Ti2Order, Ti2WordCount, Ti2Text - ) SELECT WoID, ?, ?, TiSeID, TiOrder, TiWordCount, TiText - FROM temp_word_occurrences - LEFT JOIN words - ON LOWER(TiText) = WoTextLC AND TiWordCount=1 AND WoLgID = ?" - . UserScopedQuery::forTablePrepared('words', $bindings, '') - . " ORDER BY TiSeID, TiOrder, TiWordCount", - $bindings - ); - } - } - - /** - * Get the parser registry. - * - * @return ParserRegistry Parser registry - */ - public function getRegistry(): ParserRegistry - { - return $this->registry; - } -} diff --git a/src/Modules/Language/LanguageServiceProvider.php b/src/Modules/Language/LanguageServiceProvider.php index ae26161e5..08759e013 100644 --- a/src/Modules/Language/LanguageServiceProvider.php +++ b/src/Modules/Language/LanguageServiceProvider.php @@ -47,7 +47,6 @@ // Parser Infrastructure use Lwt\Modules\Language\Infrastructure\Parser\ExternalParserLoader; use Lwt\Modules\Language\Infrastructure\Parser\ParserRegistry; -use Lwt\Modules\Language\Application\Services\ParsingCoordinator; /** * Service provider for the Language module. @@ -85,12 +84,6 @@ public function register(Container $container): void ); }); - $container->singleton(ParsingCoordinator::class, function (Container $c) { - return new ParsingCoordinator( - $c->getTyped(ParserRegistry::class) - ); - }); - // Register Repository Interface binding $container->singleton(LanguageRepositoryInterface::class, function (Container $_c) { return new MySqlLanguageRepository(); diff --git a/src/Shared/Infrastructure/Database/JapaneseTextParser.php b/src/Shared/Infrastructure/Database/JapaneseTextParser.php index ae37a920e..d420205d2 100644 --- a/src/Shared/Infrastructure/Database/JapaneseTextParser.php +++ b/src/Shared/Infrastructure/Database/JapaneseTextParser.php @@ -23,7 +23,8 @@ /** * Japanese text parsing using MeCab. * - * Handles splitting, previewing, and database insertion for Japanese text. + * Handles splitting, previewing, and tokenization for Japanese text. Produces + * ParsedToken objects consumed by TokenPersistence — no scratch table involved. * * @since 3.0.0 */ @@ -63,14 +64,13 @@ public static function displayJapanesePreview(string $text): void } /** - * Parse Japanese text with MeCab and insert into temp_word_occurrences. + * Tokenize Japanese text with MeCab into ParsedToken objects. * - * @param string $text Preprocessed text - * @param bool $useMaxSeID Whether to query for max sentence ID (true for existing texts) + * @param string $text Preprocessed text (character substitutions applied) * - * @return void + * @return ParsedToken[] */ - public static function parseJapaneseToDatabase(string $text, bool $useMaxSeID): void + public static function tokenize(string $text): array { $text = preg_replace('/[ \t]+/u', ' ', $text) ?? $text; $text = trim($text); @@ -81,7 +81,7 @@ public static function parseJapaneseToDatabase(string $text, bool $useMaxSeID): } try { - // We use the format "word num num" for all nodes + // We use the format "word\ttype\tnode" for all nodes. $mecab_args = " -F %m\\t%t\\t%h\\n -U %m\\t%t\\t%h\\n -E EOP\\t3\\t7\\n"; $mecab_args .= " -o " . escapeshellarg($file_name) . " "; $mecab = (new TextParsingService())->getMecabPath($mecab_args); @@ -93,15 +93,6 @@ public static function parseJapaneseToDatabase(string $text, bool $useMaxSeID): pclose($handle); } - Connection::execute( - "CREATE TEMPORARY TABLE IF NOT EXISTS tempword_occurrences ( - TiCount smallint(5) unsigned NOT NULL, - TiSeID mediumint(8) unsigned NOT NULL, - TiOrder smallint(5) unsigned NOT NULL, - TiWordCount tinyint(3) unsigned NOT NULL, - TiText varchar(250) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL - ) DEFAULT CHARSET=utf8" - ); $handle = fopen($file_name, 'r'); $mecabed = ''; if ($handle !== false) { @@ -112,110 +103,7 @@ public static function parseJapaneseToDatabase(string $text, bool $useMaxSeID): } fclose($handle); } - $values = array(); - $order = 0; - $sid = 1; - if ($useMaxSeID) { - $sid = (int)Connection::fetchValue( - "SELECT IFNULL(MAX(`SeID`)+1,1) as value FROM sentences" - . UserScopedQuery::forTable('sentences') - ); - } - $term_type = 0; - $last_node_type = 0; - $count = 0; - $row = array(0, 0, 0, "", 0); - $separator = mb_chr(9); - if ($separator === false) { - $separator = "\t"; - } - foreach (explode(PHP_EOL, $mecabed) as $line) { - if (trim($line) == "") { - continue; - } - $parts = explode($separator, $line); - $term = $parts[0] ?? ''; - $node_type = $parts[1] ?? ''; - $third = $parts[2] ?? ''; - if ($term_type == 2 || $term == 'EOP' && $third == '7') { - $sid += 1; - } - $row[0] = $sid; // TiSeID - $row[1] = $count + 1; // TiCount - $count += mb_strlen($term); - $last_term_type = $term_type; - if ($third == '7') { - if ($term == 'EOP') { - $term = '¶'; - } - $term_type = 2; - } elseif (in_array($node_type, ['2', '6', '7', '8'])) { - $term_type = 0; - } else { - $term_type = 1; - } - - // Increase word order: - // Once if the current or the previous term were words - // Twice if current or the previous were not of unmanaged type - $order += (int)($term_type == 0 && $last_term_type == 0) + - (int)($term_type != 1 || $last_term_type != 1); - $row[2] = $order; // TiOrder - $row[3] = $term; // TiText (no escaping needed for prepared statement) - $row[4] = $term_type == 0 ? 1 : 0; // TiWordCount - $values[] = $row; - // Special case for kazu (numbers) - if ($last_node_type == 8 && $node_type == 8) { - $lastKey = array_key_last($values); - // $lastKey is int<0, max> since we just added an element - // We need at least 2 elements to access previous - if ($lastKey > 0 && isset($values[$lastKey - 1][3])) { - // Concatenate the previous value with the current term - $values[$lastKey - 1][3] = $values[$lastKey - 1][3] . $term; - // Remove last element to avoid repetition - array_pop($values); - } - } - $last_node_type = $node_type; - } - - // Build multi-row INSERT with prepared statement - // Generate placeholders for all rows: (?, ?, ?, ?, ?), (?, ?, ?, ?, ?), ... - $placeholders = array(); - $flatParams = array(); - foreach ($values as $row) { - $placeholders[] = "(?, ?, ?, ?, ?)"; - // Flatten the row values into a single array for binding - $flatParams[] = $row[0]; // TiSeID - $flatParams[] = $row[1]; // TiCount - $flatParams[] = $row[2]; // TiOrder - $flatParams[] = $row[3]; // TiText - $flatParams[] = $row[4]; // TiWordCount - } - - if (!empty($placeholders)) { - Connection::preparedExecute( - "INSERT INTO tempword_occurrences ( - TiSeID, TiCount, TiOrder, TiText, TiWordCount - ) VALUES " . implode(',', $placeholders), - $flatParams - ); - } - // Delete elements TiOrder=@order - Connection::preparedExecute( - "DELETE FROM tempword_occurrences WHERE TiOrder=?", - [$order] - ); - Connection::query( - "INSERT INTO temp_word_occurrences ( - TiCount, TiSeID, TiOrder, TiWordCount, TiText - ) - SELECT MIN(TiCount) s, TiSeID, TiOrder, TiWordCount, - group_concat(TiText ORDER BY TiCount SEPARATOR '') - FROM tempword_occurrences - GROUP BY TiOrder" - ); - Connection::execute("DROP TABLE tempword_occurrences"); + return self::buildTokensFromMecab($mecabed); } finally { if (file_exists($file_name)) { unlink($file_name); @@ -224,29 +112,97 @@ public static function parseJapaneseToDatabase(string $text, bool $useMaxSeID): } /** - * Parse a Japanese text using MeCab and add it to the database. - * - * @param string $text Text to parse. - * @param int $id Text ID. If $id = -1 print results, - * if $id = -2 return splitted texts + * Convert raw MeCab output into ParsedToken objects. * - * @return null|string[] Splitted sentence if $id = -2 + * Kept separate from the MeCab subprocess call so it can be unit-tested with + * captured/synthetic MeCab output. Mirrors the former SQL pipeline: build + * staging rows, drop the final EOP order group, then group rows sharing a + * TiOrder into one token (text concatenated in TiCount order). * - * @psalm-return non-empty-list|null + * @param string $mecabed Raw MeCab output ("word\ttype\tnode" per line) * - * @internal Use TextParsing::splitIntoSentences(), parseAndDisplayPreview(), or parseAndSave() instead. + * @return ParsedToken[] */ - public static function parseJapanese(string $text, int $id): ?array + public static function buildTokensFromMecab(string $mecabed): array { - if ($id == -2) { - return self::splitJapaneseSentences($text); + /** @var array $values */ + $values = array(); + $order = 0; + $sid = 1; + $term_type = 0; + $last_node_type = 0; + $count = 0; + $separator = mb_chr(9); + if ($separator === false) { + $separator = "\t"; } + foreach (explode(PHP_EOL, $mecabed) as $line) { + if (trim($line) == "") { + continue; + } + $parts = explode($separator, $line); + $term = $parts[0] ?? ''; + $node_type = $parts[1] ?? ''; + $third = $parts[2] ?? ''; + if ($term_type == 2 || $term == 'EOP' && $third == '7') { + $sid += 1; + } + $tiSeID = $sid; + $tiCount = $count + 1; + $count += mb_strlen($term); + $last_term_type = $term_type; + if ($third == '7') { + if ($term == 'EOP') { + $term = '¶'; + } + $term_type = 2; + } elseif (in_array($node_type, ['2', '6', '7', '8'])) { + $term_type = 0; + } else { + $term_type = 1; + } - if ($id == -1) { - self::displayJapanesePreview($text); + // Increase word order: + // Once if the current or the previous term were words + // Twice if current or the previous were not of unmanaged type + $order += (int)($term_type == 0 && $last_term_type == 0) + + (int)($term_type != 1 || $last_term_type != 1); + $tiWordCount = $term_type == 0 ? 1 : 0; + $values[] = array($tiSeID, $tiCount, $order, $term, $tiWordCount); + + // Special case for kazu (numbers): merge consecutive number nodes. + if ($last_node_type == 8 && $node_type == 8) { + $lastKey = array_key_last($values); + if ($lastKey > 0 && isset($values[$lastKey - 1][3])) { + $values[$lastKey - 1][3] = $values[$lastKey - 1][3] . $term; + array_pop($values); + } + } + $last_node_type = $node_type; } - self::parseJapaneseToDatabase($text, $id > 0); - return null; + // Group staging rows by TiOrder, dropping the final EOP order group. + // (This replaces "DELETE WHERE TiOrder=@order" + "GROUP BY TiOrder".) + /** @var array> $groups */ + $groups = array(); + foreach ($values as $r) { + if ($r[2] === $order) { + continue; + } + $groups[$r[2]][] = $r; + } + ksort($groups); + + $tokens = array(); + foreach ($groups as $ord => $rows) { + // Concatenate merged node text in TiCount order. + usort($rows, fn(array $a, array $b): int => $a[1] <=> $b[1]); + $text = ''; + foreach ($rows as $rr) { + $text .= $rr[3]; + } + $tokens[] = new ParsedToken($rows[0][0], $ord, $rows[0][4], $text); + } + return $tokens; } } diff --git a/src/Shared/Infrastructure/Database/ParsedToken.php b/src/Shared/Infrastructure/Database/ParsedToken.php new file mode 100644 index 000000000..a2d55b6a7 --- /dev/null +++ b/src/Shared/Infrastructure/Database/ParsedToken.php @@ -0,0 +1,64 @@ + + * @license Unlicense + * @link https://hugofara.github.io/lwt/developer/api + * @since 3.2.2 + */ + +declare(strict_types=1); + +namespace Lwt\Shared\Infrastructure\Database; + +/** + * One token produced while parsing a text: a word or a non-word run + * (whitespace / punctuation). + * + * This replaces the rows that used to be written to the `temp_word_occurrences` + * scratch table during parsing. Tokens are produced fully in PHP and consumed + * by TokenPersistence, so no scratch table is needed. See ScratchTables for the + * history of why the old temp tables were removed. + * + * Field meanings mirror the former temp_word_occurrences columns: + * - $sentence: 1-based sentence index within this text (was TiSeID, but local, + * not a pre-computed SeID). + * - $order: global monotonic token order across the whole text (was TiOrder). + * - $wordCount: 1 for a word token, 0 for a non-word run (was TiWordCount). + * - $text: the token text (was TiText). + * + * @since 3.2.2 + */ +final class ParsedToken +{ + /** + * @param int $sentence 1-based sentence index within the text + * @param int $order Global monotonic token order across the text + * @param int $wordCount 1 for a word, 0 for a non-word run + * @param string $text The token text + */ + public function __construct( + public readonly int $sentence, + public readonly int $order, + public readonly int $wordCount, + public readonly string $text + ) { + } + + /** + * Whether this token is a word (as opposed to whitespace/punctuation). + * + * @return bool + */ + public function isWord(): bool + { + return $this->wordCount !== 0; + } +} diff --git a/src/Shared/Infrastructure/Database/ScratchTables.php b/src/Shared/Infrastructure/Database/ScratchTables.php index 9019dbaf5..28ae11ba5 100644 --- a/src/Shared/Infrastructure/Database/ScratchTables.php +++ b/src/Shared/Infrastructure/Database/ScratchTables.php @@ -19,59 +19,35 @@ namespace Lwt\Shared\Infrastructure\Database; /** - * Creates the per-connection scratch tables used while parsing texts and - * importing vocabulary. + * Creates the per-connection scratch table used while importing vocabulary. * - * These tables (`temp_word_occurrences`, `temp_words`, `tempexprs`) used to be - * persistent, globally-shared InnoDB tables that every parse/import `TRUNCATE`d - * and refilled. That design caused two problems: + * `temp_words` used to be a persistent, globally-shared InnoDB table that every + * import `TRUNCATE`d and refilled. That design caused two problems: * * - **Tablespace crashes.** On file-per-table InnoDB, `TRUNCATE` physically * drops and recreates the `.ibd`. When that file went missing (notably on * Windows) the table was left with a missing tablespace and every subsequent * import failed with InnoDB error 194 "Tablespace is missing for a table" * (see issue #247). - * - **Concurrency corruption.** Because the tables were shared, two parses or - * imports running at once (multiple tabs, a feed refresh overlapping a manual - * import, or any two users in multi-user mode) read and truncated each - * other's rows, silently producing scrambled sentences and word occurrences. + * - **Concurrency corruption.** Because the table was shared, two imports + * running at once (or any two users in multi-user mode) read and truncated + * each other's rows. * - * They are now created per database connection with `CREATE TEMPORARY TABLE`, so + * It is now created per database connection with `CREATE TEMPORARY TABLE`, so * each request gets its own private copy. Temporary InnoDB tables live in the * shared session temporary tablespace (there is no per-table `.ibd` to orphan), * and they are dropped automatically when the connection closes. * - * Callers must `ensure*()` the table exists before use, then clear rows with - * `DELETE FROM` — never `TRUNCATE`, which is DDL that recreates the tablespace - * and implicitly commits any open transaction. + * Callers must `ensureWords()` before use, then clear rows with `DELETE FROM` — + * never `TRUNCATE`, which is DDL that recreates the tablespace and implicitly + * commits any open transaction. + * + * (Text parsing no longer uses any scratch table — see TokenPersistence.) * * @since 3.2.2 */ final class ScratchTables { - /** - * Ensure the word-occurrence staging table exists for this connection. - * - * Holds the tokenised word occurrences of a single text while it is being - * parsed into `sentences` / `word_occurrences`. - * - * @return void - */ - public static function ensureWordOccurrences(): void - { - Connection::query( - <<<'SQL' - CREATE TEMPORARY TABLE IF NOT EXISTS temp_word_occurrences ( - TiCount smallint(5) unsigned NOT NULL, - TiSeID mediumint(8) unsigned NOT NULL, - TiOrder smallint(5) unsigned NOT NULL, - TiWordCount tinyint(3) unsigned NOT NULL, - TiText varchar(250) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci - SQL - ); - } - /** * Ensure the word-import staging table exists for this connection. * @@ -96,26 +72,4 @@ public static function ensureWords(): void SQL ); } - - /** - * Ensure the multi-word-expression staging table exists for this connection. - * - * Populated by the multi-word detection pass while parsing a text. - * - * @return void - */ - public static function ensureExpressions(): void - { - Connection::query( - <<<'SQL' - CREATE TEMPORARY TABLE IF NOT EXISTS tempexprs ( - sent mediumint unsigned, - word varchar(250), - lword varchar(250), - TiOrder smallint unsigned, - n tinyint(3) unsigned NOT NULL - ) - SQL - ); - } } diff --git a/src/Shared/Infrastructure/Database/StandardTextParser.php b/src/Shared/Infrastructure/Database/StandardTextParser.php index 4d2904697..e7486f94a 100644 --- a/src/Shared/Infrastructure/Database/StandardTextParser.php +++ b/src/Shared/Infrastructure/Database/StandardTextParser.php @@ -189,23 +189,21 @@ public static function displayStandardPreview(string $text, bool $rtlScript): vo } /** - * Parse standard text and insert into temp_word_occurrences. + * Build the tab-delimited token blob from preprocessed text. * - * @param string $text Preprocessed text + * Produces one line per token as "\t", where a term + * ending in "\r" marks the end of a sentence. This is the same + * serialization the old LOAD DATA path consumed; parseBlob() turns it + * into ParsedToken objects. + * + * @param string $text Text after word-splitting transformations * @param string $termchar Word character regex * @param string $removeSpaces Space removal setting - * @param bool $useMaxSeID Whether to query for max sentence ID - * - * @return void * - * @psalm-suppress MixedArgument + * @return string */ - public static function parseStandardToDatabase( - string $text, - string $termchar, - string $removeSpaces, - bool $useMaxSeID - ): void { + private static function buildTokenBlob(string $text, string $termchar, string $removeSpaces): string + { $qc = self::quoteChars(); $replaced = preg_replace( array( @@ -225,136 +223,120 @@ public static function parseStandardToDatabase( str_replace(array("\t", "\n\n"), array("\n", ""), $text) ); $text = trim($replaced ?? $text); - $text = StringUtils::removeSpaces( + return StringUtils::removeSpaces( preg_replace("/(\n|^)(?!1\t)/u", "\n0\t", $text) ?? $text, $removeSpaces ); + } - // It is faster to write to a file and let SQL do its magic, but may run into - // security restrictions - $use_local_infile = in_array( - Connection::fetchValue("SELECT @@GLOBAL.local_infile as value"), - array(1, '1', 'ON') - ); - // For database mode, we use a positive ID placeholder (1) since saveWithSql - // only checks if id > 0 for sentence ID calculation - $idForSql = $useMaxSeID ? 1 : 0; - if ($use_local_infile) { - TextParsingPersistence::saveWithSql($text, $idForSql); - } else { - $order = 0; - $sid = 1; - if ($useMaxSeID) { - // Get next auto-increment value from table status - // This is more reliable than MAX(SeID)+1 when there are gaps - $dbname = Globals::getDatabaseName(); - $sentencesTable = Globals::table('sentences'); - $sid = (int)Connection::preparedFetchValue( - "SELECT AUTO_INCREMENT as value FROM information_schema.TABLES - WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?", - [$dbname, $sentencesTable] - ); - // Fall back to MAX+1 if AUTO_INCREMENT is not available - if ($sid <= 0) { - $sid = (int)Connection::fetchValue( - "SELECT IFNULL(MAX(`SeID`)+1,1) as value FROM sentences" - . UserScopedQuery::forTable('sentences') - ); - } + /** + * Turn the token blob into ParsedToken objects. + * + * Replicates the semantics of the former LOAD DATA `SET` clause: each line + * is "\t"; a term ending with "\r" ends the current + * sentence (that token still belongs to the current sentence, the next one + * starts a new sentence). Order is a global 1-based counter. + * + * Unlike the old saveWithSqlFallback(), this does NOT trim the line, so the + * "\r" sentence markers and trailing-space tokens are preserved (that bug + * caused LOAD-DATA-less installs to parse every text as one sentence). + * + * @param string $blob Token blob from buildTokenBlob() + * + * @return ParsedToken[] + */ + private static function parseBlob(string $blob): array + { + $tokens = []; + $sentence = 1; + $order = 0; + foreach (explode("\n", $blob) as $line) { + $tab = strpos($line, "\t"); + if ($tab === false) { + // Blank or malformed line (e.g. a stray empty line): no token. + continue; } - $count = 0; - $rows = array(); - foreach (explode("\n", $text) as $line) { - if (trim($line) == "") { - continue; - } - list($word_count, $term) = explode("\t", $line); - $tiSeID = $sid; // TiSeID - $tiCount = $count + 1; // TiCount - $count += mb_strlen($term); - if (str_ends_with($term, "\r")) { - $term = str_replace("\r", '', $term); - $sid++; - $count = 0; - } - $tiOrder = ++$order; // TiOrder - $tiWordCount = (int)$word_count; // TiWordCount - $rows[] = array($tiSeID, $tiCount, $tiOrder, $term, $tiWordCount); + $wordCount = (int) substr($line, 0, $tab); + $term = substr($line, $tab + 1); + $sentenceEnd = str_ends_with($term, "\r"); + if ($sentenceEnd) { + $term = str_replace("\r", '', $term); } - - // Build multi-row INSERT with prepared statement - if (!empty($rows)) { - $placeholders = array(); - $flatParams = array(); - foreach ($rows as $row) { - $placeholders[] = "(?, ?, ?, ?, ?)"; - $flatParams = array_merge($flatParams, $row); - } - - Connection::preparedExecute( - "INSERT INTO temp_word_occurrences ( - TiSeID, TiCount, TiOrder, TiText, TiWordCount - ) VALUES " . implode(',', $placeholders), - $flatParams - ); + $order++; + $tokens[] = new ParsedToken($sentence, $order, $wordCount, $term); + if ($sentenceEnd) { + $sentence++; } } + return $tokens; } /** - * Parse a text using the default tools. It is a not-japanese text. - * - * @param string $text Text to parse - * @param int $id Text ID. If $id == -2, only split the text. - * @param int $lid Language ID. - * - * @return null|string[] If $id == -2 return a splitted version of the text. + * Tokenize a (character-substituted) standard text into ParsedToken objects. * - * @psalm-return non-empty-list|null + * @param string $text Preprocessed text (character substitutions applied) + * @param int $lid Language ID * - * @internal Use TextParsing::splitIntoSentences(), parseAndDisplayPreview(), or parseAndSave() instead. + * @return ParsedToken[] */ - public static function parseStandard(string $text, int $id, int $lid): ?array + public static function tokenize(string $text, int $lid): array { $settings = self::getLanguageSettings($lid); - - // Return null if language not found if ($settings === null) { - return null; + return []; } - - // Apply initial transformations (paragraph markers, trim, collapse spaces) - $text = self::applyInitialTransformations( + $text = self::applyInitialTransformations($text, $settings['splitEachChar']); + $text = self::applyWordSplitting( $text, - $settings['splitEachChar'] + $settings['splitSentence'], + $settings['noSentenceEnd'], + $settings['termchar'] ); + $blob = self::buildTokenBlob($text, $settings['termchar'], $settings['removeSpaces']); + return self::parseBlob($blob); + } - // Preview mode - display HTML BEFORE word splitting - if ($id == -1) { - self::displayStandardPreview($text, (bool)$settings['rtlScript']); + /** + * Split a (character-substituted) standard text into sentences only. + * + * @param string $text Preprocessed text (character substitutions applied) + * @param int $lid Language ID + * + * @return string[] + * + * @psalm-return non-empty-list + */ + public static function splitSentences(string $text, int $lid): array + { + $settings = self::getLanguageSettings($lid); + if ($settings === null) { + return ['']; } - - // Apply word-splitting transformations + $text = self::applyInitialTransformations($text, $settings['splitEachChar']); $text = self::applyWordSplitting( $text, $settings['splitSentence'], $settings['noSentenceEnd'], $settings['termchar'] ); + return self::splitStandardSentences($text, $settings['removeSpaces']); + } - // Split-only mode - if ($id == -2) { - return self::splitStandardSentences($text, $settings['removeSpaces']); + /** + * Echo the preview HTML for a (character-substituted) standard text. + * + * @param string $text Preprocessed text (character substitutions applied) + * @param int $lid Language ID + * + * @return void + */ + public static function echoPreview(string $text, int $lid): void + { + $settings = self::getLanguageSettings($lid); + if ($settings === null) { + return; } - - // Database insertion (for both preview mode -1 and actual save mode > 0) - self::parseStandardToDatabase( - $text, - $settings['termchar'], - $settings['removeSpaces'], - $id > 0 - ); - - return null; + $text = self::applyInitialTransformations($text, $settings['splitEachChar']); + self::displayStandardPreview($text, (bool)$settings['rtlScript']); } } diff --git a/src/Shared/Infrastructure/Database/TextParsing.php b/src/Shared/Infrastructure/Database/TextParsing.php index a64ee5650..5cb5f3819 100644 --- a/src/Shared/Infrastructure/Database/TextParsing.php +++ b/src/Shared/Infrastructure/Database/TextParsing.php @@ -23,7 +23,9 @@ /** * Text parsing and processing utilities (facade). * - * Delegates to JapaneseTextParser, StandardTextParser, and TextParsingPersistence. + * Delegates tokenization to JapaneseTextParser / StandardTextParser and + * persistence to TokenPersistence. Parsing happens entirely in PHP — there are + * no scratch tables involved. * * @since 3.0.0 */ @@ -32,9 +34,6 @@ class TextParsing /** * Split text into sentences without database operations. * - * Use this method when you only need to split text into sentences - * without saving to the database (e.g., for long text splitting). - * * @param string $text Text to parse * @param int $lid Language ID * @@ -44,15 +43,21 @@ class TextParsing */ public static function splitIntoSentences(string $text, int $lid): array { - $result = self::prepare($text, -2, $lid); - return $result ?? ['']; + $pre = self::preprocess($text, $lid); + if ($pre === null) { + return ['']; + } + [$ptext, $isMecab] = $pre; + if ($isMecab) { + return JapaneseTextParser::splitJapaneseSentences($ptext); + } + return StandardTextParser::splitSentences($ptext, $lid); } /** * Parse text and display preview HTML for validation. * - * Use this method for the text checking UI. Outputs HTML directly - * to show parsed sentences and word statistics. + * Outputs HTML directly to show parsed sentences and word statistics. * * @param string $text Text to parse * @param int $lid Language ID @@ -71,33 +76,28 @@ public static function parseAndDisplayPreview(string $text, int $lid): void } $rtlScript = (bool)$record['LgRightToLeft']; - // Parse text and display preview HTML (id=-1 triggers preview display in prepare) - self::prepare($text, -1, $lid); - - // Display sentences and word statistics - TextParsingPersistence::checkValid($lid); - - // Get multi-word expressions - $wl = TextParsingPersistence::getMultiWordLengths($lid); - - // Process multi-word expressions if any exist - if (!empty($wl)) { - TextParsingPersistence::checkExpressions($wl); + $pre = self::preprocess($text, $lid); + if ($pre === null) { + return; + } + [$ptext, $isMecab] = $pre; + + // Preview HTML is shown before word splitting. + if ($isMecab) { + JapaneseTextParser::displayJapanesePreview($ptext); + $tokens = JapaneseTextParser::tokenize($ptext); + } else { + StandardTextParser::echoPreview($ptext, $lid); + $tokens = StandardTextParser::tokenize($ptext, $lid); } - // Display statistics - TextParsingPersistence::displayStatistics($lid, $rtlScript, !empty($wl)); - - // Clean up - QueryBuilder::table('temp_word_occurrences')->delete(); + TokenPersistence::echoCheckValid($tokens, $lid); + TokenPersistence::echoStatistics($tokens, $lid, $rtlScript); } /** * Parse text and save to database. * - * Use this method when creating or updating texts. Parses the text - * and inserts sentences and text items into the database. - * * @param string $text Text to parse * @param int $lid Language ID * @param int $textId Text ID (must be positive) @@ -123,28 +123,13 @@ public static function parseAndSave(string $text, int $lid, int $textId): void throw DatabaseException::recordNotFound('languages', 'LgID', $lid); } - // Parse text into temp_word_occurrences (id>0 uses MAX(SeID)+1 for sentence IDs) - self::prepare($text, $textId, $lid); - - // Get multi-word expressions - $wl = TextParsingPersistence::getMultiWordLengths($lid); - - // Process multi-word expressions if any exist - if (!empty($wl)) { - TextParsingPersistence::checkExpressions($wl); - } - - // Register sentences and text items in database - TextParsingPersistence::registerSentencesTextItems($textId, $lid, !empty($wl)); - - // Clean up - QueryBuilder::table('temp_word_occurrences')->delete(); + $tokens = self::tokenize($text, $lid); + TokenPersistence::save($tokens, $lid, $textId); } /** * Check/preview text and return parsing statistics without saving. * - * Use this method to get text statistics for preview purposes. * Does not output any HTML or save to database. * * @param string $text Text to parse @@ -154,100 +139,45 @@ public static function parseAndSave(string $text, int $lid, int $textId): void */ public static function checkText(string $text, int $lid): array { - $settings = StandardTextParser::getLanguageSettings($lid); - - if ($settings === null) { - return [ - 'sentences' => 0, - 'words' => 0, - 'unknownPercent' => 100.0, - 'preview' => '' - ]; - } - - // Prepare text into temp_word_occurrences - self::prepare($text, -1, $lid); - - // Get sentence count - $sentences = Connection::fetchAll( - 'SELECT GROUP_CONCAT(TiText ORDER BY TiOrder SEPARATOR "") - AS Sent FROM temp_word_occurrences GROUP BY TiSeID' - ); - $sentenceCount = count($sentences); - - // Build preview from first few sentences - $preview = ''; - $previewSentences = array_slice($sentences, 0, 3); - foreach ($previewSentences as $record) { - if ($preview !== '') { - $preview .= ' '; - } - $preview .= (string) ($record['Sent'] ?? ''); - } - if (count($sentences) > 3) { - $preview .= '...'; - } - - // Get word statistics - $bindings = [$lid]; - $rows = Connection::preparedFetchAll( - "SELECT COUNT(`TiOrder`) AS cnt, IF(0=TiWordCount,0,1) AS len, - LOWER(TiText) AS word, WoTranslation - FROM temp_word_occurrences - LEFT JOIN words ON LOWER(TiText)=WoTextLC AND WoLgID=?" - . UserScopedQuery::forTablePrepared('words', $bindings, '') - . " GROUP BY LOWER(TiText)", - $bindings - ); - - $totalWords = 0; - $unknownWords = 0; - - foreach ($rows as $record) { - if ($record['len'] == 1) { - $totalWords += (int) $record['cnt']; - // Word is unknown if it has no translation - if (empty($record['WoTranslation'])) { - $unknownWords += (int) $record['cnt']; - } - } - } - - $unknownPercent = $totalWords > 0 - ? round(($unknownWords / $totalWords) * 100, 1) - : 100.0; - - // Clean up temp_word_occurrences - QueryBuilder::table('temp_word_occurrences')->delete(); - - return [ - 'sentences' => $sentenceCount, - 'words' => $totalWords, - 'unknownPercent' => $unknownPercent, - 'preview' => $preview - ]; + $tokens = self::tokenize($text, $lid); + return TokenPersistence::stats($tokens, $lid); } /** - * Pre-parse the input text before a definitive parsing by a specialized parser. + * Tokenize a text into ParsedToken objects (no database writes). * * @param string $text Text to parse - * @param int $id Text ID * @param int $lid Language ID * - * @return null|string[] If $id = -2 return a splitted version of the text + * @return ParsedToken[] + */ + private static function tokenize(string $text, int $lid): array + { + $pre = self::preprocess($text, $lid); + if ($pre === null) { + return []; + } + [$ptext, $isMecab] = $pre; + return $isMecab + ? JapaneseTextParser::tokenize($ptext) + : StandardTextParser::tokenize($ptext, $lid); + } + + /** + * Apply the language's text preprocessing (escaping + character + * substitutions) and report whether it uses the MeCab parser. * - * @psalm-return non-empty-list|null + * @param string $text Raw text + * @param int $lid Language ID * - * @internal Use splitIntoSentences(), parseAndDisplayPreview(), or parseAndSave() instead. + * @return array{0: string, 1: bool}|null [preprocessed text, isMecab] or null if language missing */ - private static function prepare(string $text, int $id, int $lid): ?array + private static function preprocess(string $text, int $lid): ?array { $record = QueryBuilder::table('languages') ->where('LgID', '=', $lid) ->firstPrepared(); - // Return null if language not found if ($record === null) { return null; } @@ -255,11 +185,6 @@ private static function prepare(string $text, int $id, int $lid): ?array $termchar = (string)$record['LgRegexpWordCharacters']; $replace = explode("|", (string) $record['LgCharacterSubstitutions']); $text = Escaping::prepareTextdata($text); - // Create the per-connection scratch table (if needed) and clear it. - // DELETE (not TRUNCATE): TRUNCATE recreates the tablespace, which crashed - // on Windows with InnoDB error 194 and implicitly commits transactions. - ScratchTables::ensureWordOccurrences(); - QueryBuilder::table('temp_word_occurrences')->delete(); // because of sentence special characters $text = str_replace(array('}', '{'), array(']', '['), $text); @@ -270,9 +195,6 @@ private static function prepare(string $text, int $id, int $lid): ?array } } - if ('MECAB' == strtoupper(trim($termchar))) { - return JapaneseTextParser::parseJapanese($text, $id); - } - return StandardTextParser::parseStandard($text, $id, $lid); + return [$text, 'MECAB' === strtoupper(trim($termchar))]; } } diff --git a/src/Shared/Infrastructure/Database/TextParsingPersistence.php b/src/Shared/Infrastructure/Database/TextParsingPersistence.php deleted file mode 100644 index 5f9ec0587..000000000 --- a/src/Shared/Infrastructure/Database/TextParsingPersistence.php +++ /dev/null @@ -1,524 +0,0 @@ - - * @license Unlicense - * @link https://hugofara.github.io/lwt/developer/api - * @since 3.0.0 - */ - -declare(strict_types=1); - -namespace Lwt\Shared\Infrastructure\Database; - -use Lwt\Shared\Infrastructure\Globals; - -/** - * Shared database operations for text parsing. - * - * Handles SQL-based text saving, sentence/text-item registration, - * statistics display, and multi-word expression checking. - * - * @since 3.0.0 - */ -class TextParsingPersistence -{ - /** - * Insert a processed text in the data in pure SQL way. - * - * @param string $text Preprocessed text to insert - * @param int $id Text ID - * - * @return void - */ - public static function saveWithSql(string $text, int $id): void - { - $file_name = sys_get_temp_dir() . DIRECTORY_SEPARATOR . "tmpti.txt"; - $fp = fopen($file_name, 'w'); - if ($fp !== false) { - fwrite($fp, $text); - fclose($fp); - } - Connection::query("SET @order=0, @sid=1, @count = 0;"); - if ($id > 0) { - // Get next auto-increment value for accurate TiSeID calculation - $dbname = Globals::getDatabaseName(); - $sentencesTable = Globals::table('sentences'); - $autoInc = (int)Connection::preparedFetchValue( - "SELECT AUTO_INCREMENT as value FROM information_schema.TABLES - WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?", - [$dbname, $sentencesTable] - ); - // Fall back to MAX+1 if AUTO_INCREMENT is not available - if ($autoInc <= 0) { - $autoInc = (int)Connection::fetchValue( - "SELECT IFNULL(MAX(`SeID`)+1,1) as value FROM sentences" - . UserScopedQuery::forTable('sentences') - ); - } - Connection::query("SET @sid = $autoInc;"); - } - // LOAD DATA LOCAL INFILE does not support prepared statements for file path - // We need to use Connection::query() here, but we escape the file path manually - $connection = Globals::getDbConnection(); - if ($connection === null) { - throw new \RuntimeException('Database connection not available'); - } - $escaped_file_name = (string) mysqli_real_escape_string($connection, $file_name); - $sql = "LOAD DATA LOCAL INFILE '$escaped_file_name' - INTO TABLE temp_word_occurrences - FIELDS TERMINATED BY '\\t' LINES TERMINATED BY '\\n' (@word_count, @term) - SET - TiSeID = @sid, - TiCount = (@count:=@count+CHAR_LENGTH(@term))+1-CHAR_LENGTH(@term), - TiOrder = IF( - @term LIKE '%\\r', - CASE - WHEN (@term:=REPLACE(@term,'\\r','')) IS NULL THEN NULL - WHEN (@sid:=@sid+1) IS NULL THEN NULL - WHEN @count:= 0 IS NULL THEN NULL - ELSE @order := @order+1 - END, - @order := @order+1 - ), - TiText = @term, - TiWordCount = @word_count"; - - // Try LOAD DATA LOCAL INFILE, fall back to INSERT if it fails - try { - Connection::query($sql); - } catch (\RuntimeException $e) { - // If LOAD DATA LOCAL INFILE is disabled, use fallback method - if (strpos($e->getMessage(), 'LOAD DATA LOCAL INFILE is forbidden') !== false) { - self::saveWithSqlFallback($text, $id); - } else { - throw $e; - } - } - unlink($file_name); - } - - /** - * Fallback method to insert text data when LOAD DATA LOCAL INFILE is disabled. - * - * @param string $text Preprocessed text to insert - * @param int $id Text ID - * - * @return void - */ - public static function saveWithSqlFallback(string $text, int $id): void - { - // Get starting sentence ID - $sid = 1; - if ($id > 0) { - // Get next auto-increment value for accurate TiSeID calculation - $dbname = Globals::getDatabaseName(); - $sentencesTable = Globals::table('sentences'); - $sid = (int)Connection::preparedFetchValue( - "SELECT AUTO_INCREMENT as value FROM information_schema.TABLES - WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?", - [$dbname, $sentencesTable] - ); - // Fall back to MAX+1 if AUTO_INCREMENT is not available - if ($sid <= 0) { - $sid = (int)Connection::fetchValue( - "SELECT IFNULL(MAX(`SeID`)+1,1) as value FROM sentences" - . UserScopedQuery::forTable('sentences') - ); - } - } - - $lines = explode("\n", $text); - $order = 0; - $count = 0; - - foreach ($lines as $line) { - $line = trim($line); - if ($line === '') { - continue; - } - - $parts = explode("\t", $line); - if (count($parts) < 2) { - continue; - } - - $word_count = (int)$parts[0]; - $term = $parts[1]; - - // Handle line breaks (increase sentence ID) - if (substr($term, -1) === "\r") { - $term = rtrim($term, "\r"); - $order++; - $count = 0; - $sid++; - } else { - $order++; - } - - $current_count = $count; - $count += strlen($term) + 1; - - Connection::preparedExecute( - "INSERT INTO temp_word_occurrences - (TiSeID, TiCount, TiOrder, TiText, TiWordCount) - VALUES (?, ?, ?, ?, ?)", - [$sid, $current_count, $order, $term, $word_count] - ); - } - } - - /** - * Echo the sentences in a text. Prepare JS data for words and word count. - * - * @param int $lid Language ID - * - * @return void - */ - public static function checkValid(int $lid): void - { - $wo = $nw = array(); - $sentences = Connection::fetchAll( - 'SELECT GROUP_CONCAT(TiText order by TiOrder SEPARATOR "") - Sent FROM temp_word_occurrences group by TiSeID' - ); - echo '

Sentences

    '; - foreach ($sentences as $record) { - echo "
  1. " . \htmlspecialchars((string) ($record['Sent'] ?? ''), ENT_QUOTES, 'UTF-8') . "
    2. "; - } - echo '
'; - $bindings = [$lid]; - $rows = Connection::preparedFetchAll( - "SELECT count(`TiOrder`) cnt, if(0=TiWordCount,0,1) as len, - LOWER(TiText) as word, WoTranslation - FROM temp_word_occurrences - LEFT JOIN words ON lower(TiText)=WoTextLC AND WoLgID=?" - . UserScopedQuery::forTablePrepared('words', $bindings, '') - . " GROUP BY lower(TiText)", - $bindings - ); - foreach ($rows as $record) { - if ($record['len'] == 1) { - $wo[] = array( - \htmlspecialchars((string) ($record['word'] ?? ''), ENT_QUOTES, 'UTF-8'), - $record['cnt'], - \htmlspecialchars((string) ($record['WoTranslation'] ?? ''), ENT_QUOTES, 'UTF-8') - ); - } else { - $nw[] = array( - htmlspecialchars((string)($record['word'] ?? ''), ENT_QUOTES, 'UTF-8'), - htmlspecialchars((string)($record['cnt'] ?? ''), ENT_QUOTES, 'UTF-8') - ); - } - } - // JavaScript moved to src/frontend/js/texts/text_check_display.ts - echo ''; - } - - /** - * Append sentences and text items in the database. - * - * TiSeID in temp_word_occurrences is pre-computed to match future SeID values. - * When parseStandardToDatabase runs with useMaxSeID=true, it sets TiSeID - * to MAX(SeID)+1, MAX(SeID)+2, etc. When we insert sentences here, they - * get those exact SeID values via auto-increment, so TiSeID = SeID. - * - * @param int $tid ID of text from which insert data - * @param int $lid ID of the language of the text - * @param bool $hasmultiword Set to true to insert multi-words as well. - * - * @return void - */ - public static function registerSentencesTextItems(int $tid, int $lid, bool $hasmultiword): void - { - // STEP 1: Insert sentences FIRST to satisfy FK constraint. - Connection::query('SET @i=0;'); - Connection::preparedExecute( - "INSERT INTO sentences ( - SeLgID, SeTxID, SeOrder, SeFirstPos, SeText - ) SELECT - ?, - ?, - @i:=@i+1, - MIN(IF(TiWordCount=0, TiOrder+1, TiOrder)), - GROUP_CONCAT(TiText ORDER BY TiOrder SEPARATOR \"\") - FROM temp_word_occurrences - GROUP BY TiSeID - ORDER BY TiSeID", - [$lid, $tid] - ); - - // STEP 1.5: Align TiSeID with actual SeID values. - // The pre-computed TiSeID may not match actual AUTO_INCREMENT values, - // so we update temp_word_occurrences to use the actual SeID from inserted sentences. - // ROW_NUMBER() maps TiSeID rank to SeOrder, which we JOIN to get SeID. - // Materialise the TiSeID->rank mapping in its own temporary table first. - // temp_word_occurrences is now a TEMPORARY table, and MySQL/MariaDB cannot - // open a temporary table twice in the same statement ("Can't reopen table"), - // so the mapping cannot be an inline self-subquery of the UPDATE. - Connection::execute("DROP TEMPORARY TABLE IF EXISTS temp_seid_map"); - Connection::execute( - "CREATE TEMPORARY TABLE temp_seid_map AS - SELECT TiSeID AS old_seid, - ROW_NUMBER() OVER (ORDER BY TiSeID) AS rn - FROM temp_word_occurrences - GROUP BY TiSeID" - ); - Connection::preparedExecute( - "UPDATE temp_word_occurrences t - JOIN temp_seid_map mapping ON t.TiSeID = mapping.old_seid - JOIN sentences s ON s.SeOrder = mapping.rn AND s.SeTxID = ? - SET t.TiSeID = s.SeID", - [$tid] - ); - Connection::execute("DROP TEMPORARY TABLE IF EXISTS temp_seid_map"); - - // STEP 1.6: Also update tempexprs.sent if multiword expressions exist. - // tempexprs was populated before sentences were inserted, so its `sent` - // field also needs to be aligned with actual SeID values. - if ($hasmultiword) { - // Same "can't reopen a temporary table" constraint as STEP 1.5: - // materialise the sent->rank mapping before the self-referencing UPDATE. - Connection::execute("DROP TEMPORARY TABLE IF EXISTS temp_sent_map"); - Connection::execute( - "CREATE TEMPORARY TABLE temp_sent_map AS - SELECT sent AS old_sent, - ROW_NUMBER() OVER (ORDER BY sent) AS rn - FROM tempexprs - WHERE sent IS NOT NULL - GROUP BY sent" - ); - Connection::preparedExecute( - "UPDATE tempexprs e - JOIN temp_sent_map mapping ON e.sent = mapping.old_sent - JOIN sentences s ON s.SeOrder = mapping.rn AND s.SeTxID = ? - SET e.sent = s.SeID", - [$tid] - ); - Connection::execute("DROP TEMPORARY TABLE IF EXISTS temp_sent_map"); - } - - // STEP 2: Insert text items. TiSeID and tempexprs.sent now equal actual SeID. - if ($hasmultiword) { - // Build SQL and bindings in lockstep. Each forTablePrepared() call - // both appends " AND WoUsID = ?" to the SQL and pushes $userId - // onto $bindings, so the bindings stay in left-to-right placeholder - // order even with two user-scope injections inside the UNION. - $bindings = [$lid, $tid]; - $sql = "INSERT INTO word_occurrences ( - Ti2WoID, Ti2LgID, Ti2TxID, Ti2SeID, Ti2Order, Ti2WordCount, Ti2Text - ) SELECT WoID, ?, ?, sent, TiOrder - (2*(n-1)) TiOrder, - n TiWordCount, word - FROM tempexprs - JOIN words - ON WoTextLC = lword AND WoWordCount = n" - . UserScopedQuery::forTablePrepared('words', $bindings, '') - . " WHERE lword IS NOT NULL AND WoLgID = ?"; - $bindings[] = $lid; - $bindings[] = $lid; - $bindings[] = $tid; - $sql .= " UNION ALL - SELECT WoID, ?, ?, TiSeID, TiOrder, TiWordCount, TiText - FROM temp_word_occurrences - LEFT JOIN words - ON LOWER(TiText) = WoTextLC AND TiWordCount=1 AND WoLgID = ?"; - $bindings[] = $lid; - $sql .= UserScopedQuery::forTablePrepared('words', $bindings, '') - . " ORDER BY TiOrder, TiWordCount"; - - $stmt = Connection::prepare($sql); - $stmt->bindValues($bindings); - $stmt->execute(); - } else { - $bindings = [$lid, $tid, $lid]; - Connection::preparedExecute( - "INSERT INTO word_occurrences ( - Ti2WoID, Ti2LgID, Ti2TxID, Ti2SeID, Ti2Order, Ti2WordCount, Ti2Text - ) - SELECT WoID, ?, ?, TiSeID, TiOrder, TiWordCount, TiText - FROM temp_word_occurrences - LEFT JOIN words - ON LOWER(TiText) = WoTextLC AND TiWordCount=1 AND WoLgID = ?" - . UserScopedQuery::forTablePrepared('words', $bindings, '') - . " ORDER BY TiOrder, TiWordCount", - $bindings - ); - } - } - - /** - * Display statistics about a text. - * - * @param int $lid Language ID - * @param bool $rtlScript true if language is right-to-left - * @param bool $multiwords Display if text has multi-words - * - * @return void - */ - public static function displayStatistics(int $lid, bool $rtlScript, bool $multiwords): void - { - $mw = array(); - if ($multiwords) { - $bindings = [$lid]; - $rows = Connection::preparedFetchAll( - "SELECT COUNT(WoID) cnt, n as len, - LOWER(WoText) AS word, WoTranslation - FROM tempexprs - JOIN words - ON WoTextLC = lword AND WoWordCount = n" - . UserScopedQuery::forTablePrepared('words', $bindings, '') - . " WHERE lword IS NOT NULL AND WoLgID = ? - GROUP BY WoID ORDER BY WoTextLC", - $bindings - ); - foreach ($rows as $record) { - $mw[] = array( - htmlspecialchars((string)($record['word'] ?? ''), ENT_QUOTES, 'UTF-8'), - $record['cnt'], - htmlspecialchars((string)($record['WoTranslation'] ?? ''), ENT_QUOTES, 'UTF-8') - ); - } - } - // JavaScript moved to src/frontend/js/texts/text_check_display.ts - echo ''; - } - - /** - * Get all multi-word expression lengths for a language. - * - * @param int $lid Language ID - * - * @return int[] Array of distinct word counts (e.g., [2, 3] for 2-word and 3-word expressions) - */ - public static function getMultiWordLengths(int $lid): array - { - $wl = []; - $rows = QueryBuilder::table('words') - ->select(['DISTINCT(WoWordCount) as WoWordCount']) - ->where('WoLgID', '=', $lid) - ->where('WoWordCount', '>', 1) - ->getPrepared(); - - foreach ($rows as $record) { - $wl[] = (int)$record['WoWordCount']; - } - - return $wl; - } - - /** - * Check a language that contains expressions. - * - * @param int[] $wl All the different expression length in the language. - * - * @return void - */ - public static function checkExpressions(array $wl): void - { - $wl_max = 0; - $mw_sql = ''; - foreach ($wl as $word_length) { - if ($wl_max < $word_length) { - $wl_max = $word_length; - } - $mw_sql .= ' WHEN ' . $word_length . - ' THEN @a' . ($word_length * 2 - 1); - } - $set_wo_sql = $set_wo_sql_2 = $del_wo_sql = $init_var = ''; - // For all possible multi-words length - for ($i = $wl_max * 2 - 1; $i > 1; $i--) { - $set_wo_sql .= "WHEN (@a$i := @a" . ($i - 1) . ") IS NULL THEN NULL "; - $set_wo_sql_2 .= "WHEN (@a$i := @a" . ($i - 2) . ") IS NULL THEN NULL "; - $del_wo_sql .= "WHEN (@a$i := @a0) IS NULL THEN NULL "; - $init_var .= "@a$i=0,"; - } - // 2.8.1-fork: @a0 is always 0? @f always '' but necessary to force code execution - Connection::query( - "SET $init_var@a1=0, @a0=0, @se_id=0, @c='', @d=0, @f='', @ti_or=0;" - ); - // Create a table to store length of each terms - Connection::query( - "CREATE TEMPORARY TABLE IF NOT EXISTS numbers( - n tinyint(3) unsigned NOT NULL - );" - ); - Connection::execute("DELETE FROM numbers"); - Connection::query( - "INSERT IGNORE INTO numbers(n) VALUES (" . - implode('),(', $wl) . - ');' - ); - // Store garbage (per-connection temporary table) - ScratchTables::ensureExpressions(); - Connection::execute("DELETE FROM tempexprs"); - Connection::query( - "INSERT IGNORE INTO tempexprs - (sent, word, lword, TiOrder, n) - -- 2.10.0-fork: straight_join may be irrelevant as the query is less skewed - SELECT straight_join - IF( - @se_id=TiSeID and @ti_or=TiOrder, - IF((@ti_or:=TiOrder+@a0) is null,TiSeID,TiSeID), - IF( - @se_id=TiSeID, - IF( - (@d=1) and (0<>TiWordCount), - CASE $set_wo_sql_2 - WHEN (@a1:=TiCount+@a0) IS NULL THEN NULL - WHEN (@se_id:=TiSeID+@a0) IS NULL THEN NULL - WHEN (@ti_or:=TiOrder+@a0) IS NULL THEN NULL - WHEN (@c:=concat(@c,TiText)) IS NULL THEN NULL - WHEN (@d:=(0<>TiWordCount)+@a0) IS NULL THEN NULL - ELSE TiSeID - END, - CASE $set_wo_sql - WHEN (@a1:=TiCount+@a0) IS NULL THEN NULL - WHEN (@se_id:=TiSeID+@a0) IS NULL THEN NULL - WHEN (@ti_or:=TiOrder+@a0) IS NULL THEN NULL - WHEN (@c:=concat(@c,TiText)) IS NULL THEN NULL - WHEN (@d:=(0<>TiWordCount)+@a0) IS NULL THEN NULL - ELSE TiSeID - END - ), - CASE $del_wo_sql - WHEN (@a1:=TiCount+@a0) IS NULL THEN NULL - WHEN (@se_id:=TiSeID+@a0) IS NULL THEN NULL - WHEN (@ti_or:=TiOrder+@a0) IS NULL THEN NULL - WHEN (@c:=concat(TiText,@f)) IS NULL THEN NULL - WHEN (@d:=(0<>TiWordCount)+@a0) IS NULL THEN NULL - ELSE TiSeID - END - ) - ) sent, - if( - @d=0, - NULL, - if( - CRC32(@z:=substr(@c,CASE n$mw_sql END))<>CRC32(LOWER(@z)), - @z, - '' - ) - ) word, - if(@d=0 or ''=@z, NULL, lower(@z)) lword, - TiOrder, - n - FROM numbers , temp_word_occurrences" - ); - } -} diff --git a/src/Shared/Infrastructure/Database/TokenPersistence.php b/src/Shared/Infrastructure/Database/TokenPersistence.php new file mode 100644 index 000000000..d4f7daf0e --- /dev/null +++ b/src/Shared/Infrastructure/Database/TokenPersistence.php @@ -0,0 +1,495 @@ + + * @license Unlicense + * @link https://hugofara.github.io/lwt/developer/api + * @since 3.2.2 + */ + +declare(strict_types=1); + +namespace Lwt\Shared\Infrastructure\Database; + +/** + * Turns a parsed token stream into `sentences` and `word_occurrences` rows, + * detecting multi-word expressions along the way — all in PHP. + * + * This replaces the old scratch-table pipeline (temp_word_occurrences + + * tempexprs + numbers + the LOAD DATA path + the stateful @-variable multi-word + * detection SQL). Sentences are inserted first (to satisfy the FK on + * word_occurrences), their real SeIDs are read back, and word occurrences — + * single words and multi-word expressions — are inserted referencing them. + * + * @since 3.2.2 + */ +final class TokenPersistence +{ + /** @var int Rows per INSERT statement. */ + private const CHUNK = 500; + + /** + * Save parsed tokens as sentences + word occurrences for a text. + * + * @param ParsedToken[] $tokens Tokens for the whole text + * @param int $lid Language ID + * @param int $textId Text ID + * + * @return void + */ + public static function save(array $tokens, int $lid, int $textId): void + { + if (empty($tokens)) { + return; + } + $bySentence = self::groupBySentence($tokens); + $seIdMap = self::insertSentences($bySentence, $lid, $textId); + + $singleMap = self::singleWordTerms($lid, self::distinctWordLowercase($tokens)); + $mwTerms = self::multiWordTerms($lid); + + // Single tokens (words + non-words): every token becomes an occurrence. + $rows = []; + foreach ($tokens as $t) { + $woId = null; + if ($t->wordCount === 1) { + $woId = $singleMap[self::lc($t->text)]['id'] ?? null; + } + $rows[] = [$woId, $lid, $textId, $seIdMap[$t->sentence], $t->order, $t->wordCount, $t->text]; + } + // Multi-word expressions overlay the single words they span. + foreach (self::detectMultiWords($bySentence, $mwTerms) as $mw) { + // Store the span text only when it differs from its lowercase form + // (matches the historical Ti2Text storage optimisation). + $stored = $mw['text'] !== self::lc($mw['text']) ? $mw['text'] : ''; + $rows[] = [$mw['id'], $lid, $textId, $seIdMap[$mw['sentence']], $mw['order'], $mw['n'], $stored]; + } + + self::insertWordOccurrences($rows); + } + + /** + * Compute preview statistics for the check-text UI (no output). + * + * @param ParsedToken[] $tokens Tokens for the whole text + * @param int $lid Language ID + * + * @return array{sentences: int, words: int, unknownPercent: float, preview: string} + */ + public static function stats(array $tokens, int $lid): array + { + if (empty($tokens)) { + return ['sentences' => 0, 'words' => 0, 'unknownPercent' => 100.0, 'preview' => '']; + } + $bySentence = self::groupBySentence($tokens); + + $counts = []; + $total = 0; + foreach ($tokens as $t) { + if ($t->wordCount === 1) { + $lc = self::lc($t->text); + $counts[$lc] = ($counts[$lc] ?? 0) + 1; + $total++; + } + } + $single = self::singleWordTerms($lid, array_keys($counts)); + $unknown = 0; + foreach ($counts as $lc => $cnt) { + if (empty($single[$lc]['tr'] ?? '')) { + $unknown += $cnt; + } + } + $unknownPercent = $total > 0 ? round($unknown / $total * 100, 1) : 100.0; + + $texts = []; + foreach ($bySentence as $sTokens) { + $texts[] = self::sentenceText($sTokens); + } + $preview = implode(' ', array_slice($texts, 0, 3)); + if (count($texts) > 3) { + $preview .= '...'; + } + + return [ + 'sentences' => count($bySentence), + 'words' => $total, + 'unknownPercent' => $unknownPercent, + 'preview' => $preview, + ]; + } + + /** + * Echo the sentence list and per-word JSON for the check-text preview. + * + * @param ParsedToken[] $tokens Tokens for the whole text + * @param int $lid Language ID + * + * @return void + */ + public static function echoCheckValid(array $tokens, int $lid): void + { + $bySentence = self::groupBySentence($tokens); + echo '

Sentences

    '; + foreach ($bySentence as $sTokens) { + echo '
  1. ' . \htmlspecialchars(self::sentenceText($sTokens), ENT_QUOTES, 'UTF-8') . '
    2. '; + } + echo '
'; + + $wordCounts = []; + $nonWordCounts = []; + foreach ($tokens as $t) { + $lc = self::lc($t->text); + if ($t->wordCount === 1) { + $wordCounts[$lc] = ($wordCounts[$lc] ?? 0) + 1; + } else { + $nonWordCounts[$lc] = ($nonWordCounts[$lc] ?? 0) + 1; + } + } + $single = self::singleWordTerms($lid, array_keys($wordCounts)); + $wo = []; + foreach ($wordCounts as $lc => $cnt) { + $wo[] = [ + \htmlspecialchars($lc, ENT_QUOTES, 'UTF-8'), + $cnt, + \htmlspecialchars($single[$lc]['tr'] ?? '', ENT_QUOTES, 'UTF-8'), + ]; + } + $nw = []; + foreach ($nonWordCounts as $lc => $cnt) { + $nw[] = [ + \htmlspecialchars($lc, ENT_QUOTES, 'UTF-8'), + \htmlspecialchars((string)$cnt, ENT_QUOTES, 'UTF-8'), + ]; + } + echo ''; + } + + /** + * Echo the multi-word statistics JSON for the check-text preview. + * + * @param ParsedToken[] $tokens Tokens for the whole text + * @param int $lid Language ID + * @param bool $rtlScript Whether the language is right-to-left + * + * @return void + */ + public static function echoStatistics(array $tokens, int $lid, bool $rtlScript): void + { + $mwTerms = self::multiWordTerms($lid); + $occ = self::detectMultiWords(self::groupBySentence($tokens), $mwTerms); + + $idInfo = []; + foreach ($mwTerms as $terms) { + foreach ($terms as $info) { + $idInfo[$info['id']] = $info; + } + } + $byWo = []; + foreach ($occ as $o) { + $byWo[$o['id']] = ($byWo[$o['id']] ?? 0) + 1; + } + $mw = []; + foreach ($byWo as $id => $cnt) { + $info = $idInfo[$id] ?? ['text' => '', 'tr' => '']; + $mw[] = [ + \htmlspecialchars(self::lc($info['text']), ENT_QUOTES, 'UTF-8'), + $cnt, + \htmlspecialchars($info['tr'], ENT_QUOTES, 'UTF-8'), + ]; + } + \usort($mw, fn(array $a, array $b): int => \strcmp((string)$a[0], (string)$b[0])); + + echo ''; + } + + // ========================================================================= + // Internal helpers + // ========================================================================= + + /** + * Group tokens by their sentence index, preserving order. + * + * @param ParsedToken[] $tokens Tokens + * + * @return array> + */ + private static function groupBySentence(array $tokens): array + { + $by = []; + foreach ($tokens as $t) { + $by[$t->sentence][] = $t; + } + ksort($by); + return $by; + } + + /** + * Concatenate a sentence's token texts. + * + * @param list $sTokens Sentence tokens + * + * @return string + */ + private static function sentenceText(array $sTokens): string + { + $s = ''; + foreach ($sTokens as $t) { + $s .= $t->text; + } + return $s; + } + + /** + * Insert sentences and return a map of local sentence index -> real SeID. + * + * @param array> $bySentence Tokens grouped by sentence + * @param int $lid Language ID + * @param int $textId Text ID + * + * @return array + */ + private static function insertSentences(array $bySentence, int $lid, int $textId): array + { + $localSids = array_keys($bySentence); + $params = []; + $seOrder = 0; + foreach ($bySentence as $sTokens) { + $seOrder++; + $firstPos = null; + $text = ''; + foreach ($sTokens as $t) { + $pos = $t->wordCount === 0 ? $t->order + 1 : $t->order; + if ($firstPos === null || $pos < $firstPos) { + $firstPos = $pos; + } + $text .= $t->text; + } + array_push($params, $lid, $textId, $seOrder, (int)$firstPos, $text); + } + self::chunkedInsert( + 'INSERT INTO sentences (SeLgID, SeTxID, SeOrder, SeFirstPos, SeText) VALUES ', + '(?, ?, ?, ?, ?)', + 5, + $params + ); + + $bindings = [$textId]; + $rows = Connection::preparedFetchAll( + 'SELECT SeID FROM sentences WHERE SeTxID = ?' + . UserScopedQuery::forTablePrepared('sentences', $bindings) + . ' ORDER BY SeOrder', + $bindings + ); + $map = []; + foreach ($localSids as $i => $localSid) { + $map[$localSid] = (int)($rows[$i]['SeID'] ?? 0); + } + return $map; + } + + /** + * Insert word-occurrence rows in chunks. + * + * @param list $rows Rows + * + * @return void + */ + private static function insertWordOccurrences(array $rows): void + { + if (empty($rows)) { + return; + } + $params = []; + foreach ($rows as $r) { + array_push($params, $r[0], $r[1], $r[2], $r[3], $r[4], $r[5], $r[6]); + } + self::chunkedInsert( + 'INSERT INTO word_occurrences ' + . '(Ti2WoID, Ti2LgID, Ti2TxID, Ti2SeID, Ti2Order, Ti2WordCount, Ti2Text) VALUES ', + '(?, ?, ?, ?, ?, ?, ?)', + 7, + $params + ); + } + + /** + * Execute a multi-row INSERT in chunks of CHUNK rows. + * + * @param string $prefix SQL up to and including "VALUES " + * @param string $rowPlaceholder Placeholder for one row, e.g. "(?, ?)" + * @param int $colsPerRow Number of columns per row + * @param list $params Flat parameter list + * + * @return void + */ + private static function chunkedInsert( + string $prefix, + string $rowPlaceholder, + int $colsPerRow, + array $params + ): void { + $rowCount = intdiv(count($params), $colsPerRow); + for ($i = 0; $i < $rowCount; $i += self::CHUNK) { + $n = min(self::CHUNK, $rowCount - $i); + $placeholders = implode(',', array_fill(0, $n, $rowPlaceholder)); + $slice = array_slice($params, $i * $colsPerRow, $n * $colsPerRow); + Connection::preparedExecute($prefix . $placeholders, $slice); + } + } + + /** + * Distinct lowercased word-token texts. + * + * @param ParsedToken[] $tokens Tokens + * + * @return list + */ + private static function distinctWordLowercase(array $tokens): array + { + $set = []; + foreach ($tokens as $t) { + if ($t->wordCount === 1) { + $set[self::lc($t->text)] = true; + } + } + return array_keys($set); + } + + /** + * Load single-word terms for the given lowercased words. + * + * @param int $lid Language ID + * @param list $lowerWords Distinct lowercased words to look up + * + * @return array + */ + private static function singleWordTerms(int $lid, array $lowerWords): array + { + if (empty($lowerWords)) { + return []; + } + $bindings = [$lid]; + // Build the IN clause manually: these are string keys (WoTextLC), not + // the integer IDs that Connection::buildPreparedInClause() expects. + $inClause = '(' . implode(',', array_fill(0, count($lowerWords), '?')) . ')'; + foreach ($lowerWords as $word) { + $bindings[] = $word; + } + $sql = 'SELECT WoID, WoTextLC, WoTranslation FROM words ' + . 'WHERE WoLgID = ? AND WoWordCount = 1 AND WoTextLC IN ' . $inClause + . UserScopedQuery::forTablePrepared('words', $bindings); + $map = []; + foreach (Connection::preparedFetchAll($sql, $bindings) as $r) { + $map[(string)$r['WoTextLC']] = [ + 'id' => (int)$r['WoID'], + 'tr' => (string)($r['WoTranslation'] ?? ''), + ]; + } + return $map; + } + + /** + * Load multi-word terms grouped by word count. + * + * @param int $lid Language ID + * + * @return array> + */ + private static function multiWordTerms(int $lid): array + { + $bindings = [$lid]; + $sql = 'SELECT WoID, WoText, WoTextLC, WoTranslation, WoWordCount FROM words ' + . 'WHERE WoLgID = ? AND WoWordCount > 1' + . UserScopedQuery::forTablePrepared('words', $bindings); + $map = []; + foreach (Connection::preparedFetchAll($sql, $bindings) as $r) { + $n = (int)$r['WoWordCount']; + $map[$n][(string)$r['WoTextLC']] = [ + 'id' => (int)$r['WoID'], + 'text' => (string)$r['WoText'], + 'tr' => (string)($r['WoTranslation'] ?? ''), + ]; + } + return $map; + } + + /** + * Detect multi-word expression occurrences in the token stream. + * + * For each sentence and each known multi-word length n, slide a window of n + * words and match the concatenated span (words + intervening separators) + * against the language's n-word terms. + * + * @param array> $bySentence Tokens by sentence + * @param array> $mwTerms Multi-word terms + * + * @return list + */ + private static function detectMultiWords(array $bySentence, array $mwTerms): array + { + if (empty($mwTerms)) { + return []; + } + $lengths = array_keys($mwTerms); + $occ = []; + foreach ($bySentence as $localSid => $sTokens) { + $wordIdx = []; + foreach ($sTokens as $idx => $t) { + if ($t->wordCount === 1) { + $wordIdx[] = $idx; + } + } + $wordCount = count($wordIdx); + foreach ($lengths as $n) { + if ($n < 2) { + continue; + } + for ($i = 0; $i + $n - 1 < $wordCount; $i++) { + $firstIdx = $wordIdx[$i]; + $lastIdx = $wordIdx[$i + $n - 1]; + $span = ''; + for ($k = $firstIdx; $k <= $lastIdx; $k++) { + $span .= $sTokens[$k]->text; + } + $lc = self::lc($span); + if (isset($mwTerms[$n][$lc])) { + $occ[] = [ + 'id' => $mwTerms[$n][$lc]['id'], + 'sentence' => $localSid, + 'order' => $sTokens[$firstIdx]->order, + 'n' => $n, + 'text' => $span, + ]; + } + } + } + } + return $occ; + } + + /** + * Lowercase a string (UTF-8). + * + * @param string $s Input + * + * @return string + */ + private static function lc(string $s): string + { + return mb_strtolower($s, 'UTF-8'); + } +} diff --git a/tests/backend/Shared/Infrastructure/Database/JapaneseTextParserTest.php b/tests/backend/Shared/Infrastructure/Database/JapaneseTextParserTest.php index 046bd224b..495bd80ec 100644 --- a/tests/backend/Shared/Infrastructure/Database/JapaneseTextParserTest.php +++ b/tests/backend/Shared/Infrastructure/Database/JapaneseTextParserTest.php @@ -128,34 +128,41 @@ public function splitJapaneseSentencesWithWhitespaceOnlyReturnsEmptyString(): vo } // ========================================================================= - // parseJapanese (split-only mode id=-2) + // buildTokensFromMecab (pure MeCab-output -> tokens, no binary needed) // ========================================================================= #[Test] - public function parseJapaneseWithIdMinus2ReturnsArrayOfSentences(): void + public function buildTokensFromMecabProducesExpectedTokens(): void { - $result = JapaneseTextParser::parseJapanese('Some text', -2); + // Captured MeCab output for "東京は大きいです。" in the parser's + // "-F %m\t%t\t%h" format, terminated with the EOP sentence marker. + $mecabed = "東京\t2\t46\nは\t6\t16\n大きい\t2\t10\n" + . "です\t6\t25\n。\t3\t7\nEOP\t3\t7\n"; - $this->assertIsArray($result); - $this->assertNotEmpty($result); - } + $tokens = JapaneseTextParser::buildTokensFromMecab($mecabed); - #[Test] - public function parseJapaneseWithIdMinus2MatchesSplitMethod(): void - { - $text = "Line A\nLine B"; - $splitResult = JapaneseTextParser::splitJapaneseSentences($text); - $parseResult = JapaneseTextParser::parseJapanese($text, -2); - - $this->assertSame($splitResult, $parseResult); + $this->assertCount(5, $tokens); + $this->assertSame( + ['東京', 'は', '大きい', 'です', '。'], + array_map(fn($t) => $t->text, $tokens) + ); + // Punctuation is a non-word; the rest are words. + $this->assertSame([1, 1, 1, 1, 0], array_map(fn($t) => $t->wordCount, $tokens)); + // All in the first sentence (the trailing EOP order group is dropped). + foreach ($tokens as $t) { + $this->assertSame(1, $t->sentence); + } + // Global order is strictly increasing. + $orders = array_map(fn($t) => $t->order, $tokens); + $sorted = $orders; + sort($sorted); + $this->assertSame($sorted, $orders); } #[Test] - public function parseJapaneseWithIdMinus2DoesNotReturnNull(): void + public function buildTokensFromMecabHandlesEmptyOutput(): void { - $result = JapaneseTextParser::parseJapanese('test', -2); - - $this->assertNotNull($result); + $this->assertSame([], JapaneseTextParser::buildTokensFromMecab('')); } // ========================================================================= @@ -204,32 +211,4 @@ public function displayJapanesePreviewCollapsesWhitespace(): void $this->assertStringContainsString('Word1 Word2', $output); } - - // ========================================================================= - // parseJapanese / parseJapaneseToDatabase (requires MeCab + DB) - // ========================================================================= - - #[Test] - public function parseJapaneseWithIdMinus1RequiresMecab(): void - { - $this->markTestSkipped( - 'MeCab installation required for display preview mode' - ); - } - - #[Test] - public function parseJapaneseWithPositiveIdRequiresMecabAndDatabase(): void - { - $this->markTestSkipped( - 'MeCab installation and database connection required for save mode' - ); - } - - #[Test] - public function parseJapaneseToDatabaseRequiresMecabAndDatabase(): void - { - $this->markTestSkipped( - 'MeCab installation and database connection required' - ); - } } diff --git a/tests/backend/Shared/Infrastructure/Database/StandardTextParserTest.php b/tests/backend/Shared/Infrastructure/Database/StandardTextParserTest.php index b8d0f4082..10cbf4069 100644 --- a/tests/backend/Shared/Infrastructure/Database/StandardTextParserTest.php +++ b/tests/backend/Shared/Infrastructure/Database/StandardTextParserTest.php @@ -288,20 +288,4 @@ public function getLanguageSettingsRequiresDatabase(): void { $this->markTestSkipped('Database connection required'); } - - // ========================================================================= - // parseStandard / parseStandardToDatabase (requires DB) - // ========================================================================= - - #[Test] - public function parseStandardRequiresDatabase(): void - { - $this->markTestSkipped('Database connection required'); - } - - #[Test] - public function parseStandardToDatabaseRequiresDatabase(): void - { - $this->markTestSkipped('Database connection required'); - } } diff --git a/tests/backend/Shared/Infrastructure/Database/TextParsingPersistenceTest.php b/tests/backend/Shared/Infrastructure/Database/TextParsingPersistenceTest.php deleted file mode 100644 index 7d50432fd..000000000 --- a/tests/backend/Shared/Infrastructure/Database/TextParsingPersistenceTest.php +++ /dev/null @@ -1,379 +0,0 @@ - - * @since 3.0.0 - */ - -declare(strict_types=1); - -namespace Tests\Backend\Shared\Infrastructure\Database; - -use Lwt\Shared\Infrastructure\Database\TextParsingPersistence; -use PHPUnit\Framework\Attributes\Test; -use PHPUnit\Framework\TestCase; -use PHPUnit\Framework\Attributes\CoversClass; - -/** - * Unit tests for TextParsingPersistence static methods. - * - * @since 3.0.0 - */ -#[CoversClass(TextParsingPersistence::class)] -class TextParsingPersistenceTest extends TestCase -{ - // ========================================================================= - // saveWithSql - signature and visibility - // ========================================================================= - - #[Test] - public function saveWithSqlIsPublicAndStatic(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'saveWithSql' - ); - - $this->assertTrue($reflection->isPublic()); - $this->assertTrue($reflection->isStatic()); - } - - #[Test] - public function saveWithSqlAcceptsStringTextAndIntId(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'saveWithSql' - ); - $params = $reflection->getParameters(); - - $this->assertCount(2, $params); - $this->assertSame('text', $params[0]->getName()); - $this->assertSame('string', $params[0]->getType()->getName()); - $this->assertSame('id', $params[1]->getName()); - $this->assertSame('int', $params[1]->getType()->getName()); - } - - #[Test] - public function saveWithSqlReturnsVoid(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'saveWithSql' - ); - - $this->assertSame('void', $reflection->getReturnType()->getName()); - } - - #[Test] - public function saveWithSqlRequiresDatabase(): void - { - $this->markTestSkipped('Database connection required'); - } - - // ========================================================================= - // saveWithSqlFallback - signature and visibility - // ========================================================================= - - #[Test] - public function saveWithSqlFallbackIsPublicAndStatic(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'saveWithSqlFallback' - ); - - $this->assertTrue($reflection->isPublic()); - $this->assertTrue($reflection->isStatic()); - } - - #[Test] - public function saveWithSqlFallbackAcceptsStringTextAndIntId(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'saveWithSqlFallback' - ); - $params = $reflection->getParameters(); - - $this->assertCount(2, $params); - $this->assertSame('text', $params[0]->getName()); - $this->assertSame('string', $params[0]->getType()->getName()); - $this->assertSame('id', $params[1]->getName()); - $this->assertSame('int', $params[1]->getType()->getName()); - } - - #[Test] - public function saveWithSqlFallbackReturnsVoid(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'saveWithSqlFallback' - ); - - $this->assertSame('void', $reflection->getReturnType()->getName()); - } - - #[Test] - public function saveWithSqlFallbackRequiresDatabase(): void - { - $this->markTestSkipped('Database connection required'); - } - - // ========================================================================= - // checkValid - signature and visibility - // ========================================================================= - - #[Test] - public function checkValidIsPublicAndStatic(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'checkValid' - ); - - $this->assertTrue($reflection->isPublic()); - $this->assertTrue($reflection->isStatic()); - } - - #[Test] - public function checkValidAcceptsSingleIntParameter(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'checkValid' - ); - $params = $reflection->getParameters(); - - $this->assertCount(1, $params); - $this->assertSame('lid', $params[0]->getName()); - $this->assertSame('int', $params[0]->getType()->getName()); - } - - #[Test] - public function checkValidReturnsVoid(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'checkValid' - ); - - $this->assertSame('void', $reflection->getReturnType()->getName()); - } - - #[Test] - public function checkValidRequiresDatabase(): void - { - $this->markTestSkipped('Database connection required'); - } - - // ========================================================================= - // registerSentencesTextItems - signature and visibility - // ========================================================================= - - #[Test] - public function registerSentencesTextItemsIsPublicAndStatic(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'registerSentencesTextItems' - ); - - $this->assertTrue($reflection->isPublic()); - $this->assertTrue($reflection->isStatic()); - } - - #[Test] - public function registerSentencesTextItemsAcceptsThreeParameters(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'registerSentencesTextItems' - ); - $params = $reflection->getParameters(); - - $this->assertCount(3, $params); - $this->assertSame('tid', $params[0]->getName()); - $this->assertSame('int', $params[0]->getType()->getName()); - $this->assertSame('lid', $params[1]->getName()); - $this->assertSame('int', $params[1]->getType()->getName()); - $this->assertSame('hasmultiword', $params[2]->getName()); - $this->assertSame('bool', $params[2]->getType()->getName()); - } - - #[Test] - public function registerSentencesTextItemsRequiresDatabase(): void - { - $this->markTestSkipped('Database connection required'); - } - - // ========================================================================= - // getMultiWordLengths - signature and visibility - // ========================================================================= - - #[Test] - public function getMultiWordLengthsIsPublicAndStatic(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'getMultiWordLengths' - ); - - $this->assertTrue($reflection->isPublic()); - $this->assertTrue($reflection->isStatic()); - } - - #[Test] - public function getMultiWordLengthsAcceptsSingleIntParameter(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'getMultiWordLengths' - ); - $params = $reflection->getParameters(); - - $this->assertCount(1, $params); - $this->assertSame('lid', $params[0]->getName()); - $this->assertSame('int', $params[0]->getType()->getName()); - } - - #[Test] - public function getMultiWordLengthsReturnsArray(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'getMultiWordLengths' - ); - - $this->assertSame('array', $reflection->getReturnType()->getName()); - } - - #[Test] - public function getMultiWordLengthsRequiresDatabase(): void - { - $this->markTestSkipped('Database connection required'); - } - - // ========================================================================= - // displayStatistics - signature and visibility - // ========================================================================= - - #[Test] - public function displayStatisticsIsPublicAndStatic(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'displayStatistics' - ); - - $this->assertTrue($reflection->isPublic()); - $this->assertTrue($reflection->isStatic()); - } - - #[Test] - public function displayStatisticsAcceptsThreeParameters(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'displayStatistics' - ); - $params = $reflection->getParameters(); - - $this->assertCount(3, $params); - $this->assertSame('lid', $params[0]->getName()); - $this->assertSame('int', $params[0]->getType()->getName()); - $this->assertSame('rtlScript', $params[1]->getName()); - $this->assertSame('bool', $params[1]->getType()->getName()); - $this->assertSame('multiwords', $params[2]->getName()); - $this->assertSame('bool', $params[2]->getType()->getName()); - } - - #[Test] - public function displayStatisticsRequiresDatabase(): void - { - $this->markTestSkipped('Database connection required'); - } - - // ========================================================================= - // checkExpressions - signature and visibility - // ========================================================================= - - #[Test] - public function checkExpressionsIsPublicAndStatic(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'checkExpressions' - ); - - $this->assertTrue($reflection->isPublic()); - $this->assertTrue($reflection->isStatic()); - } - - #[Test] - public function checkExpressionsAcceptsSingleArrayParameter(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'checkExpressions' - ); - $params = $reflection->getParameters(); - - $this->assertCount(1, $params); - $this->assertSame('wl', $params[0]->getName()); - $this->assertSame('array', $params[0]->getType()->getName()); - } - - #[Test] - public function checkExpressionsReturnsVoid(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'checkExpressions' - ); - - $this->assertSame('void', $reflection->getReturnType()->getName()); - } - - #[Test] - public function checkExpressionsRequiresDatabase(): void - { - $this->markTestSkipped('Database connection required'); - } - - // ========================================================================= - // Class-level checks - // ========================================================================= - - #[Test] - public function allPublicMethodsAreStatic(): void - { - $reflection = new \ReflectionClass(TextParsingPersistence::class); - $methods = $reflection->getMethods(\ReflectionMethod::IS_PUBLIC); - - foreach ($methods as $method) { - $this->assertTrue( - $method->isStatic(), - "Method {$method->getName()} should be static" - ); - } - } - - #[Test] - public function classHasExpectedNumberOfPublicMethods(): void - { - $reflection = new \ReflectionClass(TextParsingPersistence::class); - $methods = $reflection->getMethods(\ReflectionMethod::IS_PUBLIC); - - // saveWithSql, saveWithSqlFallback, checkValid, - // registerSentencesTextItems, displayStatistics, - // getMultiWordLengths, checkExpressions - $this->assertCount(7, $methods); - } -} From 84d25f64fc6e03094d1156231ea951f0cac4a970 Mon Sep 17 00:00:00 2001 From: HugoFara Date: Sat, 25 Jul 2026 15:43:40 +0200 Subject: [PATCH 3/3] fix(parsing): read MeCab output regardless of line endings (#247) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Windows PHPUnit jobs failed on JapaneseTextParserTest::buildTokensFromMecabProducesExpectedTokens with "actual size 0 matches expected size 5". Both MeCab output readers split their lines on PHP_EOL, but a subprocess's line terminator comes from MeCab, not from the host PHP runs on. Wherever the two disagree -- notably Windows, where PHP_EOL is "\r\n" while the output ends lines with "\n" -- explode() returned the whole output as a single element. That one pseudo-line parsed as one staging row whose order equalled the final EOP order group, which the grouping step drops, so the result was zero tokens. Real Japanese parsing on Windows was affected the same way, not just the fixture. Both readers now normalize CRLF/CR to LF before splitting, the same way SqlFileParser has since (#241): - JapaneseTextParser::buildTokensFromMecab() (the reader this branch added) - MecabParser::parseMecabOutput() (the module-layer twin, broken since it was written) Covered by tests asserting LF, CRLF and CR-only output tokenize identically. The CR-only case reproduces the Windows failure mode on a Linux host -- it yields 0 tokens instead of 5 before the fix -- so the regression is caught on every runner, not only the Windows ones. Also adds MecabParserTest, which had no coverage at all. Verified end-to-end against MeCab 0.996: both parsers tokenize "東京は大きいです。\n寿司を食べました。" into the expected words across both sentences. --- CHANGELOG.md | 14 ++ .../Infrastructure/Parser/MecabParser.php | 7 +- .../Database/JapaneseTextParser.php | 7 +- .../Infrastructure/Parser/MecabParserTest.php | 132 ++++++++++++++++++ .../Database/JapaneseTextParserTest.php | 44 ++++++ 5 files changed, 202 insertions(+), 2 deletions(-) create mode 100644 tests/backend/Modules/Language/Infrastructure/Parser/MecabParserTest.php diff --git a/CHANGELOG.md b/CHANGELOG.md index 275c2a056..40fb760fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,20 @@ ones are marked like "v1.0.0-fork". ## [Unreleased] +### Fixed + +* **Japanese (MeCab) parsing produced no tokens on Windows**: both MeCab output + readers split their lines on `PHP_EOL`, but a subprocess's line terminator + comes from MeCab, not from the host PHP runs on. Wherever the two disagree — + notably Windows, where `PHP_EOL` is `"\r\n"` while the output ends lines with + `"\n"` — the entire output stayed a single line and every token collapsed into + one, yielding zero usable tokens. Both + `JapaneseTextParser::buildTokensFromMecab()` and + `MecabParser::parseMecabOutput()` now normalize line endings before splitting, + the same way `SqlFileParser` does since #241. This is what failed the Windows + PHPUnit jobs. Verified end-to-end against MeCab 0.996 and covered by tests + that assert LF, CRLF and CR-only output tokenize identically. + ### Changed * **Word-status model is now a single source of truth** (#238, Phase 1): the diff --git a/src/Modules/Language/Infrastructure/Parser/MecabParser.php b/src/Modules/Language/Infrastructure/Parser/MecabParser.php index fe0b73b0d..1b8229755 100644 --- a/src/Modules/Language/Infrastructure/Parser/MecabParser.php +++ b/src/Modules/Language/Infrastructure/Parser/MecabParser.php @@ -216,7 +216,12 @@ protected function parseMecabOutput(string $mecabOutput): ParserResult $termType = 0; $lastNodeType = 0; - $lines = explode(PHP_EOL, $mecabOutput); + // MeCab's line terminator comes from MeCab, not from the host PHP runs + // on, so splitting on PHP_EOL broke this everywhere PHP_EOL is not + // "\n": on Windows the whole output stayed a single line and every + // token collapsed into one. Normalise first, then split on "\n". + $mecabOutput = str_replace(["\r\n", "\r"], "\n", $mecabOutput); + $lines = explode("\n", $mecabOutput); $previousTokenIndex = -1; foreach ($lines as $line) { diff --git a/src/Shared/Infrastructure/Database/JapaneseTextParser.php b/src/Shared/Infrastructure/Database/JapaneseTextParser.php index d420205d2..4a114d998 100644 --- a/src/Shared/Infrastructure/Database/JapaneseTextParser.php +++ b/src/Shared/Infrastructure/Database/JapaneseTextParser.php @@ -136,7 +136,12 @@ public static function buildTokensFromMecab(string $mecabed): array if ($separator === false) { $separator = "\t"; } - foreach (explode(PHP_EOL, $mecabed) as $line) { + // MeCab's line terminator comes from MeCab, not from the host PHP runs + // on, so splitting on PHP_EOL broke this everywhere PHP_EOL is not + // "\n": on Windows the whole output stayed a single line and every + // token collapsed into one. Normalise first, then split on "\n". + $mecabed = str_replace(["\r\n", "\r"], "\n", $mecabed); + foreach (explode("\n", $mecabed) as $line) { if (trim($line) == "") { continue; } diff --git a/tests/backend/Modules/Language/Infrastructure/Parser/MecabParserTest.php b/tests/backend/Modules/Language/Infrastructure/Parser/MecabParserTest.php new file mode 100644 index 000000000..e659df1b3 --- /dev/null +++ b/tests/backend/Modules/Language/Infrastructure/Parser/MecabParserTest.php @@ -0,0 +1,132 @@ + + * @since 3.2.2-fork + */ + +declare(strict_types=1); + +namespace Tests\Backend\Modules\Language\Infrastructure\Parser; + +use Lwt\Modules\Language\Domain\Parser\ParserResult; +use Lwt\Modules\Language\Infrastructure\Parser\MecabParser; +use PHPUnit\Framework\Attributes\CoversClass; +use PHPUnit\Framework\Attributes\Test; +use PHPUnit\Framework\TestCase; +use ReflectionMethod; + +/** + * Unit tests for MecabParser::parseMecabOutput(). + * + * Exercises the pure output-parsing step with synthetic MeCab output, so no + * MeCab binary is needed. + * + * @since 3.2.2-fork + */ +#[CoversClass(MecabParser::class)] +class MecabParserTest extends TestCase +{ + /** + * Captured MeCab output for "東京は大きいです。" in the parser's + * "-F %m\t%t\t%h" format, terminated with the EOP sentence marker. + * + * @var list + */ + private const MECAB_LINES = [ + "東京\t2\t46", + "は\t6\t16", + "大きい\t2\t10", + "です\t6\t25", + "。\t3\t7", + "EOP\t3\t7", + ]; + + /** + * Run the protected output parser. + * + * @param string $mecabOutput Raw MeCab output + * + * @return ParserResult + */ + private function parseOutput(string $mecabOutput): ParserResult + { + $method = new ReflectionMethod(MecabParser::class, 'parseMecabOutput'); + + /** @var ParserResult $result */ + $result = $method->invoke(new MecabParser(), $mecabOutput); + return $result; + } + + /** + * Note the trailing '¶': unlike JapaneseTextParser::buildTokensFromMecab(), + * which drops the final EOP order group, this parser keeps the paragraph + * marker as a token. Asserted as-is to pin current behaviour. + */ + #[Test] + public function parsesMecabOutputIntoTokens(): void + { + $result = $this->parseOutput(implode("\n", self::MECAB_LINES) . "\n"); + + $this->assertSame( + ['東京', 'は', '大きい', 'です', '。', '¶'], + array_map(fn($t) => $t->getText(), $result->getTokens()) + ); + } + + /** + * MeCab's line terminator comes from MeCab, not from the host PHP runs on, + * so splitting on PHP_EOL made this host-dependent. Whenever the output's + * terminator is not the one being split on, the whole output stays a single + * line and every token collapses — on Windows (PHP_EOL === "\r\n") an + * LF-terminated output produced no usable tokens at all. + * + * The CR-only case reproduces that failure mode on a Linux host, where + * PHP_EOL is "\n". + */ + #[Test] + public function parsingIsIndependentOfLineEndings(): void + { + $flatten = fn(ParserResult $r) => array_map( + fn($t) => [ + $t->getText(), + $t->isWord(), + $t->getSentenceIndex(), + $t->getOrder(), + ], + $r->getTokens() + ); + + $lf = $this->parseOutput(implode("\n", self::MECAB_LINES) . "\n"); + $this->assertCount(6, $lf->getTokens()); + + foreach (["\r\n" => 'CRLF', "\r" => 'CR-only'] as $eol => $label) { + $result = $this->parseOutput(implode($eol, self::MECAB_LINES) . $eol); + + $this->assertSame( + $flatten($lf), + $flatten($result), + "$label output should parse identically to LF" + ); + $this->assertSame( + $lf->getSentences(), + $result->getSentences(), + "$label output should yield the same sentences as LF" + ); + } + } + + #[Test] + public function handlesEmptyOutput(): void + { + $result = $this->parseOutput(''); + + $this->assertSame([], $result->getTokens()); + } +} diff --git a/tests/backend/Shared/Infrastructure/Database/JapaneseTextParserTest.php b/tests/backend/Shared/Infrastructure/Database/JapaneseTextParserTest.php index 495bd80ec..86f73e7f8 100644 --- a/tests/backend/Shared/Infrastructure/Database/JapaneseTextParserTest.php +++ b/tests/backend/Shared/Infrastructure/Database/JapaneseTextParserTest.php @@ -159,6 +159,50 @@ public function buildTokensFromMecabProducesExpectedTokens(): void $this->assertSame($sorted, $orders); } + /** + * MeCab's line terminator comes from MeCab, not from the host PHP runs on, + * so splitting on PHP_EOL made this method host-dependent. Whenever the + * output's terminator is not the one being split on, the whole output stays + * a single line and every token collapses — on Windows + * (PHP_EOL === "\r\n") an LF-terminated output produced zero tokens, which + * is what failed the Windows CI jobs. Every terminator must now yield the + * same tokens on every platform. + * + * The CR-only case reproduces that failure mode on a Linux host, where + * PHP_EOL is "\n": before the fix it yielded 0 tokens instead of 5. + */ + #[Test] + public function buildTokensFromMecabIsIndependentOfLineEndings(): void + { + $lines = [ + "東京\t2\t46", + "は\t6\t16", + "大きい\t2\t10", + "です\t6\t25", + "。\t3\t7", + "EOP\t3\t7", + ]; + + $flatten = fn(array $tokens) => array_map( + fn($t) => [$t->text, $t->wordCount, $t->sentence, $t->order], + $tokens + ); + + $lf = JapaneseTextParser::buildTokensFromMecab(implode("\n", $lines) . "\n"); + $this->assertCount(5, $lf); + + foreach (["\r\n" => 'CRLF', "\r" => 'CR-only'] as $eol => $label) { + $tokens = JapaneseTextParser::buildTokensFromMecab(implode($eol, $lines) . $eol); + + $this->assertCount(5, $tokens, "$label output should tokenize"); + $this->assertSame( + $flatten($lf), + $flatten($tokens), + "$label output should tokenize identically to LF" + ); + } + } + #[Test] public function buildTokensFromMecabHandlesEmptyOutput(): void {