diff --git a/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs b/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs index 6331e299..a14114f4 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs @@ -1,14 +1,12 @@ -using System.Collections.Generic; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; using System.Linq; +using System.Threading.Tasks; using SIL.Machine.Annotations; using SIL.Machine.FeatureModel; using SIL.Machine.Rules; using SIL.ObjectModel; -#if !SINGLE_THREADED -using System; -using System.Collections.Concurrent; -using System.Threading.Tasks; -#endif namespace SIL.Machine.Morphology.HermitCrab { @@ -47,18 +45,16 @@ public IEnumerable Apply(Word input) inWord.Freeze(); var output = new HashSet(FreezableEqualityComparer.Default); -#if SINGLE_THREADED - ApplySlots(inWord, _rules.Count - 1, output); -#else - ParallelApplySlots(inWord, output); -#endif + if (_morpher.MaxDegreeOfParallelism == 1) + ApplySlots(inWord, _rules.Count - 1, output); + else + ParallelApplySlots(inWord, output); foreach (Word outWord in output) outWord.SyntacticFeatureStruct.Add(fs); return output; } -#if SINGLE_THREADED private void ApplySlots(Word inWord, int index, HashSet output) { for (int i = index; i >= 0; i--) @@ -78,7 +74,7 @@ private void ApplySlots(Word inWord, int index, HashSet output) _morpher.TraceManager.EndUnapplyTemplate(_template, inWord, true); output.Add(inWord); } -#else + private void ParallelApplySlots(Word inWord, HashSet output) { var outStack = new ConcurrentStack(); @@ -126,6 +122,5 @@ private void ParallelApplySlots(Word inWord, HashSet output) output.UnionWith(outStack); } -#endif } } diff --git a/src/SIL.Machine.Morphology.HermitCrab/AnalysisScope.cs b/src/SIL.Machine.Morphology.HermitCrab/AnalysisScope.cs new file mode 100644 index 00000000..5ce8c4ad --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab/AnalysisScope.cs @@ -0,0 +1,118 @@ +using System.Collections.Generic; + +namespace SIL.Machine.Morphology.HermitCrab +{ + /// + /// Carrier for the analysis-cascade memo, threaded through clones + /// like and likewise excluded from Word.FreezeImpl/ + /// Word.ValueEquals, so dedup semantics are unchanged. + /// + /// One instance per call. A state key does not + /// encode the target surface word, so sharing a scope across parses of different words would be + /// unsound. + /// + /// + /// Not thread-safe, hence the plain collections: a scope is only installed when + /// is 1. Memoizing the parallel cascade would require + /// concurrent ones. + /// + /// + internal sealed class AnalysisScope + { + // OOM guard, since a positive entry holds Word lists rather than just a flag. Past the cap + // subtrees simply go unmemoized: only the hit rate degrades, never correctness. + private const int MaxMemoEntries = 100_000; + + public Dictionary Memo { get; } = new Dictionary(); + + // Same key space as Memo, different computation: the affix-template battery's result for a state + // (AnalysisStratumRule.ApplyTemplateBattery). Separate because a state can be memoized in one + // table but not the other. + public Dictionary TemplateMemo { get; } = + new Dictionary(); + + // Keys still under expansion somewhere on the call stack. A multiApp cascade can reach the same + // state again before its first expansion has finished (e.g. via a self-loop), which must fall + // through to unmemoized expansion rather than read a partial entry or deadlock. The template + // battery needs no equivalent: its call is eager, with no template/mrule mutual recursion inside. + public HashSet InProgress { get; } = new HashSet(); + + /// + /// Replay shared by both memo consumers. False on a miss; on a hit + /// holds the stored results grafted onto , or + /// is empty for a stored-empty ("nogood") entry. The query's non-head prefix is cloned once and + /// shared across this hit's replays, which is safe because each replay freezes immediately and + /// every non-head mutation path is CheckFrozen-guarded. + /// + public bool TryReplay( + Dictionary table, + AnalysisStateKey key, + Word query, + out List replayed + ) + { + if (!table.TryGetValue(key, out MemoEntry entry)) + { + replayed = null; + return false; + } + if (entry.Results.Count == 0) + { + replayed = new List(); + return true; + } + List queryNonHeadPrefix = query.CloneNonHeadsForReplay(); + replayed = new List(entry.Results.Count); + foreach (Word stored in entry.Results) + { + replayed.Add( + stored.ReplayOnto( + query, + entry.MruleTrailPrefixLength, + entry.NonHeadPrefixLength, + queryNonHeadPrefix + ) + ); + } + return true; + } + + /// + /// Records a fully-expanded result list against , unless the table is full. + /// + public void Store( + Dictionary table, + AnalysisStateKey key, + Word query, + List results + ) + { + if (table.Count < MaxMemoEntries) + table[key] = new MemoEntry(results, query.MorphologicalRuleTrailLength, query.NonHeadCount); + } + } + + /// + /// A memoized subtree or template-battery result. An empty means the state was + /// proved to yield nothing. The two prefix lengths are the trail/non-head counts at the moment of the + /// write, which is where splits a stored result when grafting it onto a + /// new arrival. + /// + /// There is deliberately no "incomplete" flag: only fully-explored subtrees may be recorded. Should a + /// step or time budget ever be added, an interrupted subtree must not be stored. + /// + /// + internal sealed class MemoEntry + { + public MemoEntry(IReadOnlyList results, int mruleTrailPrefixLength, int nonHeadPrefixLength) + { + Results = results; + MruleTrailPrefixLength = mruleTrailPrefixLength; + NonHeadPrefixLength = nonHeadPrefixLength; + } + + public IReadOnlyList Results { get; } + public int MruleTrailPrefixLength { get; } + public int NonHeadPrefixLength { get; } + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab/AnalysisStateKey.cs b/src/SIL.Machine.Morphology.HermitCrab/AnalysisStateKey.cs new file mode 100644 index 00000000..91a91f74 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab/AnalysisStateKey.cs @@ -0,0 +1,117 @@ +using System; +using System.Collections.Generic; +using SIL.Machine.Annotations; +using SIL.Machine.FeatureModel; + +namespace SIL.Machine.Morphology.HermitCrab +{ + /// + /// Order-independent identity of an analysis-cascade node. Two Words with an equal + /// key must make identical decisions in every analysis-side rule the cascade can invoke; that is the + /// memo's correctness contract, so this key-completeness audit of what each rule reads has to be + /// re-run whenever an Analysis*.cs rule changes: + /// + /// : Shape (FST pattern match), + /// (unifiability gate), per-rule unapplication count. + /// : adds + /// (MaxStemCount gate) -- never the non-heads' own content, only the count. + /// : adds + /// . + /// + /// No rule reads the order those rules were unapplied in, which is the redundancy this key collapses, + /// so the trail is reduced to an unordered multiset here. _isLastAppliedRuleFinal and + /// IsPartial are excluded as well: Word.ValueEquals includes them for result dedup, but + /// no analysis-side rule reads them. + /// + internal readonly struct AnalysisStateKey : IEquatable + { + private readonly Shape _shape; + private readonly Stratum _stratum; + private readonly FeatureStruct _syntacticFS; + private readonly FeatureStruct _realizationalFS; + private readonly int _nonHeadCount; + private readonly IReadOnlyDictionary _ruleCounts; + private readonly int _hashCode; + + public AnalysisStateKey(Word word) + { + // The cached hash covers live references -- notably Word.UnappliedRuleCounts, the word's own + // mutable dictionary. Keying an unfrozen word would let a later mutation invalidate a stored + // key's hash, silently causing permanent misses or entries that no longer match their bucket. + if (!word.IsFrozen) + throw new ArgumentException( + "The word must be frozen before it can be used as a memo key.", + nameof(word) + ); + + _shape = word.Shape; + _stratum = word.Stratum; + _syntacticFS = word.SyntacticFeatureStruct; + _realizationalFS = word.RealizationalFeatureStruct; + _nonHeadCount = word.NonHeadCount; + _ruleCounts = word.UnappliedRuleCounts; + + // Word.FreezeImpl deliberately leaves SyntacticFeatureStruct unfrozen, and + // AnalysisAffixTemplateRule.Apply mutates it in place on already-frozen Words -- no + // Word-level CheckFrozen guards that path. Freezing here pins the key's view of it, so a + // future rule mutating an already-keyed word throws instead of silently corrupting the table. + // Freeze is idempotent. + _shape.Freeze(); + _syntacticFS.Freeze(); + _realizationalFS.Freeze(); + + int hash = 17; + hash = hash * 31 + _shape.GetFrozenHashCode(); + hash = hash * 31 + (_stratum?.GetHashCode() ?? 0); + hash = hash * 31 + _syntacticFS.GetFrozenHashCode(); + hash = hash * 31 + _realizationalFS.GetFrozenHashCode(); + hash = hash * 31 + _nonHeadCount; + if (_ruleCounts != null) + { + // XOR rather than the usual *31 rolling combine: the multiset is unordered, so entries + // accumulated in different unapplication orders must still hash identically. + int multisetHash = 0; + foreach (KeyValuePair kvp in _ruleCounts) + multisetHash ^= (kvp.Key.GetHashCode() * 397) ^ kvp.Value; + hash = hash * 31 + multisetHash; + } + _hashCode = hash; + } + + public override int GetHashCode() => _hashCode; + + public override bool Equals(object obj) => obj is AnalysisStateKey other && Equals(other); + + public bool Equals(AnalysisStateKey other) + { + if (_hashCode != other._hashCode) + return false; + if (_nonHeadCount != other._nonHeadCount || !ReferenceEquals(_stratum, other._stratum)) + return false; + if (!_shape.ValueEquals(other._shape)) + return false; + if (!_syntacticFS.ValueEquals(other._syntacticFS) || !_realizationalFS.ValueEquals(other._realizationalFS)) + return false; + return RuleCountsEqual(_ruleCounts, other._ruleCounts); + } + + private static bool RuleCountsEqual( + IReadOnlyDictionary a, + IReadOnlyDictionary b + ) + { + int aCount = a?.Count ?? 0; + int bCount = b?.Count ?? 0; + if (aCount != bCount) + return false; + if (aCount == 0) + return true; + foreach (KeyValuePair kvp in a) + { + if (!b.TryGetValue(kvp.Key, out int otherCount) || otherCount != kvp.Value) + return false; + } + return true; + } + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab/AnalysisStratumRule.cs b/src/SIL.Machine.Morphology.HermitCrab/AnalysisStratumRule.cs index 752e0815..8ebfd2d5 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/AnalysisStratumRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/AnalysisStratumRule.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; +using System.Threading; using SIL.Machine.Annotations; using SIL.Machine.Rules; using SIL.ObjectModel; @@ -47,19 +48,18 @@ public AnalysisStratumRule(Morpher morpher, Stratum stratum) ); break; case MorphologicalRuleOrder.Unordered: -#if SINGLE_THREADED - _mrulesRule = new CombinationRuleCascade( - mrules, - true, - FreezableEqualityComparer.Default - ); -#else - _mrulesRule = new ParallelCombinationRuleCascade( - mrules, - true, - FreezableEqualityComparer.Default - ); -#endif + _mrulesRule = + morpher.MaxDegreeOfParallelism == 1 + ? (RuleCascade) + new MemoizedCombinationRuleCascade(mrules, FreezableEqualityComparer.Default) + : new ParallelCombinationRuleCascade( + mrules, + true, + FreezableEqualityComparer.Default + ) + { + MaxDegreeOfParallelism = morpher.MaxDegreeOfParallelism, + }; break; } } @@ -188,9 +188,42 @@ private IEnumerable ApplyMorphologicalRules(Word input) } } + // Counterparts to MemoizedCombinationRuleCascade's counters, for the template table. + internal static long DiagTemplateMemoHits; + internal static long DiagTemplateNogoodHits; + + // The affix-template battery, memoized by AnalysisStateKey against its own table. On + // template-heavy grammars this dominates parse time, which is why it is memoized separately from + // the mrule cascade. See AnalysisScope.InProgress for why no re-entry guard is needed here. + private IEnumerable ApplyTemplateBattery(Word input) + { + // Scope presence is the single source of truth for whether the memo is active; Morpher decides + // that once, at install time. Linear strata stay unmemoized because the key-completeness audit + // covers only the Unordered cascade. + AnalysisScope scope = input.AnalysisScope; + if (scope == null || _stratum.MorphologicalRuleOrder != MorphologicalRuleOrder.Unordered) + return _templatesRule.Apply(input); + + var key = new AnalysisStateKey(input); + if (scope.TryReplay(scope.TemplateMemo, key, input, out List replayed)) + { + if (replayed.Count == 0) + { + Interlocked.Increment(ref DiagTemplateNogoodHits); + return replayed; + } + Interlocked.Increment(ref DiagTemplateMemoHits); + return replayed; + } + + var results = new List(_templatesRule.Apply(input)); + scope.Store(scope.TemplateMemo, key, input, results); + return results; + } + private IEnumerable ApplyTemplates(Word input) { - foreach (Word tempOutWord in _templatesRule.Apply(input).Distinct(FreezableEqualityComparer.Default)) + foreach (Word tempOutWord in ApplyTemplateBattery(input).Distinct(FreezableEqualityComparer.Default)) { switch (_stratum.MorphologicalRuleOrder) { diff --git a/src/SIL.Machine.Morphology.HermitCrab/MemoizedCombinationRuleCascade.cs b/src/SIL.Machine.Morphology.HermitCrab/MemoizedCombinationRuleCascade.cs new file mode 100644 index 00000000..36f4dd24 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab/MemoizedCombinationRuleCascade.cs @@ -0,0 +1,108 @@ +using System.Collections.Generic; +using System.Threading; +using SIL.Machine.Annotations; +using SIL.Machine.Rules; + +namespace SIL.Machine.Morphology.HermitCrab +{ + /// + /// The sequential plus memoization of each + /// expanded subtree, for Unordered-order analysis strata. A node whose + /// was already searched earlier in this word's analysis, via a + /// different unapplication order, is not searched again: an empty stored result short-circuits, and a + /// non-empty one is replayed onto the current arrival (). + /// + /// The parallel cascade is left unmemoized -- its breadth-first walk never reaches a point where a + /// given subtree is known to be fully expanded, so there is nowhere to hang a memo write. + /// + /// + internal class MemoizedCombinationRuleCascade : CombinationRuleCascade + { + // Read by the equivalence tests to prove the memo actually fired: one that silently stopped + // firing would otherwise look exactly like a passing test. + internal static long DiagMemoHits; + internal static long DiagNogoodHits; + + public MemoizedCombinationRuleCascade( + IEnumerable> rules, + IEqualityComparer comparer + ) + : base(rules, true, comparer) { } + + public override IEnumerable Apply(Word input) + { + var output = new HashSet(Comparer); + ApplyRules(input, output); + return output; + } + + // Returns the results produced strictly within `input`'s subtree, at any depth, excluding `input` + // itself -- both what callers consume and what gets memoized against `input`'s key. + private List ApplyRules(Word input, HashSet output) + { + AnalysisScope scope = input.AnalysisScope; + // See Word.AnalysisScope's doc for when this is null. + if (scope == null) + return ApplyRulesRaw(input, output); + + var key = new AnalysisStateKey(input); + + if (scope.TryReplay(scope.Memo, key, input, out List replayed)) + { + if (replayed.Count == 0) + { + Interlocked.Increment(ref DiagNogoodHits); + return replayed; + } + foreach (Word replay in replayed) + { + output.Add(replay); + CheckMaxAlternatives(output.Count); + } + Interlocked.Increment(ref DiagMemoHits); + return replayed; + } + + // In-flight re-entry guard, see AnalysisScope.InProgress. + if (!scope.InProgress.Add(key)) + return ApplyRulesRaw(input, output); + + List results; + try + { + results = ApplyRulesRaw(input, output); + } + finally + { + scope.InProgress.Remove(key); + } + + scope.Store(scope.Memo, key, input, results); + return results; + } + + // Mirrors the base's multiApp expansion, including its recurse-before-add ordering: the HashSet + // keeps whichever of two comparer-equal results lands first, and Word.ValueEquals ignores + // SyntacticFeatureStruct, so which one survives is observable downstream. Delegating to the base + // is not possible -- it collects into one globally-deduped set, so a subtree result another branch + // already contributed is missing from it, yet must still be recorded here or a later replay of + // this key would return too few results. + private List ApplyRulesRaw(Word input, HashSet output) + { + var local = new List(); + for (int i = 0; i < Rules.Count; i++) + { + foreach (Word result in ApplyRule(Rules[i], i, input)) + { + // avoid infinite loop -- same guard CombinationRuleCascade uses + if (!Comparer.Equals(input, result)) + local.AddRange(ApplyRules(result, output)); + local.Add(result); + output.Add(result); + CheckMaxAlternatives(output.Count); + } + } + return local; + } + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs index 0ecd5633..4dd180ca 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs @@ -1,6 +1,9 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; +using System.Threading; +using System.Threading.Tasks; using SIL.Extensions; using SIL.Machine.Annotations; using SIL.Machine.FeatureModel; @@ -11,12 +14,6 @@ using System.IO; #endif -#if !SINGLE_THREADED -using System.Collections.Concurrent; -using System.Threading.Tasks; -using System.Threading; -#endif - namespace SIL.Machine.Morphology.HermitCrab { public class Morpher : IMorphologicalAnalyzer, IMorphologicalGenerator @@ -29,10 +26,13 @@ public class Morpher : IMorphologicalAnalyzer, IMorphologicalGenerator private readonly ReadOnlyObservableCollection _morphemes; private readonly IList _lexicalPatterns = new List(); - public Morpher(ITraceManager traceManager, Language lang) + public Morpher(ITraceManager traceManager, Language lang, int maxDegreeOfParallelism = 0) { _lang = lang; _traceManager = traceManager; + // Must be set before CompileAnalysisRule: AnalysisStratumRule picks a sequential vs. parallel + // cascade for Unordered-order analysis strata at construction time based on this value. + MaxDegreeOfParallelism = maxDegreeOfParallelism; _allomorphTries = new Dictionary(); var morphemes = new ObservableList(); foreach (Stratum stratum in _lang.Strata) @@ -84,6 +84,19 @@ public ITraceManager TraceManager /// public bool MergeEquivalentAnalyses { get; set; } + /// + /// Caps the concurrency used within a single parse. A value of 1 runs the parse fully + /// sequentially -- analysis cascade, affix-template unapplication and synthesis alike -- and is the + /// only configuration eligible for the analysis-cascade memo (see + /// ). The default of 0, like any value below 1, leaves + /// concurrency unbounded. Constructor-only, because it determines how the analysis rules compile. + /// + /// This must remain a pure performance knob: nothing that changes which analyses a parse returns + /// may be gated on it, or the memoized and unmemoized configurations stop being comparable. + /// + /// + public int MaxDegreeOfParallelism { get; } + public Func LexEntrySelector { get; set; } public Func RuleSelector { get; set; } @@ -115,6 +128,10 @@ public IEnumerable ParseWord(string word, out object trace, bool guessRoot Shape shape = _lang.SurfaceStratum.CharacterDefinitionTable.Segment(word); var input = new Word(_lang.SurfaceStratum, shape); + // Installing a scope is what enables the memo. Never while tracing: traces must stay + // byte-identical to the unmemoized engine. + if (!_traceManager.IsTracing && MaxDegreeOfParallelism == 1) + input.AnalysisScope = new AnalysisScope(); input.Freeze(); if (_traceManager.IsTracing) _traceManager.AnalyzeWord(_lang, input); @@ -279,16 +296,26 @@ Stack> permutation in PermuteRules( } } -#if SINGLE_THREADED - private IEnumerable Synthesize(string word, IEnumerable analyses) + private IEnumerable Synthesize(string word, ConcurrentQueue analyses) + { + if (MaxDegreeOfParallelism == 1) + return SynthesizeSequential(word, analyses); + return SynthesizeParallel(word, analyses); + } + + private IEnumerable SynthesizeSequential(string word, IEnumerable analyses) { var matches = new HashSet(FreezableEqualityComparer.Default); + int alternativeCount = 0; foreach (Word analysisWord in analyses) { foreach (Word synthesisWord in LexicalLookup(analysisWord)) { foreach (Word alternative in synthesisWord.ExpandAlternatives()) { + alternativeCount++; + if (MaxAlternatives > 0 && alternativeCount > MaxAlternatives) + throw new MaxAlternativesExceededException("MaxAlternatives exceeded"); foreach (Word validWord in _synthesisRule.Apply(alternative).Where(IsWordValid)) { if (IsMatch(word, validWord)) @@ -299,8 +326,8 @@ private IEnumerable Synthesize(string word, IEnumerable analyses) } return matches; } -#else - private IEnumerable Synthesize(string word, ConcurrentQueue analyses) + + private IEnumerable SynthesizeParallel(string word, ConcurrentQueue analyses) { var matches = new ConcurrentBag(); int alternativeCount = 0; @@ -346,7 +373,6 @@ private IEnumerable Synthesize(string word, ConcurrentQueue analyses throw exception; return matches.Distinct(FreezableEqualityComparer.Default); } -#endif internal IEnumerable SearchRootAllomorphs(Stratum stratum, Shape shape) { @@ -369,6 +395,9 @@ LexEntry entry in SearchRootAllomorphs(input.Stratum, input.Shape) foreach (RootAllomorph allomorph in entry.Allomorphs) { Word newWord = input.Clone(); + // Synthesis never reads the memo, and keeping the reference would pin both tables for + // as long as the caller holds the returned words. + newWord.AnalysisScope = null; newWord.RootAllomorph = allomorph; if (_traceManager.IsTracing) _traceManager.SynthesizeWord(_lang, newWord); @@ -441,6 +470,8 @@ private IEnumerable LexicalGuess(Word input) } // Create a new word that uses the root allomorph. Word newWord = input.Clone(); + // Synthesis never reads the memo; see LexicalLookup. + newWord.AnalysisScope = null; newWord.RootAllomorph = root; if (_traceManager.IsTracing) _traceManager.SynthesizeWord(_lang, newWord); diff --git a/src/SIL.Machine.Morphology.HermitCrab/Word.cs b/src/SIL.Machine.Morphology.HermitCrab/Word.cs index 9b29429e..51c3c53c 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/Word.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/Word.cs @@ -70,6 +70,11 @@ public Word(Stratum stratum, Shape shape) } protected Word(Word word) + : this(word, cloneNonHeadApps: true) { } + + // ReplayOnto passes false: it rebuilds the non-head list wholesale, so cloning it here would be + // discarded work. + private Word(Word word, bool cloneNonHeadApps) { _allomorphs = new Dictionary(word._allomorphs); Stratum = word.Stratum; @@ -84,12 +89,13 @@ protected Word(Word word) _mruleAppIndex = word._mruleAppIndex; _mrulesUnapplied = new Dictionary(word._mrulesUnapplied); _mrulesApplied = new Dictionary(word._mrulesApplied); - _nonHeadApps = new List(word._nonHeadApps.CloneItems()); + _nonHeadApps = cloneNonHeadApps ? new List(word._nonHeadApps.CloneItems()) : new List(); _nonHeadAppIndex = word._nonHeadAppIndex; _obligatorySyntacticFeatures = new IDBearerSet(word._obligatorySyntacticFeatures); _isLastAppliedRuleFinal = word._isLastAppliedRuleFinal; _isPartial = word._isPartial; CurrentTrace = word.CurrentTrace; + AnalysisScope = word.AnalysisScope; _disjunctiveAllomorphIndices = word._disjunctiveAllomorphIndices.ToDictionary( kvp => kvp.Key, kvp => new HashSet(kvp.Value) @@ -212,6 +218,16 @@ public IEnumerable MorphemesInApplicationOrder public object CurrentTrace { get; set; } + /// + /// Carrier for the analysis-cascade memo. Reference-shared like + /// and excluded from FreezeImpl/ValueEquals for the same + /// reason. Null while tracing, and for words not routed through + /// at all, so readers must fall back to + /// unmemoized behavior rather than throw. Cleared again on entry to synthesis so returned words do + /// not pin the per-parse tables. + /// + internal AnalysisScope AnalysisScope { get; set; } + public bool IsPartial { get { return _isPartial; } @@ -313,6 +329,12 @@ internal void RemoveMorph(Annotation morphAnn) /// indicates that an unknown compounding rule was unapplied. This is used when /// generating a compound word, because the compounding rule is usually not known just /// the non-head allomorph. + /// + /// The trail push and the count increment below must stay in lockstep: + /// splits a stored result on the assumption that equal unapplication multisets imply equal trail + /// lengths. Realizational rules incrementing the count without extending the trail is safe because + /// they do so on both sides of any comparison; any other divergence misaligns the graft. + /// /// internal void MorphologicalRuleUnapplied(IMorphologicalRule mrule) { @@ -338,6 +360,11 @@ internal int GetUnapplicationCount(IMorphologicalRule mrule) return numUnapplies; } + /// + /// The full multiset backing , for . + /// + internal IReadOnlyDictionary UnappliedRuleCounts => _mrulesUnapplied; + /// /// Notifies this word synthesis that the specified morphological rule has applied. /// @@ -392,6 +419,15 @@ internal int NonHeadCount get { return _nonHeadApps.Count; } } + internal IReadOnlyList NonHeads => _nonHeadApps; + + /// + /// Length of the morphological-rule trail so far. Recorded with when a + /// subtree is memoized, to mark where a replayed result's kept suffix begins; see + /// . + /// + internal int MorphologicalRuleTrailLength => _mruleApps.Count; + internal void NonHeadUnapplied(Word nonHead) { CheckFrozen(); @@ -450,6 +486,68 @@ internal IList ExpandAlternatives() return alternatives; } + /// + /// Re-parents this Word -- computed while exploring the subtree below some cascade node N -- onto + /// , which reached N's via a different + /// unapplication order. + /// + /// Sound because an equal key means N and agree on Shape, both + /// FeatureStructs, the unapplication multiset and the non-head count, so everything computed + /// inside the subtree is a function of state they share and carries over untouched. Only the two + /// ordered structures the key reduces to counts -- the rule trail and the non-head list -- can + /// differ, and only in the prefix accumulated before reaching N, which is what gets replaced. + /// + /// + /// The word that hit the memo; its trail and non-heads become the prefix. + /// + /// N's _mruleApps.Count when its subtree was memoized: this word's trail from that index on + /// is the subtree-local suffix to keep. + /// + /// Same, for _nonHeadApps. + /// + /// Pre-cloned non-heads from , so one memo hit clones them once rather + /// than per stored result; see AnalysisScope.TryReplay. Null clones them here instead. + /// + internal Word ReplayOnto( + Word queryNode, + int mruleTrailPrefixLength, + int nonHeadPrefixLength, + IReadOnlyList queryNonHeadPrefix = null + ) + { + var clone = new Word(this, cloneNonHeadApps: false); + + List mruleSuffix = clone._mruleApps.GetRange( + mruleTrailPrefixLength, + clone._mruleApps.Count - mruleTrailPrefixLength + ); + clone._mruleApps.Clear(); + clone._mruleApps.AddRange(queryNode._mruleApps); + clone._mruleApps.AddRange(mruleSuffix); + clone._mruleAppIndex = clone._mruleApps.Count - 1; + + // The clone's non-head list starts empty, so it is built as query prefix + this word's + // subtree-local suffix without ever cloning the prefix this word arrived with, which the graft + // discards anyway. + if (queryNonHeadPrefix != null) + clone._nonHeadApps.AddRange(queryNonHeadPrefix); + else + clone._nonHeadApps.AddRange(queryNode._nonHeadApps.CloneItems()); + clone._nonHeadApps.AddRange( + _nonHeadApps.GetRange(nonHeadPrefixLength, _nonHeadApps.Count - nonHeadPrefixLength).CloneItems() + ); + clone._nonHeadAppIndex = clone._nonHeadApps.Count - 1; + + clone.Freeze(); + return clone; + } + + // Hoisted out of the per-result loop by AnalysisScope.TryReplay; see ReplayOnto. + internal List CloneNonHeadsForReplay() + { + return new List(_nonHeadApps.CloneItems()); + } + public Allomorph GetAllomorph(Annotation morph) { var alloID = (string)morph.FeatureStruct.GetValue(HCFeatureSystem.Allomorph); diff --git a/src/SIL.Machine/Rules/ParallelCombinationRuleCascade.cs b/src/SIL.Machine/Rules/ParallelCombinationRuleCascade.cs index bed3b25a..b126b872 100644 --- a/src/SIL.Machine/Rules/ParallelCombinationRuleCascade.cs +++ b/src/SIL.Machine/Rules/ParallelCombinationRuleCascade.cs @@ -30,8 +30,18 @@ IEqualityComparer comparer ) : base(rules, multiApp, comparer) { } + /// + /// Maximum number of concurrent tasks used by . Values less than 1 (the + /// default is -1) do not limit the degree of parallelism. + /// + public int MaxDegreeOfParallelism { get; set; } = -1; + public override IEnumerable Apply(TData input) { + var parallelOptions = new ParallelOptions + { + MaxDegreeOfParallelism = MaxDegreeOfParallelism >= 1 ? MaxDegreeOfParallelism : -1, + }; var output = new ConcurrentStack(); var from = new ConcurrentStack>>(); from.Push(Tuple.Create(input, !MultipleApplication ? new HashSet() : null)); @@ -43,6 +53,7 @@ public override IEnumerable Apply(TData input) to.Clear(); Parallel.ForEach( from, + parallelOptions, (work, state) => { try diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisStateKeyTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisStateKeyTests.cs new file mode 100644 index 00000000..e398bb9e --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisStateKeyTests.cs @@ -0,0 +1,180 @@ +using NUnit.Framework; +using SIL.Machine.FeatureModel; +using SIL.Machine.Morphology.HermitCrab.MorphologicalRules; + +namespace SIL.Machine.Morphology.HermitCrab; + +// The memo's primitives in isolation, independent of any cascade wiring: AnalysisStateKey's +// order-invariance and field sensitivity, its frozen-word requirement, and ReplayOnto's graft. +[TestFixture] +public class AnalysisStateKeyTests : HermitCrabTestBase +{ + [Test] + public void Equals_IsInvariantOverUnapplicationOrder_ForEqualMultisets() + { + var ruleA = new AffixProcessRule { Name = "ruleA" }; + var ruleB = new AffixProcessRule { Name = "ruleB" }; + + // Same multiset {ruleA: 2, ruleB: 1} reached in two different orders. wordY touches ruleB first, + // so the backing dictionaries also differ in insertion order, not just in a repeated rule's + // position. + Word wordX = NewTestWord(); + wordX.MorphologicalRuleUnapplied(ruleA); + wordX.MorphologicalRuleUnapplied(ruleB); + wordX.MorphologicalRuleUnapplied(ruleA); + wordX.Freeze(); + + Word wordY = NewTestWord(); + wordY.MorphologicalRuleUnapplied(ruleB); + wordY.MorphologicalRuleUnapplied(ruleA); + wordY.MorphologicalRuleUnapplied(ruleA); + wordY.Freeze(); + + var keyX = new AnalysisStateKey(wordX); + var keyY = new AnalysisStateKey(wordY); + + Assert.That(keyX.GetHashCode(), Is.EqualTo(keyY.GetHashCode())); + Assert.That(keyX.Equals(keyY), Is.True); + } + + [Test] + public void Equals_False_WhenUnapplicationMultisetsDiffer() + { + var ruleA = new AffixProcessRule { Name = "ruleA" }; + + Word wordX = NewTestWord(); + wordX.MorphologicalRuleUnapplied(ruleA); + wordX.Freeze(); + + Word wordY = NewTestWord(); + wordY.MorphologicalRuleUnapplied(ruleA); + wordY.MorphologicalRuleUnapplied(ruleA); + wordY.Freeze(); + + Assert.That(new AnalysisStateKey(wordX).Equals(new AnalysisStateKey(wordY)), Is.False); + } + + [Test] + public void Equals_False_WhenNonHeadCountDiffers() + { + Word wordX = NewTestWord(); + wordX.Freeze(); + + Word wordY = NewTestWord(); + Word nonHead = NewTestWord(); + nonHead.Freeze(); + wordY.NonHeadUnapplied(nonHead); + wordY.Freeze(); + + Assert.That(new AnalysisStateKey(wordX).Equals(new AnalysisStateKey(wordY)), Is.False); + } + + [Test] + public void Equals_False_WhenSyntacticFeatureStructDiffers() + { + Word wordX = NewTestWord(); + wordX.SyntacticFeatureStruct = FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value; + wordX.Freeze(); + + Word wordY = NewTestWord(); + wordY.SyntacticFeatureStruct = FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("N").Value; + wordY.Freeze(); + + Assert.That(new AnalysisStateKey(wordX).Equals(new AnalysisStateKey(wordY)), Is.False); + } + + [Test] + public void Constructor_Throws_WhenWordIsNotFrozen() + { + Word unfrozen = NewTestWord(); + + Assert.That(() => new AnalysisStateKey(unfrozen), Throws.ArgumentException); + } + + [Test] + public void ReplayOnto_SharesHoistedQueryPrefix_AcrossOneHitsReplays() + { + Word queryNonHead = NewTestWord("32"); + queryNonHead.Freeze(); + Word query = NewTestWord("32"); + query.NonHeadUnapplied(queryNonHead); + query.Freeze(); + + Word storedNonHead = NewTestWord("32"); + storedNonHead.Freeze(); + Word subtreeNonHead = NewTestWord("33"); + subtreeNonHead.Freeze(); + Word memoized = NewTestWord("32"); + memoized.NonHeadUnapplied(storedNonHead); + memoized.NonHeadUnapplied(subtreeNonHead); + memoized.Freeze(); + + List hoisted = query.CloneNonHeadsForReplay(); + Word first = memoized.ReplayOnto(query, 0, 1, hoisted); + Word second = memoized.ReplayOnto(query, 0, 1, hoisted); + + // Query's 1 non-head prefix plus the stored subtree's 1 non-head suffix. + Assert.That(first.NonHeadCount, Is.EqualTo(2)); + Assert.That(first.CurrentNonHead.RootAllomorph, Is.SameAs(subtreeNonHead.RootAllomorph)); + // Both replays share the one hoisted clone, and it is a clone rather than the query's own instance. + Assert.That(first.NonHeads[0], Is.SameAs(second.NonHeads[0])); + Assert.That(first.NonHeads[0], Is.SameAs(hoisted[0])); + Assert.That(first.NonHeads[0], Is.Not.SameAs(queryNonHead)); + } + + [Test] + public void ReplayOnto_GraftsQueryPrefixOntoStoredSuffix_ForMruleTrail() + { + var ruleA = new AffixProcessRule { Name = "ruleA" }; + var ruleB = new AffixProcessRule { Name = "ruleB" }; + var ruleC = new AffixProcessRule { Name = "ruleC" }; + + // Trail [ruleA, ruleB] at the moment of the write: ruleA is the length-1 prefix, ruleB the + // subtree-local suffix that must survive the graft. + Word memoized = NewTestWord(); + memoized.MorphologicalRuleUnapplied(ruleA); + memoized.MorphologicalRuleUnapplied(ruleB); + memoized.Freeze(); + + // The same key reached with a different prefix, [ruleC]. + Word query = NewTestWord(); + query.MorphologicalRuleUnapplied(ruleC); + query.Freeze(); + + Word replayed = memoized.ReplayOnto(query, mruleTrailPrefixLength: 1, nonHeadPrefixLength: 0); + + Assert.That(replayed.MorphologicalRules, Is.EqualTo(new IMorphologicalRule[] { ruleC, ruleB })); + } + + [Test] + public void ReplayOnto_GraftsQueryPrefixOntoStoredSuffix_ForNonHeads() + { + // Distinct lexical entries (32 vs 33) so that a graft keeping the wrong non-head, or reversing the + // GetRange window, is distinguishable by RootAllomorph identity rather than only by count. + Word storedNonHead = NewTestWord("32"); + storedNonHead.Freeze(); + Word subtreeNonHead = NewTestWord("33"); + subtreeNonHead.Freeze(); + Word memoized = NewTestWord("32"); + memoized.NonHeadUnapplied(storedNonHead); + memoized.NonHeadUnapplied(subtreeNonHead); + memoized.Freeze(); + + // Query reached the same key with a different (empty) non-head prefix. + Word query = NewTestWord("32"); + query.Freeze(); + + Word replayed = memoized.ReplayOnto(query, mruleTrailPrefixLength: 0, nonHeadPrefixLength: 1); + + // Query's (empty) prefix + the memoized subtree's suffix (subtreeNonHead) = 1 non-head. + Assert.That(replayed.NonHeadCount, Is.EqualTo(1)); + Assert.That(replayed.CurrentNonHead.RootAllomorph, Is.SameAs(subtreeNonHead.RootAllomorph)); + Assert.That(replayed.CurrentNonHead.RootAllomorph, Is.Not.SameAs(storedNonHead.RootAllomorph)); + } + + private Word NewTestWord(string entryId = "32") + { + var word = new Word(Entries[entryId].PrimaryAllomorph, FeatureStruct.New().Value) { Stratum = Morphophonemic }; + return word; + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisStratumRuleTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisStratumRuleTests.cs new file mode 100644 index 00000000..b9913b2a --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisStratumRuleTests.cs @@ -0,0 +1,94 @@ +using NUnit.Framework; +using SIL.Machine.Annotations; +using SIL.Machine.FeatureModel; +using SIL.Machine.Matching; +using SIL.Machine.Morphology.HermitCrab.MorphologicalRules; + +namespace SIL.Machine.Morphology.HermitCrab; + +// Drives AnalysisStratumRule against a scope the test owns, which is the only way to observe whether the +// template battery memoized anything: through Morpher the scope is created and discarded inside +// ParseWord, and a Linear stratum invokes the battery at most once per distinct state, so it never +// replays there and the hit counters cannot distinguish a memoized run from an excluded one. +[TestFixture] +public class AnalysisStratumRuleTests : HermitCrabTestBase +{ + [Test] + public void Apply_MemoizesTemplateBattery_OnUnorderedStratum() + { + AddVerbTemplate(); + SetRuleOrder(MorphologicalRuleOrder.Unordered); + + AnalysisScope scope = ApplyStratumRule("sagd"); + + Assert.That( + scope.TemplateMemo, + Is.Not.Empty, + "an Unordered stratum must memoize the template battery -- otherwise the negative case below " + + "proves nothing" + ); + } + + [Test] + public void Apply_DoesNotMemoizeTemplateBattery_OnLinearStratum() + { + AddVerbTemplate(); + SetRuleOrder(MorphologicalRuleOrder.Linear); + + AnalysisScope scope = ApplyStratumRule("sagd"); + + Assert.That( + scope.TemplateMemo, + Is.Empty, + "a Linear stratum must not memoize the template battery: AnalysisStateKey's key-completeness " + + "audit covers only the Unordered cascade" + ); + Assert.That( + scope.Memo, + Is.Empty, + "nor may the mrule table be written on a Linear stratum, which runs PermutationRuleCascade" + ); + } + + // Runs one stratum rule over `word` with a scope attached, and hands the scope back for inspection. + private AnalysisScope ApplyStratumRule(string word) + { + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1); + var stratumRule = new AnalysisStratumRule(morpher, Morphophonemic); + + var input = new Word(Morphophonemic, Morphophonemic.CharacterDefinitionTable.Segment(word)); + var scope = new AnalysisScope(); + input.AnalysisScope = scope; + input.Freeze(); + + // Apply builds its result set eagerly, so this forces the battery for every state it reaches. + _ = stratumRule.Apply(input).ToList(); + return scope; + } + + private void AddVerbTemplate() + { + var any = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; + var dSuffix = new AffixProcessRule + { + Id = "TPAST", + Name = "template_d_suffix", + Gloss = "PAST", + RequiredSyntacticFeatureStruct = FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value, + }; + dSuffix.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = { Pattern.New("1").Annotation(any).OneOrMore.Value }, + Rhs = { new CopyFromInput("1"), new InsertSegments(Table3, "+d") }, + } + ); + var verbTemplate = new AffixTemplate + { + Name = "verb_template", + RequiredSyntacticFeatureStruct = FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value, + }; + verbTemplate.Slots.Add(new AffixTemplateSlot(dSuffix) { Optional = true }); + Morphophonemic.AffixTemplates.Add(verbTemplate); + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MemoCorpusVerification.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MemoCorpusVerification.cs new file mode 100644 index 00000000..b72e589b --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MemoCorpusVerification.cs @@ -0,0 +1,215 @@ +using System.Diagnostics; +using NUnit.Framework; + +namespace SIL.Machine.Morphology.HermitCrab; + +/// +/// Memo-on/memo-off equality against a real grammar, which is the only way to test +/// 's key-completeness audit against the full analysis-side rule set. The +/// synthetic unit-test grammars can force a specific redundant order deterministically but cannot reach +/// that breadth; a key missing a field some real rule reads shows up here and nowhere else. +/// +/// [Explicit] and env-var driven because this repo never commits real grammars or word lists: the test +/// embeds no grammar content and writes only TestContext lines, so no derived corpus data (signature +/// dumps included) can land in a committed path. +/// +/// +/// $env:HC_MEMO_GRAMMAR = "...\sena-hc.xml" +/// $env:HC_MEMO_WORDS = "...\sena-words.txt" +/// $env:HC_MEMO_MAX_WORDS = "60" # optional, default 60 +/// $env:HC_MEMO_TIMEOUT_MS = "5000" # optional, default 5000 (per-word watchdog) +/// dotnet test --filter "FullyQualifiedName~MemoCorpusVerification" +/// +/// +[TestFixture] +[Explicit("Manual corpus verification against a local, uncommitted real grammar; not part of CI.")] +public class MemoCorpusVerification +{ + [Test] + public void MemoOnMatchesMemoOff_AnalysisSetIdentical_OnRealCorpus() + { + (Language language, List words) = Load(); + + var memoOff = new Morpher(new TraceManager(), language); + var memoOn = new Morpher(new TraceManager(), language, maxDegreeOfParallelism: 1); + int timeoutMs = int.TryParse(Environment.GetEnvironmentVariable("HC_MEMO_TIMEOUT_MS"), out int t) ? t : 5000; + + long mruleHitsBefore = MemoizedCombinationRuleCascade.DiagMemoHits; + long mruleNogoodsBefore = MemoizedCombinationRuleCascade.DiagNogoodHits; + long templateHitsBefore = AnalysisStratumRule.DiagTemplateMemoHits; + long templateNogoodsBefore = AnalysisStratumRule.DiagTemplateNogoodHits; + + var elapsedMsPerWord = new List(); + var perWordTimes = new List<(string Word, double OnMs, double OffMs)>(); + var divergences = new List(); + var timedOut = new List(); + int noParseBoth = 0; + + foreach (string word in words) + { + List onSignatures; + List offSignatures; + double onMs; + double offMs; + try + { + var swOn = Stopwatch.StartNew(); + onSignatures = RunWithTimeout(() => Signatures(memoOn, word), timeoutMs); + swOn.Stop(); + onMs = swOn.Elapsed.TotalMilliseconds; + + var swOff = Stopwatch.StartNew(); + offSignatures = RunWithTimeout(() => Signatures(memoOff, word), timeoutMs); + swOff.Stop(); + offMs = swOff.Elapsed.TotalMilliseconds; + } + catch (TimeoutException) + { + timedOut.Add(word); + continue; + } + elapsedMsPerWord.Add(onMs + offMs); + perWordTimes.Add((word, onMs, offMs)); + + if (onSignatures.Count == 0 && offSignatures.Count == 0) + noParseBoth++; + + if (!onSignatures.SequenceEqual(offSignatures)) + { + divergences.Add( + $"{word}: memo-on={{{string.Join(",", onSignatures)}}} vs " + + $"memo-off={{{string.Join(",", offSignatures)}}}" + ); + } + } + + elapsedMsPerWord.Sort(); + double p50 = Percentile(elapsedMsPerWord, 0.50); + double p95 = Percentile(elapsedMsPerWord, 0.95); + double totalMs = elapsedMsPerWord.Sum(); + + TestContext.Out.WriteLine($"words attempted: {words.Count}, timed out (>{timeoutMs}ms): {timedOut.Count}"); + TestContext.Out.WriteLine($"words with no parse on both sides: {noParseBoth}"); + TestContext.Out.WriteLine($"aggregate wall: {totalMs:F1} ms, p50: {p50:F1} ms, p95: {p95:F1} ms"); + + // Count-based and wall-clock aggregates stay separate because a corpus is bimodal: many cheap + // words go slightly slower for want of a thread, while a few pathological ones go far faster. One + // combined ratio would hide which regime a reader is in, and both statements are true at once. + int fasterCount = perWordTimes.Count(x => x.OnMs < x.OffMs); + int slowerCount = perWordTimes.Count(x => x.OnMs > x.OffMs); + int tiedCount = perWordTimes.Count - fasterCount - slowerCount; + double totalOnMs = perWordTimes.Sum(x => x.OnMs); + double totalOffMs = perWordTimes.Sum(x => x.OffMs); + TestContext.Out.WriteLine( + $"count-based: {fasterCount}/{perWordTimes.Count} words faster under memo, " + + $"{slowerCount}/{perWordTimes.Count} slower, {tiedCount} tied" + ); + TestContext.Out.WriteLine( + $"wall-clock: memo-on total {totalOnMs:F1} ms vs memo-off total {totalOffMs:F1} ms " + + $"({(totalOnMs > 0 ? totalOffMs / totalOnMs : 0):F2}x)" + ); + if (timedOut.Count > 0) + { + // The ratio above is not a bound in either direction: one try block wraps both calls, so which + // side timed out is unrecorded, and if it was memo-on then that word's memo-off time was never + // measured at all. + TestContext.Out.WriteLine( + $"(the {timedOut.Count} timed-out word(s) above are excluded from both aggregates; " + + "re-run with a higher HC_MEMO_TIMEOUT_MS to actually measure them)" + ); + } + // Per-word attribution as well, since an aggregate dominated by cheap words hides what the + // pathological ones do. Note memo-on is sequential while memo-off is the parallel default, so + // these times measure the user-visible comparison, not the memo's contribution in isolation. + TestContext.Out.WriteLine("heaviest words (by memo-off time), memo-on vs memo-off:"); + foreach ((string w, double onMs2, double offMs2) in perWordTimes.OrderByDescending(x => x.OffMs).Take(10)) + TestContext.Out.WriteLine($" {w}: memo-on {onMs2:F1} ms, memo-off {offMs2:F1} ms"); + TestContext.Out.WriteLine( + $"mrule memo -- positive hits: {MemoizedCombinationRuleCascade.DiagMemoHits - mruleHitsBefore}, " + + $"nogood hits: {MemoizedCombinationRuleCascade.DiagNogoodHits - mruleNogoodsBefore}" + ); + TestContext.Out.WriteLine( + $"template memo -- positive hits: {AnalysisStratumRule.DiagTemplateMemoHits - templateHitsBefore}, " + + $"nogood hits: {AnalysisStratumRule.DiagTemplateNogoodHits - templateNogoodsBefore}" + ); + if (timedOut.Count > 0) + { + // Named rather than counted: these words are excluded from the equality gate, so "0 + // divergences" says nothing about them, and heavy words are exactly what the memo and the + // key-completeness audit most need checking against. + TestContext.Out.WriteLine( + $"timed-out words (excluded from the equality gate above -- re-run with a higher " + + $"HC_MEMO_TIMEOUT_MS to actually check these): {string.Join(", ", timedOut)}" + ); + } + + Assert.That( + divergences, + Is.Empty, + $"{divergences.Count} word(s) diverged between memo-on and memo-off " + + $"(showing up to 10): {string.Join(" | ", divergences.Take(10))}" + ); + Assert.That( + MemoizedCombinationRuleCascade.DiagMemoHits + AnalysisStratumRule.DiagTemplateMemoHits, + Is.GreaterThan(mruleHitsBefore + templateHitsBefore), + "the positive replay path must actually have fired somewhere in this corpus -- otherwise " + + "this run cannot distinguish a working memo from a no-op one" + ); + } + + private static List Signatures(Morpher morpher, string word) + { + try + { + return morpher + .ParseWord(word) + .Select(MorpherTests.WordAnalysisSignature) + .OrderBy(s => s, StringComparer.Ordinal) + .ToList(); + } + catch (InvalidShapeException) + { + // As Morpher.AnalyzeWord does: a real word list can contain strings the character table does + // not cover, which both sides reject identically and which tells us nothing about the memo. + return new List(); + } + } + + // Cannot cancel `action`: ParseWord has no cooperative-cancellation hook, so a timed-out word keeps + // running in the background, where it can inflate later words' counters and timings, and enough + // orphaned tasks in a row can starve the thread pool. Tolerable only because this harness never runs + // in CI, and the equality gate excludes timed-out words anyway -- but treat any run that reported + // timeouts as having approximate counts. + private static T RunWithTimeout(Func action, int timeoutMs) + { + Task task = Task.Run(action); + if (!task.Wait(timeoutMs)) + throw new TimeoutException(); + return task.Result; + } + + private static double Percentile(List sortedValues, double fraction) + { + if (sortedValues.Count == 0) + return 0; + int index = (int)Math.Ceiling(fraction * sortedValues.Count) - 1; + return sortedValues[Math.Clamp(index, 0, sortedValues.Count - 1)]; + } + + private static (Language, List) Load() + { + string? grammarPath = Environment.GetEnvironmentVariable("HC_MEMO_GRAMMAR"); + string? wordsPath = Environment.GetEnvironmentVariable("HC_MEMO_WORDS"); + if (string.IsNullOrEmpty(grammarPath) || string.IsNullOrEmpty(wordsPath)) + Assert.Ignore("set HC_MEMO_GRAMMAR and HC_MEMO_WORDS"); + + int maxWords = int.TryParse(Environment.GetEnvironmentVariable("HC_MEMO_MAX_WORDS"), out int mw) ? mw : 60; + Language language = XmlLanguageLoader.Load(grammarPath!); + List words = File.ReadAllLines(wordsPath!) + .Select(w => w.Trim()) + .Where(w => w.Length > 0) + .Take(maxWords) + .ToList(); + return (language, words); + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MemoizedCombinationRuleCascadeTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MemoizedCombinationRuleCascadeTests.cs new file mode 100644 index 00000000..32a1629e --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MemoizedCombinationRuleCascadeTests.cs @@ -0,0 +1,198 @@ +using NUnit.Framework; +using SIL.Machine.Annotations; +using SIL.Machine.FeatureModel; +using SIL.Machine.Morphology.HermitCrab.MorphologicalRules; +using SIL.Machine.Rules; +using SIL.ObjectModel; + +namespace SIL.Machine.Morphology.HermitCrab; + +// The cascade exercised directly, bypassing Morpher, so a commuting-order re-arrival at a PRODUCTIVE +// state can be forced. That matters because the end-to-end grammars in MorpherTests are small enough +// that they only ever reach the nogood table, leaving the positive replay path untested. +[TestFixture] +public class MemoizedCombinationRuleCascadeTests : HermitCrabTestBase +{ + [Test] + public void Apply_ReplaysPositiveHit_WhenTwoOrdersReachTheSameKey() + { + var ruleA = new AffixProcessRule { Name = "ruleA" }; + var ruleB = new AffixProcessRule { Name = "ruleB" }; + var ruleC = new AffixProcessRule { Name = "ruleC" }; + + // Each rule unapplies at most once, so A-then-B and B-then-A reach the same key (multiset + // {ruleA:1, ruleB:1}) by different routes. ruleC can still apply from that shared state, making + // its subtree positive rather than a nogood. + var cascade = new MemoizedCombinationRuleCascade( + new IRule[] + { + new SingleUseUnapplyRule(ruleA), + new SingleUseUnapplyRule(ruleB), + new SingleUseUnapplyRule(ruleC), + }, + FreezableEqualityComparer.Default + ); + + Word initial = NewTestWord(); + initial.AnalysisScope = new AnalysisScope(); + initial.Freeze(); + + long hitsBefore = MemoizedCombinationRuleCascade.DiagMemoHits; + List results = new List(cascade.Apply(initial)); + + Assert.That( + results, + Has.Some.Matches(w => + w.GetUnapplicationCount(ruleA) == 1 + && w.GetUnapplicationCount(ruleB) == 1 + && w.GetUnapplicationCount(ruleC) == 1 + ) + ); + Assert.That( + MemoizedCombinationRuleCascade.DiagMemoHits, + Is.GreaterThan(hitsBefore), + "this test's whole point is to force a positive replay -- it must not go vacuous" + ); + } + + [Test] + public void Apply_PositiveReplayMatchesUnmemoizedResultSet_IncludingTrailOrder() + { + // Compares MorphemesInApplicationOrder rather than rule counts: counts are order-invariant, so + // they would pass even if the graft collapsed [ruleB,ruleA,ruleC] into a duplicate of + // [ruleA,ruleB,ruleC], whereas the trail is exactly what ReplayOnto rewrites. + var ruleA = new AffixProcessRule { Id = "RULE_A", Name = "ruleA" }; + var ruleB = new AffixProcessRule { Id = "RULE_B", Name = "ruleB" }; + var ruleC = new AffixProcessRule { Id = "RULE_C", Name = "ruleC" }; + IRule[] rules = + { + new SingleUseUnapplyRule(ruleA), + new SingleUseUnapplyRule(ruleB), + new SingleUseUnapplyRule(ruleC), + }; + + Word memoized = NewTestWord(); + memoized.AnalysisScope = new AnalysisScope(); + memoized.Freeze(); + + // No AnalysisScope: takes the unmemoized fallback, the same path a tracing parse takes. + Word unmemoized = NewTestWord(); + unmemoized.Freeze(); + + var memoizedCascade = new MemoizedCombinationRuleCascade(rules, FreezableEqualityComparer.Default); + var unmemoizedCascade = new MemoizedCombinationRuleCascade(rules, FreezableEqualityComparer.Default); + + long hitsBefore = MemoizedCombinationRuleCascade.DiagMemoHits; + List memoizedSignatures = memoizedCascade + .Apply(memoized) + .Select(TrailSignature) + .OrderBy(s => s, StringComparer.Ordinal) + .ToList(); + List unmemoizedSignatures = unmemoizedCascade + .Apply(unmemoized) + .Select(TrailSignature) + .OrderBy(s => s, StringComparer.Ordinal) + .ToList(); + + Assert.That( + memoizedSignatures, + Is.EqualTo(unmemoizedSignatures), + "a positive replay must reproduce exactly the unmemoized result set, INCLUDING trail order" + ); + Assert.That( + MemoizedCombinationRuleCascade.DiagMemoHits, + Is.GreaterThan(hitsBefore), + "this test's whole point is to compare a real replay against the unmemoized result -- it " + + "must not go vacuous" + ); + } + + [Test] + public void Apply_FallsBackToUnmemoizedExpansion_WhenKeyIsAlreadyInProgress() + { + // The in-flight state is simulated by pre-populating InProgress, because single-use rules make the + // key monotonic in application count, so a genuine cyclic re-arrival cannot be forced here. + var ruleA = new AffixProcessRule { Id = "RULE_A", Name = "ruleA" }; + var cascade = new MemoizedCombinationRuleCascade( + new IRule[] { new SingleUseUnapplyRule(ruleA) }, + FreezableEqualityComparer.Default + ); + + Word initial = NewTestWord(); + var scope = new AnalysisScope(); + initial.AnalysisScope = scope; + initial.Freeze(); + + var key = new AnalysisStateKey(initial); + scope.InProgress.Add(key); + + long hitsBefore = MemoizedCombinationRuleCascade.DiagMemoHits; + List results = new List(cascade.Apply(initial)); + + Assert.That(results, Has.Some.Matches(w => w.GetUnapplicationCount(ruleA) == 1)); + Assert.That( + MemoizedCombinationRuleCascade.DiagMemoHits, + Is.EqualTo(hitsBefore), + "the in-flight fallback must not read/count a memo hit -- it never consults Memo at all" + ); + Assert.That( + scope.Memo.ContainsKey(key), + Is.False, + "the in-flight arrival's OWN key must never be written to Memo (deeper recursive calls for " + + "OTHER keys, reached via ApplyRulesRaw's normal recursion, may still memoize themselves)" + ); + } + + [Test] + public void Apply_EnforcesMaxAlternatives_ForRawAndReplayPaths() + { + var ruleA = new AffixProcessRule { Id = "RULE_A", Name = "ruleA" }; + var ruleB = new AffixProcessRule { Id = "RULE_B", Name = "ruleB" }; + IRule[] rules = { new SingleUseUnapplyRule(ruleA), new SingleUseUnapplyRule(ruleB) }; + + var rawCascade = new MemoizedCombinationRuleCascade(rules, FreezableEqualityComparer.Default) + { + MaxAlternatives = 1, + }; + Word rawInitial = NewTestWord(); + rawInitial.AnalysisScope = new AnalysisScope(); + rawInitial.Freeze(); + + Assert.Throws(() => new List(rawCascade.Apply(rawInitial))); + + var replayCascade = new MemoizedCombinationRuleCascade(rules, FreezableEqualityComparer.Default); + Word replayInitial = NewTestWord(); + replayInitial.AnalysisScope = new AnalysisScope(); + replayInitial.Freeze(); + _ = new List(replayCascade.Apply(replayInitial)); + + replayCascade.MaxAlternatives = 1; + Assert.Throws(() => new List(replayCascade.Apply(replayInitial))); + } + + private static string TrailSignature(Word word) => + string.Join("+", word.MorphemesInApplicationOrder.Select(m => m.Id)); + + private Word NewTestWord() + { + return new Word(Entries["32"].PrimaryAllomorph, FeatureStruct.New().Value) { Stratum = Morphophonemic }; + } + + // Stand-in for a compiled analysis rule: unapplies once per input, with no Shape/FeatureStruct + // matching, so commuting orders can be exercised without a real FST-backed rule. + private sealed class SingleUseUnapplyRule(IMorphologicalRule rule) : IRule + { + private readonly IMorphologicalRule _rule = rule; + + public IEnumerable Apply(Word input) + { + if (input.GetUnapplicationCount(_rule) > 0) + yield break; + + Word result = input.Clone(); + result.MorphologicalRuleUnapplied(_rule); + result.Freeze(); + yield return result; + } + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorpherTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorpherTests.cs index 3c92898e..0197e422 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorpherTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorpherTests.cs @@ -535,4 +535,354 @@ IList GetNodes(string pattern) Shape shape = new Segments(Table2, pattern, true).Shape; return shape.GetNodes(shape.Range).ToList(); } + + // A compounding rule and a commuting PAST prefix as peers in one Unordered cascade, so an equal + // AnalysisStateKey can be re-arrived at by different unapplication orders. + private void AddCompoundingAndPrefixRules() + { + var any = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; + var crule = new CompoundingRule { Name = "rule1" }; + Allophonic.MorphologicalRules.Add(crule); + crule.Subrules.Add( + new CompoundingSubrule + { + HeadLhs = { Pattern.New("head").Annotation(any).OneOrMore.Value }, + NonHeadLhs = { Pattern.New("nonHead").Annotation(any).OneOrMore.Value }, + Rhs = { new CopyFromInput("head"), new InsertSegments(Table3, "+"), new CopyFromInput("nonHead") }, + } + ); + + var prefix = new AffixProcessRule + { + Id = "PREFIX", + Name = "prefix", + Gloss = "PAST", + RequiredSyntacticFeatureStruct = FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value, + OutSyntacticFeatureStruct = FeatureStruct + .New(Language.SyntacticFeatureSystem) + .Feature(Head) + .EqualTo(head => head.Feature("tense").EqualTo("past")) + .Value, + }; + Allophonic.MorphologicalRules.Insert(0, prefix); + prefix.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = { Pattern.New("1").Annotation(any).OneOrMore.Value }, + Rhs = { new InsertSegments(Table3, "di+"), new CopyFromInput("1") }, + } + ); + } + + [Test] + public void ParseWord_SingleThreaded_MatchesParallel_WithCompounding() + { + // MaxDegreeOfParallelism must be a pure no-op on results, independent of the memo it gates. + AddCompoundingAndPrefixRules(); + + var parallel = new Morpher(TraceManager, Language); + var singleThreaded = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1); + + foreach (string word in new[] { "pʰutdidat", "pʰutdat" }) + { + List singleResult = singleThreaded.ParseWord(word).ToList(); + List parallelResult = parallel.ParseWord(word).ToList(); + Assert.That( + singleResult.Select(WordAnalysisSignature).OrderBy(s => s, StringComparer.Ordinal), + Is.EqualTo(parallelResult.Select(WordAnalysisSignature).OrderBy(s => s, StringComparer.Ordinal)), + $"single-threaded parse of '{word}' must match the parallel parse" + ); + } + } + + [Test] + public void ParseWord_MemoOnMatchesMemoOff_HitCounterGuarded_WithCompounding() + { + // The standing acceptance gate: analysis-set equality between the memoized sequential cascade and + // the unmemoized parallel default, kept non-vacuous by the hit-counter assertion at the end. + AddCompoundingAndPrefixRules(); + + var memoOff = new Morpher(TraceManager, Language); + var memoOn = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1); + + long hitsBefore = MemoizedCombinationRuleCascade.DiagMemoHits; + long nogoodHitsBefore = MemoizedCombinationRuleCascade.DiagNogoodHits; + foreach (string word in new[] { "pʰutdidat", "pʰutdat" }) + { + List onResult = memoOn.ParseWord(word).ToList(); + List offResult = memoOff.ParseWord(word).ToList(); + Assert.That( + onResult.Select(WordAnalysisSignature).OrderBy(s => s, StringComparer.Ordinal), + Is.EqualTo(offResult.Select(WordAnalysisSignature).OrderBy(s => s, StringComparer.Ordinal)), + $"memo-on parse of '{word}' must be analysis-set identical to memo-off" + ); + } + TestContext.Out.WriteLine( + $"positive hits: {MemoizedCombinationRuleCascade.DiagMemoHits - hitsBefore}, " + + $"nogood hits: {MemoizedCombinationRuleCascade.DiagNogoodHits - nogoodHitsBefore}" + ); + Assert.That( + MemoizedCombinationRuleCascade.DiagMemoHits + MemoizedCombinationRuleCascade.DiagNogoodHits, + Is.GreaterThan(hitsBefore + nogoodHitsBefore), + "the memo must actually have hit (positive or nogood) at least once on this grammar -- " + + "otherwise this test cannot distinguish a working memo from a no-op one" + ); + } + + [Test] + public void ParseWord_MemoOnMatchesMemoOff_ForSelfOpaquingSimultaneousEpenthesis() + { + // Guards the memo against a Simultaneous-mode epenthesis rule, which AnalysisRewriteRule compiles + // as ReapplyType.SelfOpaquing -- a repeat-until-fixpoint loop, and the one rule shape whose + // interaction with the nogood cache has a suspected (never reproduced) bug elsewhere. Known gap: + // no available fixture drives the loop past a single iteration, so two or more remains untested. + var highVowel = FeatureStruct + .New(Language.PhonologicalFeatureSystem) + .Symbol(HCFeatureSystem.Segment) + .Symbol("cons-") + .Symbol("voc+") + .Symbol("high+") + .Value; + var highFrontUnrndVowel = FeatureStruct + .New(Language.PhonologicalFeatureSystem) + .Symbol(HCFeatureSystem.Segment) + .Symbol("cons-") + .Symbol("voc+") + .Symbol("high+") + .Symbol("back-") + .Symbol("round-") + .Value; + + var rule4 = new RewriteRule { Name = "rule4", ApplicationMode = RewriteApplicationMode.Simultaneous }; + Allophonic.PhonologicalRules.Add(rule4); + rule4.Subrules.Add( + new RewriteSubrule + { + Rhs = Pattern.New().Annotation(highFrontUnrndVowel).Value, + LeftEnvironment = Pattern.New().Annotation(highVowel).Value, + } + ); + + var memoOff = new Morpher(TraceManager, Language); + var memoOn = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1); + + foreach (string word in new[] { "buibui", "bubu", "bibu" }) + { + List onResult = memoOn.ParseWord(word).ToList(); + List offResult = memoOff.ParseWord(word).ToList(); + Assert.That( + onResult.Select(WordAnalysisSignature).OrderBy(s => s, StringComparer.Ordinal), + Is.EqualTo(offResult.Select(WordAnalysisSignature).OrderBy(s => s, StringComparer.Ordinal)), + $"memo-on parse of '{word}' must be analysis-set identical to memo-off" + ); + } + // Pinned as an absolute value, not just on-vs-off, so a bug affecting both sides identically + // (both wrongly returning empty, say) is still caught. + Assert.That(memoOn.ParseWord("buibui").Count(), Is.EqualTo(1)); + } + + [Test] + public void ParseWord_MemoOnMatchesMemoOff_HitCounterGuarded_WithAffixTemplate() + { + // Two commuting prefixes, not one: a single rule unapplies only once, so no key would ever be + // re-arrived at and the template memo would never fire. Unapplying di-then-gu or gu-then-di + // reaches the same key by a different trail order, which is what makes the second one replay. + var any = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; + + var edSuffix = new AffixProcessRule + { + Id = "TPAST", + Name = "template_ed_suffix", + Gloss = "PAST", + RequiredSyntacticFeatureStruct = FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value, + }; + edSuffix.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = { Pattern.New("1").Annotation(any).OneOrMore.Value }, + Rhs = { new CopyFromInput("1"), new InsertSegments(Table3, "+d") }, + } + ); + var verbTemplate = new AffixTemplate + { + Name = "verb_template", + RequiredSyntacticFeatureStruct = FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value, + }; + verbTemplate.Slots.Add(new AffixTemplateSlot(edSuffix) { Optional = true }); + Morphophonemic.AffixTemplates.Add(verbTemplate); + + var diPrefix = new AffixProcessRule + { + Id = "TDI", + Name = "template_di_prefix", + Gloss = "DI", + RequiredSyntacticFeatureStruct = FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value, + }; + diPrefix.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = { Pattern.New("1").Annotation(any).OneOrMore.Value }, + Rhs = { new InsertSegments(Table3, "di+"), new CopyFromInput("1") }, + } + ); + Morphophonemic.MorphologicalRules.Add(diPrefix); + + var guPrefix = new AffixProcessRule + { + Id = "TGU", + Name = "template_gu_prefix", + Gloss = "GU", + RequiredSyntacticFeatureStruct = FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value, + }; + guPrefix.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = { Pattern.New("1").Annotation(any).OneOrMore.Value }, + Rhs = { new InsertSegments(Table3, "gu+"), new CopyFromInput("1") }, + } + ); + Morphophonemic.MorphologicalRules.Add(guPrefix); + + var memoOff = new Morpher(TraceManager, Language); + var memoOn = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1); + + long templateHitsBefore = AnalysisStratumRule.DiagTemplateMemoHits; + long templateNogoodHitsBefore = AnalysisStratumRule.DiagTemplateNogoodHits; + foreach (string word in new[] { "digusagd", "disagd", "gusagd", "sagd", "sag" }) + { + List onResult = memoOn.ParseWord(word).ToList(); + List offResult = memoOff.ParseWord(word).ToList(); + Assert.That( + onResult.Select(WordAnalysisSignature).OrderBy(s => s, StringComparer.Ordinal), + Is.EqualTo(offResult.Select(WordAnalysisSignature).OrderBy(s => s, StringComparer.Ordinal)), + $"memo-on parse of '{word}' must be analysis-set identical to memo-off" + ); + } + TestContext.Out.WriteLine( + $"template positive hits: {AnalysisStratumRule.DiagTemplateMemoHits - templateHitsBefore}, " + + $"template nogood hits: {AnalysisStratumRule.DiagTemplateNogoodHits - templateNogoodHitsBefore}" + ); + // The graft's effect on final signatures is invisible through synthesis, which re-derives rule + // orderings anyway, so this counter -- not the equality assertions above -- is what proves the + // memoized path was exercised at all. + Assert.That( + AnalysisStratumRule.DiagTemplateMemoHits + AnalysisStratumRule.DiagTemplateNogoodHits, + Is.GreaterThan(templateHitsBefore + templateNogoodHitsBefore), + "the template memo must actually have hit (positive or nogood) at least once on this " + + "grammar -- otherwise this test cannot distinguish a working memo from a no-op one" + ); + } + + [Test] + public void ParseWord_MemoOnMatchesMemoOff_OnLinearStratumWithAffixTemplate() + { + // Every other memo test runs on Unordered strata, leaving Linear -- MorphologicalRuleOrder's + // default -- with no end-to-end equivalence gate. AnalysisStratumRuleTests covers the exclusion + // that keeps the memo off this path; this covers the results it produces. + var any = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; + + var edSuffix = new AffixProcessRule + { + Id = "TPAST", + Name = "template_ed_suffix", + Gloss = "PAST", + RequiredSyntacticFeatureStruct = FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value, + }; + edSuffix.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = { Pattern.New("1").Annotation(any).OneOrMore.Value }, + Rhs = { new CopyFromInput("1"), new InsertSegments(Table3, "+d") }, + } + ); + var verbTemplate = new AffixTemplate + { + Name = "verb_template", + RequiredSyntacticFeatureStruct = FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value, + }; + verbTemplate.Slots.Add(new AffixTemplateSlot(edSuffix) { Optional = true }); + Morphophonemic.AffixTemplates.Add(verbTemplate); + + var diPrefix = new AffixProcessRule + { + Id = "TDI", + Name = "template_di_prefix", + Gloss = "DI", + RequiredSyntacticFeatureStruct = FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value, + }; + diPrefix.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = { Pattern.New("1").Annotation(any).OneOrMore.Value }, + Rhs = { new InsertSegments(Table3, "di+"), new CopyFromInput("1") }, + } + ); + Morphophonemic.MorphologicalRules.Add(diPrefix); + + var guPrefix = new AffixProcessRule + { + Id = "TGU", + Name = "template_gu_prefix", + Gloss = "GU", + RequiredSyntacticFeatureStruct = FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value, + }; + guPrefix.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = { Pattern.New("1").Annotation(any).OneOrMore.Value }, + Rhs = { new InsertSegments(Table3, "gu+"), new CopyFromInput("1") }, + } + ); + Morphophonemic.MorphologicalRules.Add(guPrefix); + + SetRuleOrder(MorphologicalRuleOrder.Linear); + var memoOff = new Morpher(TraceManager, Language); + var memoOn = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1); + + foreach (string word in new[] { "digusagd", "disagd", "gusagd", "sagd", "sag" }) + { + List onResult = memoOn.ParseWord(word).ToList(); + List offResult = memoOff.ParseWord(word).ToList(); + Assert.That( + onResult.Select(WordAnalysisSignature).OrderBy(s => s, StringComparer.Ordinal), + Is.EqualTo(offResult.Select(WordAnalysisSignature).OrderBy(s => s, StringComparer.Ordinal)), + $"Linear-stratum parse of '{word}' must be analysis-set identical with and without the memo" + ); + } + } + + [Test] + public void ParseWord_HonorsMaxDegreeOfParallelismAsACap_WithoutChangingResults() + { + // The value is a resource knob, never a semantic one, so an intermediate cap must not shift results. + AddCompoundingAndPrefixRules(); + + var unbounded = new Morpher(TraceManager, Language); + var capped = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 2); + + foreach (string word in new[] { "pʰutdidat", "pʰutdat" }) + { + List cappedResult = capped.ParseWord(word).ToList(); + List unboundedResult = unbounded.ParseWord(word).ToList(); + Assert.That( + cappedResult.Select(WordAnalysisSignature).OrderBy(s => s, StringComparer.Ordinal), + Is.EqualTo(unboundedResult.Select(WordAnalysisSignature).OrderBy(s => s, StringComparer.Ordinal)), + $"a parallelism cap must not change the analysis set for '{word}'" + ); + } + } + + // What the memo gates compare, instead of object equality: a replayed Word is not field-for-field + // identical to a freshly-computed one. MorphemesInApplicationOrder is the load-bearing part, since it + // walks the trail and non-heads that ReplayOnto rewrites; AllomorphsInMorphOrder alone would miss a + // broken graft, walking only Shape annotations that ReplayOnto never touches. The root distinguishes + // analyses that share a morpheme sequence but not a lexical entry. + internal static string WordAnalysisSignature(Word word) + { + return string.Join("+", word.AllomorphsInMorphOrder.Select(a => a.Morpheme.Id)) + + "|" + + string.Join("+", word.MorphemesInApplicationOrder.Select(m => m.Id)) + + "|root=" + + word.RootAllomorph.Morpheme.Id; + } }