diff --git a/.gitignore b/.gitignore index 2476e52091..ec18e4d2a5 100644 --- a/.gitignore +++ b/.gitignore @@ -215,5 +215,6 @@ DataTreeTimingBaselines.json /findings_*.md *scratchpad* *[Ss]cratchPad* +/.review/ Docs/migration/working/ diff --git a/CONTEXT.md b/CONTEXT.md index 475e541369..c55b89971c 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -17,6 +17,10 @@ It is intentionally not a full architecture manual. It should stay biased toward - **MSBuild project**: A `.csproj`, `.vcxproj`, `.wixproj`, or similar build unit. - **Installer project**: WiX authoring and packaging work under `FLExInstaller/`. - **Worktree**: A git worktree for isolated builds and edits. +- **`Grammar`**: overloaded — qualify when the distinction matters. + - **Grammar Area**: One of the five top-level Areas (see Core Domain section) — the Parts of Speech/Features/etc. tool group. + - **Grammar Sketch**: An existing human-readable, published linguistic-description document export (`ExportDialog.ExportGrammarSketch`); unrelated to the HC grammar below despite the shared word. + - **HC grammar**: The HermitCrab-format grammar (phonology, morphology, features, categories) produced from a language project via `HCLoader.Load` + `XmlLanguageWriter.Save` (`SIL.Machine.Morphology.HermitCrab`). Always say "HC grammar," never bare "grammar," when the distinction from the other two matters. - **`Writing system`**: overloaded — qualify when the distinction matters. - **Writing system definition**: `WritingSystemDefinition` (libpalaso / `SIL.WritingSystems`); the base writing system class, identified by a BCP-47 language tag. - **Core writing system definition**: `CoreWritingSystemDefinition` (liblcm / `SIL.LCModel.Core.WritingSystems`); extends `WritingSystemDefinition` with a `Handle` and LCM-specific features such as character sets. @@ -33,6 +37,7 @@ It is intentionally not a full architecture manual. It should stay biased toward - **IME composition**: The transient input-method editing state before text is committed. Treat composition behavior and committed text behavior as separate test and parity concerns. - **Lexicon**: The lexical data and editing experience in FLEx. - **Interlinear text**: Text annotated with multiple aligned linguistic analysis lines. +- **Word occurrence** (`IAnalysis`, in `SIL.LCModel`): One tokenized position in a paragraph — either a bare, unanalyzed **wordform occurrence** (`IWfiWordform`, `HasWordform == true` but no morphological breakdown chosen) or an **analyzed occurrence** (`IWfiAnalysis`/`IWfiGloss` — a morphological breakdown/gloss has been attached, whether by a human or an unreviewed parser guess; "analyzed" says nothing about whether the analysis is linguistically correct). Punctuation occurrences also satisfy `IAnalysis` but not `HasWordform`. This three-way distinction (punctuation / wordform-only / analyzed) recurs across `StatisticsView`, `ConcordanceControl`, and Interlinear editing — use these names rather than inventing new ones. - **Morphology**: The part of the system and data model concerned with morphemes, rules, and word analysis. - **Parser**: Morphological analysis tooling such as HermitCrab or XAmple. - **Paratext integration**: Scripture and lexicon interoperability with Paratext. Implemented across `FwParatextLexiconPlugin`, `ParatextImport` (scripture text import via `ParatextImportManager`/`ParatextSfmImporter`), and `Paratext8Plugin` (bridge to Paratext APIs). @@ -53,7 +58,7 @@ It is intentionally not a full architecture manual. It should stay biased toward - **Service locator**: `LcmCache.ServiceLocator` (`ILcmServiceLocator`). IoC container for LCModel — the primary way to retrieve repositories, factories, and services. - **Unit of work**: Groups data-model changes under `IActionHandler`. All LCModel writes must occur inside one. `UndoableUnitOfWorkHelper` (undoable) and `NonUndoableUnitOfWorkHelper` (non-undoable) are in `SIL.LCModel.Infrastructure`; use as a `using` block or via their static `.Do(...)` helpers. - **Publish/subscribe system**: Messaging system (`IPublisher` / `ISubscriber`, `SIL.FieldWorks.Common.FwUtils`) via `FwUtils.Publisher` and `FwUtils.Subscriber` singletons. Supports exact-name and prefix subscriptions; `PublishAtEndOfAction` defers delivery to end of user action. Event-based problems deserve event-based solutions — avoid state variables for event timing when a deterministic subscribe/unsubscribe solution can be used. -- **Area**: One of the five top-level navigation divisions — Lexicon, Grammar, Words & Texts, Notebook, Lists. Declared in `areaConfiguration.xml` and identified by constants in `AreaConstants`. Each Area has its own sidebar and owns a set of Tools. +- **Area**: One of the five top-level navigation divisions — Lexicon, Grammar, Texts & Words, Notebook, Lists. Declared in `areaConfiguration.xml` and identified by constants in `AreaConstants`. Each Area has its own sidebar and owns a set of Tools. (The Texts & Words area's internal `areaChoice` value is `textsWords`.) - **Tool**: A view or function within an Area, declared in `toolConfiguration.xml`. The active Tool per Area is tracked via `ToolForAreaNamed_`; switching Areas restores the last-used Tool. Navigation is property-driven through the XCore mediator system. - **Dictionary configuration**: A `.fwdictconfig` XML file (`DictionaryConfigurationModel`) defining which LCModel fields appear in a dictionary view, their order, style, and options. Scoped to one or more Publications. - **Publication**: A named output target (e.g. a print edition or web view) that dictionary configurations are scoped to. `AllPublications` applies a configuration to all current and future publications. diff --git a/DistFiles/Language Explorer/Configuration/Main.xml b/DistFiles/Language Explorer/Configuration/Main.xml index edd2bff59e..011dcf1663 100644 --- a/DistFiles/Language Explorer/Configuration/Main.xml +++ b/DistFiles/Language Explorer/Configuration/Main.xml @@ -921,6 +921,9 @@ I (RandyR) brought them into this file, just to keep track of them for the time populates the "tools" list appropriately. --> + + diff --git a/DistFiles/Language Explorer/Export Templates/AIExportInstructions.md b/DistFiles/Language Explorer/Export Templates/AIExportInstructions.md new file mode 100644 index 0000000000..74cbc2ccbe --- /dev/null +++ b/DistFiles/Language Explorer/Export Templates/AIExportInstructions.md @@ -0,0 +1,76 @@ +# Read this first: a FieldWorks grammar and texts for analysis + +This folder was exported from FieldWorks Language Explorer (FLEx) by its "Export Grammar and +Texts for AI Analysis" command. Everything in it was put here deliberately for an AI assistant +(ChatGPT, Claude, or otherwise) to read, including this file. + +## What is in this folder + +- `HCGrammar.xml` -- the project's **HC grammar**: its HermitCrab-format grammar (phonology, + morphology, features, categories), the same shape the HermitCrab parser itself consumes to + parse words. This is the parser's own input, not a human-readable grammar description. +- One `.flextext` file per exported text -- FLExText, FieldWorks' interlinear-text interchange + format. These are real corpus texts the grammar is supposed to account for. Any given text may + be fully analyzed, partly analyzed, or not analyzed at all. +- `export-instructions.md` -- this file. + +## Before you answer, read the reference + +Grammars and interlinear texts have real gotchas you will not know about on your own. Fetch and +read this first: + +``` +https://raw.githubusercontent.com/sillsdev/FieldWorks/main/Docs/ai-parser-help/README.md +``` + +That is the index for a reference covering correctness gotchas (`broken/`), performance gotchas +(`speed/`), grammar-authoring methodology (`workflow/`), and the FLExText format itself, +including the `analysisStatus` ground-truth caveat (`texts/`). Use the raw URL rather than the +`github.com/.../blob/...` page -- it returns plain markdown that fetches cleanly. Follow whichever +of its links bear on the question you were asked, and reason about this project's grammar against +the engine's documented mechanics rather than guessing. + +## How to answer: plain language, for a linguist + +The person asking is a field linguist with a real language project open in FLEx. They know their +language; they usually do not know HermitCrab's internals, and they did not ask for a tour of +them. An answer that reads like engine documentation gets abandoned, however correct it is. + +- **Accuracy first, then plain language.** An answer that misstates the engine is worse than no + answer, so never simplify to the point of being wrong. When the honest answer is complicated, + give the short version first and the detail underneath. +- **No unexplained jargon.** Use a technical term only if the answer genuinely needs it, and + define it in ordinary words the first time: "MPR feature (a label you put on a morpheme so that + rules can test for it)". Never leave an abbreviation bare -- write "morphosyntactic property", + not "MSP". +- **Use the words FLEx uses.** Name what the user actually clicks (the Grammar area, an affix + template, a slot, an allomorph, an inflection class) rather than the engine's internal class or + field name, so they can find it. Describe fixes as things to do in FLEx, not as XML to + hand-edit. +- **Lead with what to do.** Open with the change to make, in a sentence or two. Put the mechanism + -- why the engine behaves this way -- after it, for the reader who wants it. +- **Show rather than lecture.** One worked example with a real surface form beats a paragraph of + theory. +- **Short sentences.** Cut "it is important to note that", "as mentioned above", and any + restatement of what you just said. +- **Flag every guess, every time.** Name what you are unsure about and what would settle it (a + specific word to parse, a trace to look at). The user cannot see your reasoning, so an unmarked + guess reads exactly like a fact, and a confident wrong answer costs them real work. +- **Each answer stands alone.** Do not lean on earlier turns in the chat, or on file names in the + reference, as though the user had them open. + +## For the person who exported this folder + +**This is AI. It can be very wrong, and it can be wrong while sounding certain.** Treat anything +it tells you about your grammar or your texts as a suggestion to check against your own data, +never as a finding. Understand a change, and confirm it yourself by re-parsing the words you care +about, before you apply it to your project. + +Your exported files are real linguistic data about a real language project. Dropping them into a +third-party chat service sends that data to that provider (OpenAI, Anthropic, and so on), so check +your project's data-sensitivity policy before sharing an unpublished or restricted grammar or text +this way. + +A fuller human-readable walkthrough, including example questions worth asking, is at: + + diff --git a/DistFiles/Language Explorer/Export Templates/GrammarAndTextsForAI.xml b/DistFiles/Language Explorer/Export Templates/GrammarAndTextsForAI.xml new file mode 100644 index 0000000000..0e1a7c0c73 --- /dev/null +++ b/DistFiles/Language Explorer/Export Templates/GrammarAndTextsForAI.xml @@ -0,0 +1,11 @@ + + diff --git a/Docs/ai-parser-help/README.md b/Docs/ai-parser-help/README.md new file mode 100644 index 0000000000..a8a462e178 --- /dev/null +++ b/Docs/ai-parser-help/README.md @@ -0,0 +1,110 @@ +# AI Parser Help — HermitCrab reference for LLMs + +This is a living reference for asking an LLM (ChatGPT, Claude, or otherwise) questions about +**HermitCrab**, the rule-based morphological parser/generator implemented in this repository +(`sillsdev/machine`, namespace `SIL.Machine.Morphology.HermitCrab`), and about **FLExText**, +FieldWorks' interlinear-text interchange format for the connected corpus texts a grammar is +supposed to account for. It covers three kinds of question a grammar author actually asks about +HermitCrab itself, plus a fourth section for a different kind of upload entirely — real corpus +texts rather than the grammar: + +- **[`broken/`](broken/README.md)** — "why is this wrong / missing / crashing?" Correctness + gotchas: wrong parses, missing parses, crashes, and silent misconfigurations. +- **[`speed/`](speed/README.md)** — "why is this slow?" Performance gotchas: combinatorial + blowups and other parse-time costs. +- **[`workflow/`](workflow/README.md)** — "how should I approach modeling this?" Authoring + guidance for building a grammar well in the first place, grounded in HermitCrab's actual + mechanics and in H. Andrew Black's FLEx parsing methodology (primary source included verbatim + under `workflow/sources/`). +- **[`texts/`](texts/README.md)** — "I also have real corpus texts, not just a grammar." A + reference for FLExText, FieldWorks' interlinear-text interchange format: what it is, how to + extract it, and how to get an LLM to reason over it correctly, including the `analysisStatus` + ground-truth caveat and which AI products can actually run code against your uploaded file. + +Each file covers one topic in enough depth to answer questions about that topic without needing +local access to the repo — code excerpts, mechanisms, and worked examples are inlined. + +## Got a FieldWorks grammar and a question? + +Send that person **this link instead**: [`getting-started.md`](getting-started.md) — it walks +through extracting your grammar as HermitCrab XML and getting ChatGPT/Claude to reason about it +using this reference. The rest of this README is the reference material itself (for the LLM to +read), not the human-facing walkthrough. + +Got interlinear **texts** (`.flextext` files) instead of, or in addition to, a grammar? See +[`texts/getting-started.md`](texts/getting-started.md) instead — extracting and reasoning about +connected corpus texts is a different workflow from the grammar one above. + +## How to use this with an LLM + +Paste the **raw** URL of the relevant topic file into your chat, e.g.: + +``` +https://raw.githubusercontent.com/sillsdev/machine/master/docs/ai-parser-help/speed/affix-template-optional-slots.md +``` + +Then ask your question. Use the raw URL (`raw.githubusercontent.com`), not the normal +`github.com/.../blob/...` page — the raw URL returns plain markdown text with no site chrome, +which fetches cleanly for both ChatGPT (web browsing) and Claude (WebFetch) without JS rendering +or auth. If you're not sure which topic file is relevant, paste this README's raw URL first, or +whichever of `broken/README.md`, `speed/README.md`, `workflow/README.md`, `texts/README.md` best +matches your question ("why is this wrong" vs. "why is this slow" vs. "how should I model this" +vs. "I have corpus texts, not just a grammar") — an LLM that can follow links will use it as an +index; otherwise, browse the lists yourself. + +Do not use these guides as a source of real grammar or text data — see "What belongs here" below. + +## How to answer: plain language, for a linguist + +Instructions for the LLM reading this reference. The person asking is a field linguist with a +real language project open in FLEx. They know their language; they usually do not know +HermitCrab's internals, and they did not ask for a tour of them. An answer that reads like engine +documentation gets abandoned, however correct it is. + +- **Accuracy first, then plain language.** An answer that misstates the engine is worse than no + answer, so never simplify to the point of being wrong. When the honest answer is complicated, + give the short version first and the detail underneath. +- **No unexplained jargon.** Use a technical term only if the answer genuinely needs it, and + define it in ordinary words the first time: "MPR feature (a label you put on a morpheme so that + rules can test for it)". Never leave an abbreviation bare -- write "morphosyntactic property", + not "MSP". +- **Use the words FLEx uses.** Name what the user actually clicks (the Grammar area, an affix + template, a slot, an allomorph, an inflection class) rather than the engine's internal class or + field name, so they can find it. Describe fixes as things to do in FLEx, not as XML to hand-edit. +- **Lead with what to do.** Open with the change to make, in a sentence or two. Put the mechanism + -- why the engine behaves this way -- after it, for the reader who wants it. +- **Show rather than lecture.** One worked example with a real surface form beats a paragraph of + theory. +- **Short sentences.** Cut "it is important to note that", "as mentioned above", and any + restatement of what you just said. +- **Flag every guess, every time.** Name what you are unsure about and what would settle it (a + specific word to parse, a trace to look at). The user cannot see your reasoning, so an unmarked + guess reads exactly like a fact, and a confident wrong answer costs them real work. +- **Each answer stands alone.** Do not lean on earlier turns in the chat, or on file names in this + reference, as though the user had them open. + +## What belongs here + +- General HermitCrab engine mechanics: how rules, strata, templates, features, and the + analysis/synthesis engines work. This is documentation of the open-source parser itself. +- FLEx/HermitCrab grammar-authoring methodology, grounded in the engine's actual behavior. +- FLExText format documentation, grounded in the FieldWorks schema/exporter source that produces + it — see [`texts/`](texts/README.md). +- Synthetic/toy grammar snippets used purely to illustrate a mechanism (e.g. `p1`..`p12`, + `sg`/`pl` × invented cases) are fine, as are synthetic/invented interlinear-text examples. +- **Not** real grammar or text data for any specific language (e.g. Sena, Amharic, Indonesian, + Aweti). Those grammars — and any real corpus texts from those projects — are private and must + never be committed to this repo — see the project's existing grammar-privacy constraints. If a + question requires reasoning about a real grammar or text, describe the relevant structure + abstractly instead of pasting the real rules or sentences. + +## Source grounding + +Claims in `broken/`, `speed/`, and `workflow/` are grounded in the actual engine source under +`src/SIL.Machine.Morphology.HermitCrab/` and `src/SIL.Machine/` as of the commit each file was +last updated — each file's metadata header names the specific source file(s) it's grounded in. +Claims in `texts/` are grounded in the FieldWorks source that defines and produces FLExText, plus +cited external documentation — see [`texts/README.md`](texts/README.md#source-grounding) for +specifics; unlike the other three sections, `texts/` cites other repositories and external sources +directly, since explaining an interchange format requires it. File/line references may drift as +the code evolves; if something looks stale, check the live source at the paths cited. diff --git a/Docs/ai-parser-help/broken/README.md b/Docs/ai-parser-help/broken/README.md new file mode 100644 index 0000000000..925933b748 --- /dev/null +++ b/Docs/ai-parser-help/broken/README.md @@ -0,0 +1,53 @@ +# HermitCrab correctness gotchas + +Part of the [HermitCrab-for-LLMs reference](../README.md). One file per gotcha — each is +self-contained, cites the specific `machine` source file(s) (and, where the trap originates in how +FieldWorks compiles a grammar into HermitCrab's XML, the specific `FieldWorks` source file(s)) that +implement the relevant behavior, and gives a fix. This section covers *correctness* problems — wrong +parses, missing parses, crashes, and silent misconfigurations — as opposed to the sibling +[`speed/`](../speed/README.md) section, which covers performance. + +Each file starts with a metadata header: + +```yaml +--- +title: +implements: +category: +symptom: +grammar_visible: +--- +``` + +## Index + +| Gotcha | Category | Symptom | Description | +|---|---|---|---| +| [A StemName's partsOfSpeech is required — omitting it crashes, it doesn't silently reject everything](stemname-partsofspeech-required-not-silent.md) | loader | crash | An omitted `partsOfSpeech` on `` fails the grammar load outright (DTD validation or a `NullReferenceException`), not a silent empty-POS constraint. | +| [A rule's requiredStemName is checked in the forward (synthesis) pass only — including during parsing](stem-name-affix-requirement-trace-misreading.md) | lexicon | silent-misconfiguration | `AffixProcessRule.RequiredStemName` is never read during analysis-side unapplication; it's enforced later in the mandatory resynthesis/confirmation pass, so a trace reader looking only at analysis-rule entries sees no rejection. | +| [LexFamily suppletion blocking silently discards a regular derivation, and only during generation](lexfamily-blocking-generation-only.md) | lexicon | missing-parse | `Blockable` rules can substitute a suppletive family member's form for a regularly-derived one via feature-structure `Subsumes`, which is checked only on synthesis and can fire far more broadly than one paradigm cell if the irregular entry's own feature structure is under-specified. | +| [An MPR feature group's match type defaults to Any (OR), not All](mpr-group-matchtype-default-is-any.md) | feature-system | silent-misconfiguration | Omitting `matchType` on a feature group gives OR semantics (any one listed feature suffices), not AND — the opposite of what an author wanting conjunctive gating might assume. | +| [An MPR feature group's output type defaults to Overwrite, not Append](mpr-group-output-default-overwrite.md) | feature-system | silent-misconfiguration | Omitting `outputType` gives non-monotone "last rule wins" behavior by default, silently dropping an earlier rule's tag from the same group. | +| [A stem with no inflection class and no configured default silently fails every class-restricted rule](inflection-class-no-default-silent-gating.md) | feature-system | missing-parse | An untagged stem with no `DefaultInflectionClassRA` configured anywhere up its POS hierarchy gets no inflection-class MPR feature at all, so every rule gated on any inflection class fails for it — not "matches the wrong class," matches no class. | +| [RealizationalRule can only spell out features already present — it cannot assign new ones the way AffixProcessRule can](realizational-rule-cannot-add-features.md) | morphotactics | silent-misconfiguration | `RealizationalRule` has no `OutputHeadFeatures`/`outputPartOfSpeech` equivalent; it can only merge a `RealizationalFeatureStruct` into a word, never introduce a feature value the word doesn't already carry in some form. | +| [RealizationalAffixProcessRule has no MaxApplicationCount backstop, unlike AffixProcessRule](realizational-rule-no-application-cap.md) | morphotactics | wrong-parse | Unlike `AffixProcessRule`/`CompoundingRule`, this rule type has no per-rule application cap at all — only a feature-based `IsBlocked` check stands between it and reapplying indefinitely in one derivation. | +| [A null/zero-realization affix is a real rule application, not a free default value](null-affix-cannot-express-default.md) | morphotactics | wrong-parse | A "null suffix" modeling a default value still goes through the same required-feature/MPR/environment gates as any other rule in a mandatory slot — if those gates aren't unconditioned, the derivation dead-ends instead of falling back to the intended default. | +| [Independent optional slots in one template can license a combination two templates would have prevented](single-template-independent-slots-illegal-combos.md) | morphotactics | wrong-parse | Each optional slot's apply-or-skip choice is explored independently of every other slot in the same template, so two affixes meant to always co-occur can each fire without the other unless a grammar author adds an explicit feature gate or splits them into separate templates. | +| [An unclassified (or under-specified) affix rule bypasses normal template-ordering discipline](unclassified-affix-bypasses-template-ordering.md) | morphotactics | wrong-parse | FLEx's "unclassified" affix status (and an inflectional affix left with no assigned slot) compiles to `IsPartial = true`, which is specifically permitted to attach after a final template where a normal classified rule would be refused. | +| [A multi-morpheme co-occurrence exclusion requires all listed morphemes together — not any one of them](coocurrence-rule-requires-all-not-any.md) | morphotactics | wrong-parse | An ad hoc prohibition listing several "other" morphemes only blocks the key morpheme when *all* of them co-occur together in the same word, not when any single one does — a much weaker constraint than a list of independent pairwise exclusions. | +| [Modeling a discontinuous morpheme as two independent affixes loses the engine's atomic-circumfix guarantee](circumfix-as-two-affixes-loses-atomicity.md) | morphotactics | wrong-parse | HC's native circumfix process applies both parts atomically or not at all; modeling the same discontinuous morpheme as two independent affixes in two template slots gives up that guarantee and can license a one-sided (half-realized) form. | +| [Compounding is capped at one application per derivation by default, and exocentric compounding cannot be configured otherwise](compounding-max-application-count-default.md) | compounding | missing-parse | `CompoundingRule.MaxApplicationCount` defaults to 1; FieldWorks's loader only ever raises it for endocentric compound rules, so a recursive (three-or-more-element) compounding pattern modeled as exocentric can never be configured to recurse. | +| [A stem-name-restricted allomorph needs the feature explicitly assigned, not just compatible](stem-name-explicit-feature-requirement.md) | lexicon | silent-misconfiguration | `StemName.IsRequiredMatch` tests whether a feature is *explicitly present* on the word, not whether the word is merely compatible with (doesn't conflict with) the region — an unmarked form fails a stem-name-restricted allomorph even trivially. | + +## How to use this with an LLM + +Paste the raw URL of the specific gotcha file that matches your symptom, e.g.: + +``` +https://raw.githubusercontent.com/sillsdev/machine/master/docs/ai-parser-help/broken/lexfamily-blocking-generation-only.md +``` + +If you're not sure which one applies, paste this index's raw URL first and describe your grammar's +structure (not its actual rules — see the privacy note in the top-level [`README.md`](../README.md)) +and symptom (e.g. "this word parses when it shouldn't," "the grammar won't load," "an affix I marked +inflectional shows up in a position I didn't expect"). diff --git a/Docs/ai-parser-help/broken/circumfix-as-two-affixes-loses-atomicity.md b/Docs/ai-parser-help/broken/circumfix-as-two-affixes-loses-atomicity.md new file mode 100644 index 0000000000..64ef28d464 --- /dev/null +++ b/Docs/ai-parser-help/broken/circumfix-as-two-affixes-loses-atomicity.md @@ -0,0 +1,63 @@ +--- +title: "Modeling a discontinuous morpheme as two independent affixes loses the engine's atomic-circumfix guarantee" +implements: src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AffixProcessRule.cs, FieldWorks Src/LexText/ParserCore/HCLoader.cs +category: morphotactics +symptom: wrong-parse +grammar_visible: yes +--- + +## What it is + +H. Andrew Black's FLEx/HC conceptual intro (§4.3 "Circumfixes"; process-rule mechanics in §6.1.1.3 +"Circumfixation as a process") describes two ways to model a discontinuous morpheme whose prefix +part and suffix part always co-occur as a single meaning: (a) one atomic circumfix lexical entry, or +(b) two independent affix entries, one placed in a prefix slot and one in a suffix slot of the same +template. HC's engine genuinely supports circumfixation as one of its native affix processes +(`AffixProcessRule.cs:14-17` documents "prefixation, suffixation, infixation, circumfixation, +simulfixation, reduplication, and truncation" as the process types one rule can express), and +FieldWorks's loader compiles a circumfix lexical entry into a single, atomic `AffixProcessAllomorph` +whose pattern spans the stem with the prefix part on one side and the suffix part on the other +(`LoadCircumfixAffixProcessAllomorph`, FieldWorks `Src/LexText/ParserCore/HCLoader.cs:1273`). Option +(b) — two separate affixes — does not get this atomicity, and the engine has no other mechanism that +supplies it. + +## The mechanism + +A circumfix loaded as one `AffixProcessAllomorph` (option a) either applies as a whole — inserting +both the prefix and suffix material together, gated by one shared set of required +features/environment/MPR conditions — or does not apply at all. There is no way for "half a +circumfix" to appear in a derivation, because the engine only ever sees one rule/allomorph unit for +it. + +Modeled as two ordinary `AffixProcessRule`s in independent template slots (option b), each half is +an entirely separate `Morpheme`/`AffixProcessRule` with its own `RequiredSyntacticFeatureStruct`, +its own environment, its own MPR-feature gates — nothing in the engine ties their applicability +together. If the two rules' gates aren't kept in exact lockstep by the grammar author (same required +features, same MPR-feature requirements, same environment conditions restated on both), the engine +can apply one half without the other: the prefix rule's slot fires while the suffix rule's slot (in +the same optional-slot cross-multiplication described in the companion single-template gotcha) does +not, or vice versa — see `single-template-independent-slots-illegal-combos.md` for the exact +mechanism by which independent optional slots explore every combination, including one-sided ones. + +## Concrete example + +A causative meaning is realized as a circumfix `ka-...-an` in one language design. Modeled as two +independent affixes — a prefix rule for `ka-` and a suffix rule for `-an`, each in its own optional +template slot, each carrying the same `cause=true` required/output feature by the author's intent — +a later grammar edit that updates the prefix rule's required feature structure (e.g. narrowing it to +a particular verb subclass) without making the identical edit to the suffix rule's required feature +structure silently breaks the coupling: forms with `ka-` but no `-an`, or `-an` but no `ka-`, become +derivable/parseable, even though the language never realizes the causative as anything but the whole +circumfix. Modeled instead as one atomic circumfix entry, the same edit to "narrow which subclass +gets the causative" only has one place to make it, and there is no way to get a half-realized +circumfix out of the engine. + +## Fix + +Model a true discontinuous morpheme (one meaning realized as two non-adjacent surface parts that +always co-occur) as a single circumfix lexical entry/`AffixProcessAllomorph`, not as two independent +affixes in two template slots. If two independent affixes are used anyway (e.g. because the two +parts have genuinely independent distributions in some contexts), keep every gating condition +(required features, MPR features, environment) that couples them duplicated exactly and re-verified +on every edit to either rule — the engine provides no shared-identity mechanism to keep them in sync +automatically. diff --git a/Docs/ai-parser-help/broken/compounding-max-application-count-default.md b/Docs/ai-parser-help/broken/compounding-max-application-count-default.md new file mode 100644 index 0000000000..adfd7f264f --- /dev/null +++ b/Docs/ai-parser-help/broken/compounding-max-application-count-default.md @@ -0,0 +1,72 @@ +--- +title: "Compounding is capped at one application per derivation by default, and exocentric compounding cannot be configured otherwise" +implements: src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/CompoundingRule.cs, src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisCompoundingRule.cs +category: compounding +symptom: missing-parse +grammar_visible: partially +--- + +## What it is + +`CompoundingRule.MaxApplicationCount` defaults to `1` (`CompoundingRule.cs:19`), enforced the same +way an `AffixProcessRule`'s cap is: `SynthesisCompoundingRule.Apply` refuses to apply the rule again +once `input.GetApplicationCount(_rule) >= _rule.MaxApplicationCount` +(`SynthesisCompoundingRule.cs:50-62`). A grammar author modeling a language that allows more than two +elements in a compound (e.g. a three-noun compound built by applying the same compounding rule +twice) gets exactly one level of compounding per derivation unless something explicitly raises this +cap — and FieldWorks's compiler only gives *endocentric* compound rules a way to do that. + +## The mechanism + +FieldWorks's `HCLoader.LoadEndoCompoundingRule` looks up a per-rule maximum application count from a +separate `parserParams` XML document (not the main HC grammar XML), keyed by the compound rule's +GUID: + +```csharp +int maxApps = 1; +if (m_CompoundRuleLookup.TryGetValue(compoundRule.Guid.ToString(), out maxApps)) + hcCompoundRule.MaxApplicationCount = maxApps; +``` + +(`HCLoader.cs:1894-1896`), populated earlier from a `` element's `maxApps` attributes +(`HCLoader.cs:104-111`). `LoadExoCompoundingRule` (`HCLoader.cs:1922` onward) has no equivalent +lookup anywhere in its body — every `CompoundingRule` it builds keeps the C# constructor's default of +`MaxApplicationCount = 1`, with no code path that could raise it. An exocentric compound rule (the +kind that produces a category not inherited from either the head or non-head component, e.g. a +noun-noun compound that becomes a distinct new nominal category) can therefore never recurse more +than once per derivation in a FieldWorks-generated grammar, regardless of any setting a grammar +author might look for. + +## Why this is easy to miss + +The cap is invisible from the main HC grammar XML — `MaxApplicationCount` shows up on the +`CompoundingRule` element the same way it would for any other rule, and a reader checking "did the +author configure this to recurse" has to know to look in a *different*, parser-parameters document +(the one `m_CompoundRuleLookup` is built from) rather than the grammar itself, and to know that this +lookup is only ever consulted for endocentric rules. A grammar author who wants three-way (or deeper) +compounding, and who successfully configures a higher `maxApps` for an endocentric rule, can +reasonably expect the same configuration surface to work for an exocentric rule modeling the same +kind of recursive compounding — it does not, because the loading code for exocentric rules never +reads that lookup at all. + +## Concrete example + +A language allows noun compounds of arbitrary length (`N N N ... N`), modeled as one exocentric +compounding rule (the output category isn't simply "the same as the head," so it's modeled as +exocentric) meant to apply repeatedly, left-associatively, to build up longer compounds one pair at a +time. Because `LoadExoCompoundingRule` never overrides `MaxApplicationCount`, the compiled rule stays +capped at `1` — only genuine two-element compounds are generable or parseable; any three-or-more-noun +compound the grammar author expected to derive by reapplying the same rule silently fails to parse or +generate, with no configuration surface (short of switching the rule to an endocentric model, if the +category semantics allow it) to fix it. + +## Fix + +If a compounding pattern needs to recurse more than once per derivation, model it as an *endocentric* +compound rule (where the output category is inherited from the head) so FieldWorks's `maxApps` +lookup applies, and configure that count explicitly in the parser-parameters settings rather than +assuming the default is unlimited or that it matches whatever an affix rule's default would be. For a +genuinely exocentric recursive-compounding pattern, there is currently no configuration surface in +the FieldWorks-to-HC compiler to raise the cap above `1` — the underlying `CompoundingRule.MaxApplicationCount` +property itself has no such restriction (a hand-edited HC XML grammar can set it directly), only the +FieldWorks loader path for exocentric rules never populates it from anything but the default. diff --git a/Docs/ai-parser-help/broken/coocurrence-rule-requires-all-not-any.md b/Docs/ai-parser-help/broken/coocurrence-rule-requires-all-not-any.md new file mode 100644 index 0000000000..f444435276 --- /dev/null +++ b/Docs/ai-parser-help/broken/coocurrence-rule-requires-all-not-any.md @@ -0,0 +1,81 @@ +--- +title: "A multi-morpheme co-occurrence exclusion requires all listed morphemes together — not any one of them" +implements: src/SIL.Machine.Morphology.HermitCrab/MorphCoOccurrenceRule.cs, src/SIL.Machine.Morphology.HermitCrab/Allomorph.cs +category: morphotactics +symptom: wrong-parse +grammar_visible: partially +--- + +## What it is + +FLEx's "ad hoc co-occurrence" mechanism lets a grammar author pick a key morpheme (or allomorph) and +list one or more *other* morphemes/allomorphs that must or must not co-occur with it +(`IMoMorphAdhocProhib`/`IMoAlloAdhocProhib` in FieldWorks, compiled to +`MorphemeCoOccurrenceRule`/`AllomorphCoOccurrenceRule` in the HC engine). When that "others" list has +more than one entry, a grammar author modeling "the key morpheme must not co-occur with A, or with B, +or with C" (three separate, independent exclusions) by listing A, B, and C in one rule gets something +much weaker: the exclusion only fires when **all** of A, B, and C appear together with the key +morpheme in the same word. Any word containing the key morpheme plus only one or two of the listed +others is not excluded at all. + +## The mechanism + +`MorphCoOccurrenceRule.IsWordValid` (`MorphCoOccurrenceRule.cs:82-87`) is: + +```csharp +public bool IsWordValid(T key, Word word) +{ + if (_type == ConstraintType.Exclude) + return !CoOccurs(key, word); + return CoOccurs(key, word); +} +``` + +and `CoOccurs` (`MorphCoOccurrenceRule.cs:92-170`) walks the word's morphs (in the adjacency order +the rule specifies — `Anywhere`, `SomewhereToLeft/Right`, or `AdjacentToLeft/Right`) removing each +`others` entry from a working copy of the list as it's matched, and returns `others.Count == 0` at +the end — true only if **every** entry in `_others` was found. For `ConstraintType.Exclude`, +`IsWordValid` negates that: the word is invalid only when `CoOccurs` returns true, i.e. only when +*all* of the listed morphemes were present together with the key. If the list has three entries and +a word contains the key plus only one of them, `CoOccurs` returns false (not every entry was matched), +so `IsWordValid` (Exclude) returns `true` — the word is accepted, not rejected. + +This is not a quirk of how the rule happens to get built from a single-item FieldWorks list — it's +the same class regardless of list length, and FieldWorks does build multi-item lists directly into +one rule's `others` set. `HCLoader.LoadAllomorphCoOccurrenceRules`/`LoadMorphemeCoOccurrenceRules` +each take one `IMoAlloAdhocProhib`/`IMoMorphAdhocProhib`'s entire `RestOfAllosRS`/`RestOfMorphsRS` +reference collection (FieldWorks's "Rest of Allomorphs"/"Rest of Morphemes" field) and pass it as the +`others` list to a single `new AllomorphCoOccurrenceRule(ConstraintType.Exclude, others, adjacency)`/ +`new MorphemeCoOccurrenceRule(...)` call — one rule per adhoc-prohibition entry, not one rule per +listed morpheme (`HCLoader.cs:2163-2239`). A FieldWorks user who adds three morphemes to one +prohibition's "Rest of" field, expecting three independent exclusions, gets one rule whose `others` +list has three entries and whose `Exclude` semantics only fire on their joint co-occurrence. + +## Why this is easy to miss + +The FLEx UI field is a plural reference list ("Rest of Allomorphs"), which reads naturally as "any of +these" when the grammar author is modeling several independent things the key morpheme shouldn't +combine with — the field's own name doesn't distinguish "all of these together" from "any one of +these." Nothing in the exported HC XML makes the distinction more obvious either: the rule just lists +several morpheme IDs, and a reader has to already know `MorphCoOccurrenceRule`'s all-or-nothing +matching semantics to recognize that a three-item exclusion list is far weaker than three separate +one-item lists would be. + +## Concrete example + +A grammar wants to say "the passive suffix `-en` cannot co-occur with any of the three +object-agreement suffixes `-a`, `-i`, `-u`" — three independent, pairwise exclusions. Modeled as one +`IMoMorphAdhocProhib` with `FirstMorphemeRA = -en` and `RestOfMorphsRS = {-a, -i, -u}`, the compiled +rule's `others` list is `[-a, -i, -u]`, and `CoOccurs` only returns true (triggering the `Exclude`) +when a word contains `-en` together with `-a` *and* `-i` *and* `-u` all at once — a combination that +may never even be otherwise derivable. Any word with `-en` plus just `-a` (the actually-intended, +common case to reject) passes this rule with no complaint, because two of the three required +`others` entries are missing from that particular word. + +## Fix + +Model each independent pairwise (or n-ary "must not co-occur with this specific one") exclusion as +its own separate ad hoc prohibition entry, with exactly one morpheme/allomorph in its "Rest +of"/`RestOfAllosRS`/`RestOfMorphsRS` list. Only put more than one entry in a single prohibition's +"Rest of" list when the intent genuinely is "excluded only when all of these co-occur together" — +that is the only semantics `MorphCoOccurrenceRule` gives a multi-item list. diff --git a/Docs/ai-parser-help/broken/inflection-class-no-default-silent-gating.md b/Docs/ai-parser-help/broken/inflection-class-no-default-silent-gating.md new file mode 100644 index 0000000000..289e233ca6 --- /dev/null +++ b/Docs/ai-parser-help/broken/inflection-class-no-default-silent-gating.md @@ -0,0 +1,75 @@ +--- +title: "A stem with no inflection class and no configured default silently fails every class-restricted rule" +implements: src/SIL.Machine.Morphology.HermitCrab/MprFeatureSet.cs, FieldWorks Src/LexText/ParserCore/HCLoader.cs +category: feature-system +symptom: missing-parse +grammar_visible: partially +--- + +## What it is + +FieldWorks lets a grammar author set a "default inflection class" for a part of speech; H. Andrew +Black's conceptual intro (around the discussion of default inflection class, roughly line 1224 in +the full-text source) states that when this default is configured, "the FieldWorks Language Explorer +parser will use this default inflection class for any stem that is not overtly tagged" — implying +that when no default is configured, an untagged stem gets nothing, silently. FieldWorks's HC loader +confirms exactly this fallback chain, and the HC engine's MPR-feature matching confirms exactly what +happens downstream when it comes up empty. + +## The mechanism + +FieldWorks's loader resolves a stem's inflection class with: + +```csharp +protected static IMoInflClass GetInflClass(IMoStemMsa msa) +{ + if (msa.InflectionClassRA != null) + return msa.InflectionClassRA; + if (msa.PartOfSpeechRA != null) + return GetDefaultInflClass(msa.PartOfSpeechRA); + return null; +} +``` + +walking up the part-of-speech hierarchy for a configured `DefaultInflectionClassRA` if the stem +itself isn't tagged, and returning `null` if neither the stem nor any ancestor POS has one. When +`GetInflClass` returns `null`, the corresponding `MprFeatures.Add(...)` call for inflection class is +simply skipped — the compiled HC lexical entry gets **no** inflection-class MPR feature at all, not +a "default" placeholder feature. + +On the engine side, a rule restricted to specific inflection classes expresses that via +`RequiredMprFeatures`, checked through `MprFeatureSet.IsMatchRequired` +(`MprFeatureSet.cs:46-70`): for an ungrouped required feature (or a group with `MatchType.All`), the +check is `group.Any(mf => !mprFeats.Contains(mf))` — if the stem's compiled MPR-feature set doesn't +contain the required class feature at all (because it was never added), the match fails +unconditionally. There is no "unmarked stems match everything" or "unmarked stems get treated as the +elsewhere case" behavior anywhere in this check — a stem with no inflection-class feature simply +fails every rule gated on any inflection class. + +## Why this is easy to miss + +Nothing in the grammar's rule XML says "this rule requires the stem to have been assigned an +inflection class" in those words — it just lists `RequiredMprFeatures` referencing specific class +IDs, which reads the same whether or not every stem in the lexicon is guaranteed to carry one of +them. A grammar author who sets up inflection classes for the *irregular* subset of a POS and assumes +"everything else falls through to some sensible default" gets that behavior only if they remembered +to configure `DefaultInflectionClassRA` on the POS (or an ancestor POS) — if they didn't, every stem +they didn't explicitly tag silently fails every class-restricted rule, with no trace signal beyond an +ordinary MPR-feature mismatch that looks identical to "this stem really is the wrong class." + +## Concrete example + +A POS `posN` has inflection classes `classI` (a handful of irregular nouns, explicitly tagged) and no +configured default. A plural-suffix rule requires `classI` OR requires "not classI" via +`ExcludedMprFeatures`, depending on how the two paradigms were modeled; either way, the majority of +`posN` stems were never tagged with any inflection class at all (the author assumed "untagged = the +regular pattern"). Every one of those untagged stems fails the class-gated rule that was supposed to +be their regular paradigm, because their compiled MPR-feature set has no inflection-class feature to +match against — not because they matched the wrong class, but because they matched no class. + +## Fix + +Either configure an explicit default inflection class on the POS (or the relevant ancestor POS) so +every untagged stem actually receives a class feature, or design class-restricted rules so the +"regular"/default paradigm's rule has no inflection-class requirement at all (only the irregular +classes are gated), so an absent class feature can't accidentally exclude the majority case. diff --git a/Docs/ai-parser-help/broken/lexfamily-blocking-generation-only.md b/Docs/ai-parser-help/broken/lexfamily-blocking-generation-only.md new file mode 100644 index 0000000000..c2d4705d54 --- /dev/null +++ b/Docs/ai-parser-help/broken/lexfamily-blocking-generation-only.md @@ -0,0 +1,77 @@ +--- +title: "LexFamily suppletion blocking silently discards a regular derivation, and only during generation" +implements: src/SIL.Machine.Morphology.HermitCrab/Word.cs, src/SIL.Machine.Morphology.HermitCrab/LexFamily.cs, src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AffixProcessRule.cs, src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisAffixProcessRule.cs, src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisCompoundingRule.cs, src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisRealizationalAffixProcessRule.cs +category: lexicon +symptom: missing-parse +grammar_visible: partially +--- + +## What it is + +HermitCrab supports "blocking": when a regularly-derived word would be replaced by a suppletive +family member's own irregular form (classic "*goed* is blocked by *went*"). This mechanism is +controlled by `AffixProcessRule.Blockable` / `RealizationalAffixProcessRule.Blockable` / +`CompoundingRule` (each initialized to `true` in its constructor — e.g. +`MorphologicalRules/AffixProcessRule.cs:29`). It has no corresponding check on the analysis side at +all, and its trigger condition is a feature-structure subsumption test that's easy to get broader +than intended. + +## The mechanism + +After a rule derives a word, `Word.CheckBlocking` (`Word.cs:472-497`) runs: + +```csharp +LexFamily family = ((LexEntry)RootAllomorph.Morpheme).Family; +if (family == null) return false; +foreach (LexEntry entry in family.Entries) +{ + if (entry != RootAllomorph.Morpheme + && entry.Stratum == Stratum + && SyntacticFeatureStruct.Subsumes(entry.SyntacticFeatureStruct)) + { + word = new Word(entry.PrimaryAllomorph, RealizationalFeatureStruct.Clone()) { ... }; + return true; + } +} +``` + +If the just-derived word's own feature structure `Subsumes` some other family member's feature +structure, the derived word is discarded outright and replaced by that other entry's primary +allomorph. This is called from `SynthesisAffixProcessRule.cs:198`, +`SynthesisCompoundingRule.cs:192`, and `SynthesisRealizationalAffixProcessRule.cs:127`, each gated +by `if (_rule.Blockable && outWord.CheckBlocking(out Word newWord))` — and `Blockable` defaults to +`true` for every rule type unless a grammar author explicitly turns it off. + +Two things make this a correctness trap rather than just "suppletion working as intended": + +1. **It's synthesis-only.** There is no equivalent check anywhere in `AnalysisAffixProcessRule.cs`, + `AnalysisCompoundingRule.cs`, or `LexEntry.cs`/`LexFamily.cs`. Blocking suppresses a candidate + during generation but has no bearing on what parses during analysis — the two directions are not + symmetric for a family with a suppletive member. +2. **Subsumption, not equality, decides it.** `Subsumes` succeeds whenever the derived word's + feature structure is *at least as specific as* the family member's — so a family member with a + deliberately broad or under-specified feature structure (e.g. left with an unassigned feature + that was meant to narrow it to one paradigm cell) can end up blocking derivations well beyond the + single irregular cell the grammar author intended to model. + +## Concrete example + +A `LexFamily` groups `go` (regular root, produces `go+ed` via a regular past-tense +`AffixProcessRule`) and `went` (irregular root, `partOfSpeech=posV`, own feature structure just +`{tense=past}`, no other features assigned). Both entries share a stratum. Any regularly-derived +past-tense form of `go` — whatever its full feature structure ends up being — subsumes `went`'s bare +`{tense=past}` requirement (an unspecified feature structure is a subsumer of anything more specific +in HC's feature system), so `Word.CheckBlocking` fires on every one of them: the regular +`go`+`-ed` derivation is silently discarded and replaced by `went` every single time, not just for +the one paradigm cell the author meant to override. + +## Fix + +- Set `blockable="false"` on a rule if you want its output to survive regardless of family + suppletion. +- If suppletion should only block one specific paradigm cell, give the irregular family member's + own feature structure exactly that cell's features (not a bare/under-specified structure) so + `Subsumes` can't match broader derivations than intended. +- Do not assume blocking constrains parsing — a form that blocking would suppress during generation + can still be accepted as a valid analysis when parsing the same surface string, because + `CheckBlocking` never runs on the analysis side. diff --git a/Docs/ai-parser-help/broken/mpr-group-matchtype-default-is-any.md b/Docs/ai-parser-help/broken/mpr-group-matchtype-default-is-any.md new file mode 100644 index 0000000000..82b7532817 --- /dev/null +++ b/Docs/ai-parser-help/broken/mpr-group-matchtype-default-is-any.md @@ -0,0 +1,93 @@ +--- +title: "An MPR feature group's match type defaults to Any (OR), not All — the opposite of what a gating author might assume" +implements: src/SIL.Machine.Morphology.HermitCrab/MprFeatureGroup.cs, src/SIL.Machine.Morphology.HermitCrab/MprFeatureSet.cs, src/SIL.Machine.Morphology.HermitCrab/XmlLanguageLoader.cs, src/SIL.Machine.Morphology.HermitCrab/HermitCrabInput.dtd +category: feature-system +symptom: silent-misconfiguration +grammar_visible: partially +--- + +## What it is + +`MprFeatureGroup.MatchType` (`MprFeatureGroup.cs:73`) is an `MprFeatureGroupMatchType` with values +`Any` ("when any features match within the group") and `All` ("only if all features match within the +group") (`MprFeatureGroup.cs:10-21`). When a grammar's `MorphologicalPhonologicalRuleFeatureGroup` +XML omits the `matchType` attribute, the loader — and the format's own DTD — both default it to +`Any`, not `All`. A grammar author reasoning "an unmarked/omitted setting should be the strict, +conjunctive one" gets the loose, disjunctive one instead, silently. + +## The mechanism + +The DTD itself states the default explicitly: + +``` + +``` + +(`HermitCrabInput.dtd:82-87`). The C# loader's fallback agrees: + +```csharp +private static MprFeatureGroupMatchType GetGroupMatchType(string matchTypeStr) +{ + switch (matchTypeStr) + { + case "all": return MprFeatureGroupMatchType.All; + case "any": return MprFeatureGroupMatchType.Any; + } + return MprFeatureGroupMatchType.Any; +} +``` + +(`XmlLanguageLoader.cs:95-106`) — any omitted, empty, or unrecognized `matchType` string falls +through to `Any`. This matches the enum's own declared order (`Any` is the first, zero-valued +member), so even a hypothetical uninitialized `MprFeatureGroup` would default the same way. + +The consequence at match time, `MprFeatureSet.IsMatchRequired` (`MprFeatureSet.cs:46-70`): + +```csharp +if (group.Key == null || group.Key.MatchType == MprFeatureGroupMatchType.All) +{ + if (group.Any(mf => !mprFeats.Contains(mf))) { mismatchGroup = group.Key; return false; } +} +else // Any +{ + if (group.All(mf => !mprFeats.Contains(mf))) { mismatchGroup = group.Key; return false; } +} +``` + +For an `All`-type group, the check fails as soon as *any one* referenced feature is missing from the +word's accumulated set — every listed feature must be present. For an `Any`-type group (the default), +the check only fails if *every* referenced feature is missing — a single one present is enough to +satisfy the whole group. (Note this also means an MPR feature with no group at all — `group.Key == +null` — is always treated with `All` semantics, i.e. as its own singleton conjunctive requirement; +the `Any` default only applies to features that are actually placed in a declared group.) + +## Why this is the opposite trap from the intuitive one + +It would be easy to assume the risk runs the other way — that a grammar author wanting OR semantics +("any one of these MPR features suffices to license this rule") has to opt in and might forget to, +silently landing on stricter AND gating instead. The verified default runs the other direction: the +default is already `Any`. The actual silent-misconfiguration risk is for an author who *wants* +conjunctive gating — "all of these MPR features must be present together for this rule to apply" — +and doesn't realize that leaving `matchType` unset does not give them that. Any single feature in the +group being present is enough, which can silently *under*-constrain a rule that was meant to require +several co-occurring MPR features at once. + +## Concrete example + +A grammar groups MPR features `hasPrefixA` and `hasPrefixB` into one group meant to gate a rule that +should only apply when a stem carries *both* prefixes' tags (e.g. a portmanteau-blocking condition). +The group's XML omits `matchType`. Because the default is `Any`, a stem carrying only `hasPrefixA` +(not `hasPrefixB`) already satisfies `RequiredMprFeatures` for that group — the rule applies to stems +the author only meant to license when both tags co-occurred. + +## Fix + +Set `matchType="all"` explicitly whenever a group is meant to require every one of its features +together — do not rely on the omitted-attribute default, which is `any`. Conversely, if `any` +(OR) semantics really is what you want, it's already the default and you don't need to state it, but +stating it explicitly still makes the grammar's intent legible to the next person reading the XML. diff --git a/Docs/ai-parser-help/broken/mpr-group-output-default-overwrite.md b/Docs/ai-parser-help/broken/mpr-group-output-default-overwrite.md new file mode 100644 index 0000000000..583bce52b6 --- /dev/null +++ b/Docs/ai-parser-help/broken/mpr-group-output-default-overwrite.md @@ -0,0 +1,84 @@ +--- +title: "An MPR feature group's output type defaults to Overwrite, not Append" +implements: src/SIL.Machine.Morphology.HermitCrab/MprFeatureGroup.cs, src/SIL.Machine.Morphology.HermitCrab/MprFeatureSet.cs, src/SIL.Machine.Morphology.HermitCrab/XmlLanguageLoader.cs, src/SIL.Machine.Morphology.HermitCrab/HermitCrabInput.dtd +category: feature-system +symptom: silent-misconfiguration +grammar_visible: partially +--- + +## What it is + +`MprFeatureGroup.Output` (`MprFeatureGroup.cs:79`) is an `MprFeatureGroupOutput` with values +`Overwrite` ("overwrites all existing features in the same group") and `Append` ("appends +features") (`MprFeatureGroup.cs:26-37`). When a group's XML omits the `outputType` attribute, both +the DTD and the C# loader default it to `Overwrite`, not `Append` — the non-monotone, order-dependent +behavior described in `speed/mpr-overwrite-order-dependence.md` is what an omitted attribute actually +gets, not the safer accumulating behavior a reader of that file's own wording ("declared with output +policy Overwrite instead of the default Append") might assume the loader falls back to. + +## The mechanism + +The DTD states the default explicitly: + +``` + +``` + +(`HermitCrabInput.dtd:82-87`). The loader's fallback agrees: + +```csharp +private static MprFeatureGroupOutput GetGroupOutput(string outputTypeStr) +{ + switch (outputTypeStr) + { + case "overwrite": return MprFeatureGroupOutput.Overwrite; + case "append": return MprFeatureGroupOutput.Append; + } + return MprFeatureGroupOutput.Overwrite; +} +``` + +(`XmlLanguageLoader.cs:108-119`) — an omitted, empty, or unrecognized `outputType` string falls +through to `Overwrite`. That default feeds directly into `MprFeatureSet.AddOutput` +(`MprFeatureSet.cs:29-44`), which is what every rule application runs on its output MPR features +(both ordinary affix-process allomorphs and compounding subrules): for a group whose `Output` is +`Overwrite`, applying a new rule's output silently drops any of that group's features the word +already carried unless the new output restates them — "last rule to touch this group wins," not +accumulation. + +## Why this is easy to miss + +A grammar author who reasons "MPR features are just tags, and tags accumulate unless I say +otherwise" gets the *opposite* of that reasonable assumption the moment they define a +`MorphologicalPhonologicalRuleFeatureGroup` and don't set `outputType` — which is the common case, +since `outputType` only matters once a grammar author is deliberately grouping features, at which +point `Append` (the semantically "boring," accumulating choice) looks like it should need no +attribute at all. Nothing about the group's own declaration signals this; the non-monotone behavior +only becomes visible once two different rules in a derivation touch the same group and a grammar +author notices one rule's tag vanished from the word's final state. See +`speed/mpr-overwrite-order-dependence.md` for the full consequences of `Overwrite` semantics once +they're in effect (order-dependence, and its interaction with the engine's ability to collapse +otherwise-equivalent candidate derivations) — this gotcha is specifically about how easy it is to end +up with `Overwrite` unintentionally, by omission, rather than the mechanism's downstream effects. + +## Concrete example + +A grammar groups MPR features `tagX` and `tagY` under one `MorphologicalPhonologicalRuleFeatureGroup` +with no `outputType` attribute, intending them as independent accumulating flags checked later by an +`ExcludedMprFeatures` gate elsewhere. Rule A applies first and outputs `tagX`; rule B applies later +in the same derivation and outputs `tagY`. Because the group defaults to `Overwrite`, rule B's +application removes `tagX` from the word's MPR-feature set before adding `tagY` — the final word +carries only `tagY`, even though both rules fired. A downstream gate checking for `tagX` fails +silently, exactly as if rule A had never applied. + +## Fix + +Set `outputType="append"` explicitly on any `MorphologicalPhonologicalRuleFeatureGroup` whose +features are meant to accumulate across a derivation. Only rely on the omitted-attribute default when +"last rule to touch this group wins" is the actually-intended semantics (e.g. a group modeling a +paradigm cell that a later derivation step is meant to reset outright). diff --git a/Docs/ai-parser-help/broken/null-affix-cannot-express-default.md b/Docs/ai-parser-help/broken/null-affix-cannot-express-default.md new file mode 100644 index 0000000000..7a946fd1fb --- /dev/null +++ b/Docs/ai-parser-help/broken/null-affix-cannot-express-default.md @@ -0,0 +1,82 @@ +--- +title: "A null/zero-realization affix is a real rule application, not a free default value" +implements: src/SIL.Machine.Morphology.HermitCrab/AffixTemplateSlot.cs, src/SIL.Machine.Morphology.HermitCrab/SynthesisAffixTemplateRule.cs +category: morphotactics +symptom: wrong-parse +grammar_visible: yes +--- + +## What it is + +H. Andrew Black's FLEx/HC conceptual intro (§2.1.2.2, "Optional affix slots") points out that some +categories have a "default" value realized by zero marking (e.g. singular number, unmarked when +plural is overtly suffixed) and states plainly that "the current parsers do not allow us to mark +such default features" — a null/zero affix is not the same thing as an actual default. This is +verifiable directly in the engine's slot-optionality logic. + +## The mechanism + +`AffixTemplateSlot.Optional` only auto-derives `true` when the slot has **no rules at all**: + +```csharp +public bool Optional +{ + get + { + if (_rules.Count == 0) + return true; + return _isOptional; + } + set { _isOptional = value; } +} +``` + +(`AffixTemplateSlot.cs:35-45`). A slot holding one rule — even a rule whose only allomorph spells +out nothing overt (a "null suffix") — is **not** automatically optional; its `Optional` flag is +whatever the grammar author set explicitly, and if the author left it non-optional (the natural +choice for "this category is always marked, sometimes by zero"), `SynthesisAffixTemplateRule.ApplySlots` +requires that rule to actually apply, successfully, before the derivation can proceed past that slot: + +```csharp +foreach (Word outWord in _rules[i].Apply(input)) + ApplySlots(outWord, i + 1, output); +if (!_template.Slots[i].Optional) + return; // no successful application at this (mandatory) slot -> dead end +``` + +(`SynthesisAffixTemplateRule.cs:39-49`). A null-realization rule still goes through the same +application machinery as any other affix rule — its `RequiredSyntacticFeatureStruct` must unify, any +`RequiredMprFeatures`/environment/stem-name gates on it must be satisfied, and so on. If the +conditions that license the null affix's rule aren't met for some input, the mandatory slot has +nothing to apply and the whole derivation dead-ends at that slot — it does not fall through to "just +assume the default value," because there is no such fallback in the engine. + +## Why this is easy to miss + +The zero-marking case *looks* like exactly what a default feature value should be: nothing is +written, nothing seems constrained. But because the null affix is implemented as an ordinary rule +occupying a non-optional slot, every one of that rule's own gates (required features, MPR features, +environment) still has to pass. A form that should trivially take "the default" because none of the +overt alternatives apply can instead fail to derive at all, if the null rule's own required +conditions don't happen to be satisfied for that particular stem — there is no separate "otherwise +assume singular" fallback path. + +## Concrete example + +A noun category has number marked by `-s` (plural) vs. a null suffix (singular), both rules placed +in one non-optional template slot. If the null-suffix rule is written with its own +`RequiredSyntacticFeatureStruct` (e.g. requiring `num=sg` to already be set elsewhere) rather than as +the true, condition-free elsewhere case, any noun stem for which `num` hasn't already been +independently assigned by an earlier rule fails at this slot entirely — neither the `-s` rule nor the +null rule can apply, and the whole derivation dies, when the grammar author's mental model was "no +suffix just means singular by default." + +## Fix + +If a category's "default" is a zero realization, still model it as a real, unconditioned rule (no +`requiredSyntacticFeatureStruct` beyond the bare POS, no MPR-feature gate that could fail) so it acts +as a true elsewhere case within its slot — or, per Black's suggested workaround, mark the slot +optional and have every overt affix *set* the non-default feature value explicitly, leaving the +feature unset (rather than null-affixed) when the slot doesn't fire, and have downstream rules or +default unification handle the "otherwise assume the default" logic instead of relying on the +template slot mechanism to express it. diff --git a/Docs/ai-parser-help/broken/realizational-rule-cannot-add-features.md b/Docs/ai-parser-help/broken/realizational-rule-cannot-add-features.md new file mode 100644 index 0000000000..20aa713f9a --- /dev/null +++ b/Docs/ai-parser-help/broken/realizational-rule-cannot-add-features.md @@ -0,0 +1,81 @@ +--- +title: "RealizationalRule can only spell out features already present — it cannot assign new ones the way AffixProcessRule can" +implements: src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/RealizationalAffixProcessRule.cs, src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisRealizationalAffixProcessRule.cs, src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AffixProcessRule.cs, src/SIL.Machine.Morphology.HermitCrab/HermitCrabInput.dtd +category: morphotactics +symptom: silent-misconfiguration +grammar_visible: yes +--- + +## What it is + +`AffixProcessRule` and `RealizationalAffixProcessRule` look like two flavors of the same thing (both +compile from a ``-family element, both hold a list of `AffixProcessAllomorph` +subrules, both apply through a Synthesis/Analysis rule pair), but they differ in a way that isn't +obvious from either the C# API surface or a first read of the DTD: `RealizationalRule` has no +mechanism at all for introducing a new syntactic feature or part of speech into the word. It can only +"realize" (spell out morphologically) a feature bundle the word already carries. + +## The mechanism + +`AffixProcessRule` has `OutSyntacticFeatureStruct` (`MorphologicalRules/AffixProcessRule.cs:64`), +populated on synthesis via `outWord.SyntacticFeatureStruct.PriorityUnion(_rule.OutSyntacticFeatureStruct)` +(`SynthesisAffixProcessRule.cs:182`) — this is the mechanism an ordinary inflectional or derivational +affix uses to introduce a feature value (or change part of speech) that wasn't there before. + +`RealizationalAffixProcessRule` has no equivalent property. Its DTD element confirms this is not an +oversight in the C# class alone: + +``` + +``` + +(`HermitCrabInput.dtd:362`) — there is no `outputPartOfSpeech` attribute and no `OutputHeadFeatures` +child element, unlike the ordinary `MorphologicalRule` element. What it has instead is +`RealizationalFeatures`, loaded into `RealizationalFeatureStruct` +(`RealizationalAffixProcessRule.cs:54`), which `SynthesisRealizationalAffixProcessRule.Apply` merges +via `outWord.SyntacticFeatureStruct.PriorityUnion(_rule.RealizationalFeatureStruct)` +(`SynthesisRealizationalAffixProcessRule.cs:122`) — syntactically this looks like the same kind of +merge an `AffixProcessRule` does, but semantically it is gated very differently: the rule first +requires `_rule.RealizationalFeatureStruct.Subsumes(input.RealizationalFeatureStruct)` +(`SynthesisRealizationalAffixProcessRule.cs:46`) and then refuses to apply at all if +`IsBlocked(_rule.RealizationalFeatureStruct, input.SyntacticFeatureStruct, ...)` finds every one of +the rule's realizational features already present in the word's accumulated syntactic feature +structure (`SynthesisRealizationalAffixProcessRule.cs:49-59, 168-195`). A `RealizationalRule` is +designed to spell out a feature bundle some other mechanism (typically an inflectional +`AffixProcessRule`'s `OutputHeadFeatures`, or `InflFeatsOA` on the MSA in the FieldWorks compiler) has +already assigned abstractly — not to be the thing that assigns it. + +## Why this is easy to miss + +Both rule types are exposed in FieldWorks as morphological-rule-like entries with a features section +in the UI, and both end up merging a feature structure into the word during synthesis via what reads, +in the compiled C# alone, like the same `PriorityUnion` idiom. A grammar author modeling an agreement +affix as a `RealizationalRule` because "it realizes agreement features" can reasonably expect it to +also be able to introduce a feature the word didn't have yet (e.g. assigning `case=nom` for the first +time) the way an ordinary affix would — but there is no attribute or element in the `RealizationalRule` +schema that does that, and no code path in `SynthesisRealizationalAffixProcessRule` that adds a +feature the accumulated structure doesn't already carry in some form. The rule will still compile and +load without error; it simply never introduces the feature the author expected, and unification against +`RequiredHeadFeatures`/`RequiredFootFeatures` downstream continues to fail silently for words that +were supposed to receive it from this rule. + +## Concrete example + +A grammar models subject-agreement suffixes as `RealizationalRule`s, expecting each suffix to assign +the relevant person/number combination to a word that previously had no agreement features at all +(e.g. a bare verb stem). Because `RealizationalRule` has no output-feature mechanism, the suffix rule +requires `RequiredHeadFeatures` to already be compatible and merges only `RealizationalFeatureStruct` +— which, per `IsBlocked`, only proceeds when the target features are not already fully present, but +also never actually assigns them if nothing upstream (e.g. an inflectional `AffixProcessRule`'s +`OutputHeadFeatures`) already put an abstract agreement feature bundle onto the word first. A word +that never went through such an upstream rule never receives any agreement value from the +realizational suffix, no matter how many realizational rules are chained. + +## Fix + +Use `RealizationalRule` only to spell out features an inflectional `AffixProcessRule` (or another +mechanism) has already assigned abstractly elsewhere in the same derivation — never as the sole +mechanism responsible for introducing a feature value. If an affix genuinely needs to introduce a new +syntactic feature or part of speech, model it as an ordinary `AffixProcessRule` with +`OutputHeadFeatures`/`outputPartOfSpeech`, not as a `RealizationalRule`. diff --git a/Docs/ai-parser-help/broken/realizational-rule-no-application-cap.md b/Docs/ai-parser-help/broken/realizational-rule-no-application-cap.md new file mode 100644 index 0000000000..3d3f0de522 --- /dev/null +++ b/Docs/ai-parser-help/broken/realizational-rule-no-application-cap.md @@ -0,0 +1,73 @@ +--- +title: "RealizationalAffixProcessRule has no MaxApplicationCount backstop, unlike AffixProcessRule" +implements: src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AffixProcessRule.cs, src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/RealizationalAffixProcessRule.cs, src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisRealizationalAffixProcessRule.cs, src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs +category: morphotactics +symptom: wrong-parse +grammar_visible: partially +--- + +## What it is + +An ordinary `AffixProcessRule` is capped at one application per derivation by default — +`MaxApplicationCount = 1` is set in its constructor (`MorphologicalRules/AffixProcessRule.cs:28`) +and enforced on every application: `if (input.GetApplicationCount(_rule) >= _rule.MaxApplicationCount)` +(`MorphologicalRules/SynthesisAffixProcessRule.cs:46`, mirrored on the analysis side at +`AnalysisAffixProcessRule.cs:45`). `RealizationalAffixProcessRule` — the rule type meant for pure +feature-realization/spellout rules rather than form-changing affixation — has no such property at +all. Reading the whole file confirms it: no `MaxApplicationCount` field, no constructor default, no +check anywhere in `RealizationalAffixProcessRule.cs`, `SynthesisRealizationalAffixProcessRule.cs`, or +`AnalysisRealizationalAffixProcessRule.cs`. + +## The mechanism + +The only guard against a `RealizationalAffixProcessRule` reapplying itself indefinitely in the same +derivation is a feature-based recursive check, `IsBlocked` +(`SynthesisRealizationalAffixProcessRule.cs:168-195`), gated by: + +```csharp +if (!_rule.RealizationalFeatureStruct.IsEmpty + && IsBlocked(_rule.RealizationalFeatureStruct, input.SyntacticFeatureStruct, ...)) +``` + +`IsBlocked` walks the rule's `RealizationalFeatureStruct` recursively and returns true only when +every one of the rule's realizational features is *already present* in the word's accumulated +syntactic feature structure. If a rule's realizational features are only a partial or ambiguous +subset of what's already assigned — or if the rule sets a feature that the accumulated structure +doesn't yet carry in a form `IsBlocked` recognizes as "already there" — this guard does not fire, and +nothing else stops the rule from applying again in a cyclic derivation. Unlike an +`AffixProcessRule`, there is no numeric backstop underneath the feature check. + +## Why this is easy to miss + +A grammar author who reasons "every morphological rule type gets a default `MaxApplicationCount` of +1 unless I explicitly raise it" (true for `AffixProcessRule` and `CompoundingRule`, whose +constructors both set `MaxApplicationCount = 1`, e.g. `MorphologicalRules/CompoundingRule.cs:19`) +will not find that assumption holds for `RealizationalAffixProcessRule` at all — there is no +attribute to raise or lower, because the rule type doesn't have the property. Nothing in a grammar's +XML for a `RealizationalRule` (which has no `multipleApplication` attribute in the schema for this +rule kind) hints that the usual per-rule application cap doesn't exist here. + +## Concrete example + +A `RealizationalRule` meant to spell out a single agreement feature (`agr=match`) as a floating tone +or zero-marking, with a `RequiredSyntacticFeatureStruct` that doesn't itself exclude re-application +(e.g. it only requires `posV`, not "agr is not yet match"). A stratum's own rule cascade +(`LinearRuleCascade`/`CombinationRuleCascade`, see `speed/stratum-rule-ordering.md`) tracks rules +"not yet used on the current derivation path" within a single pass through that cascade, so within +one straightforward pass the rule fires at most once. But whenever a derivation reaches the same +rule's application point a second time by a different route — e.g. compounding +(`SynthesisCompoundingRule`) combines two independently-derived sub-words, each of which already ran +the realizational rule once on its own branch, or a longer, multi-stratum derivation feeds a word +back through the same stratum again later — nothing revokes or dedupes a prior application, and +`IsBlocked`'s feature-subset test is the only thing standing between that and the realizational +spellout being expressed twice in the combined result. A grammar author reasoning from +`AffixProcessRule`'s default `MaxApplicationCount = 1` would not expect this rule type to need any +extra guard against it. + +## Fix + +For a `RealizationalAffixProcessRule`, make sure `RequiredSyntacticFeatureStruct` (or the +accumulated `RealizationalFeatureStruct` state it depends on) is specific enough that a second +application is infeasible on its own terms — e.g. require the *absence* of the feature the rule sets, +not merely the presence of the features that license it. Don't rely on an implicit per-rule +application cap for this rule type; unlike `AffixProcessRule`/`CompoundingRule`, none exists. diff --git a/Docs/ai-parser-help/broken/single-template-independent-slots-illegal-combos.md b/Docs/ai-parser-help/broken/single-template-independent-slots-illegal-combos.md new file mode 100644 index 0000000000..b334fd9fe9 --- /dev/null +++ b/Docs/ai-parser-help/broken/single-template-independent-slots-illegal-combos.md @@ -0,0 +1,74 @@ +--- +title: "Independent optional slots in one template can license a combination two templates would have prevented" +implements: src/SIL.Machine.Morphology.HermitCrab/SynthesisAffixTemplateRule.cs, src/SIL.Machine.Morphology.HermitCrab/SynthesisAffixTemplatesRule.cs, src/SIL.Machine.Morphology.HermitCrab/AffixTemplate.cs +category: morphotactics +symptom: wrong-parse +grammar_visible: partially +--- + +## What it is + +H. Andrew Black's FLEx/HC conceptual intro (§2.1.2.2 and its follow-up discussion of using separate +templates to force affix co-occurrence) describes a modeling choice: two affixes that must always +co-occur (or never co-occur) can be forced into that relationship by putting them in two mutually +exclusive templates (gated by disjoint required feature structures), where putting them in +independent optional slots of *one* template cannot express that coupling — each slot's +apply-or-skip choice is made independently of every other slot in the same template. + +## The mechanism + +Within one template, `SynthesisAffixTemplateRule.ApplySlots` recurses slot by slot, and for every +*optional* slot it explores both branches — apply the slot's rule(s), and also fall through to the +next slot without applying it: + +```csharp +private void ApplySlots(Word input, int index, HashSet output) +{ + for (int i = index; i < _rules.Count; i++) + { + foreach (Word outWord in _rules[i].Apply(input)) + ApplySlots(outWord, i + 1, output); + if (!_template.Slots[i].Optional) + return; + } + output.Add(input); +} +``` + +(`SynthesisAffixTemplateRule.cs:37-55`). Nothing here couples slot `i`'s apply/skip choice to slot +`j`'s — each optional slot's two branches are explored independently, so with `n` independent +optional slots in one template, all `2^n` combinations of "applied at this slot or not" are +reachable (modulo each rule's own required-feature/environment gates), including combinations the +grammar author never intended to co-occur. + +Templates themselves, by contrast, *are* mutually exclusive per derivation in the sense that a +template is chosen by its own gate: `SynthesisAffixTemplatesRule.Apply` only enters a template when +`input.SyntacticFeatureStruct.IsUnifiable(_templates[i].RequiredSyntacticFeatureStruct)` +(`SynthesisAffixTemplatesRule.cs:37`) — so a grammar author who wants "affix A and affix B only ever +co-occur, never apply independently" can express that by putting A in one template (with a +distinguishing required feature) and B in a separate template gated on the same required feature, +rather than as two optional slots of one shared template. Note both templates that satisfy their gate +still both get tried in the same call (the loop at `SynthesisAffixTemplatesRule.cs:33-54` does not +`break` after the first applicable template) — the mutual exclusion Black describes comes from making +the templates' `RequiredSyntacticFeatureStruct`s disjoint, not from the engine picking only one +template automatically. + +## Concrete example + +A stratum's single template has two independent optional slots: slot 1 for an applicative marker +`-ap`, slot 2 for an object-agreement suffix that is only supposed to be licensed *together with* +`-ap` (the language only marks object agreement on applicativized verbs). Modeled as two independent +optional slots in one template, the engine happily explores: neither fires, only `-ap` fires, only +the agreement suffix fires, or both fire — all four are separately reachable derivations unless each +rule's own `RequiredSyntacticFeatureStruct`/MPR-feature gates independently rule out the "just +agreement, no applicative" case. If the grammar author didn't add such a gate (having assumed the +template structure itself expressed the dependency), the illegal "bare object agreement with no +applicative" form parses and generates successfully. + +## Fix + +Do not rely on template/slot placement alone to express a required co-occurrence between two +affixes. Either add explicit `RequiredMprFeatures`/`RequiredSyntacticFeatureStruct` gates so the +dependent affix's rule cannot apply without the feature the other affix's rule sets, or split the +co-occurring pair into their own template with a disjoint required feature structure from the +template(s) where they don't apply, following Black's separate-templates pattern. diff --git a/Docs/ai-parser-help/broken/stem-name-affix-requirement-trace-misreading.md b/Docs/ai-parser-help/broken/stem-name-affix-requirement-trace-misreading.md new file mode 100644 index 0000000000..49b1dcbbc5 --- /dev/null +++ b/Docs/ai-parser-help/broken/stem-name-affix-requirement-trace-misreading.md @@ -0,0 +1,75 @@ +--- +title: "A rule's requiredStemName is checked in the forward (synthesis) pass only — including during parsing" +implements: src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AffixProcessRule.cs, src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisAffixProcessRule.cs, src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs, src/SIL.Machine.Morphology.HermitCrab/Morpher.cs +category: lexicon +symptom: silent-misconfiguration +grammar_visible: partially +--- + +## What it is + +`AffixProcessRule.RequiredStemName` (`MorphologicalRules/AffixProcessRule.cs:66`) lets a grammar +author say "this affix rule may only apply to a root that already carries stem name X." This is a +different mechanism from a stem-name-restricted *allomorph* (`RootAllomorph.StemName`, checked in +`RootAllomorph.CheckAllomorphConstraints`, `RootAllomorph.cs:65-70` — see the companion gotcha in +[`stem-name-explicit-feature-requirement.md`](stem-name-explicit-feature-requirement.md) for that +mechanism). This one is a rule-level gate, and it is enforced in exactly one place in the whole +engine. + +## The mechanism + +`SynthesisAffixProcessRule.Apply` checks it directly: + +```csharp +if (_rule.RequiredStemName != null && _rule.RequiredStemName != input.RootAllomorph.StemName) +``` + +(`MorphologicalRules/SynthesisAffixProcessRule.cs:106`). This compares the rule's required stem +name against `input.RootAllomorph.StemName` — a property that is only meaningful once a specific +root allomorph has been chosen, which is a synthesis-direction concept: synthesis builds a word +forward from a chosen lexical entry and allomorph. + +`AnalysisAffixProcessRule.Apply` (`MorphologicalRules/AnalysisAffixProcessRule.cs`) never +references `RequiredStemName` at all. This isn't an oversight — it's structural: analysis unapplies +an affix rule by stripping it off a surface form *before* lexical lookup identifies which root (and +therefore which root allomorph) produced the stem. There is nothing to compare against yet at that +point in the pipeline. + +That does not mean parsing ignores `RequiredStemName` end to end. `Morpher.ParseWord` (`Morpher.cs`) +does not return raw backward-unapplication results — it feeds every analysis candidate through +`Synthesize(word, analyses)` (`Morpher.cs:283-299`), which performs lexical lookup and then reapplies +the same forward `SynthesisAffixProcessRule.Apply` used for generation, filtering the result through +`IsWordValid`. So the check at `SynthesisAffixProcessRule.cs:106` *does* run during parsing — just in +the resynthesis/confirmation half of the pipeline, not the backward-stripping half. + +## Why this is easy to miss + +A grammar author or anyone reading a parser trace one rule-application at a time sees the backward +`AnalysisAffixProcessRule` step for a `requiredStemName`-bearing rule succeed unconditionally — it +never looks at stem names at all. The actual rejection only shows up later, tagged +`FailureReason.RequiredStemName`, in what a trace reader would think of as the generation/synthesis +half of a "parse." Debugging "why didn't this word parse" by reading only the analysis-rule trace +entries shows nothing wrong; the answer is in the resynthesis-confirmation entries instead. + +## Concrete example + +POS `posV`; feature `pers` with values `sym1`/`sym2`/`sym3`. Lexical entry `eRoot` has two +allomorphs: `kap` (no stem name) and `tam` (stem name `snP1`, region `pers=sym1`). Suffix rule +`rSuf1` (spells out `-xi`) declares `requiredStemName="snP1"`. + +Parsing the surface form `kap-xi`: the backward `AnalysisAffixProcessRule` pass for `rSuf1` strips +`-xi` and reports success regardless of which root eventually gets matched. Lexical lookup then +matches the residual shape `kap` to the `kap` allomorph (no stem name). Resynthesis reapplies +`rSuf1` forward with `input.RootAllomorph` pointing at the `kap` allomorph, hits +`RequiredStemName != input.RootAllomorph.StemName` (`snP1 != null`), and the candidate is dropped — +never rejected in the initial unapplication step, only in the later confirmation step. + +## Fix + +When a parse unexpectedly fails (or unexpectedly succeeds) for a rule with `requiredStemName`, check +the resynthesis/confirmation trace entries (`FailureReason.RequiredStemName`), not just the backward +rule-application trace. If you want a stem-name requirement to behave symmetrically and visibly in +both directions, prefer modeling it as a stem-name-restricted *allomorph* region instead (or in +addition) — that mechanism is checked through the shared `Allomorph.IsWordValid` path and is at +least consistently a "final validity" check on both sides, rather than a rule-only gate that is +silently absent from one half of the pipeline's own rule trace. diff --git a/Docs/ai-parser-help/broken/stem-name-explicit-feature-requirement.md b/Docs/ai-parser-help/broken/stem-name-explicit-feature-requirement.md new file mode 100644 index 0000000000..7e69f25b69 --- /dev/null +++ b/Docs/ai-parser-help/broken/stem-name-explicit-feature-requirement.md @@ -0,0 +1,58 @@ +--- +title: "A stem-name-restricted allomorph needs the feature explicitly assigned, not just compatible" +implements: src/SIL.Machine.Morphology.HermitCrab/StemName.cs, src/SIL.Machine.Morphology.HermitCrab/RootAllomorph.cs +category: lexicon +symptom: silent-misconfiguration +grammar_visible: "no — produces silent, correct-per-the-code parse failures with no obvious cause in the XML" +--- + +## What it is + +Stem names restrict a root allomorph to only be valid when the word's accumulated syntactic +feature structure falls inside one of the stem name's declared regions; any *other* allomorph of +the same lexical entry that has no stem name (or a different one) is only valid **outside** that +region. This is the mechanism behind "principal parts" — a root with an irregular form for one +paradigm cell and a regular form everywhere else. + +## The mechanism + +A stem name's required-match check tests the word's *current* feature structure against the +stem name's declared region. Because that check tests the feature structure as it actually +stands at the point of checking — not "is this compatible with the region," but "is this +explicitly inside the region" — a bare stem (before any rule has assigned the relevant feature) +does not automatically satisfy a stem name's region, even if the region only mentions one +feature. A form with that feature totally unassigned is outside the region, not inside it. + +## Toy example + +A root `tam`/`kap` where `kap` is stem-name-restricted to require `pers=1`: + +```xml + + + + + +... + + + tam + kap + + +``` + +The bare root `kap` alone (no person feature assigned at all) has **zero** valid parses — the +stem-name-restricted allomorph requires `pers=1` to be explicitly present, and an unmarked form +doesn't count, even trivially. Only a rule that actually assigns `pers=1` (e.g. a first-person +agreement rule) makes `kap`-derived forms valid; the bare root, and any form built with a rule +assigning `pers=2`, can only surface as `tam`. + +## Gotcha and fix + +A grammar author who expects a stem-name-restricted allomorph to be usable "whenever nothing +else says otherwise" will see silent, correct-per-the-code parse failures instead — nothing in a +trace beyond a generic environment/region mismatch points at "you forgot to assign the feature +this region requires." The fix is to make sure every morphological rule that's supposed to +license a restricted stem name actually assigns the specific feature value the region requires, +not just a value that happens to be compatible with it. diff --git a/Docs/ai-parser-help/broken/stemname-partsofspeech-required-not-silent.md b/Docs/ai-parser-help/broken/stemname-partsofspeech-required-not-silent.md new file mode 100644 index 0000000000..643860800a --- /dev/null +++ b/Docs/ai-parser-help/broken/stemname-partsofspeech-required-not-silent.md @@ -0,0 +1,82 @@ +--- +title: "A StemName's partsOfSpeech is required by the loader — omitting it crashes, it doesn't silently reject everything" +implements: src/SIL.Machine.Morphology.HermitCrab/XmlLanguageLoader.cs, src/SIL.Machine.Morphology.HermitCrab/StemName.cs, src/SIL.Machine.Morphology.HermitCrab/HermitCrabInput.dtd +category: loader +symptom: crash +grammar_visible: yes +--- + +## What it is + +A `` element's `partsOfSpeech` attribute is declared `#REQUIRED` in the HC-XML DTD: + +``` + + +``` + +(`HermitCrabInput.dtd:94-99`). This is worth checking against source rather than assuming, because +the failure mode for omitting it is a crash, not — as one might guess from the compiled `StemName` +class alone — a silent "empty POS constraint that matches nothing." The compiled `StemName` class +(`StemName.cs`) doesn't even have a `PartsOfSpeech` property; part of speech is folded into each +region's feature structure at load time, and the loader never tolerates a missing value for it. + +## The mechanism + +`XmlLanguageLoader.LoadStemName` reads and immediately uses the attribute with no null-check: + +```csharp +private void LoadStemName(XElement stemNameElem) +{ + var posIDs = (string)stemNameElem.Attribute("partsOfSpeech"); + FeatureSymbol[] pos = posIDs.Split(' ').Select(id => _posFeature.PossibleSymbols[id]).ToArray(); + ... + foreach (XElement regionElem in stemNameElem.Elements("Regions").Elements("Region")) + { + var fs = new FeatureStruct(); + fs.AddValue(_posFeature, pos); + ... + } +} +``` + +(`XmlLanguageLoader.cs:323-342`). `posIDs` is a plain nullable string cast from the XML attribute +with no fallback. If the attribute is truly absent, `posIDs` is `null` and `posIDs.Split(' ')` throws +a `NullReferenceException` immediately during loading — before any word is ever parsed or generated, +and before the stem name's regions are even built. + +In practice this null case is also caught earlier by XML validation on most runtimes: +`XmlLanguageLoader.Load` sets `ValidationType = Type.GetType("Mono.Runtime") == null ? +ValidationType.DTD : ValidationType.None` (`XmlLanguageLoader.cs:212`). On a non-Mono .NET runtime, +the reader validates the document against the DTD as it parses, and a `StemName` element missing its +`#REQUIRED` `partsOfSpeech` attribute fails validation with an explicit error naming the missing +attribute — before `LoadStemName` ever runs. Only on Mono (`ValidationType.None`, no DTD validation) +would an omitted attribute reach `LoadStemName` at all, where it produces the `NullReferenceException` +described above instead. Either way, the outcome is a loud failure at load time, not a silently +unusable stem name. + +A related but distinct trap: if `partsOfSpeech` is present but contains an ID that isn't a POS the +grammar declared, `_posFeature.PossibleSymbols[id]` throws a `KeyNotFoundException` (or equivalent) — +also a crash, also at load time, also not silent. + +## Why this is worth documenting anyway + +Even though the failure is loud rather than silent, the *error message itself* doesn't say "this +stem name is unusable" in language a grammar author would immediately connect to their FLEx-side +edit — a DTD validation error or a bare `NullReferenceException`/`KeyNotFoundException` stack trace +from deep in `XmlLanguageLoader` gives no hint that the fix is "add a `partsOfSpeech` attribute to +this `StemName` element" unless the reader already knows this loader code. The practical failure +mode is "the grammar fails to load at all" (not "some words silently fail to parse"), which is an +important distinction from the "rejects everything at runtime" framing this trap is sometimes +described with — the actual failure happens at grammar-load time, for the whole grammar, not +per-word at parse time. + +## Fix + +Always give every `` an explicit `partsOfSpeech` listing every POS the stem name should +apply to. If a load fails with a `NullReferenceException` or DTD validation error mentioning +`StemName`, check for a missing or misspelled `partsOfSpeech` attribute value first — the loader +provides no default and no graceful degradation for it. diff --git a/Docs/ai-parser-help/broken/unclassified-affix-bypasses-template-ordering.md b/Docs/ai-parser-help/broken/unclassified-affix-bypasses-template-ordering.md new file mode 100644 index 0000000000..6e989198f6 --- /dev/null +++ b/Docs/ai-parser-help/broken/unclassified-affix-bypasses-template-ordering.md @@ -0,0 +1,82 @@ +--- +title: "An unclassified (or under-specified) affix rule bypasses normal template-ordering discipline" +implements: src/SIL.Machine.Morphology.HermitCrab/Morpheme.cs, src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisAffixProcessRule.cs, src/SIL.Machine.Morphology.HermitCrab/SynthesisAffixTemplatesRule.cs, FieldWorks Src/LexText/ParserCore/HCLoader.cs +category: morphotactics +symptom: wrong-parse +grammar_visible: partially +--- + +## What it is + +FieldWorks distinguishes an affix's morphosyntactic status as inflectional, derivational, or +unclassified (H. Andrew Black's FLEx/HC conceptual intro, §2.1.1 "Unclassified affixes," discusses +how an unclassified affix is "relatively unconstrained as to where it can appear" and can cause +spurious parses). This is not just a FLEx-side labeling convention: it compiles down to a real HC +engine property, `Morpheme.IsPartial` (`Morpheme.cs:45`), and the engine gives an `IsPartial` rule +genuinely different template-ordering permissions than an ordinary classified rule. + +## The mechanism + +`SynthesisAffixProcessRule.Apply` gates whether a non-template rule may apply after a template has +already fired, and the gate is different depending on `IsPartial`: + +```csharp +// if a final template was last applied, +// do not allow a non-partial rule to apply unless the input is partial +if (!_rule.IsTemplateRule && (input.IsLastAppliedRuleFinal ?? false) + && !input.IsPartial && !_rule.IsPartial) + return Enumerable.Empty(); // FailureReason.NonPartialRuleProhibitedAfterFinalTemplate + +// if a non-final template was last applied, +// only allow a non-partial rule to apply unless the input is partial +if (!_rule.IsTemplateRule && input.IsLastAppliedRuleFinal.HasValue + && !input.IsLastAppliedRuleFinal.Value && !input.IsPartial && _rule.IsPartial) + return Enumerable.Empty(); // FailureReason.NonPartialRuleRequiredAfterNonFinalTemplate +``` + +(`MorphologicalRules/SynthesisAffixProcessRule.cs:61-104`). In plain terms: an ordinary +(non-partial) rule cannot apply after a *final* template unless the word itself is already partial, +and a *partial* rule specifically can. `SynthesisAffixTemplatesRule.Apply` also skips template +matching for a partial root entirely: `&& !input.RootAllomorph.Morpheme.IsPartial` +(`SynthesisAffixTemplatesRule.cs:38`) — a partial morpheme's derivation doesn't go through the +normal template-selection gate other roots go through. + +The FieldWorks loader sets `IsPartial` from the FLEx affix classification: an inflectional affix's +`IsPartial` is `msa.SlotsRC.Count == 0` (no slot assigned — effectively still under-specified even +though nominally "inflectional"), and an unclassified affix is *always* loaded with `IsPartial = +true` regardless of anything else about it (both in `HCLoader.cs`, in the inflectional- and +unclassified-affix loading methods respectively). + +## Why this matters + +A classified, fully slotted inflectional or derivational affix is confined to the normal +template-ordering discipline: once a final template applies, no further non-partial affixation is +allowed except from other partial rules. An unclassified affix sidesteps that discipline on both +ends — it's allowed to apply where a normal rule would be refused (after a final template, if the +word itself isn't already partial... note it still needs the word to satisfy the *other* branch, but +the two branches together give a partial rule strictly more freedom around template boundaries than +a non-partial one gets). A grammar author who leaves an affix unclassified because its +morphosyntactic status genuinely doesn't matter to them can end up with an affix that combines with +stems in positions relative to a stratum's affix templates that a fully classified affix of the same +shape never could — producing analyses (or generable words) the author did not intend to license. + +## Concrete example + +A grammar has one inflectional affix template per POS, marked `final="true"` (nothing may apply +after it). A derivational suffix `-caus` is left unclassified in FLEx instead of tagged derivational +(perhaps because the modeler wasn't sure or didn't think it mattered). Because `IsPartial` is forced +`true` for any unclassified affix, `-caus` is now specifically *permitted* to apply after the final +inflectional template — stacking on top of a fully-inflected word — where a properly classified +derivational affix would have been rejected by `NonPartialRuleProhibitedAfterFinalTemplate`. The +result: words like a fully-inflected verb with `-caus` appended afterward parse (or generate) +successfully, when the grammar's actual structure (derivation happens before inflection, never +after) never intended to allow that ordering. + +## Fix + +Classify every affix as inflectional or derivational explicitly, and for inflectional affixes assign +a real template slot (`msa.SlotsRC`) rather than leaving it empty — both avoid `IsPartial` being set +in a way you didn't intend. Reserve "unclassified" for affixes that genuinely should be exempt from +template-ordering constraints (rare), and check any unclassified affix's actual generable/parseable +combinations against the templates you expect it to interact with, since it is not gated the same +way the rest of the grammar's affixes are. diff --git a/Docs/ai-parser-help/getting-started.md b/Docs/ai-parser-help/getting-started.md new file mode 100644 index 0000000000..8a2eb5ed23 --- /dev/null +++ b/Docs/ai-parser-help/getting-started.md @@ -0,0 +1,121 @@ +# Get help from ChatGPT or Claude with your HermitCrab grammar + +Part of the [AI Parser Help reference](README.md). This page is for anyone with a FieldWorks +Language Explorer (FLEx) project who wants an LLM's help with their HermitCrab parser/grammar — +whether it's broken ("why won't this word parse," "why do I get a wrong analysis"), slow ("why +is this so slow," "why do I get 500 analyses for one word"), or a modeling question ("how should +I represent this in my grammar"). + +Want help with real **interlinear texts** (`.flextext` files) instead of, or alongside, the +grammar itself? See [`texts/getting-started.md`](texts/getting-started.md) — extracting and +reasoning about connected corpus texts is a different workflow from the one below. + +## Step 1 — Extract your grammar as HermitCrab XML + +FieldWorks ships a tool, `GenerateHCConfig.exe`, that exports your project's grammar as the +HermitCrab XML format (the same format HermitCrab itself parses from). + +**First, close the project in FLEx.** The tool loads the project file directly and fails +with "currently open in another application" if FLEx (or anything else) still has it open. + +**Find the tool.** It's installed next to `FieldWorks.exe` itself, not on your PATH, and the +exact folder depends on your FieldWorks version. Easiest way to find it — paste this into +PowerShell: + +```powershell +Get-ChildItem "C:\Program Files\SIL\FieldWorks*\GenerateHCConfig.exe", ` + "C:\Program Files (x86)\SIL\FieldWorks*\GenerateHCConfig.exe" ` + -ErrorAction SilentlyContinue +``` + +That prints the full path (e.g. `C:\Program Files\SIL\FieldWorks 9\GenerateHCConfig.exe`). +If it prints nothing, your project's `RootCodeDir` registry override points somewhere else — +search for `GenerateHCConfig.exe` under wherever FieldWorks itself is installed. + +**Run it** against your project's `.fwdata` file: + +```powershell +& "C:\Program Files\SIL\FieldWorks 9\GenerateHCConfig.exe" "C:\path\to\YourProject.fwdata" "C:\path\to\YourProject-hc.xml" +``` + +(use the actual path `Get-ChildItem` printed above, and your own project's `.fwdata` path — +typically under `Documents\My FieldWorks\\`) + +This produces the second file, `YourProject-hc.xml` — that's your grammar. + +### Or export from inside FLEx, which also gets your texts + +Newer FieldWorks builds can do this without the command line, and can include your interlinear +texts alongside the grammar: **File > Export**, then choose *Export Grammar and Texts for AI +Analysis*. You pick which texts to include (the picker lists a word count and an analysis count +for each one) and a folder; FLEx writes `HCGrammar.xml`, one `.flextext` file per selected text, +and an `export-instructions.md` into that folder. + +That last file is why this route is the easy one: it carries the reference links, the answering +instructions, and the warning about trusting AI answers, so **you can drag and drop all the files +in the folder into ChatGPT (or Claude) at once and go straight to Step 4 below**. +Steps 2 and 3 below exist for the command-line route, which produces the grammar file alone. + +This export is off by default. To turn it on, set the environment variable `FLEX_AI_EXPORT` to `1` +**before** starting FieldWorks — it only sees the environment it was launched with, so setting the +variable while FLEx is running changes nothing. To set it for your Windows account permanently, +paste this into PowerShell: + +```powershell +[Environment]::SetEnvironmentVariable('FLEX_AI_EXPORT', '1', 'User') +``` + +Then start FieldWorks again. If the option still isn't in the Export list, your FieldWorks version +predates it — use the `GenerateHCConfig.exe` steps above instead. + +## Step 2 — Copy the XML into ChatGPT or Claude + +Open the exported XML file, copy its contents, and paste them into your chat with ChatGPT +or Claude as the first message (or attach the file directly if your chat supports file +uploads — for a large grammar this is more reliable than pasting inline). + +## Step 3 — Point it to the HermitCrab reference + +Paste this URL into the same chat: + +``` +https://raw.githubusercontent.com/sillsdev/machine/master/docs/ai-parser-help/README.md +``` + +This tells the LLM where to find the (non-proprietary) documentation of how the HermitCrab +engine itself works — rule ordering, affix templates, features, correctness pitfalls, complexity +pitfalls, and grammar-authoring methodology — so it can reason about *your* grammar against the +actual engine mechanics rather than guessing. If you already know your question is about +wrongness, speed, or modeling approach specifically, you can paste the more specific +`broken/README.md`, `speed/README.md`, or `workflow/README.md` URL instead. + +## Step 4 — Ask your question + +Some examples of what to ask, once both your grammar and the reference URL are in the chat: + +- "Why does the word `` fail to parse / parse with the wrong analysis?" +- "Why is parsing this word so slow? Is there a combinatorial-explosion pattern in my + affix templates like the optional-slot one described in the reference?" +- "I have a slot with N optional prefixes, most of them null — is there a better way to + model this in my grammar?" +- "How should I model a class of nouns whose plural is irregular, given HermitCrab's + mechanisms?" +- "Walk through how stratum `` would apply to the stem `
`." + +The reference asks the LLM to answer in plain language, to define any technical term it uses, and +to say plainly when it is guessing. If an answer still reads like engine documentation, ask for it +again: "explain that without the technical terms, and tell me what to change in FLEx." + +## A note on trusting the answers + +This is AI. It can be very wrong, and it can be wrong while sounding certain. Everything it says +about your grammar is a hypothesis to test, not a finding: understand the change it proposes, +re-parse the words you care about, and confirm the result yourself before you keep it. + +## A note on privacy + +Your grammar file is real linguistic data about a real language project. Pasting it into a +third-party chat service sends that data to that provider (OpenAI, Anthropic, etc.). Check +your project's data-sensitivity policy before sharing an unpublished or restricted grammar +this way — this is a separate concern from (and in addition to) the fact that real grammars +must never be committed to this repository. diff --git a/Docs/ai-parser-help/speed/README.md b/Docs/ai-parser-help/speed/README.md new file mode 100644 index 0000000000..23dcc7a001 --- /dev/null +++ b/Docs/ai-parser-help/speed/README.md @@ -0,0 +1,47 @@ +# HermitCrab performance gotchas + +Part of the [AI Parser Help reference](../README.md). One file per gotcha — each is +self-contained, cites the specific `machine` source file(s) that implement the relevant +algorithm, and gives a fix. Every gotcha here is grounded in reading the actual HermitCrab +engine source (`src/SIL.Machine.Morphology.HermitCrab/` and `src/SIL.Machine/`) — none of it +depends on any other repo or on this documentation project's own history. + +Each file starts with a metadata header: + +```yaml +--- +title: +implements: +category: +cost: +grammar_visible: +--- +``` + +## Index + +| Gotcha | Category | Cost | +|---|---|---| +| [Independently-optional affix-template slots cause exponential blowup](affix-template-optional-slots.md) | morphotactics | `O(2^n)` in slot count | +| [Unordered strata are combinatorial; linear only fixes generation, not parsing](stratum-rule-ordering.md) | morphotactics | `O(n)` gen / `O(2^n)`-ish parse (linear); `O(n!)` both ways (unordered) | +| [Kitchen-sink natural classes widen a rule's environment, not just its own cost](natural-class-feature-widening.md) | feature-system | cheap unification, widened match set downstream | +| [Environment-conditioned allomorphs aren't short-circuited until a surface form exists](disjunctive-allomorph-deferred-recheck.md) | allomorphy | multiplicative per environment-constrained allomorph | +| [MPR features and co-occurrence rules filter late, they don't prune early](mpr-cooccurrence-late-filter.md) | morphotactics | cheap per check, but runs after the full candidate is built | +| [MPR feature groups set to Overwrite are order-dependent, not accumulating](mpr-overwrite-order-dependence.md) | feature-system | blocks collapsing order-equivalent candidate derivations | +| [Iterative phonological rules rescan mutated output; simultaneous rules don't](phonological-simultaneous-vs-iterative.md) | phonology | `O(#matches)` either way; the difference is what a rule can see | +| [Iterative epenthesis can crash the engine; metathesis has no numeric backstop at all](epenthesis-metathesis-self-feeding-crash.md) | phonology | hard 256-node cap (epenthesis); unbounded (metathesis) | +| [Compounding enumerates every split point, and only the head re-derives](compounding-split-point-enumeration.md) | compounding | scales with word length × stem count | +| [Pattern-shaped root allomorphs bypass the trie and pay a linear-scan cost](root-allomorph-trie-vs-pattern.md) | lexicon | `O(word length)` trie vs. `O(#pattern allomorphs)` linear scan | + +## How to use this with an LLM + +Paste the raw URL of the specific gotcha file that matches your symptom, e.g.: + +``` +https://raw.githubusercontent.com/sillsdev/machine/master/docs/ai-parser-help/speed/affix-template-optional-slots.md +``` + +If you're not sure which one applies, paste this index's raw URL first and describe your +grammar's structure (not its actual rules — see the privacy note in the top-level +[`README.md`](../README.md)) and symptom (e.g. "parsing is slow," "I get way more analyses than +expected," "the engine crashes on one word"). diff --git a/Docs/ai-parser-help/speed/affix-template-optional-slots.md b/Docs/ai-parser-help/speed/affix-template-optional-slots.md new file mode 100644 index 0000000000..ddfb90bd97 --- /dev/null +++ b/Docs/ai-parser-help/speed/affix-template-optional-slots.md @@ -0,0 +1,148 @@ +--- +title: Independently-optional affix-template slots cause exponential blowup +implements: src/SIL.Machine.Morphology.HermitCrab/AffixTemplateSlot.cs, src/SIL.Machine.Morphology.HermitCrab/SynthesisAffixTemplateRule.cs, src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs, src/SIL.Machine/Rules/RuleBatch.cs +category: morphotactics +cost: O(2^n) in the number of independently-optional slots +grammar_visible: yes +--- + +## What it is + +An affix template models a word's inflectional morphology (e.g. a noun's number and case) as +an ordered sequence of "slots," each slot holding the rule(s) that can fill that position. If a +grammar models several mutually-exclusive realizations of the same paradigmatic position (say, +ten cases crossed with singular/plural, several of them null) as separate, independently +optional slots rather than one slot with several alternatives, parse time blows up +exponentially in the slot count. + +## The data model + +```csharp +// AffixTemplateSlot.cs +public class AffixTemplateSlot +{ + // a slot holds a LIST of rules — these are tried as alternatives, not composed + public ReadOnlyCollection Rules { get; } + + // if true, the slot may be skipped entirely (no rule in it fires) + public bool Optional { get; set; } +} +``` + +An `AffixTemplate` is just an ordered list of these slots, applied to a stem in sequence. + +## The traversal algorithm + +```csharp +// SynthesisAffixTemplateRule.cs +_rules = template.Slots.Select(slot => new RuleBatch( + slot.Rules.Select(mr => mr.CompileSynthesisRule(morpher)), + false, // disjunctive = false: try ALL rules in the slot, union their outputs + FreezableEqualityComparer.Default +)); + +private void ApplySlots(Word input, int index, HashSet output) +{ + for (int i = index; i < _rules.Count; i++) + { + foreach (Word outWord in _rules[i].Apply(input)) // batch = all rules in slot i + ApplySlots(outWord, i + 1, output); // recurse per surviving output + + if (!_template.Slots[i].Optional) + return; // mandatory slot: stop here either way + + // slot IS optional: loop continues to slot i+1 using the ORIGINAL `input`, + // i.e. "skip this slot" is tried as a separate path in addition to "apply it" + } + output.Add(input); +} +``` + +And the batch itself: + +```csharp +// RuleBatch.cs +public virtual IEnumerable Apply(TData input) +{ + var output = new HashSet(_comparer); + foreach (var rule in _rules) + { + output.UnionWith(rule.Apply(input)); + if (_disjunctive && output.Count > 0) + return output; // (not our case — slots use disjunctive: false) + } + return output; // unions ALL matching rules' outputs +} +``` + +Two facts fall out of this: + +1. **Across slots**, the recursion is per-slot, not per-rule: `ApplySlots` recurses once for + each slot's *combined* batch output, not once per individual rule inside a slot. +2. **Within a slot**, every rule in the slot is tried and *all* that structurally unify + contribute an output (union, not first-match). So a slot with `k` mutually-exclusive rules + costs `O(k)` — not `O(2^k)` — *provided* the rules' required feature structures are actually + mutually exclusive, so normally only one (or a few, for genuine ambiguity) unifies per + analysis. + +## Pattern A: one slot per affix, each optional — exponential + +Model 20 independent prefix positions (ten grammatical cases × singular/plural, several +realized as null) as 20 slots, each `optional="true"`, each holding one rule. Every optional +slot independently contributes an "applied" branch and a "skipped" branch, because the +recursion has no way to know that a null-realized rule firing looks identical, on the surface, +to the slot never having fired. Cost for a single stem: **O(2ⁿ)** — for n = 20, roughly 10⁶ +candidate paths, before any other rule interactions. A minimal worked example: 12 independent +optional slots that each insert the same single character produces `C(12,k)` equally-valid, +byte-identical-surface analyses for a form with `k` of those characters present — `C(12,6) = +924` for the midpoint case — even though nothing in the surface string distinguishes which +subset of slots actually fired. + +## Pattern B: one slot, many mutually-exclusive rules, non-optional — linear + +The fix is to recognize that the 20 cells are not 20 independent binary toggles — they are +**one paradigmatic position** (a "number+case prefix") with 20 alternative realizations, exactly +one of which applies to any given noun. Model that as: + +- **One** `AffixTemplateSlot`, containing all 20 `MorphologicalRule`s. +- Each rule gated by a required inflectional feature structure (e.g. `num=sg, case=erg` vs + `num=pl, case=abs`, ...) so the rules' domains are pairwise disjoint. +- The slot marked **non-optional** — the noun always has *some* number+case value, it's never + truly absent, it's just sometimes spelled with zero segments. Marking it optional adds a + spurious extra "skip" branch for a cell that linguistically can't actually be skipped. + +Toy example (invented `casA`/`casB`/`casC` × `sg`/`pl`, not any real language) — six +mutually-exclusive rules in one slot instead of six optional slots: + +```xml + + numberCasePrefix + + numCase + + +``` + +where e.g. `mrPlB` requires `[num:pl case:casB]` and its subrule's output is simply a copy of +the input with no inserted segments (the null realization) — the rule still "fires" and tags +the word with `[num:pl case:casB]`, it just doesn't change the string. + +With rules' feature requirements pairwise disjoint, at most one of the six unifies per analysis +→ **O(6)** for that slot, not `O(2^6)`. + +## If some null cells are genuinely homophonous + +Suppose 3 of the 20 cells really do share an identical null exponent and are truly +indistinguishable on the surface (not just under-specified features — actually ambiguous). +Analysis of such a word correctly yields 3 candidate parses, each tagged with a different +`num`/`case` feature bundle, all with identical surface text. That is **O(3)** residual +ambiguity at that node — real linguistic ambiguity, not a performance bug. It should be resolved +(if at all) by agreement elsewhere in the clause (verb/adjective concord, syntax-level +unification), not by trying to make the morphological parser guess. + +## Fix + +Collapse independently-optional slots that are really alternative fillers of one paradigmatic +position into a single non-optional slot holding all the alternatives, each gated by a distinct, +mutually-exclusive required feature structure. This changes the asymptotic behavior (`O(2ⁿ)` → +`O(n)`), not just the constant factor. diff --git a/Docs/ai-parser-help/speed/compounding-split-point-enumeration.md b/Docs/ai-parser-help/speed/compounding-split-point-enumeration.md new file mode 100644 index 0000000000..e8c243deab --- /dev/null +++ b/Docs/ai-parser-help/speed/compounding-split-point-enumeration.md @@ -0,0 +1,79 @@ +--- +title: "Compounding enumerates every split point, and only the head re-derives" +implements: src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/CompoundingRule.cs, src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingRule.cs, src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisCompoundingRule.cs, src/SIL.Machine.Morphology.HermitCrab/Morpher.cs +category: compounding +cost: "scales with word length x stem count on the analysis side" +grammar_visible: "partially — MaxStemCount is a host-application setting, not grammar XML" +--- + +## What it is + +A compounding rule combines a **head** word with a **non-head** word into one compound. +Analysis-direction compounding doesn't guess where a compound's head/non-head boundary is — it +tries every position a compounding subrule's head/non-head patterns can match against the +observed shape, at every stratum where a compounding rule is declared, and only then filters by +whether the resulting non-head substring actually resolves to a real lexical root. + +## Overview of the construct + +A compounding rule carries separate required-feature-structures for each side (head vs. +non-head), separate MPR-feature productivity restrictions per side, a maximum application count +defaulting to **1** (a compounding rule fires at most once per word by default), and is +blockable by default (so a more specific compound can block a more general one, the same +mechanism used by ordinary affix rules). + +## How the two combinatorics-limiting choices show up in the analysis-side code + +Splitting an unknown surface string into head+non-head has no a-priori split point, so the +analysis-side compounding rule compiles each subrule's combined head+non-head pattern to +enumerate every split point consistent with the pattern (anchored to both the start and end of +the input), not just one. Two explicit guards bound how far this can grow: the rule bails out if +the word already has as many stems as the host-configured stem cap allows, or if it's already +been applied as many times as its own maximum-application-count permits, or if its output +features aren't unifiable with the input's — checked before the expensive split-search work. + +The stem cap (`Morpher.MaxStemCount`) defaults to **2** — by default a word may contain at most a +head and one non-head stem; deeper nominal/verbal compounding (three or more stems) requires +explicitly raising this host-application setting, which re-opens combinatorial cost the default +exists to cap. + +## The more important, and less obvious, limit + +Analysis-direction compounding restricts each candidate non-head to a **bare lexical root** — it +looks up root allomorphs directly via a trie search, not a fully re-derived (affixed) sub-word. +The engine's own source comment states this is a deliberate complexity decision: + +> for computational complexity reasons, we ensure that the non-head is a root, otherwise we +> assume it is not a valid analysis and throw it away + +This means compounding's combinatorics are **not** symmetric — it is not +`parses(stem1) × parses(stem2)` recursively for both sides. The **head** genuinely re-enters the +normal stratum/affix-template pipeline (it's the same word continuing its ordinary derivation, so +it can carry its own inflectional morphology), but the **non-head** is capped to whatever root +allomorphs literally match its shape via the trie — no recursive re-affixing of the non-head is +attempted at all. Same-shape/same-allomorph candidates are also explicitly deduplicated before +continuing, purely to reduce the search space. + +## Gotcha and fix + +For a grammar with a large compounding lexicon and permissive head/non-head patterns (broad +optional-segment spans rather than tightly anchored shapes), the number of split points examined +before the root-lookup filter prunes them still scales with word length — and if the stem cap is +raised above its default of 2 to permit N-ary compounds, each additional stem multiplies the +number of split combinations considered, since every accepted binary split becomes a new input +that the same rule is tried against again on the next outer application (bounded by the rule's +own maximum-application-count). + +Grammars that need genuinely recursive compounding (a compound whose non-head is itself a fully +affixed compound or derived stem) run up against the root-only restriction on the analysis side +by design — raising the stem cap widens how many *flat* stems a word can contain, it does not +enable recursive non-head derivation. If deep recursive compounding is linguistically required, +expect that HC's analysis-side compounding is structured to avoid it, and budget for handling +deeply nested compounds outside the compounding mechanism (e.g. as separate lexicalized entries) +rather than assuming the grammar will discover arbitrary-depth nesting on its own. Within the +grammar itself, prefer head/non-head patterns anchored as tightly as the language allows +(fixed-length or feature-narrow contexts) over broad optional-segment spans, since a tighter +pattern reduces the number of positions the split-search has to enumerate before the root-lookup +filter gets a chance to prune them. Keep the host-configured stem cap as low as the language's +actual compounding depth requires — raising it beyond what's linguistically needed directly +multiplies the split-point search. diff --git a/Docs/ai-parser-help/speed/disjunctive-allomorph-deferred-recheck.md b/Docs/ai-parser-help/speed/disjunctive-allomorph-deferred-recheck.md new file mode 100644 index 0000000000..e16b8202ce --- /dev/null +++ b/Docs/ai-parser-help/speed/disjunctive-allomorph-deferred-recheck.md @@ -0,0 +1,82 @@ +--- +title: "Environment-conditioned allomorphs aren't short-circuited until a surface form exists" +implements: src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisAffixProcessRule.cs, src/SIL.Machine.Morphology.HermitCrab/Allomorph.cs, src/SIL.Machine.Morphology.HermitCrab/Word.cs +category: allomorphy +cost: "multiplicative per environment-constrained allomorph on a rule; stacks across derivation steps" +grammar_visible: "no — invisible from the grammar's own XML" +--- + +## What it is + +A morpheme with several allomorphs, where more than one allomorph is environment-constrained +(not the unconstrained "elsewhere" case), causes the *synthesis* search to carry more live +candidate words forward than the final grammar will actually accept — the rejection happens +later, in a second full pass, not by narrowing the search up front. + +## The engine mechanism + +A morphological rule's subrules become allomorphs, each tried in listed order. The loop over a +rule's allomorphs only `break`s out — stopping earlier alternatives from also firing — when +three conditions all hold: the allomorph isn't the last one, it does *not* free-fluctuate with +the next allomorph, it has *zero* environments, and it has an empty required syntactic feature +structure. An environment-constrained allomorph fails the "zero environments" condition, so the +loop **does not break** — every later allomorph is also tried and also contributes an output +candidate. The engine's own source comment states the reasoning directly: + +> return all word syntheses that match subrules that are constrained by environments, HC +> violates the disjunctive property of allomorphs here because it cannot check the +> environmental constraints until it has a surface form, we will enforce the disjunctive +> property of allomorphs at that time + +The actual disjunctive filtering — "did an earlier-indexed allomorph's environment also match +here, in which case this later one shouldn't have been used" — happens only once the word's full +surface form exists, in a final-validity check: it walks the set of allomorphs that were "passed +over" during synthesis and rejects the candidate if any earlier, non-free-fluctuating, +environment-satisfied allomorph should have won instead. The escape hatch: when two allomorphs' +constraint sets compare exactly equal, the loop treats them as free variation rather than +disjunctive alternatives, and neither is retroactively rejected. + +## Why this matters for performance + +For a morpheme with `k` allomorphs where `m` of the first `k-1` are environment-constrained (not +the elsewhere case), synthesis carries up to `m+1` live candidate words forward from that single +rule application — each one fully expanded through every subsequent affix template slot and +phonological rule — before the bulk of them are pruned back out at final validity. This cost is +not exponential by itself, but it **stacks multiplicatively** with every other rule application +downstream of it in the same derivation, and it is invisible from the grammar alone: nothing in +the XML says "this environment gate delays rejection instead of preventing generation." + +## Rule order is also semantically load-bearing here + +Each rule's required syntactic features must unify against whatever the *previously applied* +rules left in the accumulated feature structure, and its own output features feed forward for +the next rule to require against. So within a linearly-ordered stratum, a rule that requires a +feature another rule sets must be listed after it — a correctness dependency the engine has a +separate mechanism to catch (a word where a promised feature was never actually set by the end +of the derivation is flagged as invalid). + +## Toy example + +A rule with several phonologically-conditioned allomorphs (e.g. "insert `-i` after a consonant, +`-ni` after a vowel, elsewhere `-mi`") looks disjunctive to a grammar author — exactly one +allomorph *should* apply to any given stem — but is not treated as disjunctive internally until +each candidate surface form is checked against its environment later. If several such rules +stack across a derivation, the independently-propagated candidates multiply. + +## Fix + +This is largely an engine cost a grammar author cannot design around directly — unlike the +affix-template case, there's no "collapse into one slot" fix, because environment-conditioned +allomorphy inherently can't be resolved until synthesis produces a surface string. What *is* +actionable: + +- Minimize the number of allomorphs per morpheme that carry a genuine environment constraint, as + opposed to using a broader natural class match that could instead be pushed into a + phonological rule later in the derivation. +- Put the unconstrained "elsewhere" allomorph last, which keeps the passed-over set small per + application. +- Add MPR-feature gates (`RequiredMprFeatures`/`ExcludedMprFeatures`) to environment-conditioned + allomorphs where the choice can be made from a cheap tag lookup instead of a phonological + environment — MPR-feature gates are checked first and skip non-matching allomorphs cheaply + before the environment/pattern match is even attempted, so adding them turns wasted + pattern-matching work into a fast `continue`, not an added cost. diff --git a/Docs/ai-parser-help/speed/epenthesis-metathesis-self-feeding-crash.md b/Docs/ai-parser-help/speed/epenthesis-metathesis-self-feeding-crash.md new file mode 100644 index 0000000000..11d7852965 --- /dev/null +++ b/Docs/ai-parser-help/speed/epenthesis-metathesis-self-feeding-crash.md @@ -0,0 +1,80 @@ +--- +title: "Iterative epenthesis can crash the engine; metathesis has no numeric backstop at all" +implements: src/SIL.Machine.Morphology.HermitCrab/PhonologicalRules/EpenthesisSynthesisRewriteSubruleSpec.cs, src/SIL.Machine.Morphology.HermitCrab/PhonologicalRules/IterativePhonologicalPatternRule.cs, src/SIL.Machine.Morphology.HermitCrab/InfiniteLoopException.cs, src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisMetathesisRule.cs, src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisMetathesisRuleSpec.cs, src/SIL.Machine.Morphology.HermitCrab.Tool/SignatureFormat.cs +category: phonology +cost: "runs to a hard 256-shape-node cap, then throws (epenthesis); unbounded, no cap at all (metathesis)" +grammar_visible: "partially — visible only if you reason about whether a rule's own output can re-satisfy its own trigger" +--- + +## What it is + +An iterative-mode epenthesis rule (segment insertion with nothing consumed on the input side) +that inserts material matching its own trigger environment can re-match the segment it just +inserted on its very next scan iteration, cascading until a hardcoded safety cap fires and the +engine throws. Metathesis (swapping two matched spans) has an equivalent self-feeding risk but +**no such cap at all** — it can hang rather than crash. + +## The engine mechanism: epenthesis + +Epenthesis insertion is handled by a dedicated rewrite subrule spec that inserts the new +segment(s) directly into the word's shape. Beyond the Clean/Dirty guard that protects a rule +from re-matching a node it just touched within one rule application (see the +simultaneous-vs-iterative gotcha), there is one additional hard backstop — the **only** throw +site of `InfiniteLoopException` in the entire HermitCrab source: a cap of 256 total shape nodes +on the word. This fires if repeated epenthesis (across iterations of the enclosing iterative +loop, or via chains where each epenthesized segment satisfies a *different* subrule/rule than +the one that just inserted it, sidestepping the per-rule Clean/Dirty check) keeps growing the +word indefinitely. **There is no configurable override for this cap.** + +A minimal worked example: a rule that inserts a high-front-unrounded vowel after *any* high +vowel, with no right-environment restriction, applied to a word already containing high vowels. +Under simultaneous application this converges in one pass (every match is computed against the +original string). The same rule under iterative application risks each newly-inserted vowel +re-creating a new "after a high vowel" context for the next pass — exactly the case the +Clean/Dirty guard and the 256-node cap exist for. + +## The engine mechanism: metathesis + +Metathesis has no application-mode choice of its own — it always runs through the iterative +pattern-rule machinery, so it is always scanned iterative-style. The metathesis rule spec swaps +the two captured spans and marks every moved segment `Dirty`, and its switch-group left-hand-side +constraints are likewise cloned with a `Clean` requirement — so a rule that swaps segments A and +B cannot immediately re-match the now-`Dirty` result to swap them back within the same +application. **Unlike epenthesis, metathesis has no numeric backstop** — there is no +`InfiniteLoopException` throw anywhere in the metathesis rule classes. Oscillation prevention for +metathesis relies entirely on the Clean/Dirty mechanism; there is no cap analogous to +epenthesis's 256-node ceiling to fall back on if a grammar somehow produces a metathesis +environment the Dirty tagging doesn't cover. + +## Practical takeaway + +If a metathesis rule appears to hang rather than throw `InfiniteLoopException`, that specific +error is not the applicable diagnostic (it's epenthesis-only) — look instead at whether the +rule's environment can be satisfied again by the result of its own swap, since the only +protection there is the one-application-scoped `Dirty` tag. + +## Operational corollary: one crashing word aborts the whole batch + +This is not a parse-time complexity gotcha but a direct operational consequence of the crash +above, worth knowing before it surprises you in a batch run. The tool-level per-word parsing +entry point wraps only a single catch for the "invalid shape" exception around a word parse — +its own doc comment states explicitly that any other exception propagates, since that's treated +as a genuine engine crash, not a normal per-word outcome. `InfiniteLoopException` is exactly such +an exception. Practically: if a word list or a batch parse run hits one pathological word that +trips the 256-node epenthesis cap (or any other engine-level crash), that exception unwinds out +of the per-word handling and terminates processing of every *subsequent* word in the same run — +it does not get recorded as a per-word failure and skipped. If you're batch-parsing a large +corpus and the run stops partway with no per-word error for the remaining words, suspect a crash +on the word where it actually stopped, not silent success on everything after. + +## Fix + +For any epenthesis (insertion-only) rewrite rule whose inserted segment's own feature bundle +could satisfy the rule's own left/right environment, prefer simultaneous application over the +default iterative mode — simultaneous mode's "collect all matches against the original input, +then apply them all" semantics is structurally immune to this specific self-feeding cascade. If +iterative mode is required for some other reason (e.g. a rule that genuinely needs to see its own +prior applications to converge on a correct output), make sure the environment can never be +re-satisfied by the rule's own insertion — e.g. by writing the environment to require a feature +the inserted segment doesn't carry. For metathesis, since no numeric cap exists at all, this +verification matters even more: a self-feeding metathesis rule has no automatic backstop. diff --git a/Docs/ai-parser-help/speed/mpr-cooccurrence-late-filter.md b/Docs/ai-parser-help/speed/mpr-cooccurrence-late-filter.md new file mode 100644 index 0000000000..d859381f59 --- /dev/null +++ b/Docs/ai-parser-help/speed/mpr-cooccurrence-late-filter.md @@ -0,0 +1,79 @@ +--- +title: "MPR features and co-occurrence rules filter late, they don't prune early" +implements: src/SIL.Machine.Morphology.HermitCrab/MprFeature.cs, src/SIL.Machine.Morphology.HermitCrab/MprFeatureGroup.cs, src/SIL.Machine.Morphology.HermitCrab/MprFeatureSet.cs, src/SIL.Machine.Morphology.HermitCrab/MorphCoOccurrenceRule.cs, src/SIL.Machine.Morphology.HermitCrab/ConstraintType.cs, src/SIL.Machine.Morphology.HermitCrab/Allomorph.cs, src/SIL.Machine.Morphology.HermitCrab/Morpher.cs, src/SIL.Machine.Morphology.HermitCrab/AllomorphEnvironment.cs +category: morphotactics +cost: "each check is cheap in isolation (O(features) or O(word length)); the gotcha is when it runs, not how much it costs" +grammar_visible: "partially — the rules are visible, their late timing is not" +--- + +## What it is + +MPR features are boolean tags (e.g. a noun-class or conjugation-class agreement tag) attached to +a lexical entry and referenced from rule/allomorph gates. Morpheme/allomorph co-occurrence rules +are a separate mechanism for "morpheme X requires (or excludes) morpheme(s) Y elsewhere in the +word," with a positional-adjacency setting (anywhere / somewhere-to-left / somewhere-to-right / +adjacent-to-left / adjacent-to-right). Both mechanisms are individually cheap to evaluate — but +both are evaluated **after** a complete candidate word already exists, not as an early prune +during the search. + +## Cost mechanics + +MPR-feature matching (required/excluded checks, grouped by feature group with any/all matching +semantics) is a hash-set-style lookup — essentially free per allomorph. Co-occurrence-rule +checking takes the word's full morpheme list and does one linear scan of that list per rule +check — `O(m)` where `m` is the number of morphemes already in the word, not the size of the +grammar. Total co-occurrence cost for a candidate word is `O(rules-on-its-allomorphs × +word-length)` — cheap in isolation. + +## The real gotcha: these are a late filter, not an early prune + +Co-occurrence rules and allomorph environment checks are invoked from a word-validity check that +is only run as a final `.Where(...)` filter *after* the entire synthesis rule cascade has already +produced a complete candidate word. So a grammar author who adds obligatory co-occurrence rules +expecting them to *cut down* the combinatorial fan-out from templates, strata, or allomorph +disjunction will not see that benefit — the full candidate set is generated first, at whatever +cost the upstream combinatorics already impose, and co-occurrence rules only reject invalid +members of that already-built set. If a grammar is slow because of combinatorial +rule/template/allomorph interaction, adding co-occurrence constraints will fix *correctness* +(spurious analyses disappearing) but will not by itself fix *performance* — the fix for +performance has to happen at the source of the combinatorics (see the affix-template, +stratum-ordering, and disjunctive-allomorph gotchas). The same applies to root-allomorph +environment constraints: a broad or underspecified environment doesn't slow down unification per +se, it just means more candidates survive to be checked later — and if the environment is *too* +narrow or wrong, legitimate words silently fail to parse/generate, surfaced only through a +trace-manager failure reason rather than an upfront rejection. + +## Toy example + +A fictional toy language: a suffix rule excludes lexical entries tagged with an MPR feature. + +```xml + + Exception + +... + + ... + ...an + +... + + vokad + ... + +``` + +`eVokad`'s tag means the `-an` subrule's exclusion check fails for it — this root simply doesn't +take that suffix, an irregular-exception pattern, cheaply gated (a single tag lookup), +independent of the rest of the grammar's size. This mechanism is a fine, cheap way to encode +irregular exceptions; the gotcha above is specifically about expecting it to also bound search +cost, which it does not. + +## Fix + +Use MPR features and co-occurrence rules for what they're good at — cheaply encoding +correctness constraints and irregular exceptions on an already-generated candidate. To actually +reduce search cost, address the combinatorics at their source: collapse independently-optional +affix-template slots that are really one paradigmatic position, keep morphological-rule ordering +`linear` where possible, and minimize environment-conditioned (as opposed to MPR-feature-gated) +allomorph disjunction. diff --git a/Docs/ai-parser-help/speed/mpr-overwrite-order-dependence.md b/Docs/ai-parser-help/speed/mpr-overwrite-order-dependence.md new file mode 100644 index 0000000000..b67bc6bd51 --- /dev/null +++ b/Docs/ai-parser-help/speed/mpr-overwrite-order-dependence.md @@ -0,0 +1,59 @@ +--- +title: "MPR feature groups set to Overwrite are order-dependent, not accumulating" +implements: src/SIL.Machine.Morphology.HermitCrab/MprFeatureSet.cs, src/SIL.Machine.Morphology.HermitCrab/MprFeatureGroup.cs, src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisAffixProcessAllomorphRuleSpec.cs, src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisCompoundingRule.cs +category: feature-system +cost: "not exponential by itself, but blocks collapsing otherwise-equivalent candidate derivations that differ only in rule order" +grammar_visible: "no — reads like a plain accumulating tag set from the XML alone" +--- + +## What it is + +An MPR feature group can be declared with output policy `Overwrite` instead of the default +`Append`. Where `Append` behaves like an ordinary accumulating tag set (every rule's output MPR +features just get added to the word's running set), `Overwrite` does not accumulate — it makes +the word's final state for that group depend on which rule touched it *last*, silently dropping +any earlier rule's contribution to that same group. + +## The mechanism + +Adding a rule's output MPR features to a word's accumulated set is implemented as: for every +feature group the new output touches that is set to `Overwrite`, first remove every existing +member of that group from the word's current set unless the new output restates it, and only +then union in the new output. Concretely — if rule A sets group G to `{x}`, and a later rule B +in the same derivation sets group G to `{y}`, the word's final state for G is `{y}` alone; `x` is +gone, not merged. This is a genuinely order-dependent, non-monotone update, not an accumulation — +and it's called once per rule application (both ordinary affix-process allomorphs and +compounding subrules feed their output MPR features through the same mechanism), in whatever +order those rules actually fire during a derivation. + +## Why this is easy to miss, and why it matters beyond correctness + +It's easy to assume MPR features behave uniformly across a grammar — like plain boolean tags that +simply accumulate as rules apply, the way `RequiredMprFeatures`/`ExcludedMprFeatures` checks treat +them everywhere else. An `Overwrite` group breaks that assumption silently: nothing in a rule's +own XML declaration says "this group resets instead of accumulates" — that's a property of the +*group*, declared once, potentially far away from any of the rules that touch it. + +This also has a subtler consequence for the engine's search, not just for grammar-author +intuition. HC's search can legitimately produce multiple candidate derivations that apply the +*same set* of rules in a *different relative order* — for example, an unordered stratum's +rule-combination search, or two rules that are each independently eligible to fire in either +order relative to each other. Two such derivations would ordinarily be collapsible, or at least +treated as equivalent in their morphosyntactic outcome, if every feature mechanism accumulated +monotonically. But because an `Overwrite` group's final content depends on *which rule touched it +last*, two derivations that apply the same rules in different orders can end up with genuinely +different final MPR-feature states for that group — meaning the engine cannot treat them as +interchangeable. Downstream checks gated on that group (required/excluded MPR-feature matching, +or a compounding rule's MPR-feature productivity restriction) can then behave differently for +what looks, from the morpheme sequence alone, like "the same word." + +## Fix + +Prefer `Append` for MPR feature groups unless "last rule wins" is specifically the intended +semantics (e.g. a group modeling a paradigm cell that later derivation steps are meant to +override outright, such as a valence-changing operation resetting an argument-structure tag). If +`Overwrite` is required, keep the rules that touch that group in a fixed relative order — e.g. by +keeping the containing stratum's morphological-rule order `linear` with those rules ordered +correctly relative to each other — so the non-monotone behavior can't silently vary between +logically-equivalent derivations, and so a grammar author reasoning about the group's final state +only has to reason about one order, not every order the search might explore. diff --git a/Docs/ai-parser-help/speed/natural-class-feature-widening.md b/Docs/ai-parser-help/speed/natural-class-feature-widening.md new file mode 100644 index 0000000000..517655b5b1 --- /dev/null +++ b/Docs/ai-parser-help/speed/natural-class-feature-widening.md @@ -0,0 +1,55 @@ +--- +title: "Kitchen-sink natural classes widen a rule's environment, not just its own cost" +implements: src/SIL.Machine/FeatureModel/FeatureStruct.cs, src/SIL.Machine/FeatureModel/SymbolicFeatureValue.cs, src/SIL.Machine.Morphology.HermitCrab/SegmentNaturalClass.cs, src/SIL.Machine.Morphology.HermitCrab/SimpleContext.cs, src/SIL.Machine.Morphology.HermitCrab/NaturalClass.cs +category: feature-system +cost: "unification itself is cheap (near-constant in feature count); the real cost is a widened match set downstream" +grammar_visible: yes +--- + +## What it is + +A natural class (e.g. "vowels", "voiceless obstruents") is a named feature structure used as a +matching constraint in rule environments. `SegmentNaturalClass` builds its feature structure by +unioning every member segment's own feature structure; `SimpleContext` wraps a natural class's +feature structure (plus any pattern variables) as the actual constraint object tested against +each string position during phonological pattern matching. + +## Where the cost is, and where it isn't + +Unification recurses over each feature structure's feature dictionary, following nested complex +features and re-entrancies through a visited-node map that also acts as a cycle guard — cost is +proportional to the number of distinct features involved (typically tens), not exponential in +disjunction size. Within one feature, a symbolic value's disjunctive value-set is backed by a +bitset, and intersect/union/overlap checks are bitwise ops on that flag set. **Unifying two +symbolic values, even with large disjunctive sets, is cheap** — this is not the mechanism that +makes broad natural classes slow. + +## The actual gotcha: how a natural class is built + +The feature-structure union used to build a class from its member segments only keeps a feature +key that's present in **both** operands, narrowing the surviving value via bitset union where a +key is kept. So a natural class spanning segments that don't share many features (e.g. +"everything that isn't a vowel," lumping bilabial stops in with sibilants and laterals) +converges toward an **emptier** feature structure as more dissimilar segments are added — and an +empty feature structure renders (and behaves) as unconstrained: HC's own feature-structure +stringifier literally prints an empty structure as `"ANY"`. A pattern context built from that +natural class then matches at essentially every string position, because the constraint it wraps +has lost the features that would have restricted it. + +## Gotcha, concretely + +Defining a kitchen-sink natural class (e.g. lumping every consonant in the inventory into one +`AnyC` class instead of the specific place/manner classes a rule's environment actually needs) +does not make *unification* slow — it makes the *rule* less selective, so its environment +matches far more positions than intended, and every one of those spurious matches is a candidate +the rest of the engine (rule ordering, template slots, other phonological rules downstream) now +has to process. The blowup shows up as more candidate match/rewrite sites explored, not as +expensive feature-structure comparisons. + +## Fix + +Define natural classes at the granularity the rule's environment actually needs — a class stated +directly in terms of the relevant features is usually more precise and more legible than one +built by listing many segments and hoping their union keeps the right features — and check what +feature structure a segment-listed natural class actually reduces to when new segments are added +to it. diff --git a/Docs/ai-parser-help/speed/phonological-simultaneous-vs-iterative.md b/Docs/ai-parser-help/speed/phonological-simultaneous-vs-iterative.md new file mode 100644 index 0000000000..84ba39b5fe --- /dev/null +++ b/Docs/ai-parser-help/speed/phonological-simultaneous-vs-iterative.md @@ -0,0 +1,53 @@ +--- +title: "Iterative phonological rules rescan mutated output; simultaneous rules don't" +implements: src/SIL.Machine.Morphology.HermitCrab/PhonologicalRules/RewriteRule.cs, src/SIL.Machine.Morphology.HermitCrab/PhonologicalRules/SimultaneousPhonologicalPatternRule.cs, src/SIL.Machine.Morphology.HermitCrab/PhonologicalRules/IterativePhonologicalPatternRule.cs, src/SIL.Machine.Morphology.HermitCrab/HCFeatureSystem.cs +category: phonology +cost: "O(#matches) either way in the ordinary case; the difference is what a rule can see, not how many times it runs" +grammar_visible: yes +--- + +## What it is + +A phonological rewrite rule has an application mode — simultaneous or iterative (the default) — +and the choice determines whether later matches in the same rule application can see the effects +of earlier ones. This is not just a style preference: it changes what inputs a rule's own +environment can match against, and it's the mechanism behind a self-feeding crash risk +documented separately (see the epenthesis/metathesis gotcha). + +A rewrite rule can have multiple subrules, each with its own left/right environment; like affix +template slots, all subrules are tried and each one that matches contributes an output — `O(k)` +for `k` subrules, not exponential, provided the subrules' environments are close to mutually +exclusive. + +## The algorithmic difference that matters + +- **Simultaneous** application collects **all** matches from the matcher against the original, + unmodified word first, then applies every matched subrule's output afterward in a second pass. + No match ever sees the effect of any other match from the same application — one pass, + `O(#matches)`, no rescanning. +- **Iterative** application (the default) is a `while` loop: match, apply (or advance past a + non-match), then re-match starting just past the just-consumed match **on the already-mutated + word**. Content the rule just inserted or modified is still ahead of the scan position and can + be matched again on the next loop iteration. + +## The guard against immediate self-reapplication + +An iterative rule whose own output re-satisfies its own trigger environment can reapply into +content it just produced. HermitCrab's guard against this within a single rule application is a +feature, not a counter: a `Modified` feature with values `Dirty`/`Clean` (default `Clean`). A +rule's own left-hand-side pattern is compiled with an added requirement that the target be +`Clean`, and every node a rule inserts or modifies is marked `Dirty` immediately — so the *same +rule* cannot immediately rematch a node it just touched, within one application. The `Dirty` +marks persist for the whole scan and are only reset once, at the very end of that application — +so this protection covers one full rule application, not across separate stratum/rule +applications, and it protects against the *same rule* re-triggering on its own output, not a +*different* rule doing so. + +## Fix / practical takeaway + +If a phonological pattern is meant to look at the *original* string only (a static assimilation +table, for instance) rather than a genuinely cascading process, prefer simultaneous application +— it's a strictly one-pass operation and cannot loop by construction. Reserve iterative +application for rules that are genuinely meant to rescan (e.g. iterative stress assignment +across a whole word), and be aware that its safety against self-reapplication depends entirely +on the Clean/Dirty tagging described above, not on anything the grammar author writes explicitly. diff --git a/Docs/ai-parser-help/speed/root-allomorph-trie-vs-pattern.md b/Docs/ai-parser-help/speed/root-allomorph-trie-vs-pattern.md new file mode 100644 index 0000000000..67b2e8bd05 --- /dev/null +++ b/Docs/ai-parser-help/speed/root-allomorph-trie-vs-pattern.md @@ -0,0 +1,46 @@ +--- +title: "Pattern-shaped root allomorphs bypass the trie and pay a linear-scan cost" +implements: src/SIL.Machine.Morphology.HermitCrab/LexEntry.cs, src/SIL.Machine.Morphology.HermitCrab/RootAllomorph.cs, src/SIL.Machine.Morphology.HermitCrab/RootAllomorphTrie.cs, src/SIL.Machine.Morphology.HermitCrab/Morpher.cs +category: lexicon +cost: "O(word length) via the trie for literal-shape allomorphs; O(number of pattern allomorphs) linear scan, per analysis attempt, for the rest" +grammar_visible: yes +--- + +## What it is + +A lexical entry holds one or more root allomorphs — suppletive or phonologically-conditioned +surface forms of the same morpheme. Root lookup is normally near-free regardless of lexicon +size, but any allomorph whose own phonetic shape is written with a broad, underspecified segment +sequence (rather than a literal string) falls out of that fast path entirely. + +## Trie-indexed lookup, and what defeats it + +The engine builds one root-allomorph trie per stratum from every root allomorph in that +stratum's lexicon. The trie is a segment-by-segment automaton built by chaining each allomorph's +literal shape into shared states keyed on exact feature-structure equality per node; searching it +transduces the input shape against it, giving lookup cost proportional to the word's length, not +the number of lexical entries. + +But **not every allomorph goes into the trie**: a root allomorph is marked as a "pattern" if any +node in its shape is iterative, or is an optional, non-boundary annotation — i.e. any root +declared with a wildcard-like shape (a segment sequence using an unbounded/optional quantifier +rather than literal segments) instead of a fixed literal string. Every such pattern allomorph is +routed into a separate flat list instead of the trie. Lexical lookup then searches that flat list +with an explicit linear loop over every entry in it, once per analysis attempt, **in addition to** +(not instead of) the ordinary trie lookup for literal entries. + +## Gotcha + +An allomorph whose own shape is written with a broad, underspecified segment sequence (e.g. +matching "any string of segments" to model a maximally general root shape, as opposed to the +root's actual literal string) does not get the trie's near-free lookup — every such allomorph is +checked against every analysis input via a separate linear scan. A grammar with many such +"pattern" roots (as opposed to a handful of genuinely templatic ones, e.g. reduplication bases) +pays that linear cost on every word analyzed, on top of the normal trie lookup for its literal +entries. + +## Fix + +Reserve pattern-shaped root allomorphs for cases that are genuinely templatic (true +reduplication/prosodic templates); give ordinary roots their literal phonemic shape so they land +in the trie instead of the linear-scan fallback. diff --git a/Docs/ai-parser-help/speed/stratum-rule-ordering.md b/Docs/ai-parser-help/speed/stratum-rule-ordering.md new file mode 100644 index 0000000000..4cf83c8932 --- /dev/null +++ b/Docs/ai-parser-help/speed/stratum-rule-ordering.md @@ -0,0 +1,75 @@ +--- +title: "Unordered strata are combinatorial; linear only fixes generation, not parsing" +implements: src/SIL.Machine.Morphology.HermitCrab/Stratum.cs, src/SIL.Machine.Morphology.HermitCrab/SynthesisStratumRule.cs, src/SIL.Machine.Morphology.HermitCrab/AnalysisStratumRule.cs, src/SIL.Machine/Rules/LinearRuleCascade.cs, src/SIL.Machine/Rules/CombinationRuleCascade.cs, src/SIL.Machine/Rules/PermutationRuleCascade.cs, src/SIL.Machine/Rules/RuleCascade.cs +category: morphotactics +cost: "O(n) generation / O(2^n)-ish parsing for linear; O(n!) both ways for unordered" +grammar_visible: yes +--- + +## What it is + +A stratum groups a character-definition table, a lexicon, phonological rules, morphological +rules, and affix templates that all apply together. The per-stratum +`MorphologicalRuleOrder` attribute (`linear` or `unordered`, default `linear`) controls how the +stratum's morphological rules are applied relative to each other, and the two settings have very +different, and asymmetric, cost profiles on the generation vs. parsing side. + +## Mechanics + +The stratum's constructor picks a rule-cascade implementation from the enum: + +```csharp +case MorphologicalRuleOrder.Linear: + _mrulesRule = new LinearRuleCascade(mrules, true, ...); +case MorphologicalRuleOrder.Unordered: + _mrulesRule = new CombinationRuleCascade(mrules, true, ...); +``` + +`LinearRuleCascade.ApplyRules` walks the rule list in fixed index order; at the first index `i` +where a rule actually produces output, it recurses on that output starting at `i+1` and then +**stops trying any other rule at this level** (`if (applied) return false;`). That is a genuine +single fixed pipeline: `O(n)` rule-application attempts per derivation, not `2ⁿ` or `n!`. + +`CombinationRuleCascade.ApplyRules` instead loops over **every** rule not yet used on the current +derivation path and recurses into each one — i.e. it explores every ordering of every subset of +the stratum's morphological rules that structurally apply. Worst case that's `O(n!)` +rule-application attempts for `n` mutually-applicable rules, bounded in practice only by a +configurable alternatives cap (an exception is thrown once the alternative count exceeds the +configured limit — unbounded by default). The analysis side uses a parallelized version of the +same search for this case — same algorithm, spread across threads, not a cheaper one. + +## The gotcha most authors miss + +Switching a slow stratum from `unordered` to `linear` only fixes the *generation* (synthesis) +side. On the **analysis** (parsing) side, `linear` does not compile to a fixed-pipeline cascade +at all — it uses a permutation-style cascade instead, with the engine's own reasoning stated +directly in its source comment: + +> Use `PermutationRuleCascade` instead of `LinearRuleCascade` because morphological rules should +> be considered optional during unapplication (they are obligatory during application, but we +> don't know they have been applied during unapplication). + +`PermutationRuleCascade.ApplyRules` loops over every rule index from the current position onward +and recurses into **each one independently against the same input**, with no "stop after first +success" — i.e., during parsing, HC must consider every subset of a stratum's rules as a +candidate explanation for the surface string, because a rule that wasn't applied looks identical +(from the surface form alone) to one that doesn't fire. Because recursion only ever moves the +index forward, this restricts the search to *subsets taken in listed order* rather than full +`n!` permutations — cheaper than the unordered case's any-order-any-subset search, but still +combinatorial (`O(2ⁿ)`-ish rather than `O(n)`). + +## Practical implication + +For a stratum with more than a handful of morphological rules that can structurally co-apply, +`unordered` is expensive to parse *and* generate; `linear` is cheap to generate but still +combinatorial to parse, because parsing must guess which rules applied. There is no separate +ordering setting for phonological rules — those are always compiled into a plain fixed-pipeline +cascade regardless of stratum settings; only morphological-rule ordering is a grammar-author +choice with this cost profile. + +## Fix + +If a stratum's rules genuinely have no fixed relative order (rare, and typically small `n`), +`unordered` is correct. For anything larger, give the rules a real order and use `linear` — it +won't make parsing `O(n)`, but it keeps generation linear and keeps the parse-side search to +ordered subsets instead of full permutations. diff --git a/Docs/ai-parser-help/texts/README.md b/Docs/ai-parser-help/texts/README.md new file mode 100644 index 0000000000..ec1a0baae8 --- /dev/null +++ b/Docs/ai-parser-help/texts/README.md @@ -0,0 +1,80 @@ +# AI Parser Help — FLExText reference for LLMs + +Part of the [AI Parser Help reference](../README.md). Where [`broken/`](../broken/README.md), +[`speed/`](../speed/README.md), and [`workflow/`](../workflow/README.md) cover the HermitCrab +**grammar** — the rules a linguist authors — this directory covers a different kind of upload +entirely: **FLExText**, the XML interchange format FieldWorks Language Explorer (FLEx) uses to +export **connected texts** — real corpus sentences with word-by-word analysis — so an LLM can help +reason about actual usage data, not just the grammar that's supposed to account for it. + +- **[`flextext-format.md`](flextext-format.md)** — what FLExText actually is: the + document/paragraph/phrase/word/morph structure, the `item` element's type system, the guid + linkage back to FieldWorks objects, and a synthetic worked example. +- **[`analysis-status-and-ground-truth.md`](analysis-status-and-ground-truth.md)** — how the + `analysisStatus` attribute distinguishes a human-approved analysis from a guess, what FLEx's own + exporter and importer actually do with it, and why that distinction matters before you treat + anything in a `.flextext` file as ground truth. +- **[`llm-code-execution.md`](llm-code-execution.md)** — which AI products can actually run code + against your uploaded file (and which can't, as far as we could verify), plus ready-to-paste + Python for the ones that can, so the LLM extracts structured data instead of trying to read + potentially thousands of lines of raw XML itself. + +Each reference file starts with a metadata header: + +```yaml +--- +title: +grounded_in: +category: +when_to_use: +--- +``` + +## Got .flextext files and a question? + +Send that person **this link instead**: [`getting-started.md`](getting-started.md) — it walks +through exporting texts from FLEx as FLExText and getting ChatGPT/Claude to reason about them +using this reference. The rest of this README is the reference material itself (for the LLM to +read), not the human-facing walkthrough. + +## How to use this with an LLM + +Paste the **raw** URL of the relevant topic file into your chat, e.g.: + +``` +https://raw.githubusercontent.com/sillsdev/machine/master/docs/ai-parser-help/texts/flextext-format.md +``` + +Use the raw URL (`raw.githubusercontent.com`), not the normal `github.com/.../blob/...` page — see +the top-level [README.md](../README.md) for why. If you're not sure which topic file is relevant, +paste this README's raw URL first and describe what you're trying to do (e.g. "I have a .flextext +export and don't know what the fields mean," "how do I tell which analyses are actually +confirmed," "how do I get you to actually read this file instead of skimming it"). + +## What belongs here + +- FLExText format documentation, grounded in the FieldWorks schema and exporter source that + produces the format. +- Synthetic example texts only — an invented language, an invented sentence, invented glosses. + **Never real corpus data.** The privacy concern here is sharper than for the grammar sections: + a real `.flextext` file *is* the real corpus text of a real language project, word for word, + sentence for sentence — not a set of abstract rules that happen to describe a language. Treat it + accordingly: never commit one to this repo, and see the note on sharing it with a third-party AI + service in [`getting-started.md`](getting-started.md). +- **Not** real interlinear text data for any specific language or project (e.g. Sena, Amharic, + Indonesian, Aweti). If a question requires reasoning about a real text, describe its structure + abstractly (how many words, whether analyses are approved or guessed, what item types are + present) instead of pasting the real text. + +## Source grounding + +Claims here are grounded in the FieldWorks source that defines and produces FLExText — +`DistFiles/Language Explorer/Export Templates/Interlinear/FlexInterlinear.xsd`, +`Src/LexText/Interlinear/InterlinearExporter.cs`, `Src/LexText/Interlinear/InterlinVc.cs`, and +`Src/LexText/Interlinear/FlexInterlinModel/FlexInterlinear.cs` — as of the commit each file's +metadata header cites, plus external documentation cited inline by URL. Unlike `broken/`/`speed/`/ +`workflow/`, which avoid citing other repositories, this section's whole job is explaining an +interchange format — citing the FieldWorks source that defines it, and legitimate external +documentation of it, is expected and required here, not something to avoid. File/line references +may drift as FieldWorks evolves; if something looks stale, check the live source at the paths +cited. diff --git a/Docs/ai-parser-help/texts/analysis-status-and-ground-truth.md b/Docs/ai-parser-help/texts/analysis-status-and-ground-truth.md new file mode 100644 index 0000000000..c3b648a914 --- /dev/null +++ b/Docs/ai-parser-help/texts/analysis-status-and-ground-truth.md @@ -0,0 +1,78 @@ +--- +title: analysisStatus distinguishes confirmed analyses from guesses +grounded_in: FieldWorks DistFiles/Language Explorer/Export Templates/Interlinear/FlexInterlinear.xsd, Src/LexText/Interlinear/InterlinVc.cs, Src/LexText/Interlinear/InterlinearExporter.cs, Src/LexText/Interlinear/ITextDllTests/ImportInterlinearAnalysesTests.cs +category: provenance +when_to_use: before treating any word's morpheme breakdown, gloss, or category in a .flextext file as an established fact about the language +--- + +## What it is + +Every word in a FieldWorks interlinear text has gone through some amount of analysis -- assigning +it a morpheme breakdown, a gloss, a part of speech -- but not all of that analysis has necessarily +been reviewed by a person. FLEx's own automatic analysis-guesser can populate a word with its best +guess before a linguist has confirmed anything. The `analysisStatus` attribute is how a flextext +file records which is which. + +## Where it appears + +The XSD defines `analysisStatusTypes` as an enumeration of four values -- +`humanApproved`, `guess`, `guessByHumanApproved`, `guessByStatisticalAnalysis` +(`FlexInterlinear.xsd:222-229`) -- and allows the attribute in two places: + +- On a word's `morphemes` element (`:64`, `:912-913`) -- status of the morpheme breakdown as a + whole. +- On any `item` element (`:159`, `:1018`) -- in practice, only on `gls` and `pos` items + (`InterlinearExporter.cs:351, 617`: `WriteAnalysisStatus()` is called specifically when + `itemType == "gls" || itemType == "pos"`), i.e. on a word's gloss and category, not on arbitrary + item types. + +## What FLEx's own exporter actually emits + +Reading `InterlinVc.cs` (the view constructor that both renders the Interlinear window and drives +the exporter) shows only **one** of the four enumerated values is ever written by FLEx's own +export path: the literal string `"guess"` -- six call sites, all of the same shape: + +```csharp +// InterlinVc.cs:2562 (one of six near-identical sites: lines 2562, 2574, 2597, 2636, 2659, 2677) +m_this.SetGuessing(m_vwenv, m_this.GetGuessColor(m_defaultObj)); +// Let the exporter know that this is a guessed analysis. +m_vwenv.set_StringProperty(ktagAnalysisStatus, "guess"); +``` + +This fires when the interlinear view is displaying a *default* analysis that hasn't actually been +selected/approved by the user -- the same guessed-analysis state FLEx's UI renders with a distinct +color to flag it as unconfirmed. When a word's analysis **has** been approved, no +`analysisStatus` attribute is written at all -- **absence of the attribute means human-approved (or +the only candidate), not "no information."** So on a file exported directly from FLEx's own UI, +you should only expect to see two states in practice: attribute absent (approved) and +`analysisStatus="guess"` (FLEx's own guesser, unconfirmed). + +The other two enumerated values, `guessByHumanApproved` and `guessByStatisticalAnalysis`, are not +emitted by this exporter -- but they are real, meaningful values the *importer* understands, and +they do appear in FieldWorks' own test fixtures for round-tripping data from other sources (e.g. +`ITextDllTests/FlexTextImport/FlexTextExportOutput.flextext:11,15,19,23,27`, which uses +`analysisStatus="guessByStatisticalAnalysis"` and `="guessByHumanApproved"` on `gls` items). A +flextext file from a different tool or pipeline -- say, one that ran a statistical +tagger/analyzer over the text -- may use these two more granular "guessed, but derived from [some +other confirmed source]" states. Don't assume every flextext file you see only has the two states +FLEx's own UI produces. + +## Why it matters: the importer doesn't trust guesses either + +FieldWorks' own import logic treats this distinction the same way you should. A test named +`SkipNewGuessedWordGloss` (`ImportInterlinearAnalysesTests.cs:208-224`) imports a word gloss marked +`analysisStatus="guessByHumanApproved"` and confirms FLEx does **not** create a new approved +analysis from it -- contrast with the neighboring `ImportNewHumanApprovedWordGloss` test +(`:115-131`), which imports the identical structure but with `analysisStatus="humanApproved"` and +confirms it **does** get created as approved data. Same shape, same fields -- the only difference +that changes the outcome is this attribute. + +The practical implication for reasoning about a `.flextext` file with an LLM: if you (or the LLM) +are trying to draw a linguistic conclusion from the corpus -- "this language's plural suffix is +always `-X`," "this root never co-occurs with that affix" -- restrict that reasoning to analyses +that are actually `humanApproved` (attribute absent) or explicitly marked `humanApproved`. A +conclusion drawn from `guess`-flagged data is only as reliable as FLEx's automatic guesser, which +is exactly the caveat FieldWorks' own import code encodes by refusing to promote a guess to +approved status on its own. Extraction code that reports counts of confirmed vs. guessed analyses +(see [`llm-code-execution.md`](llm-code-execution.md)) exists specifically so this distinction +isn't silently lost when handing the file to an LLM. diff --git a/Docs/ai-parser-help/texts/flextext-format.md b/Docs/ai-parser-help/texts/flextext-format.md new file mode 100644 index 0000000000..da36b7c46d --- /dev/null +++ b/Docs/ai-parser-help/texts/flextext-format.md @@ -0,0 +1,254 @@ +--- +title: What FLExText is and how it's structured +grounded_in: FieldWorks DistFiles/Language Explorer/Export Templates/Interlinear/FlexInterlinear.xsd, Src/LexText/Interlinear/InterlinearExporter.cs, Src/LexText/Interlinear/InterlinVc.cs, Src/LexText/Interlinear/FlexInterlinModel/FlexInterlinear.cs +category: format +when_to_use: you have a .flextext file and don't know what the elements mean, or need to know which item type holds which piece of data +--- + +## What it is + +FLExText (file extension `.flextext`) is FieldWorks Language Explorer's XML interchange format for +interlinear texts — connected corpus sentences with word-by-word glossing and morpheme +breakdowns, as opposed to the grammar (rules) that HermitCrab uses. It's consumed by FLEx itself +(round-trip export/import) and by other tools, notably [ELAN](https://archive.mpi.nl/tla/elan) +(ELAN → FLEx → ELAN "round-trip" workflows are a documented use case) and +[SayMore](https://software.sil.org/saymore/). A single `.flextext` file may hold multiple texts. + +Internally, FieldWorks' generated C# serialization model renames the schema's root `` +type to `BIRDDocument` — "the names have been changed to protect the innocent developers who +wouldn't like the looooong generated type names... document was changed to BIRDDocument" +(`FlexInterlinModel/FlexInterlinear.cs:11-13`). Other FieldWorks source files call the format "BIRD +format" in comments and class names (`BIRDInterlinearImporter.cs:65`, `LinguaLinksImport.cs:372`). +Nothing in the source names what the acronym expands to, so treat "BIRD format" as an internal +synonym for FLExText rather than a documented backronym. + +The authoritative external documentation is SIL's own Ken Zook, +[*Technical Notes on FLEx Text Interlinear*](https://downloads.languagetechnology.org/fieldworks/Documentation/Technical%20Notes%20on%20FLEx%20Text%20Interlinear.pdf) +(May 2026) — it documents the same structure as the XSD below, plus the export/import UI workflow +and sample files from FLEx, ELAN, and SayMore. The format is also registered in CLARIN's Standards +Information System as +["SIL FieldWorks Language Explorer Interlinear Text"](https://standards.clarin.eu/sis/views/view-format.xq?id=fFLExText) +(media type `text/xml`, extension `.flextext`). + +**Important:** exporting to FLExText does not read the FieldWorks project database directly. It +renders the currently *configured* Interlinear Text view (`Tools > Configure > Interlinear`) and +serializes that — see [`getting-started.md`](getting-started.md). A field you haven't enabled to +display won't appear in the export, even if it exists in the project. + +## Document structure + +The schema (`FlexInterlinear.xsd`, 231 lines) defines this element hierarchy: + +``` +document (version="3") +└─ interlinear-text (guid) — one per text; a file may hold several + ├─ item (type="title", ...) — the text's title, one per writing system + ├─ paragraphs + │ └─ paragraph (guid) + │ └─ phrases + │ └─ phrase (guid, speaker, media-file, begin/end-time-offset) + │ ├─ item (type="segnum"|"txt"|"gls"|"lit"|"note") + │ └─ words + │ └─ word (guid) + │ ├─ item (type="txt"|"gls"|"pos"|"punct") + │ └─ morphemes (analysisStatus) + │ └─ morph (type, guid) + │ └─ item (type="txt"|"cf"|"hn"|"gls"|"msa") + ├─ languages + │ └─ language (lang, encoding, font, vernacular) + └─ media-files (offset-type) + └─ media (guid, location) +``` + +(`FlexInterlinear.xsd:2-138` for `document`/`interlinear-text`/`paragraphs`/`phrases`/`words`/ +`word`/`morphemes`/`morph`; `:95-126` for `languages`/`media-files`.) + +A `word` element is one of three things depending on how far its analysis got: + +- **Unanalyzed** — just `item type="txt"` (the surface wordform). No `morphemes`. +- **Partially analyzed** — has `morphemes`/`morph` breakdown, but usually no word-level `gls`. +- **Fully analyzed** — has `morphemes`, plus word-level `gls` (gloss) and `pos` (category). + +This mirrors three different FieldWorks LCM object types the `word` guid can point to — +`WfiWordform`, `WfiAnalysis`, or `WfiGloss` respectively — though the flextext file itself doesn't +label which one applies; you infer it from which children are present. A fourth pseudo-word form, +punctuation, has no morphemes or gloss at all — just `item type="punct"` — and its `word` element +has no `guid`, even though the underlying FieldWorks `PunctuationForm` object does have one. + +Word elements themselves are optional in phrases: if a phrase has a `txt` item but no `word` +children at all, FLEx will parse the baseline text into unanalyzed wordforms on import — one word +per whitespace-delimited token, which breaks for any wordform that legitimately contains a space +(FLEx supports multi-word wordforms like idioms — "kick the bucket" as one lexical unit — but only +if word elements are present to say so explicitly). (Zook's Technical Notes section 2.5 "Word.") + +## The `item` element and its type system + +Nearly every piece of actual data — text, gloss, category, translation, note — is carried by a +generic `item` element with a required `type` attribute (what kind of data) and a required `lang` +attribute (which writing system), e.g. `boy` +(`FlexInterlinear.xsd:139-161`). + +**The schema's enumerated list of item types is not exhaustive, and this is easy to miss.** The +XSD defines a `knownItemTypes` enumeration (`txt`, `cf`, `hn`, `variantTypes`, `gls`, `msa`, `pos`, +`title`, `title-abbreviation`, `source`, `comment`, `text-is-translation`, `description`, `punct` +— `:162-179`) but then unions it with plain `xs:string` +(`itemTypes = knownItemTypes ∪ xs:string`, `:180-182`) — so *any* string is schema-valid as a +`type` value. Item types FLEx actually emits that aren't in that enumeration include `segnum` +(segment number), `lit` (literal translation), and `note` — all three are real, used constantly, +and documented in Zook's Technical Notes section 2.4, but a validator or a naive reader treating +`knownItemTypes` as the complete list will miss them. + +**The same type value means something different depending on which element it's nested under.** +This is the single most important gotcha for anyone (human or LLM) writing an XPath-style query +against a flextext file: `type="txt"` and `type="gls"` both recur at multiple structural levels +with different referents: + +| Interlinear line (FLEx UI label) | Element | Item type | Meaning | +|---|---|---|---| +| Word | `word` | `txt` | the surface wordform | +| Word Gloss | `word` | `gls` | the whole word's gloss | +| Word Cat. | `word` | `pos` | the whole word's part of speech | +| Morphemes | `morph` | `txt` | one morpheme's surface form | +| Lex. Entries | `morph` | `cf` / `hn` | the lexeme (citation) form / homograph number | +| Lex. Gloss | `morph` | `gls` | that morpheme's *sense* gloss | +| Lex. Gram. Info. | `morph` | `msa` | that morpheme's grammatical category | +| Free Translation | `phrase` | `gls` | the sentence's free translation | +| Literal Translation | `phrase` | `lit` | the sentence's literal translation | +| Note | `phrase` | `note` | an annotator's note on the sentence | + +(Table per Zook's Technical Notes section 3 "Mapping FLEx interlinear to flextext fields," verified +against the exporter: `InterlinearExporter.cs:185` writes `gls` for `WfiGlossTags.kflidForm` +(word gloss); `:433-435` and `:554` open a `gls` item for `InterlinLineChoices.kflidLexGloss` and +`WfiMorphBundleTags.kflidSense` respectively (morpheme sense gloss) — same type string, two +different source fields depending on nesting.) + +A query like "get every `gls` item in the document" conflates three unrelated things: the +sentence's free translation, each word's gloss, and each morpheme's sense gloss. Always scope the +XPath to the level you mean — see [`llm-code-execution.md`](llm-code-execution.md) for concrete +scoped queries. + +The `cf` type is a naming fossil worth knowing about if you're wondering why "citation form" isn't +called `cit` or `citform`: Zook's notes explicitly say *"the abbreviation was probably for citation +form at some point, but it is really the lexeme form, so should probably be 'lf'. However it's too +hard to change at this point"* (section 2.6) — `cf` holds the FieldWorks lexeme form +(`LexEntry LexemeForm MoForm Form`), not a distinct "citation form" concept. + +## The `morph` element's `type` attribute + +A `morph` element's own `type` attribute (distinct from any `item type=` inside it) names the +morpheme's structural class, taken directly from the FieldWorks `MoMorphType` object's `Name` +property (`InterlinearExporter.cs:152-154` writes `GetText(...get_MultiStringAlt(...))` for +whichever `MoMorphType` applies — i.e. it emits that object's actual configured name string, not a +fixed code). The XSD's `morphTypes` enumeration (`FlexInterlinear.xsd:192-214`) lists FieldWorks' +standard set: `particle`, `infix`, `prefix`, `simulfix`, `suffix`, `suprafix`, `circumfix`, +`clitic`, `enclitic`, `proclitic`, `bound root`, `root`, `bound stem`, `stem`, `infixing interfix`, +`prefixing interfix`, `suffixing interfix`, `phrase`, `discontiguous phrase`. + +## guid linkage back to FieldWorks + +Elements carry `guid` attributes that map back to specific FieldWorks LCM classes (Zook's Technical +Notes section 3): + +| Flextext element | FieldWorks class | +|---|---| +| `interlinear-text` | `Text` | +| `paragraph` | `StTxtPara` | +| `phrase` | `Segment` | +| `word` | `WfiWordform` / `WfiAnalysis` / `WfiGloss` (depending on analysis depth) | +| `morph` (`type` attribute's guid) | `MoMorphType` | + +Flextext deliberately omits the guids that would link a word's analysis back to specific lexicon +entries (`WfiMorphBundle`, `LexEntry`, `LexSense`, `MoStemAllomorph`) — per Zook's notes, "the +WfiMorphBundle guid and lexical guids are not included in the flextext file. As a result... current +imports of flextext files cannot provide these linkages during import in a blank FLEx project." +Treat a flextext file as recording *what* a word's analysis is (breakdown, glosses, category), not +a queryable link to *which specific lexicon entry* produced it. + +**No engine provenance of any kind is recorded.** There is no field anywhere in the schema for +which parsing engine, rule, or trace produced a given analysis — a manually-typed analysis, an +analysis produced by FLEx's own HermitCrab-backed parser, and a statistically-guessed analysis are +structurally indistinguishable in a flextext file except via the coarse `analysisStatus` attribute +covered in [`analysis-status-and-ground-truth.md`](analysis-status-and-ground-truth.md) — which +records *whether a human confirmed it*, not *how it was produced*. + +## Media and time-alignment fields + +`phrase` elements can carry `begin-time-offset`/`end-time-offset` (millisecond strings) and a +`media-file` guid referencing a `media` element in the enclosing text's `media-files` block +(`FlexInterlinear.xsd:77-82, 113-126`). FLEx has no UI for this data but stores and round-trips it, +because ELAN and SayMore populate it for audio/video-aligned transcription — a flextext file +originating from ELAN or SayMore commonly has this, one from FLEx's own UI typically doesn't. + +## A synthetic example + +Invented language, invented words, invented sentence — not any real project's text (see this +directory's [README](README.md#what-belongs-here) for why that matters here specifically): + +```xml + + + + Toy Story 1 + + + + + 1 + Mirusi dabo. + + + Mirusi + + + miru + miru + run + v + + + -si + -si + 3sg.pst + infl + + + ran + v + + + dabo + + + dabo + dabo + quickly + adv + + + quickly + adv + + + . + + + She ran quickly. + Ran quickly. + Subject is understood from prior context. + + + + + + + + + + +``` + +Reading this against the tables above: the sentence's free translation is "She ran quickly." (the +phrase-level `gls`), the first word is fully analyzed and human-approved (root `miru` "run" + a +past-tense suffix), the second word's morpheme breakdown is only a guess (`analysisStatus="guess"` +on its `morphemes` element) even though it happens to be monomorphemic, and the final `word` is +punctuation with no morphology at all. diff --git a/Docs/ai-parser-help/texts/getting-started.md b/Docs/ai-parser-help/texts/getting-started.md new file mode 100644 index 0000000000..3bc8951988 --- /dev/null +++ b/Docs/ai-parser-help/texts/getting-started.md @@ -0,0 +1,83 @@ +# Get help from ChatGPT or Claude with your interlinear texts + +Part of the [FLExText-for-LLMs reference](README.md). This page is for anyone with a FieldWorks +Language Explorer (FLEx) project who wants an LLM's help reasoning about **connected texts** — +real corpus sentences with word-by-word glossing and morpheme breakdowns — rather than the +grammar. Typical questions: "does my corpus actually support the paradigm I think it does," "how +many of these analyses are actually human-confirmed versus the parser guessing," "summarize what +part-of-speech categories appear in this text and how often." + +If your question is about the *grammar itself* (why a word won't parse, why parsing is slow, how +to model something), see the top-level [`getting-started.md`](../getting-started.md) instead — +this page is specifically about exported texts. + +## Step 1 — Export your text(s) as FLExText + +FieldWorks exports interlinear texts via the FLEx UI itself — there is no separate command-line +tool for this (unlike `GenerateHCConfig.exe` for grammar export). The export walks the +Interlinear Text tool's own rendered view, so **the fields and writing systems you see on screen +are exactly what ends up in the file** — configure that first. + +1. **Configure what you want included.** In FLEx, go to **Tools > Configure > Interlinear** and + make sure the interlinear lines you want (baseline, word gloss, word category, morpheme + breakdown, lexical gloss, free translation, literal translation, notes, ...) and the writing + systems you care about are turned on. The export pulls from this configured view, not directly + from the underlying database — anything not currently displayed won't be exported. +2. **Open the text.** In the **Texts & Words** area, use the **Interlinear Texts** tool to open + the text you want to export, and go to its Gloss, Analyze, Tagging, or Print View tab. +3. **Start the export.** Choose **File > Export Interlinear** from the main menu. +4. **Pick the FLExText option.** In the Export Interlinear dialog, click **"ELAN, SayMore, FLEx + FLEXTEXT"**, then click **Export**. +5. **Choose which text(s).** A Choose Texts dialog opens with the current text preselected — pick + any others you want in the same file, then click OK. +6. **Save the file.** In the "Export to FLEXTEXText" dialog, pick a directory and filename, then + click **Save**. This produces a `.flextext` file — an XML file that may contain one or more + `interlinear-text` elements, one per text you selected. + +This procedure (and the underlying FLExText format) is documented by SIL in Ken Zook's +[*Technical Notes on FLEx Text Interlinear*](https://downloads.languagetechnology.org/fieldworks/Documentation/Technical%20Notes%20on%20FLEx%20Text%20Interlinear.pdf) +(May 2026), sections 2 and 4 — that document is the authoritative source for the export/import +workflow and for the schema itself; [`flextext-format.md`](flextext-format.md) in this reference +summarizes the parts most relevant to getting an LLM to reason about the file correctly. + +## Step 2 — Upload the file into ChatGPT or Claude + +Attach the `.flextext` file directly as a file upload (don't paste the XML inline — a text with +more than a few sentences quickly runs into thousands of lines, and ChatGPT/Claude's own reading +of a large raw XML file inline is unreliable — see +[`llm-code-execution.md`](llm-code-execution.md) for why, and for commands that have the LLM +extract a compact summary itself instead of trying to read the whole file at once). + +## Step 3 — Point it to the FLExText reference + +Paste this URL into the same chat: + +``` +https://raw.githubusercontent.com/sillsdev/machine/master/docs/ai-parser-help/texts/README.md +``` + +This tells the LLM where to find documentation of the FLExText format itself — its structure, its +`analysisStatus` ground-truth caveats, and (if the product supports it) code it can run directly +against your uploaded file. + +## Step 4 — Ask your question + +Some examples, once your `.flextext` file and the reference URL are in the chat: + +- "Extract every free translation and its corresponding baseline text from this file." +- "How many of the word-level analyses in this text are human-approved versus guessed? List the + guessed ones." +- "What parts of speech appear in this text, and how often does each occur?" +- "Does this text contain any words with more than N morphemes? List them with their breakdowns." +- "Summarize this text's morpheme inventory — which roots and affixes actually occur, and how + often." + +## A note on privacy + +Unlike a grammar file (a set of rules that happen to describe a language), a `.flextext` export +**is the real corpus text itself** — actual sentences, actual translations, actual speaker/note +data if present, word for word. Uploading it to a third-party chat service sends that data to that +provider. This is a stronger privacy consideration than for a grammar file, not a weaker one — +check your project's data-sensitivity policy before sharing an unpublished, restricted, or +community-sensitive text this way, independently of the fact that real corpus data must never be +committed to this repository (see [`README.md`](README.md#what-belongs-here)). diff --git a/Docs/ai-parser-help/texts/llm-code-execution.md b/Docs/ai-parser-help/texts/llm-code-execution.md new file mode 100644 index 0000000000..8c1adcbd44 --- /dev/null +++ b/Docs/ai-parser-help/texts/llm-code-execution.md @@ -0,0 +1,145 @@ +--- +title: Getting an LLM to run code against your uploaded .flextext file +grounded_in: external product documentation for ChatGPT, Claude.ai, and Google AI Mode/Gemini (cited inline) +category: llm-tooling +when_to_use: your .flextext file is too large to reliably read inline, or you want structured, checkable extraction instead of the LLM's own reading of raw XML +--- + +## Why this matters + +A `.flextext` export of even a modest text runs to hundreds or thousands of lines of nested XML +(see [`flextext-format.md`](flextext-format.md) for the structure). Asking an LLM to just "read" +that file and answer questions about it means trusting its own attention over a long, repetitive, +deeply-nested document — the same reliability problem as asking it to eyeball a large HermitCrab +grammar XML. The fix is the same: if the product can actually execute code against the uploaded +file, have it run a short, deterministic extraction script and work from that output instead of +its own reading of the raw file. + +Not every AI product we checked can do this. Below is what we verified about each, from that +product's own documentation, as of the time this was written. + +## ChatGPT — yes, Python, verified + +ChatGPT's Code Interpreter / Advanced Data Analysis feature runs Python in a stateful sandboxed +Jupyter environment and can read files uploaded to the chat directly from that environment. +Per OpenAI's own help center: *"For some data-analysis tasks, ChatGPT writes and runs Python code +in a stateful Jupyter notebook environment... The environment can use files made available to the +session"* (["Data analysis with ChatGPT"](https://help.openai.com/en/articles/8437071-data-analysis-with-chatgpt), +["File Uploads FAQ"](https://help.openai.com/en/articles/8555545-code-interpreter)). Python's +`xml.etree.ElementTree` is standard library — no install step needed. Use the script below. + +## Claude.ai — yes, Python, verified (superseded an earlier JS-only tool) + +As of this writing, Claude.ai's code execution tool runs **Python and Bash** in a sandboxed +container, available by default across Free/Pro/Max/Team/Enterprise plans, and can read files +uploaded to the conversation directly +(["Create and edit files with Claude"](https://support.claude.com/en/articles/12111783-create-and-edit-files-with-claude); +the underlying tool is documented for API use at +["Code execution tool"](https://platform.claude.com/docs/en/agents-and-tools/tool-use/code-execution-tool), +which confirms a Python sandbox with pandas/numpy/matplotlib preinstalled and direct access to +uploaded files). This **replaces** an earlier (2024) version of the tool that only ran JavaScript +in an in-browser Web Worker with a much smaller set of libraries (lodash, Papa Parse, no XML/DOM +parsing capability at all) — if you've used Claude's analysis tool before and remember it as +JS-only, that has changed. The Python script below works identically here as in ChatGPT, since +`xml.etree.ElementTree` is standard library in both. + +## Google AI Mode / Gemini — not verified; don't assume code execution here + +We could not confirm, from Google's own documentation, that either Google Search's **AI Mode** or +the consumer **Gemini app** (gemini.google.com) exposes a code-execution sandbox to the end user +over an arbitrary uploaded file the way ChatGPT and Claude do: + +- **AI Mode**'s file-upload feature is described only as reading the file directly and + cross-referencing it with web results — *"AI Mode will analyze the contents of your file and + cross-reference it with relevant information from the web to provide a helpful AI response"* + ([Google's AI Mode update announcement](https://blog.google/products/search/ai-mode-updates-back-to-school/)). + At the time of that announcement, supported upload types were images and PDFs specifically, with + more types promised later — no mention of running code against the file. +- The consumer **Gemini Apps** help page describes uploading and getting "answers, summaries, and + insights" about a file's content, but does not describe a code-execution step + (["Upload & analyze files in Gemini Apps"](https://support.google.com/gemini/answer/14903178)). +- Google **does** document a Python code-execution tool, with stdlib and numpy/pandas/matplotlib + available, that can process uploaded CSV/text files — but this is documented as a **Gemini API** + / developer-facing tool (["Code execution | Gemini API"](https://ai.google.dev/gemini-api/docs/code-execution)), + not something we could confirm is what powers file uploads in the consumer AI Mode or Gemini + chat products. + +If you're using AI Mode or the Gemini app, treat it as reading the raw XML directly rather than +running code against it — the same reliability caveat that motivates this page in the first place. +If you need the code-execution workflow, use ChatGPT or Claude for the extraction step; you can +still paste the *extracted, compact* summary those produce into a Gemini conversation afterward. + +## The extraction script + +Paste this into ChatGPT or Claude and ask it to run the script against your uploaded file +(adjust the filename to match what you uploaded): + +```python +import xml.etree.ElementTree as ET + +PATH = "YourText.flextext" # change to your uploaded file's name + +def text_of(elem, xpath): + child = elem.find(xpath) + return child.text if child is not None else None + +tree = ET.parse(PATH) +root = tree.getroot() + +approved = 0 +guessed = 0 + +for itext in root.findall("interlinear-text"): + title = text_of(itext, "./item[@type='title']") or "(untitled)" + print(f"=== {title} ===") + + for para in itext.findall("./paragraphs/paragraph"): + for phrase in para.findall("./phrases/phrase"): + baseline = text_of(phrase, "./item[@type='txt']") + free = text_of(phrase, "./item[@type='gls']") # phrase-level = free translation + literal = text_of(phrase, "./item[@type='lit']") # phrase-level = literal translation + print(f"\n{baseline!r}") + print(f" free: {free!r} literal: {literal!r}") + + for word in phrase.findall("./words/word"): + surface = text_of(word, "./item[@type='txt']") + punct = text_of(word, "./item[@type='punct']") + if punct is not None: + print(f" [punct] {punct!r}") + continue + + word_gloss = text_of(word, "./item[@type='gls']") # word-level gloss + pos = text_of(word, "./item[@type='pos']") # word-level category + morphemes = word.find("./morphemes") + status = morphemes.get("analysisStatus") if morphemes is not None else None + if status is None: + approved += 1 + else: + guessed += 1 + + morph_strs = [] + if morphemes is not None: + for morph in morphemes.findall("morph"): + m_txt = text_of(morph, "./item[@type='txt']") + m_gls = text_of(morph, "./item[@type='gls']") # morph-level = sense gloss + m_msa = text_of(morph, "./item[@type='msa']") # morph-level = category + morph_strs.append(f"{m_txt}({morph.get('type')})={m_gls}/{m_msa}") + + status_label = status or "approved" + print(f" {surface!r} [{pos}] '{word_gloss}' <{status_label}> morphs: {morph_strs}") + +print(f"\n--- {approved} word analyses approved, {guessed} guessed/unconfirmed ---") +``` + +This prints, per phrase: the baseline text, free and literal translations, and per word: surface +form, category, gloss, its `analysisStatus`, and its morpheme breakdown with each morpheme's own +gloss/category — with translation-vs-word-gloss-vs-morpheme-gloss kept separate, per the scoping +gotcha in [`flextext-format.md`](flextext-format.md#the-item-element-and-its-type-system). It +also tallies how many word analyses are approved vs. guessed, so you get the +[ground-truth caveat](analysis-status-and-ground-truth.md) as a number up front, not something you +have to remember to ask about separately. + +Ask the LLM to adapt the script for your actual question — e.g. filtering to only guessed +analyses, tallying parts of speech, or listing every distinct morpheme with its gloss — rather +than treating it as fixed. The point is to have the LLM write and run a *targeted* query against +the structure, not to make it read the whole file itself. diff --git a/Docs/ai-parser-help/workflow/README.md b/Docs/ai-parser-help/workflow/README.md new file mode 100644 index 0000000000..ea552cccdb --- /dev/null +++ b/Docs/ai-parser-help/workflow/README.md @@ -0,0 +1,58 @@ +# HermitCrab grammar-authoring workflow + +Part of the [HermitCrab-for-LLMs reference](../README.md). Where [`speed/`](../speed/README.md) +covers performance gotchas and `broken/` covers correctness gotchas, this directory covers *how to +model a language well* — the modeling decisions to make, in what order, and which HermitCrab/ +FieldWorks constructs each decision commits you to. Every mechanism described as "how HC actually +behaves" is grounded in reading the engine source under `src/SIL.Machine.Morphology.HermitCrab/`, +not just restated from the methodology guide it's paired with. + +## Primary source + +These files draw on, and cite by section number, H. Andrew Black's *A Conceptual Introduction to +Morphological Parsing for FieldWorks Language Explorer* — the methodology guide FLEx/HermitCrab +modeling follows — plus a companion workshop transcript covering the same ground in the order a +person actually works through it. Both are reproduced verbatim in +[`sources/`](sources/black-flex-conceptual-intro-fulltext.txt) for anyone who wants Black's exact +original text; the files here are original guidance that cites specific sections rather than +quoting them at length. + +Each file starts with a metadata header: + +```yaml +--- +title: +implements: +black_sections: +category: +when_to_use: +--- +``` + +## Index + +| Topic | Category | Use when | +|---|---|---| +| [Decide the typological frame and class system before authoring rules](build-order.md) | build-order | starting a new grammar, or extending one that has no rules yet | +| [Affix status (inflectional/derivational/unclassified) is a modeling commitment](affix-status-and-spurious-parses.md) | morphotactics | an affix behaves as if it has no constraints | +| [Optional slots, null affixes, and multiple templates model different facts](optional-slots-null-affixes-multiple-templates.md) | morphotactics | a paradigm cell has no overt marker in some forms | +| [Inflection classes, subclasses, and the default class are MPR features](inflection-classes-and-mpr-features.md) | feature-system | an affix's allomorph choice is lexically arbitrary | +| [Author default/exception blocks most-specific-first](ordered-rule-exception-blocks.md) | allomorphy | modeling a default form plus layered exceptions for one morpheme | +| [Circumfixes and discontinuous morphemes: one entry or two slots](circumfixes-and-discontinuous-morphemes.md) | morphotactics | a single meaning surfaces as material on both sides of the stem | + +## How to use this with an LLM + +Paste the raw URL of the specific topic file that matches your modeling question, e.g.: + +``` +https://raw.githubusercontent.com/sillsdev/machine/master/docs/ai-parser-help/workflow/build-order.md +``` + +If you're not sure which topic applies, paste this index's raw URL first and describe the modeling +decision you're facing (not your actual grammar's rules — see the privacy note in the top-level +[`README.md`](../README.md)), e.g. "I have a paradigm cell with no overt marker in half the cells, +how should I model it" or "I don't know if this affix is inflectional or derivational." + +If your question is about *why parsing is slow* or *why a parse is wrong/missing*, this isn't the +right directory — see [`speed/`](../speed/README.md) or `broken/` instead. This directory assumes +the grammar runs; it's about whether it models the language correctly. diff --git a/Docs/ai-parser-help/workflow/affix-status-and-spurious-parses.md b/Docs/ai-parser-help/workflow/affix-status-and-spurious-parses.md new file mode 100644 index 0000000000..5a0b1f37e9 --- /dev/null +++ b/Docs/ai-parser-help/workflow/affix-status-and-spurious-parses.md @@ -0,0 +1,85 @@ +--- +title: "Affix status (inflectional/derivational/unclassified) is a modeling commitment" +implements: src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AffixProcessRule.cs, src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AffixProcessAllomorph.cs, src/SIL.Machine.Morphology.HermitCrab/AffixTemplate.cs, src/SIL.Machine.Morphology.HermitCrab/SynthesisStratumRule.cs, src/SIL.Machine.Morphology.HermitCrab/HermitCrabInput.dtd +black_sections: "2.1.1 Unclassified affixes; 2.1.2.9 Underspecified inflectional affixes; 2.1.3.7 Underspecified derivational affixes; 2.1.4 Derivation outside of inflection; 2.1.5 Derivation versus inflection" +category: morphotactics +when_to_use: "an affix is behaving as if it has no constraints, or you're deciding how to classify a new affix" +--- + +## There is no "unclassified" at the engine level + +FieldWorks lets you label an affix inflectional, derivational, or unclassified (Black §2.1.1), and +treats a *partially specified* inflectional or derivational affix the same as unclassified (§2.1.2.9, +§2.1.3.7). The HermitCrab engine itself has no such attribute at all — check +`HermitCrabInput.dtd`: there is no `affixType`/`inflectionType` element or attribute anywhere in the +`MorphologicalRule`/`RealizationalRule` declarations. "Affix status" is a FieldWorks-side authoring +distinction that cashes out, by export time, into two independent engine-level facts: + +1. **Whether the rule sits in an `AffixTemplate` slot at all.** Inflectional affixes go in a slot + (order- and co-occurrence-constrained by the template); derivational affixes are stratum-level + `MorphologicalRule`s applied outside any template, gated only by their own + `RequiredSyntacticFeatureStruct`/`OutSyntacticFeatureStruct` pair. +2. **How constrained that rule's `RequiredSyntacticFeatureStruct` is.** Both `AffixProcessRule` + (see `AffixProcessRule.cs`) and `AffixProcessAllomorph` initialize this to + `FeatureStruct.New().Value` — an empty, unconstrained feature structure — unless the grammar + sets it. An "unclassified" affix, in engine terms, is simply a rule that was never given a + tighter `RequiredSyntacticFeatureStruct` and was never placed in a template slot. + +## What that actually costs at parse time + +An unconstrained `RequiredSyntacticFeatureStruct` unifies against *any* stem's feature structure — +`Subsumes` on an empty structure is trivially true. So a rule left unclassified structurally +applies to every stem whose phonological shape matches, regardless of category: a nominal-looking +suffix will be tried against verb stems, a valence-changing suffix will be tried against nouns, +and so on. This is exactly Black's own diagnosis (§2.1.1): "the affix is relatively unconstrained as +to where it can appear," producing extra incorrect parses for any word that happens to contain a +matching character sequence. + +The performance angle compounds this rather than replacing it. An unclassified rule sitting in an +`unordered` stratum multiplies into the combinatorial search described in +[`speed/stratum-rule-ordering.md`](../speed/stratum-rule-ordering.md), and if its natural-class +patterns are broad (which they often are, precisely because nobody has narrowed the rule to a +category yet), it also widens the match set the way +[`speed/natural-class-feature-widening.md`](../speed/natural-class-feature-widening.md) describes. +An unclassified affix isn't a cheap placeholder you pay for later — it's an unconstrained rule +contributing to every derivation it can structurally reach, from the moment it's added. + +## Deciding inflectional vs. derivational + +Black's criteria (§2.1.5, elaborated further in the companion workshop transcript) reduce to four +questions, and "yes" to all four means inflectional: + +- Does it belong to a set of affixes where exactly one member is used (a paradigm cell)? +- Can you state its position relative to other affixes in the word? +- Does it sit outside the less-constrained (derivational) affixes? +- Is its meaning grammatical rather than lexical/category-changing? + +If the answer trends "no," treat it as derivational: a stratum-level `MorphologicalRule` with its +own required and output syntactic feature structures, applied freely (subject to the stratum's +`Linear`/`Unordered` cascade) rather than pinned to a slot position. + +Treat "unclassified" as a temporary bookkeeping state while you're still gathering paradigm data — +not a resting state for a shipped grammar. Every unclassified affix left in place is, concretely, an +unconstrained rule contributing to every parse attempt it can reach. + +## Derivation outside of inflection + +Some languages inflect a stem, derive a new category from the inflected form, then inflect again +(Black §2.1.4's Huallaga Quechua example: verb inflected for aspect+object, nominalized, then the +resulting noun inflected for possessor+purpose). This is handled by marking the *inner* inflectional +template as one that "requires more derivation" — in engine terms, `AffixTemplate.IsFinal` is `false` +for that template instead of its default `true`. + +This is grounded directly in `SynthesisStratumRule.ApplyMorphologicalRules`/`ApplyTemplates`: after a +template applies, the stratum rule checks `mruleOutWord.IsLastAppliedRuleFinal`. If the template that +just applied is non-final, the output doesn't get treated as a complete word for this stratum — +`ApplyTemplates` recurses it back through the stratum's morphological rules (looking for the +category-changing derivational affix), and if a word reaches the end of the stratum still needing a +non-final template's obligations satisfied, `Word.HasRemainingRulesFromStratum` catches it and the +engine reports a `PartialParse` failure rather than silently accepting an under-derived form (e.g. +Black's example: a bare `see-ipfv-1.obj` verb form with no subject suffix, missing the derivation +step that would let it skip subject agreement). Black's text adds a caveat worth taking at face +value even though it wasn't independently re-traced here: a template marked as requiring more +derivation must have at least one *obligatory* slot — an all-optional template can be satisfied by +skipping every slot, which would make "this template's obligations are met" trivially true and +defeat the "must be followed by more derivation" requirement entirely. diff --git a/Docs/ai-parser-help/workflow/build-order.md b/Docs/ai-parser-help/workflow/build-order.md new file mode 100644 index 0000000000..3cc853fc43 --- /dev/null +++ b/Docs/ai-parser-help/workflow/build-order.md @@ -0,0 +1,113 @@ +--- +title: "Decide the typological frame and class system before authoring rules" +implements: src/SIL.Machine.Morphology.HermitCrab/HermitCrabInput.dtd, src/SIL.Machine.Morphology.HermitCrab/AffixTemplate.cs, src/SIL.Machine.Morphology.HermitCrab/MprFeatureGroup.cs, src/SIL.Machine.Morphology.HermitCrab/SyntacticFeatureSystem.cs +black_sections: "1.1 Key issues; 2.1.2.5 Inflection and categories considerations; 2.1.2.6.2 Inflection classes and category organization" +category: build-order +when_to_use: "starting a new grammar, or an LLM is asked to add rules to a grammar that has none yet" +--- + +## The three things to commit to, in order + +Before authoring a single morphological or phonological rule, three questions are worth answering +in this order, because each one constrains the HC constructs the next phase will use: + +1. **What's the typological frame?** Degree of synthesis, prefixing vs. suffixing vs. both, + vowel harmony, nasal assimilation, tone, noun-class/gender, case, head- vs. dependent-marking, + articles, infixation, reduplication. This is cheap to determine (a handful of paradigms usually + settles it) and it tells you which HC mechanisms you'll need at all — a language with no + noun-class agreement has no reason to design an MPR-feature-group schema; a language with no + infixation never needs to think about writing an infixation environment. +2. **What's the class system?** The noun/verb class inventory (declension classes, conjugation + classes, genders) and how membership propagates to other words via agreement (an adjective or + article tracking its head noun's class). This is a schema every later rule's vocabulary depends + on — which class IDs exist, which feature groups they belong to, which categories can carry them + — so it is worth designing deliberately rather than discovering piecemeal as each new rule turns + out to need "one more inflection class." +3. **What are the rules, in ordered default+exception blocks?** See + [`ordered-rule-exception-blocks.md`](ordered-rule-exception-blocks.md) for how that ordering maps + onto HC's actual mechanisms. + +This isn't a HermitCrab-specific idea — it's a general discipline for building any rule-based +grammar with lexically-conditioned exceptions. What follows is what each phase actually commits +you to in HC/FLEx terms, and where the commitment gets locked in. + +## Phase 1 commits you to: parts of speech and natural classes + +The typological frame determines your **part-of-speech inventory** and your **natural class +inventory** — both flat, ID-referenced lists at the engine level. Confirm this from the schema +itself: `HermitCrabInput.dtd` declares `PartsOfSpeech` as `(PartOfSpeech+)`, each `PartOfSpeech` +just an `id` and a `Name` — there is no parent/child element anywhere in that declaration. Every +place a rule or template restricts itself to a category (`requiredPartsOfSpeech` on a +`MorphologicalRule`, an `AffixTemplate`, a `PhonologicalSubrule`, `headPartsOfSpeech` on a +compounding rule, etc.) references this flat list directly, by ID. + +FieldWorks lets you organize categories hierarchically (verb, with intransitive verb and +transitive verb nested under it) and per Black §2.1.2.5/§2.1.2.6.2, a template or inflection class +defined at the parent level is inherited by every nested subcategory. That inheritance is a +FieldWorks Language Explorer authoring convenience — the exported grammar you're handed has no +category hierarchy construct to inherit through. By the time a grammar reaches the engine, the +inheritance has already been resolved: each generated rule/template's `requiredPartsOfSpeech` +already lists whatever flat set of categories it applies to. If you're reading an exported grammar +and see the same category ID (or the same list of several category IDs) repeated across many +rules, that's very likely this flattening at work, not an authoring mistake — but it also means the +hierarchy discipline (put shared structure at the highest common category) has to happen on the +FieldWorks side, before export, because there's nowhere to express it afterward. + +Natural classes (vowel, consonant, sonorant, etc.) work the same way structurally — they're a flat, +named, ID-referenced set that every environment and phonological rule pulls from. Committing to a +stable natural-class inventory during the frame phase avoids the failure mode covered in the +sibling performance reference +[`speed/natural-class-feature-widening.md`](../speed/natural-class-feature-widening.md): natural +classes invented ad hoc per-rule tend to be either too narrow (missing a segment a later rule +needs) or too broad (a "kitchen sink" class that widens every rule that references it). + +## Phase 2 commits you to: MPR feature groups and syntactic features + +The class system is realized by two genuinely different HC mechanisms, and picking the right one +per phenomenon is exactly Black's Table 9 (§2.1.2.8) decision: + +- **Inflection classes** (arbitrary lexical conditioning, no semantic difference, not visible to + agreement) → `MprFeature`/`MprFeatureGroup`. See + [`inflection-classes-and-mpr-features.md`](inflection-classes-and-mpr-features.md). +- **Agreement/inflection features** (semantically real, syntactically visible categories like + gender, person, number, case) → values in the `SyntacticFeatureSystem`, carried on + `RequiredSyntacticFeatureStruct`/`RequiredHeadFeatures` and the word's own + `SyntacticFeatureStruct`. + +The reason to design this schema before writing rules, rather than after: `MprFeatureSet` (see +`MprFeatureSet.cs`) checks membership against whichever `MprFeatureGroup` a feature happens to +belong to, and a group's `MatchType` (`Any`/`All`) and `Output` (`Overwrite`/`Append`) are +properties of the *group*, not of any individual rule that references it. Every rule that gates on +a feature in that group inherits whatever matching/output behavior the group was given. Retrofitting +the group's semantics after several rules already depend on it (for instance, discovering you need +`Overwrite` semantics — see +[`speed/mpr-overwrite-order-dependence.md`](../speed/mpr-overwrite-order-dependence.md) — after +rules were written assuming `Append`) means re-auditing every rule that touches the group, not just +adding one new rule. + +Syntactic (agreement) features are similarly global: a feature is declared once per category +(Black §2.1.2.7, "add the feature to the category's set of inflectable features") and every rule +for that category shares the same value space. A class system designed after the fact tends to +produce features invented per-rule with slightly different value sets for what's really the same +category-wide feature — the exact mismatch Black's Spanish gender example (§2.1.2.7) exists to +prevent, just discovered late instead of designed early. + +## Phase 3: rules and exceptions + +Once the frame and class schema exist, rule authoring becomes populating ordered blocks against a +fixed vocabulary of POS IDs, natural classes, and MPR features/syntactic features — rather than +inventing new ad hoc gates per rule. See +[`ordered-rule-exception-blocks.md`](ordered-rule-exception-blocks.md) for how "default, then +exception classes, then individual exceptions, most-specific-first" maps onto HC's actual +allomorph-ordering and stratum mechanisms. + +## The failure mode this order prevents + +Authoring rules opportunistically — adding a class ID the moment one affix seems to need it, adding +a natural class scoped to exactly the segments one rule cares about — tends to produce: MPR +features that are really one class split into several near-duplicates because nobody designed the +group first; natural classes that silently diverge from each other by one segment; and rules whose +`requiredPartsOfSpeech` lists were hand-copied between similar rules and drift out of sync as the +category inventory grows. None of these show up as an error — they show up later as a rule that +"mysteriously" doesn't fire for one lexical entry, or a class that behaves inconsistently between +two rules that were supposed to share it. diff --git a/Docs/ai-parser-help/workflow/circumfixes-and-discontinuous-morphemes.md b/Docs/ai-parser-help/workflow/circumfixes-and-discontinuous-morphemes.md new file mode 100644 index 0000000000..fdd0d1c968 --- /dev/null +++ b/Docs/ai-parser-help/workflow/circumfixes-and-discontinuous-morphemes.md @@ -0,0 +1,74 @@ +--- +title: "Circumfixes and discontinuous morphemes: one entry, two-sided output, or two slots" +implements: src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AffixProcessAllomorph.cs, src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/InsertSegments.cs, src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/MorphologicalOutputAction.cs, src/SIL.Machine.Morphology.HermitCrab/HermitCrabInput.dtd +black_sections: "1.1.5 Discontinuous morphemes; 2.1.2.4 Discontinuous morpheme; 4.3 Circumfixes; 6.1.1.3 Circumfixation as a process" +category: morphotactics +when_to_use: "a single meaning is realized as material on both sides of the stem at once" +--- + +## Two genuinely different strategies, not one + +Both a discontinuous tense marker (a prefix and a suffix that must co-occur, e.g. Caquinte's future +`n-...-e`) and a circumfix (Indonesian `ke-...-an` nominalizer) put material on both sides of the +stem. Black models them differently (§2.1.2.4 vs. §4.3/§6.1.1.3), and the difference is real, not +stylistic — it comes down to whether the two pieces are one morpheme or two. + +### Strategy 1: two slots, both obligatory, in one template + +For Caquinte's future tense, Black's approach is pure morphotactics: a template with a Future prefix +slot and a Future suffix slot, both non-optional. Nothing ties the two allomorphs together except +that the template requires both to be filled — they remain two separate morphemes in the analysis +(two glosses, two lexical entries), and the fact that they always co-occur is a fact about the +template, not about either morpheme's own identity. This is the same tool used for forcing +co-occurrence generally — see +[`optional-slots-null-affixes-multiple-templates.md`](optional-slots-null-affixes-multiple-templates.md) +— applied specifically to a pair of affixes that jointly spell one grammatical value. + +### Strategy 2: one lexical entry, one rule, output on both sides + +For a true circumfix — one meaning, one gloss, conventionally treated as a single morpheme — the +HC-native mechanism is a single affix-process allomorph whose output (`Rhs`) inserts segments before +*and* after the copied stem material, in the same rule. This is directly grounded in the engine: +`AffixProcessAllomorph.Rhs` is a plain `IList` (see +`AffixProcessAllomorph.cs`), and `InsertSegments` (`InsertSegments.cs`) is just one such action that +inserts a fixed shape at whatever position it occupies in that list. Nothing prevents a `Rhs` from +containing `InsertSegments("ke")`, then a copy of the stem's own matched material, then +`InsertSegments("an")` — which is exactly Black's own description of the mechanism (§6.1.1.3): "the +pattern is just whatever material is currently present (the X); the result is the prefix phonemes, +the 1, and then the suffix phonemes." The stem is one morpheme with one `MorphemeId`/gloss; the +"two parts" are just two output actions in the same rule's `Rhs`. + +Note the DTD's own morpheme-type enumeration excludes `circumfix` as a type the engine directly +recognizes (Black's footnote at §4.3 says this outright: the parser recognizes every morpheme type +"except for discontiguous phrase, simulfix, suprafix, and circumfix"). FieldWorks' lexical-entry-level +"circumfix" morpheme type (two allomorphs keyed together, one prefix-shaped and one suffix-shaped) is +therefore a FieldWorks authoring convenience that has to be translated into the single-`Rhs`, +two-sided-output form above before it means anything to the engine — if you're reading an exported +grammar and looking for "the circumfix," look for a single affix-process rule whose output list has +an insertion before *and* after the stem-copy action, not a lexical entry literally tagged +`circumfix`. + +## Deciding which one you have + +- Can the two pieces be glossed independently, and would a linguist reasonably call them two + morphemes that happen to always co-occur in this cell? → two obligatory slots in one template + (Strategy 1). The co-occurrence is a template-level fact. +- Is it conventionally analyzed as one morpheme with one meaning, whose exponent simply happens to + wrap the stem? → one lexical entry, one affix-process rule, two-sided `Rhs` (Strategy 2). + +Getting this wrong in the "should be one morpheme" direction (modeling a true circumfix as two +slots) produces a grammar that reports two morphemes and two glosses for what a reader expects to +see analyzed as one meaning unit — not a parse failure, but a wrong-shaped analysis. Getting it +wrong in the other direction (forcing two genuinely independent, separately-glossable affixes into a +single two-sided `Rhs`) loses the ability to have either piece occur without the other in some other +combination, if the language ever needs that. + +## Infixation is the same output-action mechanism, aimed inward + +The same `Rhs` list mechanism handles infixation — the inserted material lands *inside* the copied +stem's matched shape rather than outside it, using the same output-action composition, with the +insertion point defined by an infixation environment pattern (Black §3.3.1's schematic +`# [C] _ [V]`, anchoring the insertion point relative to the stem's own segments rather than to the +whole word's edges). It's the same construct as the circumfix case — output actions placed relative +to a copied-input action — just with the insertion point moved inward instead of split to both +edges. diff --git a/Docs/ai-parser-help/workflow/inflection-classes-and-mpr-features.md b/Docs/ai-parser-help/workflow/inflection-classes-and-mpr-features.md new file mode 100644 index 0000000000..1d00636ec9 --- /dev/null +++ b/Docs/ai-parser-help/workflow/inflection-classes-and-mpr-features.md @@ -0,0 +1,100 @@ +--- +title: "Inflection classes, subclasses, and the default class are MPR features" +implements: src/SIL.Machine.Morphology.HermitCrab/MprFeature.cs, src/SIL.Machine.Morphology.HermitCrab/MprFeatureGroup.cs, src/SIL.Machine.Morphology.HermitCrab/MprFeatureSet.cs, src/SIL.Machine.Morphology.HermitCrab/LexEntry.cs, src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AffixProcessAllomorph.cs, src/SIL.Machine.Morphology.HermitCrab/Word.cs +black_sections: "2.1.2.6 Inflection classes; 2.1.2.6.1 Inflection subclasses; 2.1.2.8 Inflection classes versus inflection features; 2.1.6 Exception features" +category: feature-system +when_to_use: "an affix's allomorph choice is lexically arbitrary — not phonologically or syntactically motivated" +--- + +## One mechanism, several authoring names + +Black introduces "inflection classes" (§2.1.2.6, for allomorphy that tracks the lexical stem rather +than phonology or agreement) and "exception features" (§2.1.6, for blocking an affix from stems that +lack a designated tag) as if they were separate FieldWorks features. At the engine level they are +the same primitive: `MprFeature`, grouped into `MprFeatureGroup`s, checked via `MprFeatureSet`. +A stem's tag set lives on `LexEntry.MprFeatures`; an allomorph's gates are +`RequiredMprFeatures`/`ExcludedMprFeatures` (see `AffixProcessAllomorph.cs`, and the analogous fields +on phonological subrules and compounding rules). Whether you're modeling Yalálag Zapotec's two +future-tense allomorph classes, Latin's five declensions, or Orizaba Nahuatl's absolutive-suffix +exception list, you're populating the same `MprFeature`/`MprFeatureGroup` machinery — just for a +different authoring purpose. Recognizing that they're one mechanism matters because it means the +same design discipline applies to both: decide the group's semantics once, before tagging stems and +allomorphs against it. + +## The matching mechanics, precisely + +`MprFeatureSet.IsMatchRequired` and `IsMatchExcluded` (in `MprFeatureSet.cs`) group the checked +features by their `MprFeatureGroup` and apply the group's `MatchType`: + +- `MatchType.All` (or a feature with no group at all): every feature in the group that's on the + gate must be present on the stem (for `IsMatchRequired`) or absent (for `IsMatchExcluded`). +- `MatchType.Any`: at least one feature in the group must be present/absent. + +This is the mechanical form of Black's inflection-class constraint rules (§2.1.2.6.1, rule 29): an +allomorph's `RequiredMprFeatures` is checked against the stem's `MprFeatures` set directly — there's +no separate "compatible with" notion, only "does the required set of features actually appear." + +`MprFeatureGroup.Output` (`Overwrite`/`Append`) governs what happens when a rule's `OutMprFeatures` +gets folded into a word's accumulated tag set (`MprFeatureSet.AddOutput`) — `Overwrite` removes any +sibling feature from the same group before adding the new one, `Append` just unions. This only +matters when a *rule* assigns MPR features as an output (not for a bare lexical stem's static tags), +and it's order-dependent in exactly the way +[`speed/mpr-overwrite-order-dependence.md`](../speed/mpr-overwrite-order-dependence.md) describes — +not re-derived here. + +## Subclasses: flattened, not hierarchical, at the engine level + +`MprFeatureGroup` has no parent/child relationship between its member features — it's a flat set +plus one `MatchType`. There is no engine construct corresponding to "this feature is a subclass of +that one." Black's inflection-subclass behavior (§2.1.2.6.1) — an allomorph tagged only at the main +class level also matches any stem tagged with a *subclass* of that class — therefore cannot be a +runtime hierarchy lookup; it has to be realized as multiple flat tags. Concretely, this means a stem +belonging to "Class 2, Subclass 2A" needs *both* MPR features actually present in its `MprFeatures` +set (the subclass tag and the main-class tag) for a main-class-level `RequiredMprFeatures` gate to +match it, per the mechanics above. If you're reading an exported grammar and a stem carries several +MPR-feature tags that look redundant with each other, that's very likely this flattening — the +"hierarchy" is encoded as co-occurring flat tags, not as a lookup the engine performs. + +One practical corollary: when an inflectional affix entry has allomorphs where *some* are tagged +with a subclass and *others* only with a main-level class, every allomorph in that entry needs +tagging consistently (main-level tags need to cover every subclass they're meant to reach) — an +untagged or main-level-only allomorph doesn't "inherit" narrower coverage automatically the way an +untagged natural class inherits nothing narrower either. This is Black's own warning (§2.1.2.6.1, +end) restated in engine terms: it isn't the engine being inconsistent, it's the flat-tag mechanism +having no notion of automatic narrowing. + +## The "default" class is resolved before the grammar reaches HermitCrab + +FieldWorks lets you designate one inflection class as the default, applied to any stem "not overtly +tagged." That resolution happens on the FieldWorks side, not in the engine: the exporter walks up a +part of speech's own hierarchy looking for a `DefaultInflectionClass` to fall back to for an +untagged stem (this is FieldWorks-side behavior, not part of `sillsdev/machine` — see +`GetDefaultInflClass` in FieldWorks' `Src/LexText/ParserCore/HCLoader.cs`). By the time a stem +reaches the engine, it either already carries an explicit `MprFeature` tag, or it carries none at +all — there's no "untagged means whichever class was marked default" behavior anywhere in +`MprFeatureSet`. If you're authoring a HermitCrab grammar directly (not exporting from FieldWorks), +there is no default-class shortcut available to you: model a "default"/"regular" class as a real, +explicit `MprFeature` and tag every stem that isn't a genuine exception with it, the same way you'd +tag any other class. `RequiredMprFeatures`/`ExcludedMprFeatures` are positive checks against +whatever's actually present — "nothing else claimed this stem" is not a condition the engine can +test for you. + +## Inflection classes vs. inflection (agreement) features + +Black's Table 9 (§2.1.2.8) reduces to one load-bearing distinction, and it's verifiable directly +from where each mechanism's data lives: `MprFeatureSet` is a field on `Word`/`LexEntry` entirely +separate from `SyntacticFeatureStruct` — an MPR feature never appears in the word's syntactic +feature structure, which is the only thing later syntactic/agreement reasoning (and, within +HermitCrab, any `RequiredSyntacticFeatureStruct`/`RequiredHeadFeatures` gate) can see. So: + +- If the distinction is purely about which allomorph an affix uses, with no meaning difference and + no syntax needs to know about it (declension class, conjugation class, arbitrary allomorphy) → + `MprFeature`/`MprFeatureGroup`. It's invisible outside rule-internal gating, by construction. +- If the distinction has real meaning, changes which affixes can co-occur via *agreement*, or needs + to be visible for downstream syntactic processing (gender, person, number, noun class as a + syntactic category) → a real value in the `SyntacticFeatureSystem`, carried on + `SyntacticFeatureStruct` and checked via `RequiredHeadFeatures`/`RequiredSyntacticFeatureStruct`. + +Picking `MprFeature` for something that actually needs to participate in agreement will parse +correctly in isolation but leaves nothing for a later syntax-level or cross-word agreement check to +read — the tag is real to the rule that gated on it, and invisible to everything else. diff --git a/Docs/ai-parser-help/workflow/optional-slots-null-affixes-multiple-templates.md b/Docs/ai-parser-help/workflow/optional-slots-null-affixes-multiple-templates.md new file mode 100644 index 0000000000..5574a46a14 --- /dev/null +++ b/Docs/ai-parser-help/workflow/optional-slots-null-affixes-multiple-templates.md @@ -0,0 +1,87 @@ +--- +title: "Optional slots, null affixes, and multiple templates model different facts" +implements: src/SIL.Machine.Morphology.HermitCrab/AffixTemplateSlot.cs, src/SIL.Machine.Morphology.HermitCrab/SynthesisAffixTemplateRule.cs, src/SIL.Machine.Morphology.HermitCrab/AffixTemplate.cs, src/SIL.Machine.Morphology.HermitCrab/LexEntry.cs +black_sections: "2.1.2.2 Optional affix slots; 2.1.2.3 Multiple templates; 3.6 Morphemes that may be null; 4.1.1 Null allomorphs" +category: morphotactics +when_to_use: "a paradigm cell has no overt marker in some forms and you're choosing how to model the absence" +--- + +Black lays out three options for "this paradigmatic position sometimes has nothing overt" (§2.1.2.2): +mark the slot optional, split into multiple templates, or give the affix a null allomorph. These +aren't interchangeable defaults — each licenses a different set of word forms, and picking the +wrong one either blocks legitimate words or accepts illegitimate ones. This page is about which fact +each choice actually encodes; for the *performance* cost of optional slots specifically, see +[`speed/affix-template-optional-slots.md`](../speed/affix-template-optional-slots.md) — that page's +`O(2^n)` analysis isn't repeated here. + +## Optional slot: "this position can be genuinely absent" + +Marking a slot optional means the template can produce a well-formed word with nothing in that slot +at all — the feature that slot would have set is never assigned, not "set to a default." An +optional slot is the right choice only when nothing downstream needs a definite value for whatever +feature that slot would carry, and when the slot's absence is unconditional (not tied to what other +slots in the same template did or didn't fill). + +That last condition is easy to violate silently. Black's own worked example (Orizaba Nahuatl present +intransitive, §2.1.2.3) shows the failure directly: a subject prefix and a plural suffix look, at +first glance, like a single template with the suffix slot optional (subject prefixes are obligatory, +number is optional, defaulting to singular). But `ti-` is genuinely ambiguous on its own between +2sg.subject and 1pl.subject. If Number is just one optional slot in the same template as Subject, +the engine has no way to make "the plural suffix is required *whenever the subject prefix chosen was +one of the plural set*" — an optional slot is a single, global toggle; it can't condition its own +optionality on which specific rule filled a *different* slot in the same derivation. The result is +that `timiki` (`ti-miki`, "you(sg) die") would also parse as `1pl.sbj-die`, and `timikih` would +additionally parse as `2sg.sbj-die-pl` — both spurious. + +## Multiple templates: "two branches of the paradigm have different obligation structure" + +The fix is not a smarter single slot — it's two templates, one used only for singular subjects, one +only for plural, each with its *own* slot-obligation pattern: the plural template makes the Number +slot obligatory, so a plural subject prefix can never surface without the plural suffix; the +singular template has no Number slot to omit at all. This is the general shape of the choice: reach +for multiple templates when what differs between two cells of the paradigm isn't just *which* affix +fills a slot, but *whether a slot is optional or obligatory in this branch*. A single template with +independently optional slots cannot express that distinction — it can only license or forbid a slot +uniformly across every word that reaches it, which is exactly what lets an illegitimate combination +through. + +The same reasoning applies to Black's discontinuous-morpheme case (§2.1.2.4): a future tense +realized as a prefix *and* a suffix that must co-occur is modeled as one template with both slots +obligatory, forcing "if the future prefix appears, so must the future suffix, and vice versa" — a +fact one optional slot has no way to state, because optionality is a per-slot property, not a +per-combination one. (For the alternative — modeling the discontinuous exponent as a single +morpheme instead of two co-occurring ones — see +[`circumfixes-and-discontinuous-morphemes.md`](circumfixes-and-discontinuous-morphemes.md).) + +## Null allomorph: "this cell has a feature value that must be set, spelled with zero segments" + +A null allomorph is a real lexical entry (or a real allomorph of one) whose phonetic shape is empty, +occupying an otherwise-obligatory slot. Choose this over an optional slot precisely when something +*downstream* needs the feature that slot carries to have an actual, assigned value — because an +optional slot that's skipped leaves the feature completely unset, and unset is not the same as "set +to the default value" anywhere else in the engine. This is the same distinction covered from the +allomorph side in +[`broken/stem-name-explicit-feature-requirement.md`](../broken/stem-name-explicit-feature-requirement.md): +a stem-name region (or any rule with a `RequiredHeadFeatures`/`RequiredSyntacticFeatureStruct` gate) +tests whether a feature is *explicitly present*, not whether the word is merely compatible with it. +If number agreement, a stem-name region, or a later rule needs to see `num=sg` as an actual value, +an optional-and-skipped Number slot never produces that value — only a null singular suffix +occupying a non-optional slot does. + +Black is explicit that this comes at a real running cost (§4.1.1: null allomorphs "can make the +parser run rather slowly," with `MaxNulls` as the throttle) and recommends constraining a null +allomorph with as specific an environment as possible so it doesn't get tried in places it +shouldn't. Black also notes the alternative some grammarians reach for — treating an always-null +affix as a default feature value instead of a real morpheme — isn't representable in the current +parsers at all; there is no "default feature value" construct, only rules that either assign a +feature or don't. + +## A quick decision guide + +- The position can truly have nothing there, and nothing downstream cares whether the feature was + ever assigned → optional slot. +- Two branches of the paradigm need genuinely different slot-obligation patterns, not just + different fillers for the same slot → multiple templates. +- The cell always has *some* value for a feature that something downstream must see explicitly, even + though it surfaces as zero segments → null allomorph in a non-optional slot, tightly + environment-constrained. diff --git a/Docs/ai-parser-help/workflow/ordered-rule-exception-blocks.md b/Docs/ai-parser-help/workflow/ordered-rule-exception-blocks.md new file mode 100644 index 0000000000..6d57b2ec22 --- /dev/null +++ b/Docs/ai-parser-help/workflow/ordered-rule-exception-blocks.md @@ -0,0 +1,89 @@ +--- +title: "Author default/exception blocks most-specific-first, the way HC actually orders them" +implements: src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisAffixProcessRule.cs, src/SIL.Machine.Morphology.HermitCrab/Allomorph.cs, src/SIL.Machine.Morphology.HermitCrab/MprFeatureSet.cs, src/SIL.Machine.Morphology.HermitCrab/Stratum.cs +black_sections: "4.1.2 Order of allomorphs within a lexical entry; 2.1.6 Exception features; 3.1.4 Allomorph ordering" +category: allomorphy +when_to_use: "modeling a default form plus several layers of exceptions for one morpheme" +--- + +## The general shape: default, then exception classes, then individual exceptions + +A recurring, well-grounded pattern for morphophonology is to model a rule as an ordered block: a +default (elsewhere) realization, then classes of exceptions (each with a reason — a phonological +environment, a lexical stratum, a semantic or lexically-arbitrary grouping), then individual lexical +exceptions, ordered most-specific-first — the Elsewhere principle. HermitCrab has a direct, literal +realization of exactly this ordering at the allomorph level, and a coarser one at the stratum level. +This page is about the authoring discipline of using them that way; the mechanics themselves are +covered in the sibling performance reference and not re-derived here. + +## Allomorph order is the literal mechanism + +Within one lexical entry, allomorph order is not cosmetic. `Allomorph.cs` tracks an `Index` per +allomorph, and the synthesis rule only lets an earlier allomorph pre-empt a later one, never the +reverse (see +[`speed/disjunctive-allomorph-deferred-recheck.md`](../speed/disjunctive-allomorph-deferred-recheck.md) +for the exact final-validity mechanism — the short version is that the engine's own source comment +states it can't check environment constraints until a surface form exists, so it defers the +disjunctive check and then rejects a later-indexed candidate if an earlier, more specific allomorph +should have won instead). Black's own text (§4.1.2) independently describes the same behavior from +the authoring side, and the two agree exactly: "FieldWorks Language Explorer applies the condition +of [an] Allomorph Form and, at the same time, negates the conditions of all preceding Allomorph +Forms." Concretely, Black's English plural example — + +1. `-ɪz` after strident segments +2. `-z` after voiced (non-strident) segments +3. `-s` elsewhere + +— only works because it's listed narrowest-condition-first, elsewhere-last. List them in the other +order and the elsewhere allomorph (now first) would apply everywhere, including after stridents, +before the narrower ones ever get a chance. This is the Elsewhere principle in its most literal HC +form: **most specific first, unconstrained/elsewhere last** — and it's exactly what makes the Lexeme +Form always sort last (Black notes this explicitly: the Lexeme Form field is automatically ordered +after every Allomorph Form), since the lexeme form is definitionally the least-constrained, +"nothing else applies" case. + +The same ordering discipline applies whether the "exception" is phonological (an environment +pattern), lexically arbitrary (an `MprFeature` gate — see +[`inflection-classes-and-mpr-features.md`](inflection-classes-and-mpr-features.md)), or both at +once: whichever allomorph has the narrowest, most specific gate belongs earliest in the list. + +## MPR-feature-gated exception classes follow the same discipline + +Black's "exception features" (§2.1.6) are the class-of-exceptions-with-a-reason layer: tag an affix +with an `MprFeature`, tag only the stems that are genuine exceptions with the same feature, and the +affix is now restricted to that subset instead of applying everywhere it structurally could. Layering +several such tags is exactly "classes of exceptions, most specific first" translated into MPR terms: +a narrowly-tagged allomorph (a specific loanword stratum, a specific irregular subclass) should +precede a more broadly-tagged or untagged one in the allomorph list, for the same reason the English +plural example orders `-ɪz` before `-s` — an earlier allomorph's match pre-empts a later one's, +never the reverse. + +An individual lexical exception (one specific root that behaves irregularly, not a whole class) is +the same mechanism taken to its narrowest case: a dedicated `MprFeature` (or, for stem-conditioned +allomorphy specifically, a stem name — see the stem-name gotcha referenced below) that only that one +lexical entry carries, gating an allomorph that exists only for it. + +## What ordering discipline buys you, and what it doesn't + +Getting the order right is a **correctness** discipline, not a performance optimization. Both the +environment/allomorph disjunctive check and the MPR-feature check are late filters — they run after +a candidate has already been constructed, not before +([`speed/mpr-cooccurrence-late-filter.md`](../speed/mpr-cooccurrence-late-filter.md), +[`speed/disjunctive-allomorph-deferred-recheck.md`](../speed/disjunctive-allomorph-deferred-recheck.md)). +Ordering exceptions most-specific-first doesn't make the engine skip work for the broader cases — +it determines *which candidate wins* once all of them have been tried. Getting the order backwards +doesn't just cost more; per the mechanism above, it silently changes which allomorph a form resolves +to, or produces spurious ambiguity where a listed-first "elsewhere" case pre-empts what should have +been the narrower winner. + +## Strata as the coarser block + +If an entire class of exceptions corresponds to a distinct historical layer of the lexicon (a +loanword stratum with its own phonology and morphology, rather than a handful of MPR-tagged +exceptions within one rule set), model it as a separate `Stratum` rather than MPR-gating every rule +in a single stratum to account for it. Strata are applied in a fixed sequence (each stratum's output +feeding the next), which is the natural HC analog of "this whole layer of the grammar behaves +differently, categorically, not just for a few tagged exceptions." See +[`speed/stratum-rule-ordering.md`](../speed/stratum-rule-ordering.md) for how a stratum's own +morphological-rule ordering setting (`Linear`/`Unordered`) interacts with this, which is a separate +axis from the cross-stratum sequence. diff --git a/Docs/ai-parser-help/workflow/sources/README.md b/Docs/ai-parser-help/workflow/sources/README.md new file mode 100644 index 0000000000..a8e48c9985 --- /dev/null +++ b/Docs/ai-parser-help/workflow/sources/README.md @@ -0,0 +1,23 @@ +# Primary sources + +Verbatim primary-source material this guide's `workflow/` section distills from and cites. Kept +here as full text (not just summarized) so an LLM reading this guide has the original to check +claims against, not just a paraphrase. + +- **`black-flex-conceptual-intro-fulltext.txt`** — "A Conceptual Introduction to Morphological + Parsing for FieldWorks Language Explorer," H. Andrew Black (SIL), 3 July 2025. Ships as a Help + file with FieldWorks 9 (`Helps/WW-ConceptualIntro/ConceptualIntroFLEx.pdf` under the FieldWorks + install directory). Extracted verbatim via `pdftotext -layout`; page markers are PDF page + numbers. +- **`black-parser-workshop-2026-L02-fulltext.txt`** — SIL 2026 Parser Workshop, session L02 + (H. Andrew Black). Same underlying mechanisms as the conceptual intro, but framed around the + order a person actually works through them and FLEx's UI/parse-state semantics. + +**Provenance note:** this material properly belongs alongside FieldWorks itself (it already ships +there as a Help file) — it's included in this repo for now as a practical staging point, not +because `machine` is its long-term home. If/when it's relocated to the FieldWorks repo, update the +citations in `workflow/` to point there instead. + +Nothing in `workflow/`'s own pages is a copy of this text — those pages are original guidance, +independently checked against the actual HermitCrab engine source, that cites specific sections +here for a reader who wants Black's own words. diff --git a/Docs/ai-parser-help/workflow/sources/black-flex-conceptual-intro-fulltext.txt b/Docs/ai-parser-help/workflow/sources/black-flex-conceptual-intro-fulltext.txt new file mode 100644 index 0000000000..ec03df0641 --- /dev/null +++ b/Docs/ai-parser-help/workflow/sources/black-flex-conceptual-intro-fulltext.txt @@ -0,0 +1,6129 @@ +FULL TEXT EXTRACTION (for reference; verbatim, extracted with pdftotext -layout) + +Source: "A Conceptual Introduction to Morphological Parsing for FieldWorks Language Explorer" +Author: H. Andrew Black (SIL). Document date: 3 July 2025. +Extracted from: C:\Program Files\SIL\FieldWorks 9\Helps\WW-ConceptualIntro\ConceptualIntroFLEx.pdf +Extracted: 2026-07-17. 108 pages. Page markers below use PDF page numbers (printed page = PDF page - 7). + +This is the methodology guide our grammar-induction work follows for modeling morphology with +FLEx/HermitCrab mechanisms: affix templates & slots, inflection classes, inflection features, +allomorph environments, phonological rules. See docs/plans/fable-flex-methodology.md. + + +═════ PDF page 1 ═════ + +A Conceptual Introduction to Morphological Parsing + for + + FieldWorks Language Explorer + +═════ PDF page 2 ═════ + +A Conceptual Introduction to Morphological Parsing + for + + FieldWorks Language Explorer + + H. Andrew Black + + 3 July 2025 + +═════ PDF page 3 ═════ + +Contents + +Abbreviations vi +List of Figures vii +List of Tables viii +1 Introduction 1 +1.1 Key issues 1 +1.1.1 Inflection 1 +1.1.2 Derivation 2 +1.1.3 Ambiguity 3 +1.1.4 Epenthesis 3 +1.1.5 Discontinuous morphemes 4 +1.1.6 Infixation 4 +1.1.7 Reduplication 4 +1.1.8 Root and pattern morphology 4 +1.1.9 Metathesis 5 +1.1.10 Morphemes that may be null 5 +1.2 Tasks for any morphological parser 6 +2 Morphotactics 6 +2.1 Affixation 7 +2.1.1 Unclassified affixes 7 +2.1.2 Inflectional affixes 7 + 2.1.2.1 Simple example 8 + 2.1.2.2 Optional affix slots 9 + 2.1.2.3 Multiple templates 10 + 2.1.2.4 Discontinuous morpheme 12 + 2.1.2.5 Inflection and categories considerations 13 + 2.1.2.6 Inflection classes 13 + 2.1.2.6.1 Inflection subclasses 15 + 2.1.2.6.2 Inflection classes and category organization 19 + 2.1.2.7 Agreement and other inflection features 19 + 2.1.2.8 Inflection classes versus inflection features 22 + 2.1.2.9 Underspecified inflectional affixes 23 +2.1.3 Derivational affixes 23 + 2.1.3.1 Major category-changing derivational affixes 23 + 2.1.3.2 Sub-category-changing derivational affixes 24 + 2.1.3.3 Non-category-changing derivational affixes 25 + 2.1.3.4 Inflection class and derivational affixes 26 + 2.1.3.4.1 Inflection class may change 26 + 2.1.3.4.2 Inflection class does not change 26 + 2.1.3.5 Inflection Features and Derivational Affixes 27 + 2.1.3.6 Category-changing derivational affixes and category organization 28 + 2.1.3.7 Underspecified derivational affixes 29 +2.1.4 Derivation outside of inflection 29 +2.1.5 Derivation versus inflection 31 +2.1.6 Exception “features” 32 +2.2 Stem compounding 34 +2.2.1 Headed compounds 34 +2.2.2 Non-headed compounds 35 +2.2.3 Incorporation 36 + + iii + +═════ PDF page 4 ═════ + +2.2.3.1 Incorporation as a simple headed compound 36 + 2.2.3.2 Incorporation as a headed compound with override 37 +2.2.4 Affixes between roots in compounds 38 +2.2.5 Compound rules and categories considerations 39 +2.2.6 Restricting the productivity of a compound rule 39 +2.3 Clitics 40 +2.4 Ad hoc morpheme-oriented rules 42 +2.4.1 Creating morpheme-oriented ad hoc rules 42 +2.4.2 Grouping ad hoc morpheme rules 43 +3 Morphophonemics 44 +3.1 Overview 45 +3.1.1 Phoneme sets 45 + 3.1.1.1 Phonological features 47 + 3.1.1.2 Digraphs 47 + 3.1.1.3 Tones 47 + 3.1.1.3.1 No forms conditioned by tone 47 + 3.1.1.3.2 Forms conditioned by tone 48 +3.1.2 Natural classes 48 +3.1.3 Allomorph environments 49 +3.1.4 Allomorph ordering 50 + 3.1.4.1 Free fluctuation 50 +3.2 Reduplication 51 +3.2.1 Full reduplication 51 + 3.2.1.1 Writing the pattern for full reduplication 51 +3.2.2 Partial reduplication 52 + 3.2.2.1 Writing the pattern for partial reduplication 53 +3.3 Infixation 54 +3.3.1 Writing the infixation environment 55 +3.3.2 Infixation and root and pattern morphology 55 +3.4 Epenthesis 56 +3.5 Metathesis 56 +3.6 Morphemes that may be null 57 +3.7 Non-phonologically conditioned allomorphy 57 +3.7.1 Stem allomorphs conditioned by morpho-syntactic features 58 +3.7.2 Affix allomorphs conditioned by morpho-syntactic features 60 +3.8 Irregularly inflected forms 61 +3.9 Coalescence 62 +3.10 Ad hoc allomorph-oriented rules 63 +3.10.1 Creating ad hoc allomorph-oriented rules 63 +3.10.2 Grouping ad hoc allomorph rules 65 +4 Lexical entry considerations 65 +4.1 Allomorphs 65 +4.1.1 Null allomorphs 65 +4.1.2 Order of allomorphs within a lexical entry 65 +4.2 Morpheme types 67 +4.3 Circumfixes 67 +4.4 Senses/Glosses 68 +5 Other considerations 68 +5.1 Exceptional Case for Compound Rules 69 +6 The phonological rule-based parser 69 +6.1 Item and process 69 +6.1.1 Affix process rules 70 + + iv + +═════ PDF page 5 ═════ + +6.1.1.1 Reduplication as a process 70 + + 6.1.1.1.1 Full reduplication as a process 70 + + 6.1.1.1.2 Partial reduplication as a process 71 + + 6.1.1.2 Infixation as a process 73 + + 6.1.1.2.1 Infixation and root and pattern morphology 74 + + 6.1.1.3 Circumfixation as a process 75 + +6.1.2 Phonological rules 75 + + 6.1.2.1 “Regular” phonological rules 75 + + 6.1.2.1.1 Epenthesis 76 + + 6.1.2.1.2 Glide becomes a vowel 76 + + 6.1.2.1.3 Tone processing 77 + + 6.1.2.1.4 Nasal assimilation 79 + + 6.1.2.1.4.1 Unspecified nasal in Bahasa Indonesia 79 + + 6.1.2.1.4.2 Unspecified nasal and full reduplication in Bahasa In- + + donesia 85 + + 6.1.2.1.4.3 Unspecified nasal exceptions in Bahasa Indonesia 87 + + 6.1.2.2 Constraining application of “regular” phonological rules 88 + + 6.1.2.2.1 Rule applies only with certain categories 88 + + 6.1.2.2.2 Rule applies only with certain properties 89 + + 6.1.2.3 Phonological metathesis rules 89 + +6.2 Tips for making the phonological rule-based parser work effectively. 91 + +6.2.1 Every phoneme used in the orthography must be defined as a phoneme 91 + +6.2.2 The phonological features need to uniquely identify each phoneme 91 + +6.2.3 Fully specify each phoneme 91 + +6.2.4 Features used in a rule should be explicit 91 + +6.2.5 Avoid using archiphonemes that are uppercase equivalents of a character in your + + orthography 92 + +6.2.6 Make sure every affix process rule is complete 92 + +6.2.7 Natural classes defined by phonemes may not work as expected 92 + +6.3 Known limitations 93 + +6.3.1 Affixes are tried only once per word 94 + +6.3.2 Natural classes defined by segments may or may not work as expected 94 + +6.3.3 Ambiguous digraphs and multigraphs may not work as expected 94 + +References 95 + +Language index 96 + +Subject index 97 + + v + +═════ PDF page 6 ═════ + +Abbreviations + +1 first person m masculine +2 second person neg negation, negative +3 third person nf non-future +abs absolutive nfd normalized decomposed form +acc accusative nmlz nominalizer/nominalization +act active obj object +adj adjective pas pseudo-passive +appl applicative pass passive +art article pfv perfective +auth authority pl plural +av actor voice poss possessive +caus causative pot potential +cntr contrafactive prf perfect +compl completive prs present +cont continuative prog progressive +dat dative pst past +dim diminutive ptcp participle +emph emphasis purp purposive +erg ergative r realis +f feminine rctpst recent past; same day as spoken +fam familar reg regressive +foc focus rep repetitive +fut future resp respect +goal goal s stative +hab habitual sbj subject +incl inclusive sg singular +ind indicative siml simultaneous action +inf infinitive ss same subject +int intensive t transitive verb in principal clause +intr intransitive tr transitive +ipfv imperfective un unreal +irr irrealis veri veritative +loc locative vso verb, subject, object + + vi + +═════ PDF page 7 ═════ + +List of Figures + +Figure 1. How to Create Orizaba Nahuatl Possessor Inflectional Template with Optional Number 10 +Suffix. +Figure 2. How to Create Orizaba Nahuatl Intransitive Verb Templates. 12 +Figure 3. How to Create Yalálag Zapotec Inflection Classes. 14 +Figure 4. How to Create Latin Inflection Classes. 15 +Figure 5. How to Create Isthmus Zapotec Inflection Classes. 18 +Figure 6. How to Create Inflection Features. 21 +Figure 7. How to Add Inflection Features to Roots or Stems. 22 +Figure 8. How to Add Inflection Features to Affixes. 22 +Figure 9. How to Create Category Changing Affixes. 24 +Figure 10. How to Create Sub-category Changing Affixes. 25 +Figure 11. How to Create Non-category Changing Affixes. 26 +Figure 12. How to Create Non-category Changing Affixes When the Inflection Class Changes. 26 +Figure 13. How to Create Non-category Changing Affixes When the Inflection Class Does Not Change. 27 +Figure 14. How to Create a Derivational Affix With Inflection Features. 28 +Figure 15. Huallaga Quechua Derivation Outside of Inflection. 30 +Figure 16. How to Create Derivation Outside of Inflection. 31 +Figure 17. How to Create and Use Exception “Features.” 33 +Figure 18. How to Create a Headed Compound Rule. 35 +Figure 19. How to Create a Non-headed Compound Rule. 36 +Figure 20. Orizaba Nahuatl Incorporation. 37 +Figure 21. Wanca Quechua with Interfix. 38 +Figure 22. How to Create and Use Exception “Features” in Compound Rules 40 +Figure 23. How to Create a Clitic. 42 +Figure 24. How to Create a Morpheme-oriented Ad Hoc Rule. 43 +Figure 25. How to Create a Group of Morpheme-oriented Ad Hoc Rules. 44 +Figure 26. How to Create Phonemes. 46 +Figure 27. How to Create Phonological Features. 47 +Figure 28. How to Associate Phonological Features with a Phoneme. 47 +Figure 29. How to Create Natural Classes. 48 +Figure 30. How to Create Environments. 49 +Figure 31. How to Create Full Reduplication via an Environment. 52 +Figure 32. How to Create Partial Reduplication via an Environment. 53 +Figure 33. How to Write the Pattern for Partial Reduplication. 54 +Figure 34. How to Create an Infix via an Environment. 54 +Figure 35. How to Create Root-and-Pattern Infixes. 55 +Figure 36. How to Handle Metathesis. 56 +Figure 37. How to Create a Null Morpheme. 57 +Figure 38. How to Create and Use Stem Allomorph Labels. 59 +Figure 39. How to Create Feature-Dependent Affixes. 60 +Figure 40. How to Handle Coalescence. 63 +Figure 41. How to Create Allomorph-oriented Ad Hoc Rules. 64 +Figure 42. How to Create a Group of Allomorph-oriented Ad Hoc Rules. 65 +Figure 43. How to Create a Null Allomorph. 66 +Figure 44. How to Create a Circumfix. 68 +Figure 45. Selaru Epenthesis Rule. 76 +Figure 46. Selaru Glide Vowel Rule. 77 +Figure 47. How to Handle Awngi Tone. 78 + + vii + +═════ PDF page 8 ═════ + +Figure 48. Awngi Docking Rule. 78 + +Figure 49. Awngi Deletion Rule. 78 + +Figure 50. Indonesian Nasal Assimilation Rule. 79 + +Figure 51. How to Handle the Unspecified Nasal in Indonesian. 83 + +Figure 52. Indonesian Unspecified Nasal Vowel Rule. 84 + +Figure 53. Indonesian Unspecified Nasal Sonorant Consonant Rule. 84 + +Figure 54. Indonesian Unspecified Nasal Assimilation Rule. 84 + +Figure 55. Indonesian Unspecified Voiceless Obstruent Deletion Rule. 84 + +Figure 56. Indonesian Unspecified Nasal Full Reduplication Rule. 86 + +Figure 57. How to Use an Exception “Feature” for Unspecified Nasal in Indonesian. 87 + +Figure 58. Selaru Metathesis Rule. 90 + +List of Tables + +Table 1. Orizaba Nahuatl Singular Possessor Template 8 + +Table 2. Orizaba Nahuatl Plural Possessor Template 9 + +Table 3. Orizaba Nahuatl Present Intransitive Template 11 + +Table 4. Orizaba Nahuatl Singular Present Intransitive Template 11 + +Table 5. Orizaba Nahuatl Plural Present Intransitive Template 11 + +Table 6. Caquinte Future Template 12 + +Table 7. Sample Inflection Features 20 + +Table 8. Using Features to Rule Out Parses 21 + +Table 9. Inflection Classes vs. Inflection Features 22 + +Table 10. Quechua Non-final Template (Schematic) 30 + +Table 11. Inflection vs. Derivation 31 + +Table 12. Types of Interfixes 38 + +Table 13. Morpheme Ad-hoc Rule 43 + +Table 14. Yalálag Zapotec Subject and Object Dependent Suffix Co-occurrence 44 + +Table 15. Sample Environments 50 + +Table 16. Samples of Partial Reduplication 54 + +Table 17. Silt'e Template 55 + +Table 18. Allomorph Ad-hoc Rule 63 + +Table 19. Silt'e Template (repeated) 74 + +Table 20. Silt'e Template with Phonological Rule-based Parser 75 + +Table 21. Phonological Features for Bahasa Indonesian Implementation 83 + +Table 22. Phonological Features for Selaru Implementation 90 + +Table 23. Phonological Features for Salar Implementation 93 + + viii + +═════ PDF page 9 ═════ + +1 Introduction + +Morphology is the study of word forms. Morphological parsers are computational tools that automatically +produce a morphological analysis for a given word form. Such tools have proven to be quite useful as +spelling checkers, as morphological grammar checkers, in producing interlinear text and in adaptation of a +text from one related language to another. This document is designed to help the reader do morphological +parsing using the approach allowed by the FieldWorks Language Explorer parser. We also introduce +issues related to the phonological rule-based parser, where appropriate. + + The purpose of this documentation is to provide an introduction to the key concepts and notions in +the FieldWorks Language Explorer approach to morphological parsing. It is divided into two main +sections: morphotactics and morphophonemics. The first has to do with controlling which morphemes can +co-occur with which other morphemes within a well-formed word. The second has to do with controlling +the phonological shape1 of individual morphemes. (There are two other main sections; one deals with some +issues related to lexical entries and the other to special considerations related to using compound rules.) + + As mentioned above, this document also includes the phonological rule-based parser2 that you can try in +addition to the default parser that has been available since day one of FieldWorks Language Explorer. +Both parsers are robust as far as we know. Please do report to us anything that you notice about this parser +that may help us make it be more effective. See section 6 for more on this parser, as well as the subject +index. + +1.1 Key issues + +We begin by addressing some of the key issues that any general morphological parser must face. Before +we can tell the computer what to do, we need to understand what is going on linguistically. What kinds of +language phenomena must such a computational tool be able to handle if it will indeed be a general tool? + +1.1.1 Inflection + +Many, if not most, languages inflect verbs and/or nouns. Consider the nominal Orizaba Nahuatl [nlv] forms +shown in (1) and the verbal ones shown in (2).3 + +(1) nokal mokal ikal + +no-kal mo-kal i-kal + +1sg.poss-house 2sg.poss-house 3sg.poss-house + +‘my house’ ‘your(sg) house’ ‘his/her/its house’ + +tokal amokal inkal +to-kal amo-kal in-kal +1pl.poss-house 2pl.poss-house 3pl.poss-house +‘our house’ ‘your(pl) house’ ‘their house’ + +1 In this document, “phonological shape” and “surface form” are synonymous. +2 This parser is an enhanced and updated version of Mike Maxwell's Hermit Crab parser. See Maxwell +(1998). We are deeply indebted to Mike for his pioneering work on this parser. +3 Data are from Tuggy (1991). + + 1 + +═════ PDF page 10 ═════ + +2 Parsing Conceptual Introduction + + nokalvan mokalvan ikalvan + no-kal-van mo-kal-van i-kal-van + 1sg.poss-house-PlPoss 2sg.poss-house-pl.poss 3sg.poss-house-pl.poss + ‘my houses’ ‘your(sg) houses’ ‘his/her/its houses’ + + tokalvan amokalvan inkalvan + to-kal-van amo-kal-van in-kal-van + 1pl.poss-house-pl.poss 2pl.poss-house-pl.poss 3pl.poss-house-pl.poss + ‘our houses’ ‘your(pl) houses’ ‘their houses’ + + (2) nimiki timiki miki + ni-miki ti-miki 0-miki + 1sg.sbj-die 2sg.sbj-die 3sg.sbj-die + ‘I die’ ‘you(sg) die’ ‘he/she/it dies’ + + timikih anmikih mikih + ti-miki-h an-miki-h 0-miki-h + 1pl.sbj-die-ipfv.pl 2pl.sbj-die-ipfv.pl 3pl.sbj-die-ipfv.pl + ‘we die’ ‘you(pl) die’ ‘they die’ + + Notice how each possessed noun in (1) has at least a possessor prefix. Certain nouns require this possessor +inflection. Similarly the verbs in (2) require subject markers (with the possible exception of 3rd person). +A morphological parser must account for such inflectional items. + +1.1.2 Derivation + +Consider the English [eng] forms4 in (3). What is happening here? How do you get a dumb computer to +“understand” these forms correctly? + + (3) a. institute + b. institution + c. institutional + d. institutionalize + e. institutionalization + f. institutionalizational + g. institutionalizationally + + In (3a) institute is a verb root (e.g., We need to institute some changes around here.). By adding the suffix +‑ion as in (3b), the word is changed to a noun. The suffix ‑al can be added to a noun stem to change it to +an adjective, as in (3c). The suffix ‑ize changes an adjective into a verb (3d). Further category changes +occur with the addition of each suffix in (3e-g). From this English example, we have seen that the computer +needs to be able to distinguish between roots and suffixes, with each one restricted as to what category +it attaches to and what category it changes the stem to. (Note, for example, that the suffix ‑ly cannot be +added to either a verb stem or a noun stem: *institutely, *institutionly.) + + A Huallaga Quechua [qub] example showing similar category changes along with various types of verbal +and nominal affixes is given in (4). The verb root meaning ‘to see’ has the imperfective aspect marker +added, followed by the first person object marker, yielding ‘to see me.’ The addition of the nominalizer +changes the form to a noun meaning ‘seeing me.’ The noun form can now be possessed by the second + +4 These are taken from Spencer (1991:9). + +═════ PDF page 11 ═════ + +Epenthesis 3 + +person possessive marker and then the purpose marker may optionally follow, finally giving ‘in order that +you might be seeing me.’5 + +(4) rikaykaamaanaykipaq + rika-yka:-ma:-na-yki-paq + see-ipfv-1.obj-nmlz-2.poss-purp + ‘in order that you might be seeing me’ + +A morphological parser must account for such derivational items. + +1.1.3 Ambiguity + +Ambiguity is also apparent in (3a), since institute can be either a verb, as above, or a noun, as in Australian +Institute of Marine and Power Engineers. Note that there are different types of ambiguity in natural language +as well. For example, the word bank (among other things) can mean either the side of a river or a building +that holds money. With either meaning, bank is a noun. + + Now consider the following word: + + (5) a. cooks + cook-s + person.who.prepares.food-pl + + b. cooks + cook-s + prepare.food-3sg.prs + + Note that cooks is ambiguous not only in the root meaning but also as to the suffix: the -s is a nominal +plural morpheme in (5a) but a verbal third person singular present tense morpheme in (5b). + + A morphological parser must be able to deal with the fact that individual words can legitimately be +ambiguous. That is, a morphological parser must be able to discover and report all possible analyses of +a word form. In many cases, the ambiguity is eliminated when the word is seen in context, so ideally a +morphological parser is used in the context of computational tools that look beyond a single word. + +1.1.4 Epenthesis + +There are still other types of challenges for morphological parsing. For example, consider the Caquinte +[cot] word in (6):6 + + (6) itsavetakohitiro + i-tsave-(t)-ako-hi-(t)-i-ro + 3m-tell- -dat-pas- -nf-3f.obj + ‘she is told about’ + + The (t) in two places on the second line (which shows the word broken into morphemes) are not really +morphemes at all. Instead, they are epenthetic consonants added to serve as onsets to syllables. Caquinte +does not allow vowel clusters nor syllables without onsets (in this part of the verb), so whenever two vowels +come together at a morpheme break, an epenthetic t is inserted. A morphological parser needs to be able +to correctly account for forms that include epenthetic segments inserted to preserve syllable structure. + +5 The data are from Weber, Black, and McConnel (1988:8). See also Weber (1989). +6 All Caquinte data are from Ken Swift, personal communication, and Swift (1988). + +═════ PDF page 12 ═════ + +4 Parsing Conceptual Introduction +1.1.5 Discontinuous morphemes + +Now consider the Caquinte form in (7), which is the same word as in (6), but changed to future tense: + + (7) intsavetakohitero + i-n-tsave-(t)-ako-hi-(t)-e-ro + 3m-fut-tell- -dat-pas- -fut-3f.obj + ‘she will be told about’ + + What is the challenge here? The future tense is realized as a discontinuous morpheme: it is composed of +the prefix n‑ and the suffix ‑e. The computer must be able to check these noncontiguous parts of the word +to correctly analyze the future tense in Caquinte; one part cannot be present without the other. + +1.1.6 Infixation + +The Tagalog [tgl] forms in (8) illustrate another challenge:7 + + (8) a. sulat ‘to write or writing (infinitive form)’ + b. sumulat ‘to write (with actor focus)’ + c. sinulat ‘to write (with object focus)’ + + What is happening here? This is a case of infixation, where the root sulat splits into two parts so that +one of the focus morphemes, ‑um‑ or ‑in‑, can be inserted. A parser must correctly recognize the root even +though it is broken apart by the infix. + +1.1.7 Reduplication + +Look at the additional Tagalog forms in (9) to determine how the imperfective aspect is marked: + +(9) a. susulat ‘to write (imperfective)’ + +b. magpasulat ‘to make someone write (perfective)’ + +c. magpapasulat ‘to make someone write (imperfective)’ + + We know from (8a) that sulat means ‘to write’. So in (9a) it appears that the imperfective marker is su, +but we cannot tell if it is a prefix or an infix without looking at other forms. In example (9b) the causative +‘to make someone’ is the prefix pa‑. The mag‑ is what some call the actor focus or actor voice morpheme. +But the imperfective of this causative form is not *sumagpasulat, *magsupasulat, nor *magpasusulat as we +would expect from either prefixing or infixing su. Instead, we have magpapasulat in (9c) where it is clear +that the marker for imperfective is the extra pa. The correct analysis is therefore that imperfective aspect is +marked in Tagalog by reduplicating either the first syllable of the stem or the initial consonant and vowel +of the first syllable of the stem. + + A morphological parser must be able to recognize reduplication within a word form. + +1.1.8 Root and pattern morphology + +Semitic [sem] languages pose a special challenge with their root and pattern morphology. These languages +have roots composed of three consonants, as exemplified in the Silt'e [stv] data in (10), where ‘buy’ is +the root wkb. The aspect markers are composed of vowel patterns that fit between or around the root +consonants, such as the a-a vowel pattern indicating the perfective aspect shown in (10). The parser + +7 The data are from Spencer (1991:12-13). + +═════ PDF page 13 ═════ + +Morphemes that may be null 5 + +needs to be able to find the root consonants and corresponding vowels of the aspect, even though they are +intermingled in the surface form of the word.8 + +(10) wakaba + a-a-wkb-a + pfv-buy-3sg.pfv + ‘he bought’ + +1.1.9 Metathesis + +Now study the following Caquinte word. + + (11) ihikekehai + i-hi-k-e-kea-hi + 3m-think.mistakenly-prog-nf-foc-neg + ‘he thought mistakenly’ + + What change takes place at the juncture between the final two morphemes? Notice that where one might +expect the sequence keahi, what surfaces is kehai, where the h and a switch positions.9 Such a transposition +of phonemes is called metathesis. Furthermore, notice that the metathesis process in (11) crosses morpheme +boundaries. + + Such data imply that a morphological parser must be able to correctly identify morphemes even when +some segments within the morphemes may have switched positions. + +1.1.10 Morphemes that may be null + +For a final challenge, consider these Caquinte forms (you do not need to understand all the morpheme +glosses here; just concentrate on the initial subject prefixes): + + (12) a. anehero + a-∅-neh-e-ro + 1incl-fut-see-fut-3f.obj + ‘we will see her’ + +b. okeekake + o-keek-ak-e + 3f-dig-pfv-nf + ‘she had dug’ + + c. oasanomahakemparime + ∅-∅-o-(a)-sano-maha-k-e-Npa-ri-me + 1incl-fut-eat- -veri.m-veri-prog-fut-r-3m.obj-cntr + 3f-fut-eat- -veri.m-veri-prog-fut-r-3m.obj-cntr + ‘we/she will not really be eating it’ + + What is the problem with the subject prefixes? In (12a) we see that the first person inclusive subject +marker is a‑, and in (12b) the third person feminine subject marker is o‑. Yet, in (12c), the gloss shows +ambiguity between ‘we’ and ‘she’ as the subject, and both of these are represented as null. This is because +both subject prefixes are vowels and the stem in (12c) is vowel-initial, yielding two vowels together. Recall + +8 The data are from Gardner (1994). +9 This metathesis process is actually optional. The word is from Swift (1988:133). + +═════ PDF page 14 ═════ + +6 Parsing Conceptual Introduction + +from (6) that Caquinte generally does not allow vowel clusters, and therefore adds an epenthetic ‑t‑ when +necessary to avoid such clusters. It turns out that epenthesis is only used in the suffixes. Within the prefixes, +the initial vowel of a cluster deletes, causing the ambiguity seen in (12c). + + This means that a morphological parser must be able to identify a morpheme even when the morpheme +has no overt segments. + +1.2 Tasks for any morphological parser + +Given the challenges of morphological parsing exemplified in the preceding section, how can a computer +program go about analyzing words into their constituent morphemes? Let's say that the task of a morpho- +logical parser is to take a form like itsavetacojitiro from (6) above and + + • see if it is a legitimate word, + • give an indication of which characters correspond to what morpheme, and + • produce an ordered list of its constituent morphemes (e.g., give glosses).10 + + What are some of the things our parser is going to have to know and what are some of the things that it +is going to have to do? + + Things the parser needs to KNOW: + + • the forms and glosses of prefixes, infixes, roots, and suffixes, + • which of the affixes go with which roots, + • the relative ordering of the affixes, and + • when a particular shape of a form is legitimate and when it is not (allomorphy/phonology). + + Things the parser needs to DO: + + • find the affixes and root(s) in a given word, and + • check to see whether each affix can go with the rest of the word (that is, apply a set of constraints). + + Clearly, properly using and controlling the constraints is the major task in implementing a parser for a +given language. Since a morphological parser must model linguistic reality, it is a good idea to use con- +straints that model appropriate linguistic notions. Two major concepts for morphology are morphotactics +and morphophonemics. Morphotactics deal with which morphemes can co-occur with which other mor- +phemes. Morphophonemics deal with what shape a given morpheme will have in various phonological +and morphological environments. The next two major sections outline the constraints available with the +FieldWorks Language Explorer parser and how to use them. + +2 Morphotactics + +Morphotactics has to do with controlling the order of the morphemes in a well-formed word and controlling +which morphemes can co-occur with which other morphemes. As examples of the former, one would not +expect to find a prefix at the end of a word or a suffix at the beginning of a word. As an example of the +latter, while one would expect a tense affix to appear with a verb root in a verbal word, one would not +expect a tense affix to show up on a pronoun. The morphotactic mechanisms described in this section +delineate what one can do within the FieldWorks Language Explorer model to control such things. + +10 Another thing a parser could produce would be the actual word structure which could be shown via a tree +diagram. While the default FieldWorks Language Explorer parser actually produces such a structure, +we do not make it visible. + +═════ PDF page 15 ═════ + +Inflectional affixes 7 + +The idea is to use the morphotactic mechanisms to correctly describe the facts of the language and thereby +not only provide correct parses, but also rule out false parses. + + By the way, correctly describing the facts of the language also provides the basis for a grammatical +description, something that FieldWorks Language Explorer provides. By making a correct description +of the facts we can both generate a description that people can read to learn about the language and we +can feed the information to a parser that can put our description to work checking spellings, adapting to +other languages, and verifying the fit of our description. + + Note that for words which consist solely of a single morpheme, there are no special morphotactic con- +siderations. One merely adds appropriate lexical entries for these and ensures that the morpheme type of +the allomorph(s)11 in the entry is(are) set to a root or stem type. + + This section has four major sub-sections. The first deals with handling affixation to stems (section 2.1). +The second deals with stem compounding (section 2.2). The third discusses issues related to clitics (section +2.3). The fourth is for those cases where the parser is producing parses that are incorrect, but the parser +mechanisms do not allow any other way to eliminate the false parses (section 2.4). + +2.1 Affixation + +This section discusses issues relating to adding affixes to stems. Linguists typically divide affixes into two +major categories: inflectional and derivational. Therefore, FieldWorks Language Explorer allows +you to declare a given affix as being either inflectional or derivational. In the process of analyzing a +language, however, sometimes one does not yet know whether a given affix is inflectional or derivational. +There are certain affixes which are truly difficult to classify in this fashion. For this reason, FieldWorks +Language Explorer also allows you to label a given affix as being unclassified with respect to inflection +and derivation. As you study the language more, you should eventually figure out whether such affixes are +inflectional or derivational and then you can change their status from being unclassified to the appropriate +one. + +2.1.1 Unclassified affixes + +You can label an affix as “unclassified” when you do not know if it is derivational or inflectional. Please +understand, though, that when you do this, the affix is relatively unconstrained as to where it can appear. +As a result, the FieldWorks Language Explorer parser may return a number of incorrect parses for +some word forms which happen to contain a sequence of characters that match one or more allomorphs of +an unclassified affix. One partial solution to this is to indicate the category of the stem to which the affix +may attach. The best solution, of course, is to classify the affix as being either inflectional or derivational12 +so it will only show up where it should. See section 2.1.5 for more on how to determine if an affix is +derivational or inflectional. + +2.1.2 Inflectional affixes + +Inflectional affixes typically reflect what some call “grammatical meaning.” These are things like person, +number, case, gender, tense, aspect, etc. One can also typically create a paradigm of word forms with the +various inflectional categories as labels on a chart.13 + +11 We use the term “allomorph” here as a cover term for any form in a lexical entry. Typically, a lexical +entry contains a single morpheme. Some morphemes have more than one allomorph. +12 Note that it is possible to classify an affix as being inflectional or derivational, but still not fully specify the +required information. Such partially specified inflectional and derivational affixes are effectively treated as +“unclassified” affixes by the FieldWorks Language Explorer parser. See sections 2.1.2.9 and 2.1.3.7. +13 For more on this, see Bickford (1998:113ff). + +═════ PDF page 16 ═════ + +8 Parsing Conceptual Introduction +2.1.2.1 Simple example + +For example, consider the information for a possessed noun in Orizaba Nahuatl given in (1) above, but this +time displayed in a different fashion: + +(13) house singular possessed noun + + 1st Person Singular Possessive nokal + 2nd Person Singular Possessive mokal + 3rd Person Singular Possessive ikal + 1st Person Plural Possessive tokal + 2nd Person Plural Possessive amokal + 3rd Person Plural Possessive inkal + + What are the inflectional affixes here? Given that every form has the sequence kal, it appears that there +are six possessor prefixes which occur before the noun stem. Similar paradigms for other singular possessed +nouns would show the same situation (ignoring any morphophonology). Therefore we could posit that the +singular possessed noun has an inflectional template that consists of a possessor prefix followed by the +stem. We could diagram this as in table 1. + + Table 1. Orizaba Nahuatl Singular Possessor Template + + Possessor Stem + no- 1sg.poss + mo- 2sg.poss + i- 3sg.poss + to- 1pl.poss + amo- 2pl.poss + in- 3pl.poss + +Now consider the plural possessed noun data from (1) above, but displayed in a similar fashion to (13). + +(14) house plural possessed noun + + 1st Person Singular Possessive nokalvan + 2nd Person Singular Possessive mokalvan + 3rd Person Singular Possessive ikalvan + 1st Person Plural Possessive tokalvan + 2nd Person Plural Possessive amokalvan + 3rd Person Plural Possessive inkalvan + + What are the inflectional affixes here? Notice that there is the same stem (kal) and the same set of six +possessor prefixes as in (13). In addition, there is a plural suffix ‑van. Similar paradigms for other plural +possessed nouns would show the same situation (ignoring any morphophonology). Therefore we could +posit that the plural possessed noun has an inflectional template that consists of a possessor prefix followed +by the stem which, in turn, is followed by a plural suffix. Since plural is an instance of the notion of number, +we could diagram this as an inflectional template as shown in table 2. + +═════ PDF page 17 ═════ + +Optional affix slots 9 + + Table 2. Orizaba Nahuatl Plural Possessor Template + + Possessor Stem Number + no- 1sg.poss -van pl + mo- 2sg.poss + i- 3sg.poss + to- 1pl.poss + amo- 2pl.poss + in- 3pl.poss + + Notice what we have described here: for a particular category (possessed noun), we have an inflectional +template with one prefix slot (for possessor) and one suffix slot (for number). The possessor slot can be +filled by any of the inflectional prefixes listed in table 2. The number slot can be filled by the plural suffix. + + Be aware that if you define a template that has no slots, then the parser will ignore that template. A +template must have slots for the parser to use it. Similarly, if a slot has no affixes in it, that slot will also +be ignored. + +2.1.2.2 Optional affix slots + +Now you may well have noticed that there is a potential problem here with the template in table 2. If +we treat each slot in the template as being obligatory, then the template says we must have a number +suffix in order for the template to be satisfied. This means that a possessed singular noun will not meet +the requirements of this template because it does not have a suffix in the number slot. It turns out that +FieldWorks Language Explorer actually does treat each slot as being obligatory unless it is overtly +marked as being optional. + + What can we do about this? There are at least three options available within the FieldWorks Language +Explorer approach: + + 1. Treat the Number slot as being optional so that for the singular case, there would not be any suffix in + the Number slot. + + 2. Create two distinct templates for the possessed noun category: one for singular and one for plural. + 3. Create a null singular number suffix which could then satisfy the requirement of something being in + + the Number slot. + + Which of these three should we use? Options 1 and 2 will effectively give the same result, although +option 1 is definitely simpler. Following the general principle known as Occam's Razor,14 option 1 is thus +better. + + Option 3 requires us to posit a null suffix and some argue that if an affix is always null (as it would be +here) then what we really have is a default feature: unless there is an overt number suffix, assume that the +number is singular. The current parsers do not allow us to mark such default features. + + Therefore, from a long term perspective, we recommend following option 1. + This means that to model this inflectional template, we will need to do what is shown in figure 1 below. + Once we have done this, we will have successfully set up the inflectional morphotactics for possessed +nominals in Orizaba Nahuatl. + +14 Occam's Razor states “one should not increase, beyond what is necessary, the number of entities required +to explain anything.”. See Principia Cybernetica Web (1997) for more detail. + +═════ PDF page 18 ═════ + +10 Parsing Conceptual Introduction + + 1. Create or at least make sure we have a possessed noun category. + a. Use Grammar / Category Edit. + b. Click on Noun. + c. Use Insert menu item / Subcategory. + d. In the ensuing dialog box, click on The category I need is not shown in the Catalog. Create + a custom category. + e. Fill in its name, abbreviation, and description. + + 2. Create an inflectional template within the possessed noun category. + a. Click on Affix Templates and then Insert Affix Template. + b. Fill in a name for it. + + 3. Give that template one prefix slot (for possessor). + a. Right-click on STEM and then Insert Slot before Stem. + b. Click on Add an obligatory slot to Noun. + c. Type “Possessor” in the highlighted area. + + 4. Give that template one suffix slot (for number). + a. Right-click on STEM and then Insert Slot after Stem. + b. Click on Add an optional slot to Noun. + c. Type “Number” in the highlighted area. + + 5. For the possessor prefix slot, put in it the six possessor prefixes listed in the first column of table + 2. If these possessor prefixes do not already exist, then we need to create the lexical entries and + mark them as inflectional. + a. Right-click on Possessor and then Add inflectional affix(es) to Possessor . + b. In the ensuing dialog, either select the already existing prefixes or click on the Create new + inflectional affix link. + + 6. For the number suffix slot, put the plural suffix in it. If the number suffix does not already exist, + then create the lexical entry for it and mark it as inflectional. + a. Right-click on Number and then Add inflectional affix(es) to Number . + b. In the ensuing dialog, either select the already existing number suffix or click on the Create + new inflectional affix link. + + . + +Figure 1. How to Create Orizaba Nahuatl Possessor Inflectional Template with Optional Number Suffix. + +2.1.2.3 Multiple templates + +In the previous section we suggested that using optional affix slots in a template was a good choice for +handling Orizaba Nahuatl nominal possession. Since we noted that within the FieldWorks Language +Explorer approach, one could add more than one template to a category, one might wonder when it +would be appropriate to choose such an option. + + Orizaba Nahuatl happens to provide such a case. Consider the information for an intransitive, present +tense verb given in (2) above, but this time displayed in a fashion more conducive to our purposes here: + + (15) die, present tense 1st Person Subject 2nd Person Subject 3rd Person Subject + + Singular nimiki timiki miki + Plural timikih anmikih mikih + +═════ PDF page 19 ═════ + +Multiple templates 11 + + What are the inflectional affixes here? At least under one analysis, there are four subject prefixes and a +plural suffix. Third person subject is the default or is null. Similarly, singular number is the default or null. + + Where do these inflectional affixes appear? Notice that all the subject ones appear just before the stem +and that the plural suffix appears right after the stem. Similar paradigms for other intransitive verbs would +show the same situation (ignoring any morphophonology). Therefore we could posit that the present tense, +intransitive verb has an inflectional template that consists of a subject inflectional affix followed by the stem +which is followed by a number inflectional suffix. We could diagram this as in table 3. + + Table 3. Orizaba Nahuatl Present Intransitive Template + + Subject Stem Number + ni- 1sg.sbj -h pl + ti- 2sg.sbj + ti- 1pl.sbj + an- 2pl.sbj + + At first glance, this is very much like what we saw for possessed nominals in table 2 above. We might +think initially that we can do exactly what we did for possessed nominals and merely mark the Number +slot as optional for these intransitive verbs. If we were to do that, however, notice what would happen +for a form like timiki which is supposed to only mean ‘‘you(sg) die.’’ Because the Number slot would be +optional, the FieldWorks Language Explorer parser would allow a parse of ‘1pl.sbj-die’ as well (this, +of course, is because both ‘2sg.sbj’ and ‘1pl.sbj’ have the same shape: ti‑). At this point, we would have +nothing to prevent this incorrect parse.15 + + To eliminate this problem (as well as to eliminate the possibility of the parser allowing a parse for an +ill-formed word such as *anmiki), we can create two inflectional templates: one for singular and one for +plural. The singular one will be like this: + + Table 4. Orizaba Nahuatl Singular Present Intransitive Template + + SgSubject Stem + ni- 1sg.sbj + ti- 2sg.sbj + +The plural one will be like this: + + Table 5. Orizaba Nahuatl Plural Present Intransitive Template + + Subject Stem Number + + ti- 1pl.sbj -h pl + + an- 2pl.sbj + + Notice how this method places the singular subject markers in the singular template and puts the plural +subject markers in the plural template. This way we force the presence of the plural suffix for the plural +subject prefixes. + +15 In addition, the form timikih could parse as ‘2sg.sbj‑die‑pl.’ This, too, is incorrect. If we used the +Inflectional Affix Gloss Builder tool available in FieldWorks Language Explorer to create the glosses, +then this parse would not appear: the subject number agreement feature would have a value of ‘singular’ +which would conflict with the number agreement feature value of the suffix; namely ‘plural.’ The current +parsers, however, do not have any way to indicate default features for a category (e.g., marking ‘singular’ +as the default) in order to prevent the form timiki from parsing as ‘1pl.sbj‑die.’ + +═════ PDF page 20 ═════ + +12 Parsing Conceptual Introduction + + What needs to be done to handle the 3rd person cases? We will need to mark the subject slot as optional +in both templates in order to allow for the 3rd person cases. + + This means that to model this inflectional template, we will need to do what is shown in figure 2. + + 1. Create or at least make sure we have an intransitive verb category. + a. Use Grammar / Category Edit. + b. Click on Verb. + c. Use Insert menu item / Subcategory. + d. In the ensuing dialog box, open up Verb and choose Intransitive verb. + + 2. Create two inflectional templates within the intransitive verb category (see figure 1 for how to + do this): + a. For the singular template: + i. Give it one optional prefix slot (for singular subject). + ii. Put the 1sg.sbj‑ and 2sg.sbj‑ prefixes in this slot. If these prefixes do not already exist, + create them and mark them as inflectional. + b. For the plural template: + i. Give it one optional prefix slot (for plural subject). + ii. Put the 1pl.sbj‑ and 2pl.sbj‑ prefixes in this slot. If these prefixes do not already exist, + create them and mark them as inflectional. + iii. Give it a obligatory suffix slot (for number). + iv. Put the ‑pl suffix in this slot. If this suffix does not already exist, create it and mark it as + inflectional. + + . + + Figure 2. How to Create Orizaba Nahuatl Intransitive Verb Templates. + +2.1.2.4 Discontinuous morpheme + +In section 1.1.5 above, we noted that in Caquinte, the future tense is realized as a discontinuous morpheme: +it is composed of the prefix n‑ and the suffix ‑e. We repeat the example here: + + (16) intsavetacojitero (=7) + + i-n-tsave-(t)-ako-hi-(t)-e-ro + + 3m-fut-tell- -dat-pas- -fut-3f.obj + + ‘she will be told about’ + + How do we fulfill this requirement that both the future prefix and future suffix appear? One way is to +create a future tense inflectional template which has both the prefix and the suffix required. The template +might look like this: + + Table 6. Caquinte Future Template + + Subject Future Stem Future Object + no- 1.sbj N- fut -e fut -na 1.obj + a- 1incl.sbj -ahi 1incl.obj + pi- 2.sbj -Npi 2.obj + i- 3m.sbj -ri 3m.obj + o- 3f.sbj -ro 3f.obj + +═════ PDF page 21 ═════ + +Inflection classes 13 + + Another possible way to treat discontinuous morphemes when one part appears before the stem and the +other appears after the stem is to treat them as a single circumfix entry. See section 4.3. + +2.1.2.5 Inflection and categories considerations + +The categories in FieldWorks Language Explorer are organized in a hierarchical fashion. For example, +one can have a major category of verb and then nest other verb types underneath it (e.g., intransitive +verb, transitive verb, etc.) One can even nest other types under these if one so wishes (e.g., one +might put bitransitive verb under transitive verb.). + + The exact hierarchy one uses can make a difference for how the FieldWorks Language Explorer +parser handles the inflectional templates and their slots. For templates, when you define an inflectional +template for a given category, that template will be tried for any stem of that category or a stem of any of +its nested categories. If, for example, you have intransitive verb and transitive verb nested +under verb, then any inflectional template you define on verb will also be tried by the FieldWorks +Language Explorer parser for any intransitive verb or transitive verb stem. On the other +hand, in this scenario, any inflectional template defined under intransitive verb will only be applied +to intransitive verb stems and any inflectional template defined under transitive verb will only +be applied to transitive verb stems. + + Thus, you can capture generalizations about the inflectional templates by placing common inflectional +templates higher in the hierarchy. + + Inflectional affix slots behave similarly with respect to the hierarchy: when one defines the slots for a +given category, those slots may be used in any template for this category and any of its nested categories. +For example, if all of your verbs share a common subject slot, then you can define this subject slot at the +main verb category. This slot will then be available for any affix templates in all sub-categories of verb.16 + + You may well need to keep this in mind as you design your category hierarchy. + +2.1.2.6 Inflection classes + +We now turn to something that is actually about morphophonemics, not morphotactics. We include it here, +though, because it relates to inflectional affixes. + + Consider the Yalálag Zapotec [zpu] data given in (17)‑(18):17 + + (17) a. utecho + u-te-cho + fut-pass(tr)-1pl.incl + +b. u:ke'nia'cho + u-:ke'nia'-cho + fut-limp(intr)-1pl.incl + +16 Please note that you can use the same name for slots at different points in the hierarchy (e.g., use “Subject” +at both the top verb level and also under a nested intransitive verb category). The FieldWorks +Language Explorer parser will always know which one is which. You and others who look at your +implementation, however, may find it confusing. Therefore, we do not recommend that you do this. +17 The data are from López y Newberg (1990). + + The orthography used here is slightly different from what is used in López y Newberg (1990). In particu- +lar, fortis consonants are preceded by a colon (:). Lenis consonants are not (and use the voiceless equivalent +instead of the voiced one). + +═════ PDF page 22 ═════ + +14 Parsing Conceptual Introduction + + (18) a. :techo + :-te-cho + fut-pass(intr)-1pl.incl + + b. :ti:pla':chcho + :-ti:pla':ch-cho + fut-encourage(tr)-1pl.incl + + What is the phonological shape of the Future marker? It appears to be u‑ in (17) but the “fortifier” +segment/feature :‑ (i.e., a colon) in (18). Notice that there do not appear to be any phonological reasons +for the different allomorphs. In fact, the stem has the same phonological shape in (17a) and in (18a). (The +difference in future allomorphy is not due to transitivity.) This problem is not isolated to these pairs of +forms; it turns out that verb stems in general divide into two groups, those that take the u‑ future and those +that take the :‑ future. + + How do we handle this kind of allomorphy when the choice of allomorphs is not motivated by the +phonological environment but by the choice of the lexical stem? The FieldWorks Language Explorer +approach is to use inflection classes. An inflection class is “a set of lexemes whose members each have the +same type of inflectional forms” (Aronoff 1994:64). They correspond to the traditional idea of declension +classes or conjugation classes. For Yalálag Zapotec, we would create two inflection classes at the top-level +verb category (so that it applies to verb and all sub-categories of verb; see section 2.1.2.6.2). One class +would be for stems that select the u‑ allomorph and the other would be for those that take the “fortifier” :‑ +allomorph. + + This means that to model these inflection classes, we will need to do what is shown in figure 3. + + 1. Create two inflection classes within the verb category. + a. Use Grammar / Category Edit. + b. Click on Verb. + c. Click on Inflection Class Info and then Insert Inflection Class. + d. Fill in its name, abbreviation, and description. + + 2. Create the future inflectional prefix (using Lexicon / Lexicon Edit) and within it + a. Create the u‑ allomorph and tag it as belonging to the first inflection class. + i. To see the Inflection Classes field, you may have to check the Show Hidden Fields box. + ii. Click in the Inflection Classes field. + iii. Click on the chooser button. + iv. In the ensuing dialog box, check the first inflection class. + b. Create the “fortifier” :‑ allomorph and tag it as belonging to the second inflection class. + + 3. For each verb root (or stem), tag it as belonging to either the first or the second inflection class, + whichever is correct for that verb. + a. Use Lexicon / Lexicon Edit and find each verb, one at a time. + b. To see the Inflection Class field, you may have to check the Show Hidden Fields box. + c. Click in the Inflection Class field under Grammatical Info. Details. + d. Click on the chooser button. + e. In the ensuing dialog box, select the appropriate inflection class. + f. Alternatively, use Lexicon / Bulk Edit Entries with the List Choice tab to set the values for + many verbs at once. See the help files in FieldWorks Language Explorer for more. + +. + + Figure 3. How to Create Yalálag Zapotec Inflection Classes. + +═════ PDF page 23 ═════ + +Inflection subclasses 15 + +Now consider the following Latin [lat] data which also illustrates the use of inflection classes.18 + +(19) Declension Citation Form Gloss Dative Plural + + I causa reason caus-is + II annus year ann-is + III civis citizen civ-ibus + IV manus hand man-ibus + V dies day di-ebus + + Note that while there are five distinct declensions in Latin, there are only three forms for the dative plural: +‑is, ‑ibus, and ‑ebus. In particular, notice that ‑is is used for both declension class I and II and, similarly, +‑ibus is used for both declension class III and IV. So to model this Latin data in FieldWorks Language +Explorer, we will need to do the following.19 See figure 3 for details on the steps given. + + 1. Create five inflection classes within the noun category. + 2. Create the dative plural inflectional suffix and within it + + a. Create the ‑is allomorph and tag it as belonging to both the first and second inflection classes. + b. Create the ‑ibus allomorph and tag it as belonging to both the third and fourth inflection + + classes. + c. Create the ‑ebus allomorph and tag it as belonging to the fifth inflection class. + 3. For each noun root (or stem), tag it as belonging to the appropriate inflection class, whichever + is correct for that noun. + . + + Figure 4. How to Create Latin Inflection Classes. + + Note that one can also set the default inflection class to be one of the inflection classes. If you do this, +the FieldWorks Language Explorer parser will use this default inflection class for any stem that is not +overtly tagged with an inflection class. + + In addition, if an affix entry has any inflection classes and at least some of the allomorphs are constrained +with environments (as described in section 3.1.3), one should be careful to tag all allomorphs in the en- +try with the inflection class(es) they go on. Otherwise, some allomorphs without environments may be +incorrectly constrained. + +2.1.2.6.1 Inflection subclasses + +Now we consider one more situation where inflection classes are appropriate. Like Yalálag Zapotec, Isthmus +Zapotec [zai] also has verbal inflection classes.20 There is a distinction, however. First, consider the data +in (20)-(23), paying attention to the aspect prefixes. + +18 The data are from http://www.thelatinlibrary.com/decl.html and https://web.archive.org/web/ +20000525045116/https://www.slu.edu/colleges/AS/languages/classical/latin/tchmat/grammar/ +decl-c.html. +19 Of course, one would want to model the full nominal paradigm if one were working on Latin, but this +limited usage here illustrates the point about letting a given allomorph refer to more than one inflection +class. +20 The data are taken from Pickett, Black & Marcial (2001) and follow the practical orthography. + +═════ PDF page 24 ═════ + +16 Parsing Conceptual Introduction + + Habitual Progressive + (20) a. rucaa (21) a. cucaa + + ru-caa cu-caa + hab-write prog-write + + b. riree b. caree + ri-ree ca-ree + hab-leave prog-leave + + c. richesa c. cachesa + ri-chesa ca-chesa + hab-jump prog-jump + + d. rizá d. cazá + ri-zá ca-zá + hab-walk prog-walk + + Unreal nucaa Future zucaa + (22) a. nu-caa (23) a. zu-caa + un-write fut-write + + b. niree b. zaree + ni-ree za-ree + un-leave fut-leave + + c. nichesa c. zachesa + ni-chesa za-chesa + un-jump fut-jump + + d. nizá d. zazá + ni-zá za-zá + un-walk fut-walk + + Notice that based on this data, there are two inflection classes as summarized in (24).21 + + (24) Aspect Class 1 Class 2 + + Habitual ru- ri- + Progressive cu- ca- + Unreal nu- ni- + Future zu- za- + + Second, when we consider two other aspects, things are not so straightforward. The stems are presented +in the same order in (25)-(26) as they were above in (20)-(23): + +21 I am swapping the class numbering found in section 7.3 of Pickett, Black & Marcial (2001) for pedagogical +reasons. + +═════ PDF page 25 ═════ + +Inflection subclasses 17 + +Completive Potential +(25) a. bicaa (26) a. gucaa + + bi-caa gu-caa + compl-write pot-write + + b. biree b. guiree + bi-ree gui-ree + compl-leave pot-leave + + c. guchesa c. guichesa + gu-chesa gui-chesa + compl-jump pot-jump + + d. guzá d. sa' + gu-zá ∅-sa' + compl-walk pot-walk + + For example, while the habitual prefix in (20b) differs from the one in (20a), they are the same for +completive aspect in (25b) and (25a). Further, the potential aspect is quite different in (26d). How can we +understand this data? + + At least one way to understand this data is to posit two main inflection classes where one of these has +three subclasses. We can summarize the affix allomorphy as shown in (27). + +(27) Aspect Class 1 Class2A Class 2 Class2C + bi- gu- + Habitual ru- gui- ri- ∅- + Progressive cu- ca- + Unreal nu- ni- + Future zu- za- + Class2B + Completive bi- gu- + Potential gu- gui- + +Finally, there is the Perfect aspect which has the same shape for all verbs as illustrated in (28).22 + +(28) a. huacaa + hua-caa + prf-write + +b. huaree + hua-ree + prf-leave + +c. huachesa + hua-chesa + prf-jump + +d. huazá + hua-zá + prf-walk + +22 Some of these aspects also have some simple phonologically-based allomorphy which I am not showing +here for pedagogical reasons. + +═════ PDF page 26 ═════ + +18 Parsing Conceptual Introduction + + To model this Isthmus Zapotec data in FieldWorks Language Explorer, we will need to do what is +shown in figure 5. See figure 3 for details on the steps not overtly described. + + 1. Create two inflection classes within the verb category. + 2. Within the second one, create three subclasses. + + a. In Grammar / Category Edit / Verb, click on Subclasses under the second inflection class. + b. Click on Insert Inflection Class. + c. Enter the name, abbreviation, and description for the subclass. + d. Repeat for all three subclasses. + 3. For each verb stem, tag it as belonging to the appropriate inflection class or subclass, whichever + is correct for that verb. + 4. For the Perfect aspect, merely create the inflectional prefix entry with its single form. + 5. For the Habitual, Progressive, Unreal, and Future aspects, create the inflectional prefix entries, + including their two allomorphs. (You will need to decide which form to use as the lexeme form + and which as the affix allomorph form.) + a. For the Class 1 allomorph, tag it as belonging to Class 1. + b. For the Class 2 allomorph, tag it as belonging to Class 2. + 6. For the Completive aspect, create the inflectional prefix entry, including its two allomorphs. + a. For the bi- allomorph, tag it as belonging to Class 1 and Subclass 2A. + b. For the gu- allomorph, tag it as belonging to Subclass 2B and 2C. + 7. For the Potential aspect, create the inflectional prefix entry, including its three allomorphs. + a. For the gu- allomorph, tag it as belonging to Class 1. + b. For the gui- allomorph, tag it as belonging to Subclass 2A and 2B. + c. For the null allomorph, tag it as belonging to Subclass 2C. + + . + + Figure 5. How to Create Isthmus Zapotec Inflection Classes. + + In general terms, here is how the FieldWorks Language Explorer morphological parser will constrain +an inflectional affix allomorph tagged for inflection classes when there are both main level classes and +subclasses for at least one main level class: + + (29) 1. If the inflectional affix entry has only one form, then that form will always be used, no matter + what inflectional class or subclass the stem is tagged with. + + 2. If the inflectional affix entry has at least one allomorph tagged with a subclass: + a. any allomorph tagged with a subclass will only go on stems which are also tagged with + that subclass. + b. any allomorph tagged with a main level class will only go on stems which are also tagged + with that main level class + + 3. If the inflectional affix entry has allomorphs tagged only at the main level, then + a. an allomorph can go on a stem tagged with the same main level inflection class as it is + tagged with or + b. an allomorph can go on a stem tagged with an inflection subclass that is a subclass of + the main level inflection class that it is tagged with. This implies that the FieldWorks + Language Explorer parser pays attention to the inflection class hierarchy. Even if a stem + is tagged with a subclass, the inflectional affix needs to only be tagged at the main level. + This is the case even if there are subclasses within subclasses. + +═════ PDF page 27 ═════ + +Agreement and other inflection features 19 + + Finally, please recall that if an affix entry has any inflection classes and at least some of the allomorphs +are constrained with environments, one should be careful to tag all allomorphs in the entry with the in- +flection class(es) they go on. This may need to include subclasses. Otherwise, some allomorphs without +environments may be incorrectly constrained. For example, if the allomorph conditioned with an environ- +ment goes on a subclass and an unconditioned allomorph is tagged with a main level inflection class, you +will need to change the unconditioned one to go on all subclasses. This is because of condition 2b in (29) +above. + +2.1.2.6.2 Inflection classes and category organization + +As we noted in section 2.1.2.5, the categories in FieldWorks Language Explorer are organized in a +hierarchical fashion. + + The exact hierarchy one uses can make a difference for how the FieldWorks Language Explorer parser +handles inflection classes. When you define an inflection class (or an inflection subclass) at a particular +category in the hierarchy, then that class is available to be used for any lexical item associated with that +category or any of its nested categories. Thus, you will probably want to define your inflection classes at +the highest appropriate level in the hierarchy in order to capture generalizations. + +2.1.2.7 Agreement and other inflection features + +Consider the Spanish [spa] noun data given in (30) below: + + (30) a. casa + kas-a + house-f + +b. caso + kas-o + case-m + +c. casita + kas-it-a + house-dim-f + + d. casito + kas-it-o + case-dim-m + + Notice that the main difference between these nouns is the gender agreement suffix. If the ‑a ‘f’ suffix +is used, then the cas root means ‘house.’ On the other hand, if the ‑o ‘m’ suffix is used, then the cas root +means ‘case.’ + + For a human, it is not necessarily difficult to keep these facts straight, but for a morphological parser, we +need some way to prevent it from thinking that casa has the masculine root cas that means ‘case.’ Similarly +we need a way to keep the parser from thinking that caso has the feminine root cas that means ‘house.' +That is, we need a way to prevent the parser from giving “analyses” such as the ones shown in (31), where +the asterisk (*) indicates that the analysis is incorrect. + +═════ PDF page 28 ═════ + +20 Parsing Conceptual Introduction + + (31) a. casa + kas-a + *case-f + + b. caso + kas-o + *house-m + + With the FieldWorks Language Explorer parser we use inflection features to deal with this issue. +Inflection features are typically characteristics of a morpheme that play a role in the inflection of a word +and/or play a role in the syntax (such as agreement within a noun phrase or agreement between a verbal +affix and the noun phrase it agrees with). Note that if you use the Inflectional Affix Gloss Builder tool for +glossing inflectional affixes, then FieldWorks Language Explorer will automatically add some inflection +features for you. + + Coming back to the Spanish data in (30) and (31) above, how exactly does one use inflection features to +rule out incorrect parses such as the ones in (31)? The problem here is that there is a mismatch between +the gender of the root and the gender of the affix. If we can mark the root for the correct gender and also +mark the suffixes for the gender they agree with, then the FieldWorks Language Explorer parser will +only produce the correct parses. + + Many languages will use one or more of the inflection features listed in the chart shown in table 7 below. + + Table 7. Sample Inflection Features + +Feature Type Feature Name Sample Values +Agr + Person 1st, 2nd, 3rd +Infl Number Singular, Dual, Plural + Gender Masculine, Feminine, Neuter + Class 1, 2, …, 20 (or by shape or other classification system) + Animacy Animate, Inanimate + Case Nominative, Accusative, Dative, Locative, Genitive, Ergative, Absolutive + Aspect Completive, Continuative, Habitual, Perfective, Progressive, Stative + Tense Past, Present, Future + Mood Declarative, Imperative, Interrogative, Irrealis, Realis + + These are just some examples. Your language may use these or may need others. You may want to check +with a linguistic consultant who is familiar with your language family for ideas as to which inflection +features are appropriate for your language. Or you may just want to add them only when you find a need +for them, such as when the FieldWorks Language Explorer parser gives incorrect parses for forms. + + The features shown in table 7 are all simple features. There are times when a given word could contain +more than one such set of simple features. This is where complex features are important. For example, +for cases where a noun has noun class, say, and in addition, has a possessive affix which has a different +noun class, then we must be careful to avoid the two noun classes from clashing with each other. If we +merely use a simple inflection feature of “Class” for both the noun and the possessive affix, then the values +will differ and the parser will not analyze the word. Instead, we need to use separate noun agreement and +possessor agreement complex features. Within each of these complex features, we use the “Class” feature +and its values. In this way, not only does the parser correctly analyze the word (because the two complex +features do not clash), it also will have the correct features demarcated for eventual syntactic analysis. + + Another possible example for the use of complex features is when a verbal word has both subject and +object agreement markers in it. If the person features are different for subject and object, then we need to +be sure and use two complex features, one for the subject agreement features and the other for the object +agreement features. + +═════ PDF page 29 ═════ + +Agreement and other inflection features 21 + + The Spanish data illustrates how we can use gender inflection features to rule out incorrect parses when +a gender affix shows up incorrectly on a root. Some possible situations where inflection features could play +a similar role in ruling out incorrect parses include those shown in table 8. + +Table 8. Using Features to Rule Out Parses + +Situation Possible Inflection Features + to use +Gender mismatch between affix and stem +Noun class mismatch between affix and stem Gender agreement features +Animacy mismatch between affix and stem Noun class agreement features +Two or more aspect markers showing on a verb, when there should only Animacy agreement features +be one Aspect features +Two or more tense markers showing on a verb, when there should only +be one Tense features + + How does one create and use an inflection feature in FieldWorks Language Explorer? Figures 6–8 +below explain how. + + 1. Determine the inflection feature involved, including its type,a name, and possible values. (You + may need to check with a linguistic consultant on this.) + + 2. Use Grammar / Inflection Features. + a. Use Insert menu item / Feature. + b. This brings up the “Add Inflection Features from Catalog” dialog box. + c. Explore the catalog and try to find the feature you need. + d. If so, add the feature via the catalog (it's much easier this way). + e. If not, then in the dialog, click on the The inflection feature I need is not shown in the Catalog. + Create a custom inflection feature link. + i. Fill in the name, abbreviation, and description fields. + ii. For each value this feature needs, click on Values and then on Insert Feature Value. + iii. Fill in the value's name, abbreviation, and description. You'll probably want to check the + Use Abbreviation as label box. + + 3. For each category which will use the feature, + a. Add the feature to the category's set of inflectable features. + i. Use Grammar / Category Edit. + ii. Click in the Inflectable Features field. + iii. Click on the chooser button. + iv. In the ensuing dialog box, check the box by the feature. + b. Make sure that the category has appropriate affix templates (see section 2.1.2). If there are + no templates for the category, then the FieldWorks Language Explorer parser will ignore + the features. + + a We recommend using only two types: “Agr” for agreement features and “Infl” (= Inflection) for +. all others. + + Figure 6. How to Create Inflection Features. + +═════ PDF page 30 ═════ + +22 Parsing Conceptual Introduction + + 1. For each root (or stem) needing the feature, add the feature and its appropriate value to the + stem's grammatical information details. + a. Use Lexicon / Lexicon Edit. + b. To see the Inflection Features field, you may have to check the Show Hidden Fields box. + c. Click in the Inflection Features field. + d. Click on the chooser button. + e. In the ensuing dialog box, select the feature and value you need. + f. Alternatively, use Lexicon / Bulk Edit Entries with the List Choice tab to set the values for + many entries at once. See the help files in FieldWorks Language Explorer for more. + + . + + Figure 7. How to Add Inflection Features to Roots or Stems. + + 1. For each inflectional affix needing the feature, add the feature and its appropriate value to the + inflectional affix's grammatical information details. + a. Use Lexicon / Lexicon Edit. + b. To see the Inflection Features field, you may have to check the Show Hidden Fields box. + c. Click in the Inflection Features field. + d. Click on the chooser button. + e. In the ensuing dialog box, select the feature and value you need. + + . + + Figure 8. How to Add Inflection Features to Affixes. + +2.1.2.8 Inflection classes versus inflection features + +When modeling a given language, one may well wonder if a given phenomenon should be handled by +inflection classes or by inflection features. Here are some guidelines to help one decide: + + Table 9. Inflection Classes vs. Inflection Features + + If the various affixes involved … then use … + have no semantic differences (i.e., have the same meanIng), have inflection class + non-phonologically motivated shape differences, and are not + involved in (syntactic) agreement inflection features + inflection features + have semantic differences (i.e., actually have different meaning) inflection classes + inflection features + are involved in (syntactic) agreement + + are really declension classes or conjugation classes + + are noun classes or gender + +═════ PDF page 31 ═════ + +Major category-changing derivational affixes 23 + +2.1.2.9 Underspecified inflectional affixes + +In the above, we discussed how one can fully specify inflectional affixes. Sometimes it is the case, though, +that you are confident that a particular affix is inflectional, but you just do not yet know the category it +goes on. Or it might be the case that you know the category, but you do not yet know what the template +looks like so you cannot put it in an inflectional affix slot. + + FieldWorks Language Explorer allows you to model what you know. That is, you can still label +such an affix as being inflectional, but only partially specify the rest of the information about it. If you +know the category, but not the slot, you can say so. Be advised, though, that when you do this, the +FieldWorks Language Explorer parser will treat such underspecified inflectional affixes just like it +does for “unclassified” affixes (see section 2.1.1). + +2.1.3 Derivational affixes + +Derivational affixes typically reflect what some call “lexical meaning.” They go on a stem to produce a new +stem. The new stem may then be inflected (if the category of the new stem has inflection). Derivational +affixes often change syntactic category. See Bickford (1998:135ff) for more on this. + +2.1.3.1 Major category-changing derivational affixes + +The English data from example (3) is repeated below with more information: + +(32) Form Derivational Affix Category + + institute (none) verb + institution -ion noun + institutional -al adjective + institutionalize -ize verb + institutionalization -ation noun + institutionalizational -al adjective + institutionalizationally -ly adverb + + What do we have here? We have five derivational suffixes, each of which changes the major category of +the resulting stem. Recall that these suffixes only go on stems of a certain category. For example, the ‑al +suffix only goes on noun stems. It does not go on other stems (*institutal, *institutionalal, and *quicklyal). +These affixes are summarized in (33) below. + +(33) Form “attaches to category” “changes to category” Gloss + + ‑ion verb noun Nominalizer + ‑al noun adjective Adjectivizer + ‑ize adjective verb Verbalizer + ‑ation verb noun Nominalizer2 + ‑ly adjective adverb Adverbializer + + How do we model these category changing affixes in FieldWorks Language Explorer? We need to +do what is shown in figure 9. + +═════ PDF page 32 ═════ + +24 Parsing Conceptual Introduction + + 1. Add each affix as a lexical entry and mark it as being derivational. + a. Use Lexicon / Lexicon Edit. + + b. Use Insert menu item / Entry. + c. In the ensuing “New Entry” dialog box, set the Affix Type to “Derivational.” + 2. For the “Attaches to Category” piece of information, use the category of the stem to which this + affix attaches (see section 2.1.3.6 for more on this). + 3. For the “Changes to Category” piece of information, use the category of the stem that results + when this affix is attached (see section 2.1.3.6 for more on this). + . + + Figure 9. How to Create Category Changing Affixes. + +2.1.3.2 Sub-category-changing derivational affixes + +Now consider the pairs of data in (34)-(36) from Turkish [tur]:24, 25 + + (34) a. Çocuğu yıkadı + Çocuğ-u yıka-dı + child-acc wash-pst + ‘(S)he washed the child’ + + b. Çocuk yıkandı + Çocuk yıka-n-dı + child wash-pass-pst + ‘The child was washed’ + + (35) a. Bu işi yapmaya başlıyorlar + + Bu iş-i yap-ma-ya başl-ıyor-lar + + this work-acc do-inf-dat begin-prog-3pl + + ‘They are beginning to do this work’ + + b. Bu iş yapılmaya başlanıyor + + Bu iş yap-ıl-ma-ya başla-n-ıyor + + this work do-pass-inf-dat begin-pass-prog + + ‘This work is beginning to be done’ + + (36) a. O adamlar sigara içiyor + + O adam-lar sigara iç-iyor + + Those man-pl cigarette drink-prog + + ‘Those men are smoking cigarettes’ + + b. Sigara içilmez + + Sigara iç-il-mez + + cigarette(s) drink-pass-neg + + ’Cigarettes are not smoked here’ (= no smoking) + +24 The data are from Inkelas (2001). +25 (I wish I had access to a more standard Turkish grammar to get examples, but this is the best I could +find on the net. I also changed the glosses of two items per my Turkish Ample files which were based on +Underhill's grammar.) + +═════ PDF page 33 ═════ + +Non-category-changing derivational affixes 25 + + What is the key difference in each pair? It is the addition of the passive morpheme. Notice how the +number of arguments changes from two (subject and object) to one (just subject) with the addition of the +passive. + + Is passive, then, a category changing derivational affix? While it does not change major category (i.e., +it does not change a verb into a noun, say), it does change a transitive verb into an intransitive verb. That +is, passive is a case where the sub-category is changed. Many languages have other such sub-category +changing derivational affixes such as causatives, applicatives, and transitivizers. As far as FieldWorks +Language Explorer is concerned, these are category changing derivational affixes since the result of the +derivation produces a different sub-category that potentially requires a different inflectional template to +complete the word form. + + How do we model these sub-category changing affixes in FieldWorks Language Explorer? We need +to do what is shown in figure 10. See figure 9 for details on the first step. + + 1. Add each affix as a lexical entry and mark it as being derivational. + 2. For the “Attaches to Category” piece of information, use the (sub-)category of the stem to which + + this affix attaches (see section 2.1.3.6 for more on this). + 3. For the “Changes to Category” piece of information, use the (sub-)category of the stem that + + results when this affix is attached (see section 2.1.3.6 for more on this). + . + + Figure 10. How to Create Sub-category Changing Affixes. + +2.1.3.3 Non-category-changing derivational affixes + +Now consider the following Yalálag Zapotec data:26 + + (37) a. :xopcho + :-xop-cho + fut-drag-1pl.incl + + b. waxopcho + w-a-xop-cho + fut-rep-drag-1pl.incl + + (38) a. uchi:chcho + u-chi:ch-cho + fut-laugh-1pl.incl + + b. wachi:chcho + w-a-chi:ch-cho + fut-rep-laugh-1pl.incl + + The addition of the repetitive prefix does not change either the major category or the sub-category of the +words in (37)-(38). One might wonder, then, if the repetitive in Yalálag Zapotec is actually an inflectional +prefix. The evidence that it is derivational is that it actually changes the inflection class of the resulting + +26 The data are from López y Newberg (1990). + The orthography used here is slightly different from what is used in López y Newberg (1990). In particu- + +lar, fortis consonants are preceded by a colon (:). Lenis consonants are not (and use the voiceless equivalent +instead of the voiced one). + +═════ PDF page 34 ═════ + +26 Parsing Conceptual Introduction + +stem. As we saw in section 2.1.2.6, Yalálag Zapotec verbs have two inflection classes. In (37a) the stem is +inflection class 2 (because it takes the “fortifier” :‑ allomorph of the future prefix). After the a‑ repetitive +prefix is added in (37b), the resulting stem uses the inflection class 1 allomorph of future (u/w‑). + + How do we model these non-category changing affixes in FieldWorks Language Explorer? We need + +to do what is shown in figure 11. See figure 9 for details on the first step. + + 1. Add each affix as a lexical entry and mark it as being derivational. + 2. For the “Attaches to Category” piece of information, use the category of the stem to which this + + affix attaches (see section 2.1.3.6 for more on this). + 3. For the “Changes to Category” piece of information, use the same category as for the “attaches + + to category”. + . + + Figure 11. How to Create Non-category Changing Affixes. + + Notice that in this case the from‑ and to‑ categories will be the same, but we do need to deal with the +change in inflection class. This leads us to the next topic below. + +2.1.3.4 Inflection class and derivational affixes + +If the language you are studying has inflection classes (see section 2.1.2.6), then what happens when +derivational affixes are attached? Does the inflection class of the stem stay the same or does it change? + +2.1.3.4.1 Inflection class may change + +As we saw from the Yalálag Zapotec data in section 2.1.3.3, the inflection class can indeed change. How +do we model this? In addition to what we've done for the categories, we need to do what is shown in figure +12. + + 1. Also indicate the resulting inflection class in the “To Inflection Class” piece of information in + the lexical entry for the appropriate affix. + a. Use Lexicon / Lexicon Edit. + b. To see the To Inflection Class field, you may have to check the Show Hidden Fields box. + c. Under Grammatical Info. Details, click in the To Inflection Class field. + d. Click on the chooser button. + e. In the ensuing dialog box, select the inflection class you need. + + . + + Figure 12. How to Create Non-category Changing Affixes When the Inflection Class Changes. + + Note that rarely, if ever, does one need to indicate the “from inflection class” information. We include it +in case you do find that you need it. + +2.1.3.4.2 Inflection class does not change + +There are cases, though, where a derivational affix is attached and it does not change the inflection class +of the resulting stem. For example, consider the following data from Atzingo Popoloca [poe]:27 + +27 Data are from Austin, Kalstrom, and Hernández (1995). + +═════ PDF page 35 ═════ + +Inflection Features and Derivational Affixes 27 + +(39) a. tjanchia + t-janchi-a + prs-ask-1sg.auth.sbj.act + + b. tjáncháhā + t-jánchá-h-ā + prs-ask-appl-1sg.auth.sbj.act + +(40) a. nínkaon + ∅-nínkaon + prs-get.angry + + b. nínkakonhen + ∅-nínkakon-hen + prs-get.angry-appl + + The applicative suffix ‘appl’ adds an argument to the verb, but it does not change the inflection class of +the resulting stem. The root in (39) belongs to inflection class 1 and so takes the t‑ allomorph of the present +tense morpheme. Adding the applicative does not change this (39b). Similarly, the root in (40) belongs to +inflection class 2 and so takes a null allomorph of the present tense. Once again, adding the applicative +does not change the inflection class of the resulting stem (40b). + + To model this in FieldWorks Language Explorer, one does what is shown in figure 13. + + 1. Merely leave the “To Inflection Class” information blank in the lexical entry for the appropriate + affix. + + . + + Figure 13. How to Create Non-category Changing Affixes When the Inflection Class Does Not Change. + +2.1.3.5 Inflection Features and Derivational Affixes + +If the language you are studying has inflection features (see section 2.1.2.7), then what happens when +derivational affixes are attached to a stem with, say, agreement features? Or what happens when a deriva- +tional affix changes the category of the stem to a category that has agreement features? For example, +consider the Spanish data in (41) and (42):28 + + (41) a. apretar + apret-ar + press-inf + + b. apretón + apret-ón + press-nmlz + +28 This data are taken from Velásquez (1974:16). + +═════ PDF page 36 ═════ + +28 Parsing Conceptual Introduction + + (42) a. trasquilar + traskil-ar + shear-inf + + b. trasquilón + traskil-ón + shear-nmlz + + Here we have a verb (e.g., apretar) and a noun derived from that verb (e.g., apretón). Recall from section +2.1.2.7 that Spanish nouns are marked for gender (masculine or feminine). While Spanish verbs are not +marked for gender, a noun derived from a verb will have gender. In the case of the ‑ón derivational suffix, +the resulting noun has masculine gender. To properly model this, we would need to indicate that the +resulting noun has this gender. + + How does one mark a derivational affix for inflection features in FieldWorks Language Explorer? +See figure 14. + + 1. Determine the inflection feature involved, including its type,a name, and possible values. + 2. Create them if needed. See figure 6. + 3. For each derivational affix needing a feature: + + a. If the derivational affix requires the stem to have such a feature, add the feature and its + appropriate value to the derivational affix From Inflection Features field. + i. Use Lexicon / Lexicon Edit. + ii. To see the From Inflection Features field, you may have to check the Show Hidden Fields + box. + iii. Click in the From Inflection Features field. + iv. Click on the chooser button. + v. In the ensuing dialog box, select the features and values you need. + + b. If the stem that results from adding the derivational affix has such a feature, add the feature + and its appropriate value to the derivational affix To Inflection Features field. + i. Use Lexicon / Lexicon Edit. + ii. To see the To Inflection Features field, you may have to check the Show Hidden Fields + box. + iii. Click in the To Inflection Features field. + iv. Click on the chooser button. + v. In the ensuing dialog box, select the features and values you need. + + a We recommend using only two types: “Agr” for agreement features and “Infl” (= Inflection) for + . all others. + + Figure 14. How to Create a Derivational Affix With Inflection Features. + +2.1.3.6 Category-changing derivational affixes and category organization + +As we noted in sections 2.1.2.5 and 2.1.2.6.2, the categories in FieldWorks Language Explorer are +organized in a hierarchical fashion. + + The exact hierarchy one uses can make a difference for how the FieldWorks Language Explorer parser +handles the categories of derivational affixes. When one indicates the “from category”, the FieldWorks + +═════ PDF page 37 ═════ + +Derivation outside of inflection 29 + +Language Explorer parser will allow the derivational affix to apply to stems of this category and any of +its nested categories. You may well need to keep this in mind as you design your category hierarchy. + + You can use the hierarchy to capture some generalizations. For example, suppose your language has a +nominalizing derivational affix that can attach to any verb stem, resulting in a noun stem. Further, suppose +that the top-level verb category has two sub-categories: intransitive verb and transitive verb. If +you mark the “from category” as being verb, then this affix can attach to a verb stem, an intransitive +verb stem, or a transitive verb stem. + + Sometimes, however, the hierarchy implies that one will need to have more than one mapping for a given +derivational affix. For example, one might need a causative to map as follows if the inflectional templates +are different for intransitive verb, transitive verb, and ditransitive verb: + +(43) “from category” “to category” + + intransitive verb transitive verb + transitive verb ditransitive verb + noun transitive verb + + To do this, you need to add a separate mapping for each possible from/to pair. You do that by adding +distinct senses and associating each sense with the appropriate mapping. + + If a derivational affix only changes meaning (i.e., it does not change the category or the sub-category), +then one can use the highest level category for both the “from category” and the “to category”. In this +case, the FieldWorks Language Explorer parser will pass on the (sub-)category of the stem to which +the derivational affix attaches as the resulting category of the new stem. For example, if one chooses to +model an adverbial affix on a verb as being derivational, then if one marks both the “from category” and +the “to category” as "verb," then when this affix attaches to an intransitive verb, the resulting stem will still +be intransitive. If it attaches to a transitive verb, then the resulting stem will still be transitive. + +2.1.3.7 Underspecified derivational affixes + +In the above, we discussed how one can fully specify derivational affixes. Sometimes it is the case, though, +that you are confident that a particular affix is derivational, but you just do not yet know the category it +goes on or the resulting category after it attaches. Or it might be the case that you know either the category +it attaches to or the category it results in, but not both. + + FieldWorks Language Explorer allows you to model what you know. That is, you can still label such +an affix as being derivational, but only partially specify the rest of the information about it. If you know the +category it attaches to, but not the resulting category, you can say so. If you know the category it results in, +but not the category it attaches to, you can say so. Be advised, though, that when you specify the category +it attaches to, but not what the resulting category is, the FieldWorks Language Explorer parser will +treat such an underspecified derivational affix just like it does an “unclassified” affix (see section 2.1.1). If, +on the other hand, you do not say what category it attaches to, but do say what the resulting category is, +the FieldWorks Language Explorer parser will treat it as if you had said that the derivational affix can +go on every category. + +2.1.4 Derivation outside of inflection + +Derivational affixation tends to be close to the root. Since derivation sometimes changes the category of a +stem, this is not surprising. Derivational affixes, then, normally occur inside of inflectional affixes. + + However, there are cases in some languages where a stem will be inflected, then a category changing +derivational affix will be attached and the resulting stem will be inflected. + + The Quechua example we saw in (4) is such a case. It is repeated below in (44). + +═════ PDF page 38 ═════ + +30 Parsing Conceptual Introduction + + (44) rikaykaamaanaykipaq (=4) + + rika-yka:-ma:-na-yki-paq + + see-ipfv-1obj-nmlz-2.poss-purp + + ‘in order that you might be seeing me’ + + At least under one analysis, the verb root meaning ‘to see’ has the imperfective aspect marker added, +followed by the first person object marker, yielding ‘to see me.’ We thus have a verb stem inflected with +an aspect and an object marker. To this inflected form, the nominalizer derivational affix is attached, +resulting in a noun meaning ‘seeing me.’ The noun form then has the second person possessive marker and +the purpose marker added, finally giving ‘in order that you might be seeing me.’ That is, the resulting noun +stem is now inflected by a possessive and a (kind of) case marker. We could diagram this process as in +figure 15. + + Figure 15. Huallaga Quechua Derivation Outside of Inflection. + + In figure 15 the Infl nodes represent inflected forms. Note how the derivational suffix ‑na changes the +inflected verb into a noun stem (Stem[n]). This stem is then inflected. + + It turns out that while the Infl[n] node is a fully inflected noun, the Infl[v] is actually only a partially +inflected verb: It lacks a required subject suffix. That is, a form such as rikaykaamaa with the analysis of +‘see‑Ipgv‑1.obj’ is ill-formed. Thus, the verbal inflectional template given in table 10 is a special kind of +template. It does not represent a fully inflected form. Rather, it requires that there be a derivational affix +attached outside of the template in order for the word to be well-formed. When you have such templates, +you will need to mark them as requiring additional derivation. The default situation is for the FieldWorks +Language Explorer to assume that an inflectional template does not require additional derivation outside +of the template. + + Table 10. Quechua Non-final Template (Schematic) + + Stem Aspect Object + + How does one handle such derivation outside of inflection in FieldWorks Language Explorer? One +needs to perform the steps shown in figure 16. (See figures 1–2 for details on how to create templates.) + +═════ PDF page 39 ═════ + +Derivation versus inflection 31 + + 1. Add all the inflectional and derivational affix entries. + 2. Create the "inside" inflectional template and mark it specially as a template that requires ad- + + ditional derivation by checking the box after Requires more derivation. Please note that this + template must have at least one slot that is required. If all of the slots are marked as being + optional, the FieldWorks Language Explorer parser will arbitrarily treat them as if they are + all required. The reason is that when all slots are optional, the implication is that the template + is not needed. It also results in invalid instructions for the parser. + 3. Create the "outside" inflectional template. + +. + + Figure 16. How to Create Derivation Outside of Inflection. + +2.1.5 Derivation versus inflection + +Determining if a given affix is derivational or inflectional can sometimes be quite a challenge. Arguably, +the range from derivational to inflectional is a continuum and there are some affixes which seem to “float” +somewhere in the middle. Nonetheless, there are recognized criteria one can use to try and help one figure +out which kind a given affix might be. These are not hard and fast rules, however. + + Albert Bickford offers the following guidelines in helping one to decide (taken from Bickford 1998:139, +including the note on productivity).29 + + Table 11. Inflection vs. Derivation + +Characteristic Inflectional Derivational + +Changes one lexical entry into another no yes +Changes syntactic category no often +Productivity virtually totala partial at best +Organized in paradigms yes no +Distance from root farther closer +Type of meaning grammatical usually lexical +Meaning predictable from parts usually yes often not +Relevant to syntax yes no + +aAlthough some words may be inflected irregularly, they will almost always have +some form for every position in the inflectional paradigm. On rare occasions, +there may be words whose paradigms are defective, or missing certain forms, +such as English troops soldiers,’ a noun which has only a plural form with no +singular. + +29 Bickford (1998:138) uses the term “Conventionalized semantics” and explains this term as follows: + + In semantics, when the meaning of the whole is not fully predictable from the meaning of its parts, + we say that the meaning is conventionalized. One characteristic of derivational morphology is that + its meaning is often conventionalized, while the meaning of inflectional morphology is almost always + fully predictable. + +Here in this table, I've chosen to use “meaning predictable from parts” instead in order to avoid a double +negative. + +═════ PDF page 40 ═════ + +32 Parsing Conceptual Introduction + + Tom Payne also has some suggestions about characteristics of derivational affixes. The following quote +is taken from T.Payne (1997:42): + + According to Bybee (1985) derivational operations tend to be more RELEVANT to the situation ex- + pressed in the root than do inflectional operations. Derivational operations consist primarily of the + following: + + 1. Operations that change the grammatical category of a root, e.g., denominalization (changing a noun + into some other category) and nominalization (changing a form of any grammatical category into a + noun…). + + 2. Operations that change the valence (transitivity) of a verb root, e.g., detransitivization, causativiza- + tion and desiderative… + + 3. Operations which in other ways significantly change the basic concept expressed by the root, e.g., + distributive, diminutive… + + Characteristics of derivational operations include: + + 1. They are 'non-obligatory' insofar as they are employed in order to adjust the basic semantic content + of roots and are not themselves determined by some other operation or element in the syntactic + structure. + + 2. They tend to be idiosyncratic and non-productive. + + 3. They tend not to occur in well-defined paradigms. + +2.1.6 Exception “features” + +Even when one has correctly classified the affixes in a language as being derivational or inflectional, some- +times a morphological parser will find combinations of stem and affix that are simply incorrect. This may +be due to historical or some other seemingly arbitrary reasons. + + For example, consider the following Orizaba Nahuatl data: + + (45) nitlakuika tlakuikatl + ni-tlakuika tlakuika-tl + 1sg.sbj-sing sing-abs + ‘I sing’ ‘song’ + + kiavi kiavitl + kiavi kiavi-tl + rain rain-abs + ‘it rains’ ‘rain’ + + Notice that in this data, the “Absolutive” suffix (which normally goes on singular, unpossessed nouns) +appears to derive a noun from a verb. When one models this, one may find that other nouns which have +the absolutive suffix now analyze as derived nouns. For example, one might get these: + +═════ PDF page 41 ═════ + +Exception “features” 33 + +(46) komitl komitl + kom-itl *kom-i-tl + jug-abs *jug-drink-abs + ‘jug’ + +kakavatl kakavatl +kakava-tl *ka~kava-tl +peanut-abs *int-leave-abs +‘peanut’ + + The FieldWorks Language Explorer parser allows one to rule out such incorrect combinations via +what have sometimes been called exception “features.”30 The basic idea is to tag the affix with an exception +“feature.” The only time the FieldWorks Language Explorer parser will then allow this affix to occur +is when the stem to which it attaches also has been tagged with the same exception “feature.” Thus you +can restrict the productivity of the affix to only occur on certain stems. Note that this is only possible for +affixes which have been fully classified as either being derivational or inflectional. Exception “features” +are not available for unclassified affixes. + + 1. If necessary, create the exception “feature.” We recommend you give as meaningful a description + as you are able. For example, if you happen to know that there are historical reasons for this + situation, then go ahead and say what they are. + a. Use Grammar / Exception “Features.” + b. Use Insert menu item / Exception “Feature.” + c. Enter name, abbreviation, and description. + + 2. Tag the affix with that exception “feature.” + a. Use Lexicon / Lexicon Edit. + b. To see the From Exception “Features” field, you may have to check the Show Hidden Fields + box. + c. Click in the From Exception “Features” field under Grammatical Info. Details. + d. Click on the chooser button. + e. In the ensuing dialog box, check the box for each appropriate exception “feature.” + + 3. For each root or stem to which the affix may attach, tag the root or stem with the exception + “feature.” + a. Use Lexicon / Lexicon Edit. + b. To see the Exception “Features” field, you may have to check the Show Hidden Fields box. + c. Click in the Exception “Features” field under Grammatical Info. Details. + d. Click on the chooser button. + e. In the ensuing dialog box, check the box for each appropriate exception “feature.” + + . + + Figure 17. How to Create and Use Exception “Features.” + + If a given affix has two or more exception “features,” then the stem to which it attaches must be tagged +with all of the exception “features” that the affix has. Note that if an affix does not have any exception + +30 The way we have implemented these in FieldWorks Language Explorer is to create a separate list +for these objects. Technically, these are not true features. Internally, we are calling these “productivity re- +strictions” because they restrict the productivity of an affix. Another way of looking at them is as restricting +the distribution of an affix. + +═════ PDF page 42 ═════ + +34 Parsing Conceptual Introduction + +“features” but the stem to which it is being attached does have one or more exception “features,” then the +affix will still be allowed to attach (as far as the exception “features” are concerned). + + To tag affixes and stems with exception “features,” follow what is in figure 17 above. + +2.2 Stem compounding + +This section relates to the compounding of two or more stems within a single orthographic word.31 + There are two basic kinds of compounds: headed compounds (section 2.2.1) and non-headed compounds + +(section 2.2.2). We also discuss issues relating to incorporation (section 2.2.3), issues relating to compound- +ing when stems contain affixes (section 2.2.4), issues relating to the organization of categories (section +2.2.5), and issues relating to controlling the productivity of compound rules (section 2.2.6). + +2.2.1 Headed compounds + +Consider the following Orizaba Nahuatl data:32 + + (47) adjective noun Compound + + tliltik kowatl tlilkowatl + tlil-tik kowa-tl tlil-kowa-tl + black-adj snake-abs black-snake-abs + ‘black snake’ + + weyi atl weyatl + weyi a-tl wey-a-tl + big water-abs big-water-abs + ‘river’ + + weyi ohtli wéyohtli + weyi oh-tli wéy-oh-tli + big road-abs big-road-abs + ‘highway’ + + What are the categories of the two members of the compound? The left one is an adjective and the right +one is a noun. What is the category of the compound? It is a noun. Thus the examples in (47) show an +adjective compounding with a noun where the result is the right member of the compound. Thus, we can +say that the “head” of the compound is the right member of the compound. + + Now consider the following Orizaba Nahuatl data: + + (48) a. tlaxkálsolli + tlaxkál-sol-li + tortilla-old-abs + ‘tortilla from the day before’ + + b. nomahppagebrilweyi + no-mah-pil-weyi + 1sg.poss-hand-projection-big + ‘my thumb’ + +31 Compounds involving more than one orthographic word (e.g., student film society) are not dealt with here +since they are properly outside the realm of morphology. +32 The data are taken from Tuggy (1991:76-77). + +═════ PDF page 43 ═════ + +Non-headed compounds 35 + + In (48) the left member is a noun and the rightmost member is an adjective. Like in (47), the result is a +noun. Thus the “head” of the compound is the left member in the cases in (48). + + Both of these are instances of headed compounds. Either the left or the right member of the compound +is the head of the compound. That is, the category of the resulting compound is the same as either the left +or the right member of the compound. + + How do we model these kinds of rules for FieldWorks Language Explorer? See the steps in figure +18. + + 1. Create a headed compound rule. + a. Use Grammar / Compound Rules. + b. Use Insert menu item / Headed Compound. + c. Enter its name and description. + + 2. Indicate what the left member's category is by clicking on the drop-down box and choosing the + category. + + 3. Indicate what the right member's category is by clicking on the drop-down box and choosing + the category. + + 4. Indicate whether the resulting compound is left-headed or right-headed by checking the Right + Headed check box if it is right-headed; leave the box unchecked if it is left-headed. + +. + + Figure 18. How to Create a Headed Compound Rule. + +2.2.2 Non-headed compounds + +Now consider the following Spanish data: + +(49) Word Meaning Source + paracaidas from the third person singular present indicative verb + parachute (m; para ‘stops’ and the feminine plural noun caidas ‘falls’ + ambiguous for + number) from the third person singular present indicative verb + saca ‘removes’ and the feminine plural noun muelas +sacamuelas dentist (m or f; ‘teeth’ + singular) + + Which member of the compound is the head? Clearly it is not the left member since the resulting com- +pound in both cases is a noun and the left member is a verb. But is the head really the right member of +the compound? While the right member is a noun, this noun is not inflected for the correct gender and/or +number. Thus, these examples show the need for the other kind of compound rule: non-headed compounds. +In non-headed compounds, the category and/or agreement features of the resulting stem are not merely +the same as the head. Instead, the new stem may be something different. + + To model this in FieldWorks Language Explorer, we do what is shown in figure 19. + +═════ PDF page 44 ═════ + +36 Parsing Conceptual Introduction + + 1. Create a non-headed compound rule. + a. Use Grammar / Compound Rules. + b. Use Insert menu item / Non-headed Compound. + c. Enter its name and description. + + 2. Indicate what the left member's category is by clicking on the drop-down box and choosing the + category. + + 3. Indicate what the right member's category is by clicking on the drop-down box and choosing + the category. + + 4. Indicate what the category of the resulting compound is by clicking on the drop-down box and + choosing the category. + + 5. If the language you are modeling has inflection classes (see section 2.1.2.6) and the resulting + category has inflection classes, then also indicate the inflection class of the resulting category. + a. Click in the Inflection Class field. + b. Click on the chooser button. + c. In the ensuing dialog box, select the inflection class. + + . + + Figure 19. How to Create a Non-headed Compound Rule. + +2.2.3 Incorporation + +Some languages allow the incorporation of lexical roots within the stem. The resulting stem may or may not +differ from the non-incorporated stem in terms of category and/or features. This means that if the language +you are modeling has incorporation, you will need to consider whether to use a headed or a non-headed +compound rule for it. + +2.2.3.1 Incorporation as a simple headed compound + +Consider the following Yalálag Zapotec data:33 + + (50) a. sejpe' + s-ej-pe' + s-go-3fam + ‘He left.’ + + b. sejtope' + s-ej-to-pe' + s-go-rapidly-3fam + ‘He left quickly.’ + + (51) a. chpipe' + ch-pip-e' + prs-bite-3resp + ‘He bites.’ + +33 The data are from López y Newberg (1990). The orthography used here is slightly different from what +is used in López y Newberg (1990). In particular, fortis consonants are preceded by a colon (:). Lenis +consonants are not (and use the voiceless equivalent instead of the voiced one). + +═════ PDF page 45 ═════ + +Incorporation as a headed compound with override 37 + +b. chpip:cha:che' + ch-pip-:cha:ch-e' + prs-bite-repeatedly-3resp + ‘He bites repeatedly.’ + + At least under one analysis, in (50b) the adverb to is incorporated onto the verb ej. In (51b), a different +adverb, :cha:ch, is incorporated. + + Notice that the resulting stem appears to have all of the characteristics of the verbal stem which is the +left member of the compound as indicated by (50a) and (51a). Therefore, this kind of data can be modeled +as a left-headed compound rule. + +2.2.3.2 Incorporation as a headed compound with override + +Now consider the following Orizaba Nahuatl data:34 + +(52) a. pahtli (53) a. nakatl + pah-tli naka-tl + medicine-abs meat-Abs + ‘medicine’ ‘‘meat’’ + + b. niktolova b. nikkua + ni-k-tolova ni-k-kua + 1sg.sbj-3.obj-swallow 1SgSubj-3Obj-to.eat + ‘I swallow it’ ‘‘I eat it’’ + + c. nipahtolova c. ninakakua + ni-pah-tolova ni-naka-kua + 1sg.sbj-medicine-swallow 1SgSubj-meat-to.eat + ‘I medicine-swallow’ ‘‘I meat-eat’’ + + What is happening here? Notice how the nouns in (52a) and (53a) replace the ‘3.obj’ marker in (52b) +and (53b) to produce the forms in (52c) and (53c). In particular notice that the resulting stem no longer +requires a transitive verb inflectional template, but rather an intransitive verb one. We can say that this is +because the noun has been incorporated as the object and the result is an intransitive stem. We can model +this in FieldWorks Language Explorer as a headed compound, but override the category of the head +stem. We could diagram it something like this (where “[vt]” means a transitive verb stem and “[vi]” means +an intransitive verb stem): + + Figure 20. Orizaba Nahuatl Incorporation. +34 Data are from Tuggy (1991:77-8). + +═════ PDF page 46 ═════ + +38 Parsing Conceptual Introduction + + That is, we create a right-headed compound rule and set the “Overriding category” in the rule to be an +intransitive verb. The rule will use all the characteristics of the head stem except for the category. It will +override the category of the head stem with the specified “Overriding category”. + +2.2.4 Affixes between roots in compounds + +Consider the Wanca Quechua [qvw] form given in (54) below:35 + + (54) wasin-wasin + wasi-n=wasi-n + house-3.poss=house-3.poss + ‘from house to house’ + + Here we have a (reduplicated) compound consisting of a root, a suffix, the same root, and the same suffix. +This forms a compound as shown in figure 21: + + Figure 21. Wanca Quechua with Interfix. + + In FieldWorks Language Explorer, we must treat suffixes like ‑n in a special way. Affixes which +can appear between roots in compounds we call “interfixes.” In order to tell the FieldWorks Language +Explorer parser that a suffix like ‑n can appear in compounds like it does in (54), we must give it a +morpheme type of “suffixing interfix.” This tells the FieldWorks Language Explorer parser that this +suffix can appear either as a “regular” suffix (merely after a root) or as a suffix before another root in a +compound. Note that it is the leftmost instance of ‑n that is crucial here. + + There are three varieties of interfixes as shown in table 12. + + Table 12. Types of Interfixes + + Type Description + + infixing interfix An infixing interfix is an infix that can occur between two roots or stems. + prefixing interfix A prefixing interfix is a prefix that can occur between two roots or stems. + suffixing interfix A suffixing interfix is an suffix that can occur between two roots or stems. + + If the language you are modeling has these kinds of compounds and you want the parser to analyze them +via a compound rule, then you will need to mark any affixes which can appear between roots with these +special morpheme types. + +35 The data are from Rick Floyd, personal communication + +═════ PDF page 47 ═════ + +Restricting the productivity of a compound rule 39 + +2.2.5 Compound rules and categories considerations + +As we noted in sections 2.1.2.5, 2.1.2.6.2 and 2.1.3.6 above, the categories in FieldWorks Language +Explorer are organized in a hierarchical fashion. + + The exact hierarchy one uses can make a difference for how the FieldWorks Language Explorer +parser handles the categories in compound rules. When one indicates the information for a left or right +member of a compound, FieldWorks Language Explorer will consider stems of this category and any +of its nested categories to match. For example, if the main level verb category has two sub-categories +of intransitive verb and transitive verb, then if a verb stem, an intransitive verb stem, +or a transitive verb stem may be the left member, say, of a compound, you only need to say that the +left member must be of category verb. The FieldWorks Language Explorer parser will allow the left +member to be a verb stem, an intransitive verb stem, or a transitive verb stem. You can thus +capture a generalization. + + You may well need to keep this in mind as you design your category hierarchy. + +2.2.6 Restricting the productivity of a compound rule + +Sometimes merely specifying the categories of the left and right members of a compound opens the door +for many unwanted parses. For example, consider the following data from Ket [ket].36 + + The nominal compound in example (55) is licit as are many other nominal compounds. + + (55) ís al + is̄ al̄ + fish soup + noun noun + ‘fish soup’ + + There are cases, however, where a nominal compound is given as a parse when it is not licit. For example, +the word áses could give the three parses shown in example (56). + + (56) a. áses + áses + what.kind + adjective + ‘what kind’ + +b. ā ses̄ + +a ses̄ ̄ + +heat river + +noun noun + +‘*’ + +c. ā seˀs + +a seˀs ̄ + +heat larch + +noun noun + +‘*’ + +Only the first one is a valid analysis. What can one do in such a situation? + +36 The Ket data are from a personal email from Alexandre Arkhipov dated June 4, 2025. + +═════ PDF page 48 ═════ + +40 Parsing Conceptual Introduction + + One solution is to use what we call exception “features” (see section 2.1.6). The idea is to require +that a stem filling the left and/or right member of a compound rule must not only need to be of a specified +category, but must also have at least one of any of these exception “features”37 associated with that member. + + For example, to deal with the situation in examples (55–56), we follow the steps outlined in figure 22. +We might create an exception “feature” called “Right-headed nominal compound” and then tag the right +member of the nominal compound rule to require this exception “feature.” We then set the al̄ ‘soup’ stem +to have this exception “feature.” When the FieldWorks Language Explorer parser tries the nominal +compound rule with the is̄ ‘fish’ stem on the left and the al̄ ‘soup’ stem on the right, the compound will +succeed. The compounds in (56b–c), however, will fail because neither ses̄ ̄ ‘river’ nor seˀs̄ ‘larch’ have this +exception “feature.” + + To reiterate: a member of a given compound rule can have more than one exception “feature” associated +with it. When the FieldWorks Language Explorer parser tries a given stem in that member's position of +the compound rule, that stem must have at least one of those exception “features” for the rule to successfully +apply. The stem does not have to have all of the exception “features,” just one. + + 1. If necessary, create the exception “feature.” We recommend you give as meaningful a description + as you are able. + a. Use Grammar / Exception “Features.” + b. Use Insert menu item / Exception “Feature.” + c. Enter name, abbreviation, and description. + + 2. Tag the compound rule member with that exception “feature.” + a. Use Grammar / Compound Rules. + b. To see the Exception “Features” fields, you may have to check the Show Hidden Fields box. + c. Click in the Exception “Features” field. + d. Click on the chooser button. + e. In the ensuing dialog box, check the box for each appropriate exception “feature.” + + 3. For each root or stem that can be a part of this compound rule, tag the root or stem with the + exception “feature.” + a. Use Lexicon / Lexicon Edit. + b. To see the Exception “Features” field, you may have to check the Show Hidden Fields box. + c. Click in the Exception “Features” field under Grammatical Info. Details. + d. Click on the chooser button. + e. In the ensuing dialog box, check the box for each appropriate exception “feature.” + + . + + Figure 22. How to Create and Use Exception “Features” in Compound Rules + +2.3 Clitics + +We turn now to consider clitics. Consider the Shipibo [shp] data below38 and notice the ‑ra morpheme. +Where does it occur and on what kinds of words does it appear? + +37 The way we have implemented these in FieldWorks Language Explorer is to create a separate list +for these objects. Technically, these are not true features. Internally, we are calling these “productivity +restrictions” because they restrict the productivity of a compound rule. +38 Data are from Black (1992). + +═════ PDF page 49 ═════ + +Clitics 41 + +(57) Enra binon beque. + + e-n-ra binon be-que + + 1sg-erg-ind aguaje bring-compl + + ‘I brought aguajes.’ + +(58) Binonra en beque. + + binon-ra e-n be-que + + aguaje-ind 1sg-erg bring-compl + + ‘Aguajes I brought.’ + + In both (57) and (58), the indicative ‑ra morpheme appears at the end of the first word. In (57) it attaches +to a subject and in (58) it attaches to the object. + + This morpheme can also attach to other categories as the following examples demonstrate: + +(59) Yahuish rabecanra pique. + yahuish rabe-can-ra pi-que + armadillo two-erg-ind eat-compl + ‘The two armadillos have eaten.’ + +(60) Nii meranra ea catanhuanque. + nii meran-ra ea ca-tan-huan-que + forest in-ind 1sg go-reg-rctpst-compl + ‘I went into the forest today (and have come back).’ + +(61) Caquinra en pique. + + ca-quin-ra e-n pi-que + + go-siml.t.ss-ind 1sg-erg eat-compl + + ‘When I came, I ate.’ + +(62) Ramabira en janquenhati shinanai. + + rama-bi-ra e-n janquenha-ti shinan-ai + + now-emph-ind 1sg-erg finish-inf think-cont + + ‘Now I'm thinking of finishing it.’ + + The ‑ra morpheme attaches to an adjective in (59), a postposition in (60), a verb in (61), and an adverb +in (62). Notice that it actually appears at the end of the first constituent (a noun phrase in (59) and a +postposition phrase in (60)). + + Morphemes like this are often analyzed as being clitics. Orthographically, such clitics may be written +attached to another word (like in Shipibo) or they may be written independently. In my experience, ortho- +graphic conventions vary on this point. If the clitic is written as attached, then it should be classified as a +proclitic if it “prefixes” and as an enclitic if it “suffixes.” If the clitic is written as an independent +word, then one may classify it as a clitic. Some orthographic conventions are such that what the analyst +considers to be a proclitic or enclitic is also written as an independent word. In such cases, one may still +give these a morpheme type of proclitic or enclitic. The FieldWorks Language Explorer parser +will correctly handle a clitic that is labeled as being either a proclitic or enclitic whether it is written +attached or as a separate word. + + How do we model such clitics in FieldWorks Language Explorer? See figure 23. + FieldWorks Language Explorer will do the rest: such morphemes will be allowed to appear at the +end (for enclitics which attach) or at the beginning (for proclitics which attach) of words. More than one +clitic may appear on a single word. There is no ordering restriction between sequences of attached clitics +(other than ad hoc rules; see sections 2.4 and 3.10). + +═════ PDF page 50 ═════ + +42 Parsing Conceptual Introduction + + 1. Create the clitic lexical entry and mark it as being a clitic, proclitic or enclitic as + appropriate. + + 2. Give it the category that is appropriate for the clitic itself. + 3. If the clitic is written attached and can go on only certain sets of categories, then enter these sets + + of categories in the Attaches to Categories field under the Grammatical Info. Details section. As + with other items that refer to categories, FieldWorks Language Explorer will allow such + clitics to attach to any word whose category or sub-category is in one of the sets listed in this + field. That is, the category hierarchy is taken into account. Thus, if the clitic can attach to any + kind of verb, you can just list the top verb category and FieldWorks Language Explorer will + allow the clitic to attach to any of the sub-categories of verb. + 4. If the clitic is written attached and can go on any category, merely leave the Attaches to Cate- + gories field blank. + . + + Figure 23. How to Create a Clitic. + + A final note: if your orthographic convention permits several clitics to be written together as a single +word (where every morpheme in that orthographic word is a clitic of some kind), then only one of the +clitics may be marked as clitic. The others must be marked as proclitic or enclitic and these +must be in the proper order (proclitics before the clitic and enclitics after the clitic). + +2.4 Ad hoc morpheme-oriented rules + +When one uses a morphological parser, it is not unusual for the parser to sometimes return a parse that is +simply incorrect. These are sometimes due to allomorphs matching in places one would not have expected +them to match. When one has used all the mechanisms provided by the parser to the best of one's ability +and such incorrect parses continue to surface, one may well wish for some kind of mechanism to rule them +out. FieldWorks Language Explorer provides “Ad hoc Rules” for such situations. + +2.4.1 Creating morpheme-oriented ad hoc rules + +There are two main types of ad hoc rules: morpheme-oriented ones and allomorph-oriented ones. This +section deals with morpheme-oriented ones (see section 3.10 for allomorph-oriented ones). The basic idea +is to list a key morpheme and then to list one or more other morphemes that cannot co-occur with the key +one. One can constrain these other morphemes to never occur in one of the following ways with respect to +the key morpheme:39 + + Note that when there are two or more morphemes listed for “other morphemes,” the rule only applies +when all of them co-occur in the same word with the key morpheme. In addition, their relative order is +significant. They should be listed in the same linear order they occur in a word. + +39 One approach to this is to strive to make the tightest constraint possible (i.e., use one of the adjacency +ways first if possible; if not, then try the somewhere case; if that does not work, then try the anywhere +case). That way, should you encounter another case involving these particular morphemes, then you will +know more: it is now clear that you need looser constraints. You can then add some comments/annotations +to document what you have learned (or put the information in the description). + +═════ PDF page 51 ═════ + +Grouping ad hoc morpheme rules 43 + + Table 13. Morpheme Ad-hoc Rule + +Manner Meaning +Anywhere + The morphemes in question are constrained from appearing anywhere together in the +Somewhere same word. +before The key morpheme is constrained from appearing anywhere in the word before the +Somewhere after other morphemes. + The key morpheme is constrained from appearing anywhere in the word after the +Adjacent before other morphemes. + The key morpheme is constrained from appearing immediately before the other +Adjacent after morphemes in the same word. + The key morpheme is constrained from appearing immediately after the other + morphemes in the same word. + + How does one create a morpheme-oriented ad hoc rule in FieldWorks Language Explorer? See figure +24 below. + +2.4.2 Grouping ad hoc morpheme rules + +Occasionally one finds a situation where a set of ad hoc constraints have a common theme. Perhaps they +all relate to a particular morpheme or to particular morphemes of a certain variety. This may be a hint as +to what is really happening and may lead you to discover a linguistically-motivated way to model them. +Or it could be that the FieldWorks Language Explorer model just does not happen to provide the +appropriate linguistic mechanism to model the phenomenon correctly. + + 1. Determine the morphemes involved. + 2. Determine the most appropriate way to constrain them (see table 13). + 3. Create a morpheme-oriented ad hoc rule. + + a. Use Grammar / Ad hoc Rules. + b. Use Insert menu item / Rule to prevent morpheme co-occurrence. + c. Indicate the key morpheme involved. + + i. Click in the Key Morpheme field. + ii. Click on the chooser button. + iii. In the ensuing dialog box, key the form of the morpheme in the Find box. + iv. Select the correct one in the middle pane. + v. Check the Grammatical Info. drop-down box to make sure this is the one you want. + d. Indicate the way or manner in which to constrain this morpheme (see table 13) by using the + drop-down box in the Cannot Occur field. + e. Indicate the non-key (i.e., other) morpheme(s) involved. + i. Click in the Other Morpheme(s) field. + ii. Click on the chooser button. + iii. In the ensuing dialog box, key the form of the morpheme in the Find box. + iv. Select the correct one in the middle pane. + v. Check the Grammatical Info. drop-down box to make sure this is the one you want. + vi. Repeat the above steps for any other morphemes needed. + +. + + Figure 24. How to Create a Morpheme-oriented Ad Hoc Rule. + +═════ PDF page 52 ═════ + +44 Parsing Conceptual Introduction + + Yalálag Zapotec dependent pronominal suffixes exemplify such a situation (see López y Newberg 1990:9). +In Yalálag Zapotec, a verb may have both a subject and an object person suffix on it. Being a vso language, +the subject occurs before the object. What is different here is that there is a pronominal hierarchy among +these dependent pronominal suffixes. Given the subject suffix, the only dependent object suffixes which +may follow are those that are lower down on the person hierarchy. This is illustrated in table 14. + + Table 14. Yalálag Zapotec Subject and Object Dependent Suffix Co-occurrence + + OBJECT + 1st, 2nd 3rd respect 3rd familiar 3rd animate 3rd thing + + S 1st, 2nd No YES YES YES YES + + U 3rd respect No No YES YES YES + B + J 3rd familiar No No No YES YES + + E 3rd animate No No No No YES + C + T 3rd thing No No No No No + + How would one model such a hierarchy in FieldWorks Language Explorer? Well, one could create +a number of different transitive verb inflectional templates in order to force the hierarchy to come out. But +this does not really capture the facts all that well and also complicates and obscures what is common in the +transitive verb template. (By the way, neither the subject nor the object is required to be filled by a suffix.) +Probably the better approach is to create a morpheme ad hoc rule group and place the set of appropriate +ad hoc rules for the hierarchy in that group. This way one can document the fact of the hierarchy and have +it all in one place. It also documents the fact that the FieldWorks Language Explorer model does not +have an overt mechanism to deal with such a hierarchy. + + How does one create such a group? See figure 25. + + 1. Create an ad hoc rule group. + a. Use Grammar / Ad hoc Rules. + b. Use Insert menu item / Group of ad hoc rules. + + 2. For each ad hoc morpheme rule in the group: + a. Create the ad hoc morpheme rule (see figure 24) and include it in the group. + + . + + Figure 25. How to Create a Group of Morpheme-oriented Ad Hoc Rules. + + Finally, note that FieldWorks Language Explorer allows one to group both allomorph and morpheme +ad hoc rules together. Please be sure to only do so if these rules truly do have something in common. + +3 Morphophonemics + +Besides constraining the overall positions where morphemes can occur (i.e., deal with morphotactics), we +need to be able to account for the surface forms that the morphemes have and the particular environments +where an allomorph is legitimate. + +═════ PDF page 53 ═════ + +Phoneme sets 45 + +3.1 Overview + +Consider the following Orizaba Nahuatl data: + +(63) 1st Singular Subject 2nd Singular Subject + + ni-miki ti-miki + 1sg.sbj-die 2sg.sbj-die + n-ahsi t-ahsi + 1sg.sbj-arrive 2sg.sbj-arrive + + What are the shapes of the ‘1sg.sbj’ and the ‘2sg.sbj’ allomorphs? The first person singular subject +marker appears to be ni‑ before consonants and n‑ before vowels. Similarly, the second person singular +subject marker alternates between ti‑ and t‑. + + How can we encode this information? There are at least two ways to deal with such phonological infor- +mation: + + 1. Give the underlying form along with a set of rules to create the surface forms; or + 2. List the surface allomorphs and condition each one to appear in the appropriate surface environment. + + Generative phonology uses the first approach (also known as the item and process approach, Hockett +1954). For example, given the data in (63), one might consider the underlying forms of the two subject +prefixes would be ni and ti, respectively. We would then write a phonological rule to delete the first vowel +when it is followed by a second vowel. + + The default parser of FieldWorks Language Explorer, however, chooses the second approach (also +known as the item and arrangement approach, Hockett 1954). For example, once again considering the +data in (63), we would need two forms for each subject prefix entry. We could make the Lexical Form be +the longer one (ni and ti, respectively) and have an allomorph for the shorter one (n and t, respectively) +that would be conditioned to have an environment saying that it must be followed by a vowel. + + As noted in section 1, we now also have the phonological rule-based parser which allows for both item +and arrangement and item and process. See section 6 for more. + + For the default parser of FieldWorks Language Explorer, the basic mechanism available is to list +surface allomorphs and then have the option to constrain individual surface allomorphs by their environ- +ment. To define an environment, one may well want to use natural classes of segments (e.g., consonant, +vowels, voiceless stops, nasals, etc.). To define such natural classes, we need to know what the possible +segments are. + +3.1.1 Phoneme sets + +In order to use environments which refer to phonemes or which have natural classes, you need to create a list +of all the phonemes in your language. For each phoneme, you need to indicate one or more representations +that represent them. For example, in Greek, the /s/ phoneme has two such representations: ς (which is +used word finally) and σ (which is used everywhere else). + + In addition to these phonemes, you may also need to refer to word boundaries in an environment. For +this reason, FieldWorks Language Explorer comes with a predefined word boundary marker: the # +symbol. + + FieldWorks Language Explorer also comes with a potential set of phonemes already defined. That +is, you do not need to start from scratch when building the list of phonemes for your language. However, +you may well need to edit the list of phonemes initially included for a new language project. This initial +set of phonemes is given in (64) below. + +═════ PDF page 54 ═════ + +46 Parsing Conceptual Introduction + + (64) Name Description Grapheme + + i high front unrounded vowel i + e mid front unrounded vowel e + a low central unrounded vowel a + o mid back rounded vowel o + u high back rounded vowel u + p voiceless bilabial stop p + b voiced bilabial stop b + t voiceless alveolar stop t + d voiced alveolar stop d + k voiceless velar stop k + g voiced velar stop g + f voiceless labiodental fricative f + v voiced labiodental fricative v + s voiceless alveolar fricative s + z voiced alveolar fricative z + x voiceless velar fricative x + m bilabial nasal m + n alveolar nasal n + ŋ velar nasal ŋ + l alveolar lateral l + r alveolar flap r + w labiovelar approximant w + j palatal approximant j + + To define the set of phonemes for the language you are modeling, do what is shown in figure 26. + + 1. Determine the phonemes in your language. + 2. Use Grammar / Phonemes. + 3. Remove any phonemes that you do not need. + + a. Click on the phoneme in the middle pane. + b. Use Edit menu item / Delete Phoneme.... + 4. For each phoneme not already present, + a. Create a phoneme entry. + + i. Use Insert menu item / Phoneme. + ii. Enter the basic symbol in the Refer to as field. + iii. Enter the description. + iv. Optionally, enter the IPA symbol. + b. For each symbol or sequence of symbols that represent that phoneme, create a grapheme. + i. Click on In Orthography as. + ii. Click on Insert Grapheme. + iii. Key the grapheme. + 5. For each phoneme in the list given in (64), + a. Edit the name (or “Refer to as”), description, and graphemes if need be. + b. Add any new graphemes needed. + + . + + Figure 26. How to Create Phonemes. + +═════ PDF page 55 ═════ + +No forms conditioned by tone 47 + +3.1.1.1 Phonological features + +The default FieldWorks Language Explorer parser does not use phonological features, but the phono- +logical rule-based parser does. This section mentions how to create phonological features and associate +them with phonemes in case you wish to use the phonological rule-based parser. + + To create the set of phonological features, follow the steps in figure 27. + + 1. Use Grammar / Phonological Features. + 2. Use Insert menu item / Phonological Feature. + 3. In the ensuing dialog box, check all features that you need. + 4. If the dialog box does not have a feature you need, click on the The phonological feature I need + + is not shown in the Catalog. Create a custom phonological feature. + a. Enter the name, abbreviation, and description of the phonological feature. + b. Edit the values if they are not correct. You can add a new value by clicking on Values and + + then clicking on Insert Feature Value. + . + + Figure 27. How to Create Phonological Features. + + Once you have created your phonological features, you can associate them with individual phonemes. +See figure 28. + + 1. Click in the Phonological Features field. + 2. Click on the chooser button. + 3. In the ensuing dialog box, select the value needed for each phonological feature. + . + + Figure 28. How to Associate Phonological Features with a Phoneme. + +3.1.1.2 Digraphs + +If you have orthographic digraphs (or trigraphs) as phonemes, you follow the same basic steps outlined in +figure 26 steps 3 and 4. Merely use the appropriate digraph for the phoneme and also use the digraph as +the grapheme. For example, if you have an aspirated voiceless alveolar stop written as th, then use th for +the information. + +3.1.1.3 Tones + +While many languages have tones, not all mark the tone in the practical orthography. If the language you +are modeling includes tone symbols marked on vowels as accents, say, then you need to decide which of +several ways to go in indicating tone. + +3.1.1.3.1 No forms conditioned by tone + +This is the simplest case where no forms are ever conditioned by surrounding tone. For example, it is never +the case that you have an affix which must be preceded or followed by a high tone (or low tone). Because +of this, you never need to write an environment that refers to tone, only to natural classes of segments. + +═════ PDF page 56 ═════ + +48 Parsing Conceptual Introduction + + If this is the situation you have, then for each vowel that bears tone, add a distinct representation for +how that vowel with tone is written. For example, when the vowel a has high tone, it is written as á, then +you need to add á to the list of graphemes for the vowel a. + +3.1.1.3.2 Forms conditioned by tone + +The second situation is more complicated. This is where there are forms in your language that must be +conditioned by surrounding tone. For example, suppose there is an affix form which is licit only if it is +preceded or followed by a particular tone (high, say). This means that you will need to be able to write an +environment that contains all phonemes that bear high tone. To do this, there are two options. + + The first option is to have distinct phonemes for each high toned vowel, say. If this is your case, then +you need to make distinct phonemes for each vowel that bears different tones. For example, if a low tone a +is written as a while a high tone a is written as á, then you will need two phonemes: one for the low toned +a and one for the high toned a. You can then create a natural class that contains all high-toned phonemes +and write your environment in terms of this natural class. + + The second option is to create phonemes that consist merely of the tone diacritic itself. Since FieldWorks +Language Explorer always processes its data in “decomposed” form (nfd), any accent marks or other +diacritics will be stored after the main symbol. Thus, á is stored as two characters aˊ, the /a/ and then +the acute accent. Taking advantage of this, you could create a phoneme that is for the acute accent ˊ and +call it something like “High tone.” You could then condition the forms to occur only when followed by this +acute accent phoneme. + +3.1.2 Natural classes + +Once you have the phonemes defined, then you can create natural classes of phonemes. Some common +ones include such things as consonants, vowels, voiceless stops, back vowels, etc. To do this in FieldWorks +Language Explorer do what is shown in figure 29. + + 1. Determine the natural classes you need. + 2. Use Grammar / Natural Classes. + 3. If any of the phonemes in any of these natural classes are not already in your list of phonemes, + + add them to the list of phonemes. See figure 26. + 4. Create any other natural classes you need. + + a. Use Insert menu item / Natural Class (Phonemes). + b. Enter a name, abbreviation, and description. + c. Click in the Phonemes field. + d. Click on the chooser button. + e. In the ensuing dialog box, check every phoneme that is a part of this natural class. + + . + + Figure 29. How to Create Natural Classes. + + We highly recommend that you seek to give unique abbreviations for these. While it is possible to have +two or more natural classes with abbreviations spelled exactly the same way, we do not recommend that +you do so on purpose. Having two or more natural classes with the same abbreviation will not confuse +FieldWorks Language Explorer because FieldWorks Language Explorer uniquely identifies every +natural class internally. That does not imply, however, that either you or a reader of your grammar will +not be confused as a result. + +═════ PDF page 57 ═════ + +Allomorph environments 49 + +3.1.3 Allomorph environments + +Once the set of phonemes and natural classes are defined for the language you are modeling, you can define +environments for allomorphs. You can add them either in the environments editor or with a given lexeme +form or allomorph. To use the environments editor, use Grammar / Environments. To add an environment +within a lexeme form or allomorph, click in the Environments field. (To see the Environments field, you +may have to check the Show Hidden Fields box.) See the help files in FieldWorks Language Explorer +for more. + + In FieldWorks Language Explorer, you key these environments using a special notation. This nota- +tion is one that is reminiscent of what is used in many generative-style rules. The basic rules of thumb are +given in figure 30. + + 1. Begin each environment with the forward slash character /. + 2. The location of the allomorph itself is indicated by an underscore character _. + 3. Any phonemes or natural classes that must come before the allomorph are typed in before this + + underscore character. Type them in the order in which they must appear. + 4. Any phonemes or natural classes that must come after the allomorph are typed in after this + + underscore character. Type them in the order in which they must appear. + 5. Any phoneme is indicated by typing in the representation/grapheme of the phoneme. + 6. Any natural classes are indicated by + + a. typing a left square bracket [, + b. typing the abbreviation of the natural class,a and + c. then typing a right square bracket ]. + 7. Optional phonemes or natural classes are indicated by + a. typing an opening parenthesis (, + b. typing the phoneme or natural class as above, and + c. then typing a closing parenthesis ). + d. Note that one should not nest optional items. The FieldWorks Language Explorer parser + + will not handle these properly. You must enter each optional item after the other. + + a This is another reason why you should use unique abbreviations for natural classes. If you have + two or more natural classes with the same abbreviation, it is not clear which one you mean. Field- + . Works Language Explorer will automatically select one, but it may not be the one you intended. + + Figure 30. How to Create Environments. + + Table 15 gives some sample environments along with what they mean. + A given allomorph may have more than one environment, in which case the various environments are +logically ORed with each other. That is, if any one of the environments for the allomorph are found, then +the allomorph is considered to be valid (as far as its environments are concerned). For example, if a given +allomorph can appear either before a consonant or word finally, then you can list both an environment for +“before a consonant” and one for “before a word boundary.” Example (65) shows what this might look like, +assuming that you have a natural class of consonants with an abbreviation of C. + +═════ PDF page 58 ═════ + +50 Parsing Conceptual Introduction + +Environment Table 15. Sample Environments +/m_ +/ [V] _ Meaning +/#i_ +/ # [V] _ after an m phoneme +/ [V] y _ after a vowel (assuming there is a natural class of vowels called V) +/_i after a word initial i phoneme +/ _ [C] after a word initial vowel (assuming there is a natural class of vowels called V) +/_y# after a vowel (assuming there is a natural class of vowels called V) and a y phoneme +/ _ [C] # before an i phoneme + before a consonant (assuming there is a natural class of consonants called C) +/m_w before a y phoneme which is word final +/ [C] _ [C] before a word final consonant (assuming there is a natural class of consonants called +/ ai _ C) +/ _ ai between an m and an w phoneme +/ _ (a)i between two consonants (assuming there is a natural class of consonants called C) +/ _ ([C]) # after an a and an i phoneme + before an a and an i phoneme + before an optional a and an i phoneme; that is, either before ai or before i + before an optional word final consonant (assuming there is a natural class of + consonants called C); that is, either before a word final consonant or word finally + +(65) / _ [C] /_# + + Finally, if an affix entry has inflection classes as well as at least some allomorphs with environments, +you should be careful to tag all allomorphs in the entry with the inflection class(es) they go on. Otherwise, +some allomorphs without environments may be incorrectly constrained. + +3.1.4 Allomorph ordering + +It is crucial to note that allomorphs are ordered in the sense that their respective environments are dis- +junctively ordered. For example, for the Orizaba Nahuatl ‘1sg.sbj’ allomorphs above in example (63), we +could list the two allomorphs in any of the ways shown in (66)-(69). + +(66) Overt 1 (67) Overt 2 (68) Implicit 1 (69) Implicit 2 + + ni / _ [C] n / _ [V] ni / _ [C] n / _ [V] + n / _ [V] ni / _ [C] n ni + + Note in particular that for the two implicit methods, one does not have to overtly state the environment for +the last allomorph. This is because each allomorph automatically inherits the negation of the environments +of any preceding allomorphs. Thus, for the Implicit 1 method, the n is automatically treated as having an +environment of “not before a consonant.” Similarly, for the Implicit 2 method, the ni is automatically treated +as having an environment of “not before a vowel.” For more on this, see section 4.1.2. +3.1.4.1 Free fluctuation +Sometimes a language has free fluctuation between two allomorphs of a morpheme. In such cases, one +should create both allomorphs and condition them exactly the same way (in terms of environments and +inflection classes). One should also order them one after the other. The FieldWorks Language Explorer +default parser will try both forms in such cases. + +═════ PDF page 59 ═════ + +Writing the pattern for full reduplication 51 + +3.2 Reduplication + +In the next five sections, we will address five issues brought up in section 1.1. First, we deal with reduplication. + +3.2.1 Full reduplication + +Consider the following data from Bahasa Indonesia [ind]:40 + (70) a. pel ‘mop’ + b. pel-pel ‘mops’ + +(71) a. buku ‘book’ + +b. buku-buku ‘books’ + +(72) a. komik ‘clown’ + +b. komik-komik ‘clowns’ + +(73) a. orang ‘person’ + +b. orang-orang ‘people’ + +(74) a. perpustakaan ‘library’ + +b. perpustakaan-perpustakaan ‘libraries’ + + In examples (70)-(74) note that the entire word is reduplicated, no matter what its syllabic shape might +be. This is what is often called full reduplication.41 + + In examples like (70)-(74), one cannot tell whether the reduplication morpheme is a prefix or a suffix. +However, sometimes a stem will reduplicate and other affixes may be adjoined. For example, consider +(75)‑(76): + +(75) a. tangan ‘hand’ + +b. tangan-tangannya ‘his hands’ + +(76) a. perpustakaan ‘library’ + +b. perpustakaan-perpustakaannya ‘his libraries’ + + Notice the ‑nya suffix which comes after the reduplicated stem. The way we are implementing full +reduplication in the FieldWorks Language Explorer default parser, the root must be at one end and +then any affixes (including the reduplication morpheme) must either be all prefixes or be all suffixes.42 +Thus, in modeling examples (75)‑(76), we would make the reduplication morpheme be a suffix. + +3.2.1.1 Writing the pattern for full reduplication + +How do we indicate full reduplication for the default parser of FieldWorks Language Explorer? See +figure 31 below. (See section 6.1.1.1.1 for how to write an affix process rule for full reduplication using +the phonological rule-based parser.) + +40 The data are from Howard Shelden, personal communication, and Jonathan Coombs, personal commu- +nication. +41 It is also called total reduplication and sometimes general reduplication. +42 There is a technical reason for this. The default parser matches the entire rest of the word (for a prefix) +or the entire beginning of the word (for a suffix). It cannot match if there is additional material. This is +not the case for the phonological rule-based parser. See section 6.1.1.1.1. + +═════ PDF page 60 ═════ + +52 Parsing Conceptual Introduction +3.2.2 Partial reduplication + +As we saw in the Tagalog data from (9) from section 1.1.7, it is not always the case that the entire stem is +reduplicated. The Tagalog data is repeated here. + +(77) a. susulat ‘to write (ipfv)’ (=9) + +b. magpasulat ‘to make someone write (pfv)’ + +c. magpapasulat ‘to make someone write (ipfv)’ + + Recall that we saw that this is a case where the imperfective aspect is realized by reduplicating the first +CV syllable of the stem to which it attaches. (The mag- prefix is what some call actor focus or actor voice.) + + 1. Determine that the reduplication is indeed full reduplication. Note that if both prefixes and + suffixes attach to a reduplicated sequence, then you will need to use partial reduplication (see + section 3.2.2). + + 2. Determine whether the reduplication morpheme is derivational or inflectional. + 3. Determine if the reduplication affix is a prefix or a suffix: + + a. If you do get either prefixes or suffixes (but not both) on a fully reduplicated stem, make the + affix type be the same as the additional material. That is, if it can take additional prefixes, + make the reduplicant morpheme also be a prefix. If it can take suffixes, make the reduplicant + morpheme also be a suffix. + + b. If it is not clear, we suggest making it be a suffix (this will probably be more efficient for the + parser). + + 4. Create a lexical entry for the reduplication morpheme. + a. Mark it as derivational or inflectional according to what you determined. + b. Add a form which contains the special full reduplication sequence indicator: [...] (that's + a left square bracket, three periods, and a right square bracket)a in the lexeme form or an + allomorph form. If there is constant segmental material that precedes or follows the redu- + plicated material, put that segmental material before or after the full reduplication pattern + indicator. For example, for the Bahasa Indonesia data above, we would include a hyphen + before the indicator.b + c. Optionally label the morpheme using the Citation Form field (e.g., RDP- or CVC-). + + a This is the same notation as used in Shoebox and Toolbox. AMPLE uses <...>. + b Thus, it would be keyed as -[...] where we would put the hyphen before the indicator because + the hyphen would be part of the suffix. You can put anything before or after the indicator. For + example, if you used t[...]- and made it be a prefix, then this would match a full reduplication + morpheme in a form such as tabrak-menabrak ‘keep on running into,’ where we would model the +. men as an infix and the abrak would be the truncated allomorph of the stem tabrak ‘collide.’ + + Figure 31. How to Create Full Reduplication via an Environment. + +Now consider the following Orizaba Nahuatl data:43 + +43 Data are from Tuggy (1991:41). The hyphens are not part of the orthography but are included to clarify +the relevant morphemes involved. + +═════ PDF page 61 ═════ + +Writing the pattern for partial reduplication 53 + +(78) a. tone (the sun) shines + b. toh-tone (the sun) burns + +(79) a. chipin-tok it is dripping + +b. chih-chipin-tok it is dripping and dripping + + What is the reduplication pattern here? It is the initial CV of the stem followed by an h. Thus we see that +in this Nahuatl case of reduplication, there is not only the copied material, but also some fixed segmental +material. + + The kind of reduplication illustrated in (77)-(79) above is often referred to as partial reduplication. How +do we model such partial reduplication for the default parser of FieldWorks Language Explorer? See +figure 32. (See section 6.1.1.1.2 for how to write an affix process rule for partial reduplication using the +phonological rule-based parser.) + + 1. Determine whether the reduplication morpheme is derivational or inflectional. + 2. Determine what the reduplication pattern is (see section 3.2.2.1). + 3. Create any needed natural classes of segments that are in the pattern (see section 3.1.2). + 4. Create an environment that has the reduplication pattern either before or after the reduplication + + morpheme (depending, of course, on whether what is reduplicated comes after or before the + “stem”). See section 3.1.3. + 5. Create a lexical entry for the reduplication morpheme. + a. Mark it as derivational or inflectional according to what you determined. + b. In the lexeme form (or in an allomorph form), type in the reduplication pattern. + c. Condition that allomorph to use the environment you created above. + d. Optionally label the morpheme using the Citation Form field (e.g., RDP- or CVC-). + . + + Figure 32. How to Create Partial Reduplication via an Environment. + +3.2.2.1 Writing the pattern for partial reduplication + +For the default parser of FieldWorks Language Explorer, we use a special notation to indicate a partial +reduplication pattern.44 The idea is to list a sequence of specially marked natural class names. The special +marking consists of the steps outlined in figure 33. + + 1. [ (i.e., a left square bracket) + 2. the abbreviation of the natural class + 3. ˆ (i.e., the caret character - a shift-6 on many keyboards) + 4. a positive integer number (usually 1, 2, or 3) which is an index indicating matching items + + between the allomorph and the environment + 5. ] (i.e., a right square bracket) + +. + +44 It is the same notation as used in AMPLE. + +═════ PDF page 62 ═════ + +54 Parsing Conceptual Introduction + + Figure 33. How to Write the Pattern for Partial Reduplication. + + Suppose we have a natural class for consonants with an abbreviation of C and one for vowels abbreviated +as V. Then the reduplication patterns for our Tagalog and Orizaba Nahuatl reduplication examples above +in (77) and (78)‑(79) would be as in table 16. + + Table 16. Samples of Partial Reduplication + + Language Allomorph Environment + [Cˆ1][Vˆ1] / _ [Cˆ1][Vˆ1] + Tagalog [Cˆ1][Vˆ1]h / _ [Cˆ1][Vˆ1] + Orizaba Nahuatl + + For the Orizaba Nahuatl case, notice the use of the h (the fixed segmental material) in the allomorph +pattern. It is not included in the environment pattern for the simple reason that the h does not show up in +the environment. + + Note that if a language has a CVC reduplication pattern, then one would want to use a pattern of +[Cˆ1][Vˆ1][Cˆ2], where the distinct indices on the consonant natural classes makes it clear that they +can be different. + +3.3 Infixation + +We now address another issue from section 1.1: infixation. We repeat here the Tagalog data from example +(8) in section 1.1.6. + + (80) a. sulat ‘to write or writing (infinitive form)’ (=8) + b. sumulat ‘to write (with actor focus)’ + c. sinulat ‘to write (with object focus)’ + + Recall that there are two focus morphemes here, ‑um‑ and ‑in‑, both of which are infixes. + How does one create such infixes for the default parser of FieldWorks Language Explorer? See +figure 34. (See section 6.1.1.2 for how to write an affix process rule for infixation using the phonological +rule-based parser.) + + 1. Determine the environment(s) of the infix with respect to the stem in which it infixes (see section + 3.3.1) below. + + 2. Add the environment(s) if needed (and any natural classes, too) in the Grammar area (see section + 3.1.3). + + 3. Create the lexical entry and its allomorph. + 4. Mark the allomorph as being an infix. + 5. Set the infix position to the environment(s). + + a. To see the Infix Positions field, you may have to check the Show Hidden Fields box. + b. Editing the Infix Positions field is the same as editing an Environments field. See section + + 3.1.3. + . + + Figure 34. How to Create an Infix via an Environment. + + Note that infix allomorphs may be conditioned by regular environments just like any other allomorph. See +section 3.1.3. With the default parser, these environments should be with respect to what the environment +is before the infix has been pulled out of the stem. + +═════ PDF page 63 ═════ + +Infixation and root and pattern morphology 55 + + If you need to have two or more infix forms, make sure that all of them use the same infixation environ- +ment. If some need a different infixation environment, you will need to split the entry into several entries +where the forms in each entry all use the same infixation environment. + +3.3.1 Writing the infixation environment + +Infix environments describe the location within the sequence of characters where the infix is to go. For +example, in (80), it would be within sulat between the initial s and ulat. The environment would then be / +# [C] _ [V] where # indicates the beginning of the sequence within the stem, [C] is the natural class of +consonants and [V] is the natural class of vowels. Note that with the infixation environment, the # does +not indicate word boundary, but rather the beginning of the stem. + +3.3.2 Infixation and root and pattern morphology + +In section 1.1.8 we noted the Silt'e data repeated here from (10): + +(81) wakaba (=10) + +a-a-wkb-a + +pfv-buy-3sg.pfv + +‘he bought’ + + We noted that such Semitic languages have roots composed of three consonants, as exemplified in the +Silt'e data in (81), where ‘buy’ is the root wkb. The aspect markers are composed of vowel patterns that +fit between or around the root consonants, such as the a-a vowel pattern indicating the perfective aspect +shown in (81). + + How does one model this in the default parser of FieldWorks Language Explorer? The basic idea is +to treat each vowel as an infix. See figure 35. (See section 6.1.1.2.1 for how to write an affix process rule +for this using the phonological rule-based parser.) + + 1. One needs to model each vowel as a separate infix. Thus, the perfective aspect has to be treated + as two parts: The first ‑a‑ is part one and the second ‑a‑ is part two. The result would then “pull + out” the infixes in front of the root. That is, the resulting analysis would look something like + what is in (82). + + 2. Assuming that perfective aspect is inflectional, then one would build an inflectional template + that had both parts as slots in the template. The template might look like the one in table 17. + +. + + Figure 35. How to Create Root-and-Pattern Infixes. + +(82) wakaba + a-a-wkb-a + pfv1-pfv2-buy-3sg.pfv + + Table 17. Silt'e Template + + Aspect1 Aspect2 Stem Subject + + -a- pfv1 -a- pfv2 -a 3sg.pfv + + etc. etc. etc. + +═════ PDF page 64 ═════ + +56 Parsing Conceptual Introduction + +3.4 Epenthesis + +In section 1.1.4 we noted the Caquinte data repeated here from (6). + +(83) itsavetakohitiro (=6) + +i-tsave-(t)-ako-hi-(t)-i-ro + +3m-tell- -dat-pas- -nf-3f.obj + +‘she is told about’ + + Recall that this is an instance of epenthesis. Many languages have certain syllable well-formedness +constraints that require the insertion of either a vowel or a consonant to preserve syllable structure (see Itô +1989 for an interesting discussion). In the data above it is a consonant t. + + How can one model such epenthetic segments using the default parser of FieldWorks Language Ex- +plorer? There are at least two ways: + +1. The first method is to treat the epenthetic segment as a kind of pseudo morpheme. +2. The second is to create extra allomorphs containing the epenthetic segment(s) for every morpheme that + + might possibly be involved with epenthesis and using environments to condition them appropriately. + + The advisability of the use of the first method is debatable. If the epenthetic segment is rather common, +then one might want to model it as a pseudo-morpheme. Such an approach allows you to use the output +of FieldWorks Language Explorer to explore where it occurs and perhaps glean some insights about +its true nature. The second approach captures the fact that epenthesis has no meaning whatsoever (as one +would expect with a true morpheme) but it misses the generalization that the presence of the segment +is due to syllabification considerations (by adding otherwise unnecessary allomorphs to many dictionary +entries). FieldWorks Language Explorer does not model syllables. + +3.5 Metathesis + +Another morphophonemic issue we noted in section 1.1 was metathesis. We repeat the Caquinte word in +(11) given in section 1.1.9. + +(84) ihikekehai (=11) + +i-hi-k-e-kea-hi + +3M-to.think.mistakenly-PROG-NF-FOC-NEG + +‘he thought mistakenly’ + + Recall that the h and a in the final two morphemes switch positions. + How does one model such metathesis processes in the default parser of FieldWorks Language Ex- +plorer? Since this parser does not have any way to model processes, one must use allomorphy. For data +like that in Caquinte, one would do what is shown in figure 36. (See section 6.1.2.3 for how to write a +metathesis rule using the phonological rule-based parser.) + + 1. In the lexical entry for kea, create an allomorph of keh and condition it to be followed by an a + (i.e., write an environment of / _ a). + + 2. In the lexical entry for hi, create an allomorph of ai and condition it to be preceded by a h (i.e., + write an environment of / h _ ). + +. + + Figure 36. How to Handle Metathesis. + +═════ PDF page 65 ═════ + +Non-phonologically conditioned allomorphy 57 + +3.6 Morphemes that may be null + +Recall that in section 1.1.10 we noted some other Caquinte data in (12) repeated here (you do not need to +understand all the morpheme glosses here; just concentrate on the initial subject prefixes): + +(85) a. anehero (=12) + +a-∅-neh-e-ro + +1incl-fut-see-fut-3FO + +‘we will see her’ + +b. okeekake + o-keek-ak-e + 3f-dig-PERF-nf + ‘she had dug’ + + c. oasanomahakemparime + ∅-∅-o-(a)-sano-maha-k-e-Npa-ri-me + 1incl-fut-eat- -veri.m-veri-prog-fut-r-3MO-cntr + 3f-fut-eat- -veri.m-veri-prog-fut-r-3MO-cntr + ‘we/she will not really be eating it’ + + What is the issue with the subject prefixes? In (85a) we see that the first person inclusive subject marker +is a‑, and in (85b) the third person feminine subject marker is o‑. Yet, in (85c), the gloss shows ambiguity +between ‘we’ and ‘she’ as the subject, and both of these are represented as null. This is because both subject +prefixes are vowels and the stem in (85c) is vowel-initial, yielding two vowels together. Recall from (83) +that Caquinte generally does not allow vowel clusters, and therefore adds an epenthetic ‑t‑ when necessary +to avoid such clusters. It turns out that epenthesis is only used in the suffixes. Within the prefixes, the +initial vowel of a cluster deletes, causing the ambiguity seen in (85c). + + How does one model such allomorphy in the default parser of FieldWorks Language Explorer? See +figure 37. + + 1. Ensure that you have a natural class for vowels. + 2. Create or check that you have an environment that is before a vowel. + 3. Create the lexical entries for the two prefixes. + 4. In each one, create two forms: + + a. Create one for the null allomorph (see section 4.1.1) and condition it as occurring before a + vowel (i.e., use the “before a vowel” environment). This should be in an allomorph form. + + b. Create one for the overt form (a or o) in the lexeme form. It does not need to be conditioned + by an environment if it is in the lexeme form because it will be the “elsewhere” allomorph. + + . + + Figure 37. How to Create a Null Morpheme. + +3.7 Non-phonologically conditioned allomorphy + +Sometimes there is unpredictable allomorphy in either stems or in affixes. How does one deal with these? +We have already seen how to deal with affix allomorphy determined by inflection classes in section 2.1.2.6. +The following two subsections explain how to use morpho-syntactic features to control both stem allomor- +phy and affix allomorphy. + +═════ PDF page 66 ═════ + +58 Parsing Conceptual Introduction +3.7.1 Stem allomorphs conditioned by morpho-syntactic features + +The next morphophonemic issue we address relates to dealing with various inflectional stems which can +appear in word paradigms. For example, consider the following data from Orizaba Nahuatl.46 + +(86) want 1st Person Singular Subject 1st Person Plural Subject + Present tense + nikneki tiknekih + Past tense ni-k-neki ti-k-neki-h + 1sg.sbj-3.obj-want 1pl.sbj-3.obj-want-ipfv.pl + + oniknek otiknekkeh + o-ni-k-nek o-ti-k-nek-keh + pst-1sg.sbj-3.obj-want pst-1pl.sbj-3.obj-want-pfv.pl + + Notice the shape of the root. It is neki in present tense, but merely nek in the past tense. A more +complete look at the rest of the verbal paradigm would show that the shorter nek form also occurs with the +pluperfect, the durative, and some special aspectuals. The longer neki form occurs everywhere else in the +verbal paradigm.47 + + Further note that this truncation of the final vowel does not appear to be phonologically conditioned. +Rather, it is conditioned by the inflectional features of the word itself. If the word is in past tense (or is +pluperfect or is durative or has one of the special aspectuals), then the truncated allomorph of the root is +used. Otherwise, the longer form of the root is used. + + So what we would need is “something” that allows us to define the sets of inflection features that must +be present in order for a particular stem allomorph to be licit. We would then associate that particular +stem allomorph with that “something” so that we and the parser both know that the allomorph can only +occur when one of those feature sets is present. Further, since we are talking about inflection features, this +“something” should be associated with the appropriate category which can inflect for the features contained +in these sets. In the Nahuatl case, this would be the verb category. + + So what is this “something” in FieldWorks Language Explorer? It is what we call Stem Allomorph +Labels (see Bonami & Boyé (2002:55), Hippisley et al. (2004) where they are referred to as “stem names”). +Each Stem Allomorph Label is defined in a particular category. Each Stem Allomorph Label has one or +more sets of inflection features associated with it. Whenever a stem allomorph is tagged with such a Stem +Allomorph Label, then the FieldWorks Language Explorer parser will only allow that allomorph to be +valid if one of the sets of inflection features is present. + + For example, for the Nahuatl case in (86), we could define a Stem Allomorph Label of, say, Truncates +and then make at least one feature set. But how many feature sets would we need? Remember that the +truncated allomorph occurs whenever the word is past tense, pluperfect, durative, or has one of the special +aspectuals. Suppose we made one feature set and put all four of these features in it. This would mean that +the truncated allomorph would only be valid when the word had all four features. But that never happens +because some of these features are mutually exclusive. This means we would need as many feature sets as +there are mutually exclusive features. In this Nahuatl case, we would need to give it four feature sets, one +each for the past tense, pluperfect, durative, and the special aspectuals. Then in the lexical entry for neki +we would add an allomorph of nek and tag it as belonging to the Stem Allomorph Label Truncates. + + In this example, what should we do with the neki allomorph? We have tagged the nek allomorph with +the Truncates Stem Allomorph Label so we do not need to do anything to tag the neki allomorph. The +FieldWorks Language Explorer parser will automatically constrain the neki allomorph so that it will + +46 Data are from Tuggy (1991:102). +47 This variation of stem shape illustrated in (86) occurs only for verbs in inflection class II. Verbs in +inflection class III also have a similar, but slightly different, allomorphy that occurs in different parts of the +verbal paradigm. Verbs in classes I and IV do not have any such allomorphy. See Tuggy (1991:102-104) +for examples. + +═════ PDF page 67 ═════ + +Stem allomorphs conditioned by morpho-syntactic features 59 + +not occur with any of the inflection features defined in the Truncates Stem Allomorph Label. Similarly, if +a lexical entry has three allomorphs, two of which need to be tagged with distinct Stem Allomorph Labels, +then the third, untagged, one will automatically be constrained to never occur with any of the feature sets +defined for the other two Stem Allomorph Labels associated with the other two allomorphs in that entry. + + How does one create and use a Stem Allomorph Label in FieldWorks Language Explorer? See figure +38. + + 1. Unless you have already done so, set up the needed inflection features. See figure 6. + 2. Create a Stem Allomorph Label in the appropriate category (sub-categories will inherit any Stem + + Allomorph Labels which their parent categories have, so you will probably want to create the + Stem Allomorph Label on the highest level category). + a. Use Grammar / Category Edit. + b. Click on Stem Allomorph Labels. + c. Click on Insert Stem Allomorph Label. + d. Give the Stem Allomorph Label a name, abbreviation, and description. + e. Create the sets of inflection features that are relevant to this Stem Allomorph Label. That is, + + create one feature set for each mutually exclusive set of features that go on words in which + this allomorph occurs. + + i. Click in the Feature Set field. + ii. Click on the chooser button. + iii. In the ensuing dialog box, select all feature values that apply. + 3. Tag all appropriate stem allomorphs with the Stem Allomorph Label. (Remember that you do + not need to tag the other allomorphs unless they, too, have a limited set of inflection features. + In this latter case, you will need to define yet another Stem Allomorph Label for them.) + a. To see the Stem Allomorph Label field, you may have to check the Show Hidden Fields box. + b. Click on the Stem Allomorph Label field. + c. Click on the chooser button. + d. In the ensuing dialog box, select the Stem Allomorph Label. + 4. Make sure that you have at least one inflectional template defined for the category (or categories) + that use Stem Allomorph Labels.a See section 2.1.2 for more on inflectional templates. + + a The FieldWorks Language Explorer parser will only be able to correctly constrain stem al- + lomorphs tagged with a Stem Allomorph Label if an inflectional template has been defined which + has slots containing the inflectional affixes which bear the appropriate inflection features. For ex- + ample, if you have verbal stems which have one shape when inflected in the past tense and another + shape in all other tenses, then you will need to have at least one verbal inflectional template that + has one or more slots containing these tense inflectional affixes. And, of course, these inflectional + affixes will have to be overtly tagged with the appropriate tense feature (e.g., past tense, present + . tense, etc.). + + Figure 38. How to Create and Use Stem Allomorph Labels. + + To summarize, one should use Stem Allomorph Labels to control stem allomorphy that is dependent not +on phonological issues, but on the presence of certain inflection features. Note that Stem Allomorph Labels +can only be used for stem allomorphs (not affix allomorphs). + + Finally, note that derivational affixes may also be constrained to only occur with a particular Stem +Allomorph Label. + +═════ PDF page 68 ═════ + +60 Parsing Conceptual Introduction +3.7.2 Affix allomorphs conditioned by morpho-syntactic features + +In section 3.7.1 we discuss how to control stem allomorphy based on sets of morpho-syntactic (inflection) +features.49 This section deals with something similar, but for affixes, not for stems. For example, consider +the following data from Axininca Campa [cpu].50 + +(87) a. itʃʰikana (88) a. itʃʰikakina + i-tʃʰik-a-na i-tʃʰik-ak-i-na + 3m-cut-nf-1.obj 3m-cut-prf-nf-1.obj + ‘he cut me’ ‘he has cut me’ + + b. itʃʰikimi b. itʃʰikakimi + i-tʃʰik-i-mi i-tʃʰik-ak-i-mi + 3m-cut-nf-2.obj 3m-cut-prf-nf-2.obj + ‘he cut you’ ‘he has cut you’ + + c. itʃʰikiri c. itʃʰikakiri + i-tʃʰik-i-ri i-tʃʰik-ak-i-ri + 3m-cut-nf-3m.obj 3m-cut-prf-nf-3m.obj + ‘he cut him’ ‘he has cut him’ + + d. itʃʰikiro d. itʃʰikakiro + i-tʃʰik-i-ro i-tʃʰik-ak-i-ro + 3m-cut-nf-3f.obj 3m-cut-prf-nf-3f.obj + ‘he cut her’ ‘he has cut her’ + + Notice that in (87), the non-future suffix is ‑i except when it comes before the first person object marker +‑na where the non-future is ‑a (87a). Example (88) shows that when the non-future follows the perfect +suffix, it is ‑i even when it comes before the first person object (88a). + + 1. Add the appropriate inflectional features. See figure 6. + 2. Create the appropriate inflectional template. + 3. Create an entry for each morpheme (assigning the inflectional affixes to the appropriate slots in + + the template). + 4. Assign the appropriate inflectional features to those affixes which bear them. See figure 8. + 5. For those entries with allomorphy, create the Allomorph Form(s). + 6. For each Allomorph Form, + + a. To see the Required Features field, you may have to check the Show Hidden Fields box. + b. Click on the Required Features field. + c. Click on the chooser button. + d. In the ensuing dialog box, select the inflectional features that this allomorph must have in + + order to be licit. + e. You can leave the Lexeme Form without any “Required Features” if it is the “elsewhere” case. + +. + + Figure 39. How to Create Feature-Dependent Affixes. + +49 These kinds of features are to be contrasted with phonological features. +50 The Axininca Campa data are from Payne (1981:234; see also page 36). + +═════ PDF page 69 ═════ + +Irregularly inflected forms 61 + + How does one model such allomorphy in FieldWorks Language Explorer? Since only allomorphy +is available, one must create allomorphs for both forms of the non-future suffix and condition them to +co-occur with the appropriate set of features.51 See figure 39. + + For the Axininca Campa data above, the verbal inflection features are for perfect aspect and for first +person object agreement. We would need to create the appropriate inflectional suffixes and assign perfect +aspect to the perfect ‑ak suffix and first person object agreement to the first person object ‑na suffix. For +the non-future suffix, we would make the Lexeme Form be ‑i and we would create an Allomorph Form of +‑a. We would set the “Required Features” field of the ‑a allomorph to be for first person object agreement. +One implication of this is that the default form ‑i of the Lexeme Form will automatically be conditioned to +not occur when the first person object agreement feature is present. This means that a form like (88a) will +fail to analyze correctly. Currently, a way to compensate for this is to add another Allomorph Form for ‑i, +setting its “Required Features” field to be for both first person object agreement and perfect aspect. + + While this example is for inflectional affixes, the FieldWorks Language Explorer parsers also allow +constraining allomorphs of derivational affixes. The process is similar: create the allomorph and constrain +it via the “Required Features” field. + + Also please note that “Required Features” only apply between two affixes. If there is only one affix, it +does not work. You have to have two or more affixes. + +3.8 Irregularly inflected forms + +Next consider the following small set of data for Turka [tuz], a language spoken in Burkina Faso.52 + + (89) a. duu ‘house’ + b. don ‘house.pl’ + c. fuga ‘field’ + d. fugasǝ ‘field.pl’ + e. gbaflǝ ‘pot’ + f. gbafiɛ ‘pot.pl’ + g. kuu ‘idol’ + h. kii ‘idol.pl’ + + Notice that there is not an obvious way that plurals are formed. That is, the plural forms appear to be +portmanteau morphemes: a combination of the singular morpheme plus plural. + + At least one way to deal with this is to create irregularly inflected form variants for each of the plural +forms, tagging them as being the plural variant type, and linking them to their respective singular form. If +you make sure that the plural variant type53 has its “Append to Gloss” field set to ‘.pl’ (if you are going to +use small caps) or ‘.PL’ (if you are not), then the FieldWorks Language Explorer parser will parse these +plural variant forms, using the gloss of the singular form plus what is in the “Append to Gloss” field. When +you use this method, the irregularly inflected form must have no sense information or it will not work as +intended. + + You can also set the “Inflection Features” field of the irregularly inflected variant type to those inflection +features borne by the variant. The FieldWorks Language Explorer parser will then prevent parses +where the indicated inflection features conflict with similar inflection features of other affixes that have +different values. + +51 In cases like Axininca Campa, merely using environments (see section 3.1.3) will not work properly due +to the fact that one must identify the exact morpheme, not merely its surface form. Other morphemes may +well share the same form, but the allomorphy of the non-future does not occur with them. It is only with +the perfect suffix and the first person object suffix. +52 The data are from Colin Suggett, personal communication. +53 Irregularly inflected variant types are located in the “Lists” area under “Variant Types.” + +═════ PDF page 70 ═════ + +62 Parsing Conceptual Introduction + + Suppose, now, that there are some inflectional templates that have required slots and that the irregularly +inflected form portmanteau morpheme has in it the information needed to fill one or more of those required +slots. What will happen? Since the FieldWorks Language Explorer parser requires these slots to be +filled, the parse will fail when it should not. + + The solution is to set the “Slots” field of the irregularly inflected variant type to refer to all required slots +in templates that it is taking the place of. The FieldWorks Language Explorer parser will then correctly +parse such forms.54 In addition, it will prevent parses where the indicated slots are filled by some other +affix. + +3.9 Coalescence + +Another morphophonemic phenomenon can be illustrated by the following data from Menya [mcr].55 + + (90) a. mapŋque + ma-p=ŋqä=i + get-23.irr=goal=ind + ‘they're going to get’ + + b. metuŋque + ma-atu=ŋqä=i + get-1.irr=goal=ind + ‘we're going to get’ + + (91) a. ikäpŋque + ikä-p=ŋqä=i + tour-23.irr=goal=ind + ‘they're going to tour’ + + b. ikituŋque + ikä-atu=ŋqä=i + tour-1.irr=goal=ind + ‘we're going to tour’ + + In example (90), the root is ma, but when this vowel-final root is followed by a vowel initial suffix as in +(90b), the two vowels coalesce so it surfaces as me. In this case, the coalesced vowel also changes quality. +A similar coalescence occurs in (91b) for the vowel-final root ikä. + + How does one model such allomorphy in the default parser of FieldWorks Language Explorer? Since +only allomorphy is available, one must create allomorphs for both the root and the suffixes and condition +them to have the correct environments.56 See figure 40. + +54 This works by FieldWorks Language Explorer automatically creating a null prefix (for required slots +that occur before the stem) or a null suffix (for required slots that occur after the stem) and giving it the +inflection features of the variant type. +55 The data are from Whitehead (2004). +56 It may be the case that the stem allomorphy will depend on the morphosyntactic features of the affixes. +If so, you may be able to use Stem Allomorph Labels. See section 3.7.1. + +═════ PDF page 71 ═════ + +Creating ad hoc allomorph-oriented rules 63 + + 1. Create an entry for each morpheme. + 2. For those entries with allomorphy, create the Allomorph Forms. + 3. For each Allomorph Form, + + a. Create or check that you have an environment that is correct for the form. + b. Assign the correct environment for the form. +. + + Figure 40. How to Handle Coalescence. + +3.10 Ad hoc allomorph-oriented rules + +When one uses a morphological parser, it is not unusual for the parser to sometimes return a parse that is +simply incorrect. These are sometimes due to allomorphs matching in places one would not have expected +them to match. When one has used all the mechanisms provided by the parser to the best of one's ability +and such incorrect parses continue to surface, one may well wish for some kind of mechanism to rule them +out. FieldWorks Language Explorer provides “Ad hoc Rules” for such situations. + +3.10.1 Creating ad hoc allomorph-oriented rules + +There are two main types of ad hoc rules: morpheme-oriented ones and allomorph-oriented ones. This +section deals with allomorph-oriented ones (see section 2.4 for morpheme-oriented ones). The basic idea +is to list a key allomorph and then to list one or more other allomorphs that cannot co-occur with the key +one. One can constrain these other allomorphs to never occur in one of the following ways with respect to +the key allomorph as shown in table 18.57 + + Table 18. Allomorph Ad-hoc Rule + +Manner Meaning +Anywhere + The allomorphs in question are constrained from appearing anywhere together in the +Somewhere same word. +before The key allomorph is constrained from appearing anywhere in the word before the +Somewhere after other allomorphs. + The key allomorph is constrained from appearing anywhere in the word after the +Adjacent before other allomorphs. + The key allomorph is constrained from appearing immediately before the other +Adjacent after allomorphs in the same word. + The key allomorph is constrained from appearing immediately after the other + allomorphs in the same word. + + Note that when there are two or more allomorphs listed for “other allomorphs,” the rule only applies +when all of them co-occur in the same word with the key allomorph. In addition, their relative order is +significant. They should be listed in the same linear order they occur in a word. + +57 One approach to this is to strive to make the tightest constraint possible (i.e., use one of the adjacency +ways first if possible; if not, then try the somewhere case; if that does not work, then try the anywhere +case). That way, should you encounter another case involving these particular allomorphs, then you will +know more: it is now clear that you need looser constraints. You can then add some comments/annotations +to document what you have learned (or put the information in the description). + +═════ PDF page 72 ═════ + +64 Parsing Conceptual Introduction + + The English plurals in (92) show some cases where we might choose to use an allomorph ad hoc rule for +the default parser of FieldWorks Language Explorer.58 + + (92) a. book-s + b. dog-s + c. ox-en + + The exceptional case, of course, is the ‑en allomorph (there are other exceptional plurals in English, but +this one will do for our example here). Suppose you have these allomorphs in your dictionary and that you +also have the noun molt as well as the verb molt in your dictionary. Then the word form molten would be +parsed at least two ways as shown in (93). + + (93) a. molt-en + molt-pst.ptcp + + b. molt-en + molt-pl + + The parse in (93b), of course, is incorrect. To rule out this incorrect parse, one could create an allomorph +ad hoc rule for the en allomorph of the plural with the molt allomorph of the noun molt. + + 1. Determine the allomorphs involved. + 2. Determine the most appropriate way to constrain them (see table 18). + 3. Create an allomorph-oriented ad hoc rule. + + a. Use Grammar / Ad hoc Rules. + b. Use Insert menu item / Rule to prevent allomorph co-occurrence. + c. Indicate the key allomorph involved. + + i. Click in the Key Allomorph field. + ii. Click on the chooser button. + iii. In the ensuing dialog box, key the form of the allomorph in the Find box. + iv. Select the correct one in the middle pane. + v. Check the Allomorph drop-down box to make sure this is the one you want. + d. Indicate the way or manner in which to constrain this morpheme (see table 18) by using the + drop-down box in the Cannot Occur field. + e. Indicate the non-key (i.e., other) allomorph(s) involved. + i. Click in the Other Allomorph(s) field. + ii. Click on the chooser button. + iii. In the ensuing dialog box, key the form of the allomorph in the Find box. + iv. Select the correct one in the middle pane. + v. Check the Allomorph drop-down box to make sure this is the one you want. + vi. Repeat the above steps for any other allomorphs needed. + + . + + Figure 41. How to Create Allomorph-oriented Ad Hoc Rules. + +58 Admittedly, this is not the greatest example. One could use inflection classes for these or, perhaps better, +one could merely use an environment to constrain the exceptional allomorphs for the roots to which they +attach. + +═════ PDF page 73 ═════ + +Order of allomorphs within a lexical entry 65 + + How does one create an allomorph-oriented ad hoc rule in FieldWorks Language Explorer? See +figure 41. + + By the way, when you are indicating the allomorph, be sure that the particular allomorph is for the +correct morpheme, too. FieldWorks Language Explorer maintains a distinction between identically +shaped allomorphs; only those for the particular morpheme will actually be constrained. + +3.10.2 Grouping ad hoc allomorph rules + +Occasionally one finds a situation where a set of ad hoc constraints have a common theme. Perhaps they +all relate to a particular allomorph or to particular allomorphs of a certain variety. This may be a hint as +to what is really happening and may lead you to discover a linguistically-motivated way to model them. +Or it could be that the FieldWorks Language Explorer model just does not happen to provide the +appropriate linguistic mechanism to model the phenomenon correctly. + + One can group such ad hoc rules together. How does one create such a group? See figure 42. + + 1. Create an ad hoc rule group. + a. Use Grammar / Ad hoc Rules. + b. Use Insert menu item / Group of ad hoc rules. + + 2. For each ad hoc allomorph rule in the group: + a. Create the ad hoc allomorph rule (see figure 41) and include it in the group. + + . + + Figure 42. How to Create a Group of Allomorph-oriented Ad Hoc Rules. + + Finally, note that FieldWorks Language Explorer allows one to group both allomorph and morpheme +ad hoc rules together. Please be sure to only do so if these rules truly do have something in common. + +4 Lexical entry considerations + +This section lists a few items that one should keep in mind while adding lexical entries. + +4.1 Allomorphs + +There are two things to keep in mind while keying Lexeme Forms and Allomorph Forms. + +4.1.1 Null allomorphs + +Generally speaking, one wants to avoid having null allomorphs if for no other reason than that they can +make the parser run rather slowly. If having a null allomorph is indeed the best analysis, then please keep +what is shown below in figure 43 in mind. + +4.1.2 Order of allomorphs within a lexical entry + +In FieldWorks Language Explorer, the order of Lexeme Forms and Allomorph Forms is quite significant. +Consider the following English data. + +═════ PDF page 74 ═════ + +66 Parsing Conceptual Introduction + + (94) a. pisɪz ‘pieces’ + b. dagz ‘dogs’ + c. kaɪts ‘kites’ + +Under one possible analysis, we can say that the allomorphs for the English plural are: + +1. ‑ɪz after strident segments +2. ‑z after voiced (but non-strident) segments +3. ‑s elsewhere + + If we have a natural class for stridents and one for voiced segments (including stridents) and create +two environments (one for “after stridents” and one for “after voiced segments”), then we can order and +condition the allomorphs as follows: + +1. ɪz to occur after stridents +2. z to occur after voiced segments +3. s + + 1. One indicates a null in the allomorph's form field by typing either ˆ0 (a caret - shift-6 on many + keyboards - followed by the zero character), &0 (an ampersand followed by the zero character), + *0 (an asterisk followed by the zero character), or the empty set character (if the font you are + using has it).a Ideally, we would always just use the empty set character. Unfortunately, many + fonts do not happen to include it, so we offer these other ways. Note that if you happen to use + an asterisk (or caret or ampersand) as a leading or trailing token for some morpheme type, then + there is a potential conflict between indicating the morpheme type and indicating a null. If you + discover such a conflict, try using a different alternative for indicating a null (or change the + leading or trailing token for the morpheme type; see the help files in FieldWorks Language + Explorer for more.). + + 2. Attempt to constrain the null with as specific an environment as possible. This is to avoid having + it show up in undesired places in incorrect parses. + + 3. You may also need to adjust the Parser Parameters to allow for more than one null allomorph per + word. That is, determine what the maximum number of null allomorphs a legitimate word could + have and set the MaxNulls parameter to that number. Remember, that the lower the number, + the faster the parser will run. + a. Use Parser menu item / Edit Parser Parameters... + b. Click in the box below MaxNulls and enter the number you need. + c. Click on OK. + + . a The empty set character is Unicode hex code 2205. + + Figure 43. How to Create a Null Allomorph. + + Because of the ordering and the fact that the first two are conditioned, the third (elsewhere) case will +automatically be constrained to not occur after stridents as well as to not occur after voiced segments. The +second allomorph will be conditioned to not only occur after voiced segments, but also to not occur after +stridents. + + Do you see how it works? For a given Allomorph Form, FieldWorks Language Explorer applies the +condition of this Allomorph Form and, at the same time, negates the conditions of all preceding Allomorph +Forms. This is why the ordering of allomorphs is crucial. + +═════ PDF page 75 ═════ + +Circumfixes 67 + + Having said that, please note that the Lexeme Form field is always automatically ordered last after all of +the Allomorph Forms listed in the Allomorphs section of Lexicon Edit. + +4.2 Morpheme types + +Morpheme types are things like “root,” “prefix,” “clitic,” etc. FieldWorks Language Explorer keys on +certain ones of these in order to tell the parser how to handle the particular form. The types in the following +list are significant to the parser.59 + +• bound root +• bound stem +• enclitic +• infix +• infixing interfix +• particle +• phrase +• prefix +• prefixing interfix +• proclitic +• root +• stem +• suffix +• suffxing interfix + + One should keep this in mind when applying a type to a form. + In particular, note that if a form is tagged as a particle, then it must stand alone (i.e., it will not be +allowed to take affixes or to participate in compounds). Clitics, though, may attach to a morpheme tagged +as a particle. + One may also tag a form as a phrase, although there are limitations. The FieldWorks Language +Explorer parser treats such phrases like a stem. That is, any derivational or inflectional affixes must +attach around the phrase, not within it (unless such an affix is an infix, of course). In particular, the +FieldWorks Language Explorer parser cannot handle something like kicked the bucket where the past +tense inflection occurs as a suffix on the first word of the phrase. + +4.3 Circumfixes + +In some languages, there is a special class of affixes. The segmental material represented by these affixes +appears at both ends of the stem at the same time. It is as if there are two parts of such an affix: one part +is typically a prefix and the other part is a suffix. These are called circumfixes. Consider the following data +from Bahasa Indonesia.60 + + (95) kebangunan + ke-bangun-an + nmlz-wake.up-nmlz + ‘awakening’ + +59 Another way of saying this is that the parser recognizes all morpheme types except for discontiguous +phrase, simulfix, suprafix, and circumfix. For circumfix, however, see section 4.3. +60 The data are from Howard Shelden, personal communication (December 15, 2006 email) + +═════ PDF page 76 ═════ + +68 Parsing Conceptual Introduction + + The ke‑ prefix and the ‑an suffix act together to form a single morpheme even though they are on opposite +ends of the bangun stem. Another way of looking at this is to say that the nominalizer (‘nmlz’) morpheme +is realized by a circumfix whose left member is the ke‑ prefix and whose right member is the ‑an suffix. + + How does one create such circumfixes in the default parser of FieldWorks Language Explorer? +See figure 44 below. (See section 6.1.1.3 for how to write an affix process rule for circumfixes using the +phonological rule-based parser.) + + Note the following characteristics about circumfix entries created this way: + + 1. The only place this entry is marked as being a circumfix is in the morpheme type field of the lexeme + form. + + 2. There is at least one allomorph for the prefix member. + + 3. There is at least one suffix allomorph for the suffix member. + + When one keys a circumfix in this manner, the FieldWorks Language Explorer parser will require +both the left and right member affixes to appear simultaneously for them to be parsed as an instance of +this entry. Circumfixes may be classified as derivational, inflectional, or as unclassified. The FieldWorks +Language Explorer parser handles all three varieties correctly. + + 1. Add a new lexical entry for the circumfix. + 2. Indicate the lexeme form in such a way that it is clear that it is a circumfix. + + a. One way is to indicate the prefix portion (with a trailing hyphen), an ellipsis marker (...), + and then the suffix portion (with a preceding hyphen). + + b. Another way is to merely separate the prefix and suffix portions by a space (but still using + the hyphen to indicate the appropriate type). + + 3. Two allomorphs will be created automatically: + a. One is for the left member (the prefix in our example). Its morpheme type defaults to prefix + (if yours is an infix, you will need to change the morpheme type by hand). + b. The other is for the right member (the suffix in our example). Its morpheme type defaults to + suffix (if yours is an infix, you will need to change the morpheme type by hand). + c. If either of these have allomorphs, key those as well. + d. Include all environments as appropriate for each member. For example, if a prefix allomorph + only occurs before labials, indicate that for that prefix. Remember that the environments for + the different members are independent of each other. + + . + + Figure 44. How to Create a Circumfix. + +4.4 Senses/Glosses + +While it is possible to have two or more glosses somewhere in your lexicon spelled exactly the same way, +we do not recommend that you do so on purpose. Having two or more morphemes with the same gloss +will not confuse FieldWorks Language Explorer because FieldWorks Language Explorer uniquely +identifies every gloss internally. That does not imply, however, that either you or a reader of your glossed +texts will not be confused as a result. + +═════ PDF page 77 ═════ + +Item and process 69 + +5 Other considerations + +The FieldWorks Language Explorer approach has been purposely designed to allow you to incremen- +tally build up the morphological description piece by piece (with one exception; see section 5.1 below). For +example, you can add inflectional templates as you discover them. It is not the case that once you start to +use inflectional templates, you must define inflectional templates for all categories at once. You can define +them one by one if you need to or all at once (if you happen to already know what they are). + +5.1 Exceptional Case for Compound Rules + +The exception to this general case is compound rules. Once you define your first compound rule, the +FieldWorks Language Explorer parser will then only allow compounds for which there are rules. In +particular, this means that you may have a number of word forms that will suddenly fail to analyze once +you write your first compound rule. To get them to analyze, you will need to define appropriate compound +rules for them. We wish we could allow the discovery and development of compound rules to also be +incremental, but we have not figured out how to do it. + +6 The phonological rule-based parser + +As mentioned in section 1, FieldWorks Language Explorer has a phonological rule-based parser. This +section outlines some of its capabilities in section 6.1, gives some tips for making it work in section 6.2, +and lists some known limitations in section 6.3. + +6.1 Item and process + +The phonological rule-based parser offers an item and process approach for both affixation and morpho- +phonemics. That is, one can describe affixes as processes (such as infixation or reduplication)61 and one +can also describe phonological rules. It is also designed to work with the item and arrangement approach +of the default FieldWorks Language Explorer parser. This means that one should be able to move +from an item and arrangement description to an item and process description as one determines what these +processes are. + + When you get to the point of trying this parser and want to use processes, how might you describe such +a process? That is, what are the pieces of a process? The approach we take is to think of what the input +pattern is and then what the resulting output of the process should be. Therefore, all process rules will +have an input pattern component and an output result component. + + Further, given the way the phonological rule-based parser works for both affixes and phonology, one +needs to think generatively. For phonological rules, the rules are applied in order from the underlying form +to the surface form. For affixation, the word is built from the root out. That is, each affix is attached to what +has been built so far during the derivation. Contrast this with how the default FieldWorks Language +Explorer parser works: the default parser searches for surface forms from the beginning of the word to +the end of the word; that is, it basically looks for prefixes first, then roots, then suffixes.62 + + The rest of this section illustrates these process rules. Examples of how one writes special affix process +rules are in section 6.1.1 and how one writes phonological rules are in section 6.1.2. + +61 “Normal” affixation (prefixation and suffixation) are actually also treated as processes internally in the +phonological rule-based parser, but the user does not need to do anything special. By merely saying that +the affix is a prefix or a suffix, the phonological rule-based parser will treat them appropriately. +62 If there are interfixes (see section 2.2.4), then it will also look for those between roots. In addition, it +looks for infixes within prefixes and roots. + +═════ PDF page 78 ═════ + +70 Parsing Conceptual Introduction +6.1.1 Affix process rules + +With the phonological rule-based parser, one can write an affix process rule that describes what the affix +does to the material it “attaches” to. Any such affix process rule is entered in the lexical entry for the affix +as a special kind of form (e.g., Lexeme Form or Affix Allomorph).63 As mentioned above, the rule has the +general form of a pattern followed by a result. The pattern describes a sequence of items, where each item +in the pattern is assigned an index number (starting with 1 for the left-most item). Then the result indicates +how the pattern items are to be arranged and/or how new material is to be added. The following sections +have examples of such rules.64 + +6.1.1.1 Reduplication as a process + +Recall from section 3.2 that there are two kinds of reduplication: full and partial. The phonological rule- +based parser can handle both kinds. + +6.1.1.1.1 Full reduplication as a process + +Let's begin with full reduplication. Section 3.2.1 had the following data from Bahasa Indonesia (repeated +here from (70-74) for your convenience). + + (96) a. pel ‘mop’ + b. pel-pel ‘mops’ + +(97) a. buku ‘book’ + +b. buku-buku ‘books’ + +(98) a. komik ‘clown’ + +b. komik-komik ‘clowns’ + +(99) a. orang ‘person’ + +b. orang-orang ‘people’ + +(100) a. perpustakaan ‘library’ + +b. perpustakaan-perpustakaan ‘libraries’ + + Recall that in this data, the entire stem material is copied, no matter how long it is. In addition, an +orthographic hyphen is inserted between the reduplicated material and the original stem. + + To handle full reduplication data like this using the phonological rule-based parser, what do we do? +Remember that these processes are described by two parts: a pattern and a result. + + So, what is the pattern here for full reduplication? Since it is full reduplication, we want to describe a +pattern that is the entire stem. That is, we will want to copy everything in the input. + +63 Since these special forms include information about the context (the pattern), it does not make sense +to also have an environment for an affix process rule. Therefore, if you are changing an environmentally- +conditioned affix form to an affix process rule, any environments you previously had for this form will no +longer be associated with that form. +64 Please be aware that while the phonological rule-based parser is able to handle any entry created for +the default parser, the default parser does not know what to do with an affix process rule. Therefore, you +cannot create affix process rules and then expect the default parser to use them. This is one area where it +is not possible to switch back and forth between the two parsers. The other area is phonological rules since +the default parser does not handle phonological rules. + +═════ PDF page 79 ═════ + +Partial reduplication as a process 71 + + What would the result look like? The result would be to copy the entire stem, then insert a morpheme +boundary symbol, the (orthographic) hyphen, and then another copy of the entire stem. It might look like +this: + +(101) Pattern Result + 1+-1 +X ⇒ +1 + + So in (101), the pattern is quite simple: it is just the stem (indicated in this case by X). The result refers +to the index number of the stem material (1), inserts a morpheme boundary symbol (+), the phonological +shape of the hyphen (- in this case), and then a copy of the stem (indicated by the index number 1). Note +that in order to include the orthographic hyphen in this process rule, we have to define it as a phoneme. +See table 21 below and the discussion before it for more on this “phoneme;” also see sections 6.2.1 and +6.2.2. + + Like we did in section 3.2.1, we are treating the reduplication affix as a suffix. With the phonological +rule-based parser, we can actually treat it either as a prefix or as a suffix. Either one will work fine. Note +that this is different from the default FieldWorks Language Explorer parser where it is important to +determine which kind of affix it is. + + By the way, if we chose not to use something like an orthographic hyphen to separate the two parts of +the full reduplication, the affix process rule would be written like this: + +(102) Pattern Result + 1+1 +X ⇒ +1 + + Note how this rule merely says to make a copy of the stem and insert a morpheme boundary between +the two copies. Because this rule is a copy of the entire stem, the rule itself does not give any indication of +whether the affix is a prefix or a suffix (like the other process rules below do). One implication of this is +that you must be sure to set the morpheme type within the lexical entry for a full reduplication affix. + +6.1.1.1.2 Partial reduplication as a process + +Turning now to partial reduplication, recall the Tagalog data from section 3.2.2. The data in (103) are +repeated here from (77) for your convenience. + +(103) a. susulat ‘to write (ipfv)’ + +b. magpasulat ‘to make someone write (pfv)’ + +c. magpapasulat ‘to make someone write (ipfv)’ + + Recall that in this data, the imperfective aspect is realized by reduplicating the first CV syllable of the +stem to which it attaches. (The mag- prefix is what some call actor focus or actor voice.) + + In order to treat this reduplication as a process using the phonological rule-based parser, we need to first +determine the pattern and the result of the process. What is the pattern? Since what reduplicates is the +first consonant and vowel of the stem, we will need to specify the first consonant and also the first vowel. +We then can specify “whatever else is left over." + + What might the result be? We will need to have a copy of the initial consonant, then a copy of the initial +vowel, a morpheme boundary, and then the initial consonant, the initial vowel, and finally, “whatever else +is left over." It might look like this: + +═════ PDF page 80 ═════ + +72 Parsing Conceptual Introduction + + (104) Pattern X ⇒ Result + [C] [V] 3 12+123 + 12 + + Thus, in (104) the pattern has three parts: the initial consonant ([C]), the vowel that follows it ([V]),65 +and “whatever else is left over" (the X). Notice that these each have their respective index number, 1 through +3. The result is to copy the initial consonant (indicated by the first 1), copy the initial vowel (indicated +by the first 2), the morpheme boundary (indicated by the +), and then to keep the original stem itself: the +initial consonant (indicated by the last 1), the initial vowel (indicated by the last 2), and “whatever else is +left over" (indicated by the 3). + + Note that in the pattern the “whatever else is left over" (the X) can be empty or have any positive number +of segments in it. One thing this implies is that this rule will apply to a simple CV stem. In some languages +this may be incorrect. That is, there may be a requirement that the stem be at least CVC or CVV in shape. +How could we modify this rule to allow for this extra requirement? One way is to create a natural class that +consists of all consonants as well as all vowels. Let's call it [A] (for all segments). Then we could rewrite +the above rule as follows: + +(105) Pattern X ⇒ Result + [C] [V] [A] 4 123+1234 + 123 + + Since the pattern now requires a consonant, a vowel, plus one more segment, the rule can only be applied +to stems that are at least CVC or CVV in shape. + + For another example, consider the Orizaba Nahuatl data in (106-107), repeated from (78-79) (see section +3.2.2). + + (106) a. tone (the sun) shines + b. tohtone (the sun) burns + + (107) a. chipintok it is dripping + b. chihchipintok it is dripping and dripping + + Recall that the reduplication pattern here is to copy the first CV of the stem and then append an h. That +is, there is copied material plus some fixed material. In order to treat this as a process, we will need to +write the pattern. What is it? Well, it is actually just like what we had for Tagalog above in (104): we need +to specify the first consonant and the first vowel. These two items are the material that will be copied from +the stem. Since every pattern must include it, we also need to then specify “whatever else is left over." + + What will the result be? Like with the Tagalog case, we need to copy the initial consonant and copy the +initial vowel. This time, however, we need to then insert an h. After this we need the morpheme boundary, +then the first consonant, the first vowel, and finally “whatever else is left over." The process rule might look +like this: + +65 It does not matter whether the C and V natural classes are defined via segments or via phonological +features. What is important is to be sure that these natural classes do indeed correctly identify the set of +phonemes you have in mind. + +═════ PDF page 81 ═════ + +Infixation as a process 73 + +(108) Pattern X ⇒ Result + [C] [V] 3 12h+123 + 12 + + Thus, in (108) the pattern has three parts: the initial consonant ([C]), the vowel that follows it ([V]), +and “whatever else is left over" (the X). Notice that these each have their respective index number, 1 through +3. The result is to copy the initial consonant (indicated by the first 1), copy the initial vowel (indicated +by the first 2), insert the phoneme h, the morpheme boundary (indicated by the +), and then to keep the +original stem itself: the initial consonant (indicated by the last 1), the initial vowel (indicated by the last +2), and “whatever else is left over" (indicated by the 3). + +6.1.1.2 Infixation as a process + +Now let's consider another kind of affix process for which we can write an affix process rule: infixation. + Suppose one has an infix -um- that inserts after the first consonant of the stem like we saw above for + +Tagalog in section 1.1.6 (this is part of (8) repeated here for your convenience): + + (109) a. sulat ‘to write or writing (inf)’ + b. sumulat ‘to write (with actor focus)’ + + Once again, to treat this as a process, we need to identify the pattern and then identify the result. What +is the pattern? The pattern begins with the initial consonant and is followed by whatever else the stem +might have. + + What is the result? The result would be to copy the initial consonant, insert the form of the infix (um), +and then copy whatever the rest of the stem was. It might look like this: + +(110) Pattern Result + 1um2 +[C] X ⇒ + 1 2 + + So in (110), the pattern consists of the consonant natural class (indicated by [C]) and whatever else +is in the stem (indicated by X). The result refers to the index number of the consonant natural class (1), +inserts the phonemes of the infix (first u and then m in this case), and then it refers to the rest of the stem +(indicated by the index number 2). + + Some languages have an infix very much like the -um- of Tagalog, but with another situation to consider. +For example, Yu (2007:32) gives the following data from Toba Batak [bbc]: + + (111) a. b-al-átuk ‘ladder’ + b. al-ógo ‘wind’ + + What is special about the al affix? It infixes after an initial consonant in (111a) whereas it prefixes before +an initial vowel in (111b). How can we deal with this case using the phonological rule-based parser? + + First, we can handle the consonant-initial case similarly to what we did for Tagalog. That is, we can +write an affix process rule just like this: + +(112) Pattern Result + 1al2 +[C] X ⇒ + 1 2 + +═════ PDF page 82 ═════ + +74 Parsing Conceptual Introduction + + This works just like what we had for Tagalog in (110) except that it inserts different segmental material +(al instead of um). + + But how can we deal with the vowel initial case where the al affix prefixes before the vowel? All we +need to do is to add a prefix allomorph to this lexical entry which is conditioned to occur before a vowel. + +6.1.1.2.1 Infixation and root and pattern morphology + +Yet another way that infixation can be treated as a process was discussed in section 3.3.2. Consider the +Silt'e data repeated here for your convenience: + + (113) wakaba (=81) + + a-a-wkb-a + + pfv-buy-3sg.pfv + + ‘he bought’ + + Recall that such Semitic languages often have roots composed of three consonants, as exemplified in the +Silt'e data in (113), where ‘buy’ is the root wkb. The aspect markers are composed of vowel patterns that +fit between or around the root consonants, such as the a-a vowel pattern indicating the perfective aspect. + + Recall that in order to model this using the default FieldWorks Language Explorer parser, we had +to treat each vowel as a distinct infix. So the analysis would look like what is in (114), assuming the +inflectional template given in table 19. + + (114) wakaba + a-a-wkb-a + pfv1-pfv2-buy-3.sg.pfv + + Table 19. Silt'e Template (repeated) + + Aspect1 Aspect2 Stem Subject + + -a- pfv1 -a- pfv2 -a 3sg.pfv + + etc. etc. etc. + + What is bothersome about this analysis? It is that what is conceptually one morpheme had to be treated +as two morphemes. That is, the -a- -a- perfective aspect morpheme had to be split into two distinct entries. + + With the phonological rule-based parser, however, we can write an infix process rule for the single +perfective aspect morpheme and our inflectional template can also include just a single slot for Aspect. + + What would the pattern be for this perfective aspect? We would need the three consonants of the root, +plus “whatever else.” + + What would the result look like? It would be the first consonant, then the first a, the second consonant, +the second a, the third consonant, and “whatever else.” So one way to do it would be like this: + + (115) Pattern X ⇒ Result + [C] [C] [C] 4 1a 2a 3 4 + 123 + + We could also simplify the inflectional template to look like this: + +═════ PDF page 83 ═════ + +“Regular” phonological rules 75 + + Table 20. Silt'e Template with Phonological Rule-based Parser + + Aspect Stem Subject + + -a- -a- pfv -a 3sg.pfv + + etc. etc. + +6.1.1.3 Circumfixation as a process + +Yet another use of a process affix rule is with circumfixes. Recall from section 4.3 the following data: + +(116) kebangunan (=95) + +ke-bangun-an + +nmlz-wake.up-nmlz + +‘awakening’ + + To model this, we can use an affix process rule as in (117). The pattern is just whatever material is +currently present (the X). The result is the prefix phonemes, the 1 and then the suffix phonemes. + +(117) Pattern Result + ke1an +X ⇒ +1 + +6.1.2 Phonological rules +Turning now to phonological processes, this section discusses how to write phonological rules using the +phonological rule-based parser. The basic capabilities of phonological rules are very similar to what are +given in the classic generative phonology textbook Kenstowicz and Kisseberth (1979). Since it is not really +appropriate to review all of that material here, I will be assuming that you are familiar with the basic +concepts. I will primarily illustrate such rules using several examples below. + + Note that “regular” phonological rules used in the phonological rule-based parser follow the classic rule +formula that describes the rule as a change from this to that when this occurs after C and before D. + + The basic formula is given in (118): + (118) this → that / C _ D +where each of this, that, C, and D are either natural classes, segments or phonological feature structures. +The first two are empty or a single natural class, single segment or single feature structure,66 whereas the +latter two may contain more than one of these in a sequence. The latter two may also be word boundary +symbols. Neither C nor D are required. For deletion rules, that is empty. Similarly, for insertion rules, +this is empty. +Another major characteristic of these rules is that they are ordered. That is, the output of rulei becomes +the input to rulei+1. +6.1.2.1 “Regular” phonological rules +This section provides several examples of how one might write non-metathesis or regular phonological +rules. Section 6.1.2.3 discusses an example of a metathesis rule. For how to add phonological features to +FieldWorks Language Explorer, see section 3.1.1.1. + +66 Using two or more items in either the this or the that portion will not do what you hope it will. Each +this and that must be empty or a single item. + +═════ PDF page 84 ═════ + +76 Parsing Conceptual Introduction +6.1.2.1.1 Epenthesis + +One of the issues mentioned in section 1.1.4 was epenthesis. We can handle this as allomorphy (as in +section 3.4) or as a phonological rule. One case where a phonological rule is appropriate is in Selaru [slu]. +Consider the following data:67 + + (119) a. roban + r-oban + 3pl-hit + ‘they hit’ + +b. raknam + r-knam + 3pl-eat + ‘they eat’ + + c. rba + r-ba + 3pl-go + ‘they go’ + + Notice that the underlying form of the ‘3pl’ prefix is r-. Where does the first a come from in (119b), +then? Under at least one analysis, we can say that the a is inserted or epenthesized as a default vowel in +order to break up an unacceptable consonant cluster. If the a were not there, then the word would begin as +rkn which is an invalid sequence of consonants in Selaru. So to make this word legitimate, we can say that +there is a phonological rule that inserts the default vowel a after the first of three consecutive consonants. +It might be written as shown in figure 45. + + Figure 45. Selaru Epenthesis Rule. + +That is, when there are three consonants in a row, an epenthetic a is inserted between the first two conso- +nants. (I use the empty set symbol here to indicate that this is an insertion process; the user interface of +FieldWorks Language Explorer uses a blank to indicate an insertion site.) + + If you have an insertion rule and you need to indicate a morpheme boundary in the immediate context, +then where should the morpheme boundary go? Should it be immediately before the underscore or imme- +diately after the underscore? The way the phonological rule-based parser works, it depends on the direction +of the rule. If the direction is “left-to-right iterative” or “simultaneous,” then put the morpheme boundary +at the beginning of the right-hand context. Alternatively, if the direction is “right-to-left iterative,” then +put the morpheme boundary at the end of the left-hand context. + +6.1.2.1.2 Glide becomes a vowel + +Here is some more data from Selaru. Notice what happens to the underlying glides w and y. + + (120) a. mwohut + mw-ohut + 2sg-discern + ‘you discern’ + +67 The data presented here are from Coward & Coward (2000) and Coward (2005). + +═════ PDF page 85 ═════ + +Tone processing 77 + +b. muknam + mw-knam + 2sg-eat + ‘you eat’ + +(121) a. yoban + y-oban + 3sg-hit + ‘he hit’ + +b. iknam + y-knam + 3sg-eat + ‘he eats’ + + These glides will sometimes surface as a vowel and sometimes as a glide. What are the conditioning +factors for when they are a vowel and when they are a glide? Under at least one analysis, the glide becomes +a vowel when it is followed by a consonant in a following morpheme. We can write a phonological rule to +perform this process. It might look like what is in figure 46. + + Figure 46. Selaru Glide Vowel Rule. + +That is, a glide becomes a vowel when it is followed by a morpheme boundary and another consonant. + In section 6.1.2.1.1 there was a rule for epenthesizing an a after the first of three consonants (see figure + +45). What would happen with (120b) and (121b) if the epenthesis rule applied before the rule in figure +46? We would get *mawaknam and *yaknam, respectively. Therefore, we either must be careful to order +the rule in figure 46 before the epenthesis rule or else we need to exclude the glides from the class of +consonants. + +6.1.2.1.3 Tone processing + +Another potential use of phonological rules is for handling tone changes. Here is one simple illustration. + Consider the following word from Awngi [awn]:68 + + (122) ɨskɨɣ́ wa + ɨsk↗-ɨɣwa + be.there-pst.pfv + ‘I was there’ or ‘he was there’ + + The root ɨsk has a final floating high tone (or, if you prefer to think of it in a different way, it causes a +following tone to be raised). This is indicated here by the ↗ symbol. The suffix then gets the floating high +tone (or it has the tone of its initial vowel raised). + + How do we handle something like this with the phonological rule-based parser? One way is to do what +is shown in figure 47. + +68 The Awngi data comes from Andreas Joswig, personal communication (emails of November 19, 2008 +and May 12, 2009). + + The ɨsk↗ root is actually the allomorph for first person singular or third person masculine or plural. The +-ɨɣwa suffix is the past perfective form for all persons except second person plural and third person plural. + +═════ PDF page 86 ═════ + +78 Parsing Conceptual Introduction + + 1. Create a custom tone phonological feature that has three values: high, low, and floating + high. See section 3.1.1.1 for how to do this. + + 2. Create distinct phonemes for each low-tone vowel and each high-tone vowel. (I assume Awngi + marks high tone vowel with an acute accent and leaves low tone vowels unmarked.) + + 3. Give each low tone vowel a tone feature with a value of low. + 4. Give each high tone vowel a tone feature with a value of high. + 5. Also add a phoneme for the floating high tone (the ↗ symbol). + 6. For the ↗ phoneme, give it a tone feature with a value of floating high. + 7. Write two phonological rules and order them as follows: + + a. Dock the floating high tone. + b. Delete the floating high tone. + + . + + Figure 47. How to Handle Awngi Tone. + + Here is how the docking rule might look: + + Figure 48. Awngi Docking Rule. + +This rule uses [+:syllabic] to indicate a vowel (or tone bearing unit). It means a low tone vowel becomes +a high tone vowel when the low tone vowel is preceded by a floating high tone and where there may be +zero or more consonants in between. The use of the subscript 0 and the superscript ∞ after the consonant +class [C] follows a standard notational device used in these kinds of phonological rules (see Kenstowicz +and Kisseberth (1979:64) and Halle & Clements (1983:93)). The subscript indicates the minimum number +of occurrences and the superscript indicates the maximum number.69 + + Here is how the deletion rule might look: + + Figure 49. Awngi Deletion Rule. + +That is, delete a floating high tone. (I use the empty set symbol here to indicate deletion; the user interface +of FLEx uses a blank to indicate a deleted item.) Normally, deletion rules only apply once. If you find that +you need a deletion rule to apply more than once, you will need to set the DelReapps Parser Parameter to +more than zero. You will want to make this parameter be small or else the parser will run very slowly. + + Note that it is crucial for the docking rule to be ordered before the deletion rule. If we ordered them +the other way around, then the docking rule would never apply because the floating high tone would have +already been deleted. + +69 Note that in order to create the iteration (zero or more) on the consonant class in the FieldWorks +Language Explorer user interface for phonological rules, you may need to select the consonant class and +then perform a right click. Please see the Help files on setting occurrence values (min. and max.) for how +to add these to a rule. + +═════ PDF page 87 ═════ + +Unspecified nasal in Bahasa Indonesia 79 + +6.1.2.1.4 Nasal assimilation + +Now let's look at some Bahasa Indonesia data for one of its “voice” or “focus” morphemes.70 + Consider the following forms, where the first line is the orthographic form and the second line is the + +underlying form. + +(123) a. membantu + məⁿ-bantu + av-help + +b. mendengar + məⁿ-deŋar + av-hear + + c. menggambar + məⁿ-gambar + av-draw.a.picture + + What is the shape of the Actor Voice prefix? It appears that the final nasal assimilates to the point of +articulation of the following voiced stop. We could write a rule for this as shown in figure 50. + + Figure 50. Indonesian Nasal Assimilation Rule. + +That is, a nasal unspecified for place of articulation will assimilate to the place of articulation of a following +obstruent (when the nasal follows a vowel).71 + + The rule in figure 50 exemplifies a process that happens in a number of languages. + +6.1.2.1.4.1 Unspecified nasal in Bahasa Indonesia + +The situation in Bahasa Indonesia, however, is actually much more complicated. For example, consider +what happens with alveo-palatal affricates: + +70 The data presented here are from Halle & Clements (1983:125) and Sneddon (1996). Since the latter +does not always include stem glosses, some stem glosses were obtained via http://www.babylon.com/ +define/108/Indonesian-English-Dictionary.html from May 21-28, 2009. The gloss for pijit was gleaned from +http://www.kamus.net/result.php?w=id-indonesia&q=pijit&submit=Search&e=0 on May 26, 2009. + + Any errors in assigning glosses are mine. + We also use ⁿ for the unspecified nasal instead of the more normal N. See section 6.2.5 for an explanation. +71 Please see the Help files to learn how to create these “feature variables” (also known as “alpha variables” +since the first such variable is commonly represented by the Greek letter alpha α). + +═════ PDF page 88 ═════ + +80 Parsing Conceptual Introduction + + (124) a. menjahit + məⁿ-dʒahit + av-sew + + b. mencatat + məⁿ-tʃatat + av-write + + Notice that while one might expect a palatal nasal ɲ here, at least orthographically an alveolar n is used. +There are some other possibly orthographic conventions that appear to apply as well. Notice how things +are written for a voiceless palatal fricative and a voiceless glottal fricative: + + (125) a. mensyaratkan + məⁿ-ʃarat-kan + av-condition-caus + + b. menghitung + məⁿ-hituŋ + av-count + + The palatal fricative uses the alveolar nasal n where the glottal fricative uses the velar nasal ŋ. That +the glottal fricative takes the back-most nasal should not be surprising. There is no such thing as a glottal +nasal, of course. It is surprising, however, that the palatal fricative does not use a palatal nasal when the +orthography does have one (ny). + + We are not done with the interesting things that happen here. Consider what happens when a stem +begins with a vowel: + + (126) a. mengambil + məⁿ-ambil + av-take + +b. mengerang + məⁿ-eraŋ + av-groan + +c. mengisi + məⁿ-isi + av-fill.up + +d. mengolah + məⁿ-olah + av-process + + e. mengundang + məⁿ-undaŋ + av-invite + +Notice how vowel-initial stems all take the velar nasal ng. There is more. Consider the data below: + +═════ PDF page 89 ═════ + +Unspecified nasal in Bahasa Indonesia 81 + +(127) a. melempar + məⁿ-lempar + av-throw + +b. menikah + məⁿ-nikah + av-marry + +c. mengaco + məⁿ-ŋatʃo + av-chat + +d. menyanyi + məⁿ-ɲaɲi + av-sing + +e. merasa + məⁿ-rasa + av-feel + +f. mewakili + məⁿ-wakil-i + av-proxy-appl + + g. meyakini + məⁿ-yakin-i + av-certain-caus.i + + Notice how the məⁿ- prefix surfaces as just me. What is common among the initial segments of all the +stems in (127)? These are all sonorant consonants, so it appears that this affix loses its final nasal when it +is followed by a sonorant consonant. + + There is even more. Look at the following data: + + (128) a. memukul + məⁿ-pukul + av-hit + +b. menulis + məⁿ-tulis + av-write + + c. mengirim + məⁿ-kirim + av-send + + What is unusual here? Notice that while the unspecified nasal assimilates to the point of articulation +of the following voiceless stop in (128a)-(128c), that stop does not surface. So we might suppose that the +nasal assimilates and then the voiceless stop deletes. What happens with other voiceless obstruents? Look +at the following to see: + +═════ PDF page 90 ═════ + +82 Parsing Conceptual Introduction + + (129) a. memfitnah + məⁿ-fitnah + av-slander + +b. mengkhawatirkan + məⁿ-xawatir-kan + av-worry-caus + +c. menyewa + məⁿ-sewa + av-lease + +d. mensyaratkan (=125a) + +məⁿ-ʃarat-kan + +av-condition-caus + +e. menziarahi + məⁿ-ziarah-i + av-make.a.pilgrimage-loc + + f. menghitung (=125b) + məⁿ-hituŋ + av-count + + Notice that there is only one fricative that deletes: s. None of the other fricatives delete, even the voiceless +ones like f , sy, and h. Further, s not only deletes, the nasal that is written in the practical orthography is +palatal ny instead of the expected n. + + To summarize, this is what we have seen: + + (130) 1. The unspecified nasal becomes ng before a vowel. + 2. The unspecified nasal deletes before a sonorant consonant. + 3. The unspecified nasal assimilates to the point of articulation of a following obstruent with the + following exceptions: + a. h takes ng. + b. s takes ny. + c. sy takes n. + 4. The following phonemes assimilate and then delete: p, t, k, and s. + + How might we deal with such an intricate combination of arguably unexpected factors and successfully +implement them in FieldWorks Language Explorer using the phonological rule-based parser? One +possible way is shown in figure 51. + + How might we do the first step? Given that there appear to be some orthographic conventions employed +in the practical orthography, one way to deal with some of the unusual assimilations is to create a special +“orthographic place” feature and assign the phonemes to use the place involved in the practical orthography. + +═════ PDF page 91 ═════ + +Unspecified nasal in Bahasa Indonesia 83 + + 1. Create the phonological features to use. See section 3.1.1.1. + 2. Make sure every phoneme is defined. + 3. Assign these features to each phoneme. + 4. Write a number of ordered phonological rules to deal with each case. + . + + Figure 51. How to Handle the Unspecified Nasal in Indonesian. + + In addition, there is the problem with the segments that delete as illustrated above in (128) and (129c). +To my knowledge, there is no true natural set of features that can encompass p, t, k, and s without also +including some of the other voiceless fricatives. So one way to deal with this is to create a custom feature +where p, t, k, and s have the + value and all other segments have the - value. + + Table 21. Phonological Features for Bahasa Indonesian Implementation + + approx back cons cont dr high lat low nas NO→N OrthPlace son syl voice + +- + + + - - - -- - - - -- - + +a + - - ++ - -+ - - - ++ + + +b - - + - - - - - - - labial - - + + +c - - + - ++ - - - - alveolar - - - + +d - - + - - - - - - - alveolar - - + + +e + - - +- - -- - - - ++ + + +f - - + +- - - - - - labial -- - + +g - + + - -+ - - - - velar - - + + +h - + + +- - - - - - velar -- - + +i + - - +-+ - - - - - ++ + + +j - - + +++ - - - - alveolar - - + + +k - - + - - - - - - + velar -- - + +kh - + + +-+ - - - - velar -- - + +l + - + +- - +- - - alveolar + - + + +m - - + - - - - -+ - labial +- + + +n - - + - - - - -+ - alveolar + - + + +ⁿ - - + - - - - -+ - ? +- + + +ng - - + - - - - -+ - velar +- + + +ny - - + - - - - -+ - palatal +- + + +o + + - +- - - - - - - ++ + + +p - - + - - - - - - + labial -- - + +r + - + +- - - - - - alveolar + - + + +s - - + +- - - - - + palatal -- - + +sy - - + +- - - - - - alveolar - - - + +t - - + - - - - - - + alveolar - - - + +u + + - +-+ - - - - - ++ + + +w + + + +-+ - - - - velar +- + + +y + - + +-+ - - - - palatal +- + + +z - - + +- - - - - - alveolar - - + + +═════ PDF page 92 ═════ + +84 Parsing Conceptual Introduction + + The phoneme-feature matrix shown in table 21 above illustrates one way that the first three steps of figure +51 could be done (see the OrthPlace feature for the assimilation place and the custom NO→N feature for +the four segments which delete).72 + + Having devised a phonological feature system and assigned the appropriate feature values to the phonemes, +the next step from figure 51 is to write the phonological rules. What should they be? One way is to write +rules for each of the four items given in (130) above. + + The rule in figure 52 addresses the case for the vowels: + + Figure 52. Indonesian Unspecified Nasal Vowel Rule. + +That is, an unspecified nasal becomes a velar nasal before a vowel-initial morpheme. + The next rule handles the case for the sonorant consonants: + + Figure 53. Indonesian Unspecified Nasal Sonorant Consonant Rule. + +That is, an unspecified nasal deletes before a morpheme-initial sonorant consonant. + The third rule does the nasal assimilation. Note how it is similar to the rule in figure 50 above. + + Figure 54. Indonesian Unspecified Nasal Assimilation Rule. + +That is, an unspecified nasal assimilates to the place of articulation of a following obstruent when that +obstruent begins another morpheme within the word. Note that by using the special OrthPlace feature +and by setting its values to the implied place of articulation used in the orthography for segments like c, j, s, +sy, and h that orthographically assimilate to unexpected values, we avoid having to write special exception +rules. The rule in figure 54 will take care of these cases, too. + + The fourth and final rule is the one that takes care of the special case of the four segments that delete +after assimilation. + + Figure 55. Indonesian Unspecified Voiceless Obstruent Deletion Rule. + +That is, delete a voiceless obstruent with the special [+:NO→N] feature when it occurs after the unspecified +nasal across a morpheme boundary. Since my understanding is that the only time Bahasa Indonesia has an + +72 The first “phoneme” is a hyphen which is used in full reduplicated forms. See section 6.1.1.1.1. Note +that its features uniquely identify it so that it cannot be confused with any other phoneme. Another way +to deal with this would be to create a custom feature for which the hyphen is the only phoneme that has +a + value; all other phonemes would have a - value for this custom feature. (See section 6.2.2.) The +OrthPlace feature is a custom feature whose possible values indicate the place of articulation (labial, +alveolar, palatal, velar, and glottal) or that the segment is unspecified for place (?). (The name +reflects the observation that how this works seems to be more orthographic conventions than true place of +articulation.) The NO→N feature's name is an attempt to imply that these segments (the O) delete after an +unspecified nasal (the N). + +═════ PDF page 93 ═════ + +Unspecified nasal and full reduplication in Bahasa Indonesia 85 + +unspecified nasal, that unspecified nasal is preceded by the vowel e, I've added this to the environment. It +may make the phonological rule-based parser run a bit more efficiently. + + What would happen if we reversed the order of the last two rules of figures 54 and 55? The p, t, k, and +s would all delete before there was an opportunity for the unspecified nasal to assimilate. Therefore, it is +crucial that we order these rules correctly. + +6.1.2.1.4.2 Unspecified nasal and full reduplication in Bahasa Indonesia + +There is actually yet another complication in Bahasa Indonesia with the four segments that can assimilate +and then delete. Consider the following forms where the verb stem is fully reduplicated as the continuative +morpheme.73 + + (131) a. mengurut-urut + məⁿ-urut-R + av-stroke-cont + +b. menanti-nanti + məⁿ-nanti-R + av-wait-cont + +c. membagi-bagi + məⁿ-bagi-R + av-divide-cont + +d. memijit-mijit + məⁿ-pijit-R + av-massage-cont + +e. menulis-nulis + məⁿ-tulis-R + av-write-cont + + f. mengayuh-ngayuh + məⁿ-kayuh-R + av-pedal-cont + + What material is actually reduplicated? In (131a-c) it is clear that the original verb stem is reduplicated. +In (131d-f), however, what is reduplicated is the result of the assimilation rule as well as the deletion rule. +For example, while one might expect to get *memijit-pijit for (131d); instead the assimilated m is what is +actually reduplicated. + + Using the phonological rule-based parser, it is possible to write a phonological rule that will handle this +complex situation. Let's assume that we have created a reduplication entry for the continuative suffix which +uses an affix process rule identical to what we had in (101). Also recall that when using the phonological +rule-based parser we need to think generatively. For the case of (131d-f), this means that we need to assume +that the continuative suffix process rule will apply to the verb stem, resulting in the reduplication of the +underlying form of the verb stem. After this process, the meⁿ- prefix will attach. Finally, we apply our +phonological rules to the result. Thus, for the case of (131d), the input to the phonological rules would be + +73 Thanks to Jonathan Coombs (personal communication, email of May 30, 2012) for pointing out that the +form in (131b) in earlier versions of this document (menunda-nunda) is actually from the root tunda, not +the root *nunda. He also suggested using menanti-nanti whose root is indeed nanti. + +═════ PDF page 94 ═════ + +86 Parsing Conceptual Introduction + + (132) meⁿ+pijit+pijit + + We then need a phonological rule that will match the “deletable” voiceless obstruent at the beginning of +the reduplication suffix and change it to its corresponding nasal, when somewhere to the left of it, there is +an unspecified nasal followed by the same “deletable” voiceless obstruent. In the case of (132), we need to +match the p just after the reduplication hyphen and change it to an m. Here is a rule that will do this: + + Figure 56. Indonesian Unspecified Nasal Full Reduplication Rule. + +What does all this mean? Remember that all phonological rules follow the basic formula of (118). The +this of (118) is the first feature structure. The that is the second feature structure. In addition, there is +only the left environment (the C of (118)). + + One way to read this is that when there is a voiceless obstruent (indicated by [-:voice] and [- +:sonorant]) which can undergo the special deletion (indicated by [+:NO→N]), change this voiceless ob- +struent to a nasal (indicated by [+:nasal]) at the same place of articulation (indicated by [α:OrthPlace] +in both the first and second features).74 Two of the other four features mentioned in the that part of the +rule ([+:sonorant] and [+:voice]) are needed because nasals have these features while the obstru- +ents involved have their opposite values. Also, since the nasals do not undergo the special deletion, we +need to include the feature [-:NO→N]. We need to overtly mention these nasal-oriented values in order +for the parser to correctly identify the nasal segment. If we left these out, then the parser would have a +set of features that did not match any known phoneme and therefore, the rule would not work for us. The +reason that the [-:continuant] feature is also used in the that part of the rule is that while the p, t, +and k segments already have the [-:continuant] feature (since they are stops), the s segment does not +(because it is a fricative and fricatives are [+:continuant]). + + Turning now to the context of this rule, first note that it uses a natural class [A]. This is a custom +natural class I added to this language project that consists of all the phonemes in the language (except for +the orthographic hyphen). Having noted this, what does this context mean? I'll explain it working from +where the this is found back to the left toward the front of the word: there must be an orthographic +hyphen immediately before the this. To the left of this there must be zero or more segments, indicated +by the custom natural class [A] demarcated by the subscript 0 along with the superscript ∞. This follows +a standard notational device used in these kinds of phonological rules (see Kenstowicz and Kisseberth +(1979:64) and Halle & Clements (1983:93)). The subscript indicates the minimum number of occurrences +and the superscript indicates the maximum number.75 In this case, we do not know how many there may +be so we use any possible number (we could be more specific, especially at the maximum, but using infinity +means that we never need to worry about what that maximum might be). So we use this special natural +class with this notational device to indicate the “somewhere to the left of the hyphen.” What we want to +find is the exact same segment that is in the this match. So note that to the immediate left of the [A] +natural class is the same set of features used for the this match. Since we do not want to match just + +74 Please see the Help files to learn how to create these “feature variables” (also known as “alpha variables” +since the first such variable is commonly represented by the Greek letter alpha α). +75 Note that in order to create the iteration (zero or more) on the consonant class in the FieldWorks +Language Explorer user interface for phonological rules, you may need to select the consonant class and +then perform a right click. Please see the Help files on setting occurrence values (min. and max.) for how +to add these to a rule. + +═════ PDF page 95 ═════ + +Unspecified nasal exceptions in Bahasa Indonesia 87 + +any such segment, but only the one that immediately follows the unspecified nasal, we put the unspecified +nasal ⁿ just before it. Finally, since it just so happens that all instances of the unspecified nasal in Bahasa +Indonesia (as far as I know) are preceded by the vowel e, we include that vowel, too. + +6.1.2.1.4.3 Unspecified nasal exceptions in Bahasa Indonesia + +Having addressed how to model most of the complications of the unspecified nasal in Bahasa Indonesia, +it should be noted that there are at least three exceptional cases where the p, t, k, and s segments do not +delete. + + First, when the meⁿ- prefix is followed immediately by the derivational per- prefix, the p of the per- prefix +does not delete as illustrated in (133). + + (133) a. memperlebar + məⁿ-per-lebar + av-caus-wide + + b. memperoleh + məⁿ-per-oleh + av-caus-by + +We can use an exception “feature” to handle this exception. We do the steps shown in figure 57. + + 1. Create the exception “feature” per figure 17. We could call it ‘Does not undergo voiceless ob- + struent deletion” because this is the rule that does not apply (see figure 55). + + 2. Create or find the per- entry. + 3. Mark it as having this exception "feature” in its list of To Exception “Features.” + 4. Find the phonological rule which deletes a voiceless obstruent (see figure 55) and add this + + exception “feature” to its list of excluded properties. + a. Use Grammar / Phonological Rules. + b. Select the rule you need in the middle pane. + c. Click in the Excluded Properties field. + d. Click on the chooser button. + e. In the ensuing dialog box, check the exception “feature.” + . + + Figure 57. How to Use an Exception “Feature” for Unspecified Nasal in Indonesian. + + When the phonological rule-based parser finds the per- morpheme, it will no longer try to apply the delete +voiceless obstruent rule because it will be excluded. (Please note that you will only see that this rule is not +applied during the synthesis process; during analysis, the rule will still be “unapplied.”) + + The second exception is for loan words. Consider the following words: + + (134) a. memproklamasikan + məⁿ-proklamasi-kan + av-proclamation-caus + + b. mentraktir + məⁿ-traktir + av-treat.someone + +═════ PDF page 96 ═════ + +88 Parsing Conceptual Introduction + + c. menswadayakan + məⁿ-swadaya-kan + av-self.supporting-caus + + d. mengklasifikasikan + məⁿ-klasifikasi-kan + av-classification-caus + + The crucial thing to notice here is that in none of these does the initial consonant of the stem delete. To +handle these, add the exception “feature” created above to each stem entry. Sneddon (1996:12) notes that +as a loan word ceases to be considered as foreign and becomes thought of as part of the native vocabulary, +the expected deletion will occur. If you find that happening for some word, merely remove the exception +“feature” from it and the deletion rule will then apply. + + The third exception is for monosyllabic stems, which Sneddon (1996:13) says are all loan words. These, +too, can be handled via our exception “feature” by marking these stems as having this exception "feature.” + +6.1.2.2 Constraining application of “regular” phonological rules + +There are times when the application of a “regular” rule needs to be limited to a set of categories or to +some other properties of the word. + +6.1.2.2.1 Rule applies only with certain categories + +Consider the following data from Payne (1981:108). + + (135) a. iNkomataati + i-N-koma-aa-i + 3pl.m-fut-paddle-rep-fut + ‘he will paddle again’ + + b. iNtʃʰikaati + i-N-tʃʰik-aa-i + 3pl.m-fut-cut-rep-fut + ‘he will cut again’ + + Notice that when two vowels occur across morpheme boundaries, an epenthetic /t/ is inserted. + That is not the entire story, however. Consider what happens in the data in example (136) (also from +Payne (1981:110)): + + (136) a. hitoiriki + hito-iriki + spider-dim + ‘small spiders’ + + b. mapiiriki + mapi-iriki + rock-dim + ‘small rocks’ + + In these words, even though two vowels occur across a morpheme boundary, no epenthetic /t/ is inserted. + What is the difference between the data in (135) and (136)? Notice that all the forms in (135) are verbs +while the ones in (136) are nouns. So if we had a way to say that the phonological rule which inserts the +epenthetic /t/ only applies on verbs, then we can deal with this kind of data. + +═════ PDF page 97 ═════ + +Phonological metathesis rules 89 + + With the phonological rule-based parser, it is possible to constrain a phonological rule so that it will apply +only when certain categories are present in the word. Like many other situations involving categories in +the FieldWorks Language Explorer parsers, the category hierarchy is respected. That is, if you specify +that a particular phonological rule should only apply when a particular category is present, then that rule +will only apply if that category or any of its subcategories (or their subcategories) are present. + +6.1.2.2.2 Rule applies only with certain properties + +The phonological rule-based parser also allows one to constrain the application of a given phonological +rule by limiting its application to when specified exception “features” or inflection classes are present or +are not present. If one says that this set of exception “features” and/or inflection classes are required, then +every one of them must be present before the phonological rule will be applied. If one says that this set of +exception “features” and/or inflection classes are excluded, then the phonological rule will be applied only +if none of these are present. + + See section 6.1.2.1.4.3 for an example of using an exception “feature” which must not be present in order +for a phonological rule to apply. + +6.1.2.3 Phonological metathesis rules + +Now let's look at an example of phonological metathesis. Consider the following data from Selaru,76 paying +special attention to the position of the glides in the underlying form (second line) versus the surface form +(first line). + + (137) a. sit + cat + + b. sitke + sit-ke + cat-art + +(138) a. sihy + chicken + + b. sihkye + sihy-ke + chicken-art + +(139) a. asw + dog + + b. askwe + asw-ke + dog-art + + (140) mtwaklulw + mw-taklulw + 2sg-sit + + What happens to the glides w and y? They exchange places with a following consonant whenever the +consonant comes before a vowel. (More data would show that this only happens with obstruent consonants.) +This kind of exchange of segments is known as metathesis. + +76 The Selaru data is from David Coward, personal communication (emails of November 15, 2008, Novem- +ber 17, 2008, and January 12, 2009). + +═════ PDF page 98 ═════ + +90 Parsing Conceptual Introduction + + How do we model this metathesis using the phonological rule-based parser? Since this kind of a change is +not possible to write using the standard rule notation of (118), we need to create a special metathesis rule. +These rules can be described very much like what we saw above for affix processes (e.g., see Kenstowicz +and Kisseberth (1979:371)). The user interface of FieldWorks Language Explorer, however, follows a +different notation that we think is clearer. It might look like what is in figure 58 for the Selaru metathesis +case: + + Figure 58. Selaru Metathesis Rule. + + The row labeled Input indicates the pattern of the input. Glides are indicated by the [? : consonantal] +feature.77 Obstruents are indicated by the combination of [+ : consonantal] and [- : sonorant] features. +The following vowel is indicated by the V in the right environment. Notice how the two middle column +contents are switched in the final row (the result). + + Table 22 below shows a phoneme-feature matrix one might use for Selaru. + + Table 22. Phonological Features for Selaru Implementation + + back cg cons cont cor front high lat nas rnd son voice + + a+- - - - - - -- -+ + + + b- -+ - - - - -- - - + + + d- -+ -+ - - -- - - + + + e - - - - - + - -- -+ + + + f - -+ + - - - -- - - - + + h+-+ + - - - -- - - - + + i - - - - - + +- - -+ + + + j - - ? - ++ +- - -+ + + + k+-+ - - - +- - - - - + + l - -+ - - - - +- -+ + + + m- -+ - - - - -+ - + + + + n- -+ -+ - - -+ - + + + + o+- - - - - - - - ++ + + + r - -+ - - - - -- -+ + + + s - - + ++ - - -- - - - + + t - -+ - + - - -- - - - + + u+- - - - - + - - ++ + + + w+ - ? - - - + - - ++ + + + ʔ +++ - - - - -- - - - + +77 In the analysis used, glides are unspecified for the consonantal feature, indicated by the question mark. +If the glide occurs between consonants and ends a morpheme, then the glide becomes a vowel. + +═════ PDF page 99 ═════ + +Features used in a rule should be explicit 91 + +6.2 Tips for making the phonological rule-based parser work effectively. + +When using the phonological rule-based parser, there are several key things one should keep in mind. These +are given here. + +6.2.1 Every phoneme used in the orthography must be defined as a phoneme + +The first thing to keep in mind when you use the phonological rule-based parser is that every phoneme +used in the orthography must be listed among the phonemes in your language project.78 If a word or an +affix or a phonological rule has even just one phoneme that is not listed in the list of phonemes, then the +phonological rule-based parser is guaranteed to not give an analysis for any word containing that phoneme. +This is because this parser crucially depends on phonemes being defined. This should not be surprising, +really, for a parser that is able to handle phonological rules: it simply must know what the phonemes are +in order to deal with such rules. + +6.2.2 The phonological features need to uniquely identify each phoneme + +Whenever you use phonological features with the phonological rule-based parser, it is very important that +each phoneme have distinct values for its phonological features. If two or more phonemes share the exact +same set of phonological feature-value pairs, then the parser may not be able to correctly determine the +morphemes involved. You may want to double check that every phoneme has a unique set of feature-value +pairs. One way to do this is via the “Bulk Edit Phoneme Features” tool; another is via the “Grammar +Sketch.” Both of these tools are in the Grammar area. + +6.2.3 Fully specify each phoneme + +While some phonological theories promote feature underspecification and while some theories also consider +it to be good form to minimally specify features in a phonological rule, the way the phonological rule-based +parser works, you really need to be as specific as practical. So if a rule is not working, try to be more specific +about the features. + +6.2.4 Features used in a rule should be explicit + +Whenever you use a phonological feature structure in a phonological rule, be sure to use as many features +as necessary to correctly identify the phonemes involved. If you are not explicit enough, the parser will +not be able to identify the correct phonemes and your rule may well not work as you expect. + + For example, in working on the Bahasa Indonesia data, I wrote a complicated rule (see figure 56) where I +wanted to change certain voiceless obstruents to their co-articulated nasal. Since these particular obstruents +all shared a common feature value, I thought I could just use that feature and the place feature. That did +not work. I had to be sure to include all features that differ between the phonemes involved. In particular, +I needed to include sonorant, voice, and nasal feature values. + + The lesson here is that while we humans may easily make correct assumptions about the sound system +of a language, the parser cannot. + +78 The default FieldWorks Language Explorer parser does not require this. For the default parser, you +only need to define phonemes that are used in natural classes or in environments. + +═════ PDF page 100 ═════ + +92 Parsing Conceptual Introduction + +6.2.5 Avoid using archiphonemes that are uppercase equivalents of a character in your orthography + +In testing the Bahasa Indonesia data, I had an affix that ended with an unspecified nasal (this affix has +nasal assimilation to the point of articulation of a following obstruent as explained in section 6.1.2.1.4). So +I wrote a phonological rule to do the assimilation. To represent this affix, I used an archiphoneme of capital +N. That is, I made it have the form meN-. I also added a new phoneme whose representation was also this +archiphoneme N. Since the language already had an alveolar nasal (n), it turned out that the phonological +rule-based parser did not successfully apply the rule. + + The reason is that the phonological rule-based parser automatically treated N as the uppercase equivalent +of the alveolar nasal n. The solution I used was to change the representation of the unspecified nasal +phoneme to a superscript n (ⁿ). So while I could continue to use the N for the name of the unspecified +nasal phoneme, I did need to be careful to change the representation/grapheme and all lexical forms to use +the superscript n. For example, the affix I mentioned above became meⁿ-. Once I made these changes, the +phonological rule-based parser was able to correctly parse words containing this morpheme. Note that it +is not a problem to type forms such as meN- into the Citation Form field of an affix's entry for publication +purposes, since the parser does not look at the Citation Form at all. + +6.2.6 Make sure every affix process rule is complete + +Whenever you create an affix process rule, be sure to fill in the morph type field and also put something in +the form field. If the morph type field is empty, the rule may not work. The reason for putting something +in the form field is so you will not just get “???” for the shape of the morpheme. If there are several affix +process rules in a lexical entry, you can make the form field be slightly different for each one. This can +make it easier to know which form/affix process rule was actually tried. + +6.2.7 Natural classes defined by phonemes may not work as expected + +This one is a bit tricky and not easy to explain. + When you have phonological features defined and have fully and uniquely specified each phoneme, you + +may still run into some odd behaviors with natural classes. Consider the set of phonemes and features +shown in table 23 below. + + One would expect that while these are not fully specified, they are unique. The problem came when a +consonant natural class was defined as consisting of the following phonemes + + (141) b, c, ç, ch, d, f, g, ğ, h, k, l, m, n, ñ, p, q, r, s, ş, sh, t, v, x, y, z, zh +and there was an affix process rule whose match portion ended in two consonants. One naturally would +expect that only those phonemes defined in the consonant natural class would be considered. But this is +not how the phonological rule-based parser works during its synthesis process. During synthesis, it uses +only phonological features. When it was attempting to see if a sequence like madi ended in two consonants, +it said “yes, it does.” How did this happen? How could it think that the final i was a consonant? + + The answer is primarily related to the multi-valued nature of the art feature. Since the various phonemes +that are part of the consonant natural class (see (141)) have different values for the art feature, the +phonological rule-based parser has to ignore this feature when considering phonological features. Looking +at the feature chart in table 23 and ignoring the art column, notice that the /i/ phoneme and the /l/ +phoneme have compatible phonological features. Because of this, the phonological rule-based parser said +that the final i of the base madi matched the consonant natural class. + + The solution was to add a binary feature of ±consonantal. This was + for all consonants and - for all +vowels. + + The moral of this story, then, is to make sure that the phonological feature system is such that each +natural class, when defined via features, contains just the phonemes you want. Using binary features can +certainly make this easier. + +═════ PDF page 101 ═════ + +Known limitations 93 + + Table 23. Phonological Features for Salar Implementation + + *** art cont cor front high rnd son str voice + + -+ - - - - - -- - + + a vowel - - -+ + + b lab - -+ - -- + + + c pal -+ - + ++ + + + ç pal -+ - + -+ - + + ch ret -+ - - -+ - + + d alv - ++ + -- + + + e vowel + - -+ + + f lab + - + - -+ - + + g vel - - - + -- + + + ğ vel + - - + -+ + + + h uvu + - - - -- - + + i vowel + + -+ + + ı vowel - + -+ + + k vel - - - + -- - + + l alv + + + + +- + + + m lab + - + - -- + + + n alv + + + + -- + + + ñ vel + - - + -- + + + o vowel - - ++ + + ö vowel + - ++ + + p lab - -+ - -- - + + q uvu - - - - -- + + r alv - ++ + +- + + + s alv + + + + -+ - + + ş pal ++ - + -+ - + + sh ret ++ - - -+ - + + t alv - ++ + -- - + + u vowel - + ++ + + ü vowel + + ++ + + v lab + - + - -+ + + + x vel + - - -+ - + + y pal ++ - + +- + + + z alv + + + + -+ + + + zh ret ++ - - -+ + + + We have added warning messages about this situation in the Try-A-Word tool. The message tells you +what the natural class is, what the implied phonological features are, what the predicted phonemes are and +what the actual phonemes in the class are. We hope this helps you figure out how to avoid this potential +problem. + +6.3 Known limitations + +There are several known limitations of the current implementation of the phonological rule-based parser. + +═════ PDF page 102 ═════ + +6.3.1 Affixes are tried only once per word + +While the default FieldWorks Language Explorer parser will try a given affix as many times as its form +is found within a single word, the phonological rule-based parser tries a given form (or affix process rule) +only once per word. This is normally not an issue since it is quite rare for an affix to be repeated several +times within a word. There are cases, however, where this is an issue. For example, Coward & Coward +(2000) note that in Selaru, “It is possible to reduplicate /nini/, /soso/ and others basically without limit. +As many as eight reduplication levels have been encountered in natural text.” + + If you run into this limitation, a possible work-around is to add extra allomorphs for the affix involved or +to add the form as a distinct lexical entry. You also, of course, have the option of just allowing this parser +to fail to parse such words and then parsing them by hand. + +6.3.2 Natural classes defined by segments may or may not work as expected + +When you define a natural class by listing the segments (as opposed to using phonological features), the +phonological rule-based parser may not treat this natural class exactly as you expect (see section 6.2.7 for +an example). If you do not have any phonological features defined, then this parser will treat the class as +consisting solely of the segments listed in the class. + + If, on the other hand, you have defined phonological features, then the phonological rule-based parser +converts all the segments listed in the natural class into their respective feature sets. It then takes the +set intersection of all those features and uses that to determine if a given segment is in that natural class. +Normally, this is not an issue. In one case, however, when I was trying to deal with the recalcitrant case of +the meN- prefix in Bahasa Indonesia (see section 6.1.2.1.4) where a following p, t, k, or s, deletes, I knew +that I was not aware of a real natural class that would cover these segments and not also include the other +voiceless obstruents that do not delete. So I tried to by-pass this by creating a segment-based natural class +that just included these four segments. Since I had also defined phonological features, this approach did +not work for me. I had to create a special phonological feature whose value was + for these four segments +and - for all other segments. + + We have added warning messages about this situation in the Try-A-Word tool. The message tells you +what the natural class is, what the implied phonological features are, what the predicted phonemes are and +what the actual phonemes in the class are. We hope this helps you figure out how to avoid this potential +problem. + +6.3.3 Ambiguous digraphs and multigraphs may not work as expected + +When an orthography contains ambiguous sequences for phonemes, the phonological rule-based parser +may not parse the sequence the way you expect. + + For example, suppose your orthography includes the orthographic sequences shown in example (142). + +(142) Grapheme Phoneme + + i high front vowel + u high back rounded vowel + k voiceless velar stop + ki palatalized voiceless velar stop + ku labialized voiceless velar stop + + When there are a sequences like kut and kuat, what will happen? The first should find k, u, and t. The +second should find ku, a and t. The problem is that currently the phonological rule-based parser determines +what the phonemes are by finding the longest match first. This means whenever it finds ku, it will always +treat it as the ku phoneme. Thus, it thinks that kut consists of two phonemes: ku and t, which is incorrect +in this orthography. + + 94 + +═════ PDF page 103 ═════ + +References 95 + + Unfortunately, the only currently available solution is to change the orthography to remove the ambiguity +(such as using kʷ instead of ku and using kʲ instead of ki). + + References + +Aronoff, Mark. 1994. Morphology by Itself. Linguistic Inquiry Monograph Twenty-Two. Cambridge, + Massachusetts: The MIT Press. + +Austin Krumholz, Jeanne, Marjorie Kalstrom Dolson, and Miguel Hernández Ayuso. 1995. Diccionario + poploca de San Juan Atzingo Puebla. Tucson, AZ: Instituto Lingüístico de Verano, A.C. + +Bickford, J. Albert. 1998. Tools for Analyzing the World's Languages. Dallas: The Summer Institute of + Linguistics. + +Black, H. Andrew. 1992. South American Verb Second Phenomena: Evidence from Shipibo. Syntax at + Santa Cruz 1:35-63. + +Bonami, Olivier and Gilles Boyé. 2002. Suppletion and dependency in inflectional morphology. Frank + van Eynde, Lars Hellan and Dorothee Beermann, , eds. Proceedings of the 8th International HPSG + Conference, Norwegian University of Scie 10,nce and Technology (3-5 August 2001). 51-70. CSLI + Publications. https://www.researchgate.net/publication/2949350_Suppletion_and_Dependency_in_ + Inflectional_Morphology, accessed May 10, 2024. + +Coward, David Forrest. 2005. An Introduction to the Grammar of Selaru. SIL International. Manuscript. +Coward, David and Naomi Coward. 2000. A phonological sketch of the Selaru language. In Charles E. + + Grimes, ed. Spices from the east: papers in languages of eastern Indonesia., Pacific Linguistics, 503:9-54. + Canberra: Research School of Pacific and Asian Studies, Australian National University. +Gardner, Simon. 1994. A Problem in Boundary Morphophonemics for Computer Analysis. Notes on + Computing 13.6:44-48. +Halle, Morris and G.N.Clements. 1983. Problem Book in Phonology: A Workbook for Introductory Courses in + Linguistics and Modern Phonology. Cambridge, Massachusetts: Bradford. +Hippisley, Andrew, Marina Chumakina, Greville Corbett, and Dunstan Brown. 2004. Suppletion: + Frequency, categories and distribution of stems. Studies in Language 28(2). + https://www.researchgate.net/publication/228916973_Suppletion_Frequency_categories_and_ + distribution_of_stems, accessed May 10, 2024. +Hockett, Charles. 1954. Two models of grammatical description. Word 10:210-231. +Inkelas, Sharon. 2001. Derivational Morphology Handout. + http://ist-socrates.berkeley.edu/~aclyu/ling115/handout07.pdf, accessed ca. 2004. +Itô, Junko. 1989. A prosodic theory of epenthesis. Natural Language and Linguistic Theory 7:217-259. +Kenstowicz, Michael and Charles Kisseberth. 1979. Generative Phonology: Description and Theory. Orlando, + Florida: Academic Press, Inc. +López L., Filemón y Ronaldo Newberg Y. 1990. La Conjugación del Verbo Zapoteco; Zapoteco de Yalálag. + México, D.F: Instituto Lingüístico de Verano, A.C. +Maxwell, Mike. 1998. Two Theories of Morphology, One Implementation. SIL Electronic Working Papers + 1998-001. https://www.sil.org/system/files/reapdata/12/92/93/ + 129293197146083661515535735515235171004/SILEWP1998_001.pdf, accessed December 26, + 2023. +Payne, David L. 1981. The Phonology and Morphology of Axininca Campa. Summer Institute of Linguistics + Publications in Linguistics, 66. Texas: Summer Institute of Linguistics and the University of Texas at + Arlington. +Payne, Thomas E. 1997. Describing morphosyntax: a guide for field linguists. New York: Cambridge + University Press. +Pickett, Velma B., Cheryl Black and Vicente Marcial Cerqueda. 2001. Grammatica Popular del Zapoteco del + Istmo. 2nd Edition Edition. Juchitán, Oaxaca, Mexico and Tucson, Arizona: Centro de Investigación y + +═════ PDF page 104 ═════ + +Desarrollo Binnizá A.C. and Instituto Lingüístico de Verano A.C. + https://www.sil.org/resources/archives/35304, accessed December 19, 2023. +Principia Cybernetica Web. 1997. Occam's Razor. http://pespmc1.vub.ac.be/OCCAMRAZ.html, accessed + December 19, 2023. +Sneddon, James Neil. 1996. Indonesian: A Comprehensive Grammar. New York: Routledge. +Spencer, Andrew. 1991. Morphological Theory. Cambridge: Basil Blackwell. +Swift, Kenneth. 1988. Morfología del Caquinte. Serie Lingüística Peruana, No. 25. Yarinacocha, Péru: + Instituto Lingüístico de Verano. +Tuggy T., David. 1991. Curso del Nájuatl Moderno. Puebla, México: Universidad de las Américas. +Velásquez de la Cadena, Marciano, Edward Gray, Juan L. Iriba, Ida Navarro Hinojosa, Manuel + Blanco-González, and Richard John Wiezell. 1974. New Revised Velásquez Spanish and English + Dictionary. Chicago: Follett Publishing Company. +Weber, David John. 1989. A Grammar of Huallaga (Huánuco) Quechua. Linguistics Volume 112. Berkeley: + University of California Press. +Weber, David J., H. Andrew Black, and Stephen R. McConnel. 1988. AMPLE: A Tool for Exploring + Morphology. Occasional Publications in Academic Computing No. 12. Dallas, Texas: Summer Institute + of Linguistics. +Whitehead, Carl R. 2004. A reference grammar of Menya, an Angan language of Papua New Guinea. Ph.D. + dissertation. University of Manitoba. https://www.sil.org/resources/archives/9799, accessed + December 19, 2023. +Yu, Alan C. L. 2007. A Natural History of Infixation. New York: Oxford University Press. + + Language index + +Atzingo Popoloca 26 +Awngi 77 +Axininca Campa 60, 88 +Bahasa Indonesia 51, 52, 67, 70, 79-88, 91, 92 +Caquinte 3, 4, 5, 12, 56, 57 +English 2, 23, 64, 65 +Huallaga Quechua 2, 3, 29 +Indonesian, see Bahasa Indonesia. +Isthmus Zapotec 15-18 +Ket 39 +Latin 15 +Menya 62 +Orizaba Nahuatl 1, 8-12, 32, 34, 37, 45, 50, 52, 58, 72 +Popoloca, see Atzingo Popoloca. +Quechua + + Huallaga Quechua, see Huallaga Quechua. + Wanca Quechua, see Wanca Quechua. +Selaru 76, 89 +Semitic 4, 55, 74 +Shipibo 40 +Silt'e 4, 55, 74 +Spanish 19, 27, 35 +Tagalog 4, 52, 54, 71, 73 +Toba Batak 73 +Turka 61 + + 96 + +═════ PDF page 105 ═════ + +Turkish 24 +Wanca Quechua 38 +Yalálag Zapotec 13, 25, 26, 36, 44 +Zapotec + + Isthmus, see Isthmus Zapotec. + Yalálag, see Yalálag Zapotec. + + Subject index + +Ad hoc constraints 42, 63 + Allomorph-oriented 63-65 + Grouping 43, 65 + Key Allomorph 63 + Key Morpheme 42 + Morpheme-oriented 42-44 + Other Allomorph(s) 63 + Other Morpheme(s) 42 + +Affixes 7-34 + Circumfix, see Circumfix. + Clitics, see Clitic. + Derivational Affix, see Derivation, Derivational Affix. + Inflectional Affix, see Inflection, Inflectional Affix. + Interfixes, see Compounding, Affixes in Middle of Compounds. + Non-productive Affix 33 + Process 70, 92 + Unclassified Affix 7, 23, 29, 33 + Underspecified 23, 29 + +Allomorph Forms Field 67 +Allomorphy 14, 65 + + Affix Allomorphy (conditioned by features) 60 + Allomorphs Field, see Allomorph Forms Field. + Environments 45, 49, 50, 54, 63, 68, 70 + Free fluctuation 50 + Lexeme Form Field, see Lexeme Form Field. + Null Allomorphs, see Null Allomorph. + Ordering of Allomorphs 65 + Stem Allomorphy (Stem Allomorph Labels), see Stem Allomorph Labels. +Ambiguity 3 +Analyses + Ambiguity, see Ambiguity. + Eliminating Incorrect Parses, see Ad hoc constraints. +Approximant 76, 89 +Bound Root 67 +Bound Stem 67 +Category + Category Hierarchy 13, 14, 19, 28, 39, 59 + Changing Category 2, 23, 25, 27 + Inflectable Features 21, 58 +Circumfix 13, 67, 75 + + 97 + +═════ PDF page 106 ═════ + +98 Parsing Conceptual Introduction + +Class + Conjugation Class, see Conjugation Class. + Declension Class, see Declension Class. + Inflection Class see Inflection, Inflection Class. + Subclass, see Inflection, Inflection Class, Subclass. + Natural Class, see Natural Class. + Noun Class, see Noun Class. + +Clitic 40, 67 + Enclitic 67 + Proclitic 67 + +Co-occurrence Constraints 12 +Coalescence 62-63 +Complex Feature, see Inflection, Inflection Feature, Complex Features. +Compounding 34-39, 69 + + Affixes in Middle of Compounds 38 + Endocentric Compound, see Compounding, Headed Compound. + Exception “features” in 39 + Exocentric Compound, see Compounding, Non-headed Compound. + Headed Compound 34, 36, 37 + Incorporation 36-38 + Non-headed Compound 35 +Conjugation Class 14. See also Inflection, Inflection Class. +Declension Class 14, 15. See also Inflection, Inflection Class. +Default Feature, see Inflection, Inflection Feature, Default Features. +Default Inflection Class, see Inflection, Inflection Class, Default Inflection Class. +Derivation 2, 23-31 + Category Changing 23, 25, 27, 28 + Derivational Affix 7, 23-31, 59 + + Underspecified 7, 29 + Inflection Classes, and 25, 26-27 + Inflection Features, and 27 + Outside of Inflection 29 + Versus Inflection 25, 31 +Digraphs in Hermit Crab 94 +Discontinuous Morpheme 4 +Empty Set Symbol 66, 76 +Enclitic, see Clitic. +Environments + Allomorph Environments, see Allomorphy, Environments. + And Inflection Classes, see Inflection, Inflection Class, And Allomorph Environments. + Infix Environments 55 + Reduplication Environments 53 + Word Boundary, see Word Boundary. +Epenthesis 3, 56, 76 +Exception Features 32 +Features, see Inflection, Inflection Feature. +Free fluctuation, see Allomorphy, Free fluctuation. +Gender 19, 28. See also Inflection, Inflection Feature. +Glide, see Approximant. +Hermit Crab Parser 1, 69 +Infix 67. See also Infixation. + +═════ PDF page 107 ═════ + +Subject index 99 + + Infixing Interfix 67. See also Interfixes. +Infixation 4, 54-55, 73 +Inflection 1 + + Inflection Class 13, 22, 25, 26-27 + And Allomorph Environments 15, 19, 50 + Default Inflection Class 15 + Subclass 15-18 + And Allomorph Environments 19 + Versus Inflection Features 22 + + Inflection Feature 19, 22, 27, 58 + Agreement Features 20, 27 + And Allomorphy 58, 60 + Complex Features 20 + Default Features 9, 11 + Feature Catalog 21 + Feature Type 21, 28 + Using Features to Rule Out Incorrect Parses 19, 21, 58, 60 + Versus Inflection Class 22 + + Inflectional Affix 7 + Underspecified 7, 23 + + Inflectional Slot 9-13, 59 + Optionality 9 + + Inflectional Template 8-13, 30, 37, 55, 59 + Versus Derivation, see Derivation, Versus Inflection. +Inflectional Affix Gloss Builder 11, 20 +Interfixes 38 +Irregularly inflected forms 61 +Item and Arrangement 45 +Item and Process 45, 69 +Lexeme Form Field 67, 68, 70 +Metathesis 5, 56, 89 +Morpheme Types 67 +Morphological Parser 6, 42, 63, 65, 67 +Morphophonemics 1, 6, 44-65 +Morphotactics 1, 6 +Natural Class 48, 92, 94 + Abbreviations 48, 49 +Non-productive Affix, see Affixes, Non-productive Affix. +Noun Class 20. See also Inflection, Inflection Feature. +Null Allomorph 5, 9, 57, 65 +Occam's Razor 9 +Order of Allomorphs in an Entry 50. See also Allomorphy, Ordering of Allomorphs. +Parser see Morphological Parser. + Phonological rule-based Parser 45, 69 +Part of Speech, see Category. +Particle 67 +Phonemes 45, 91 + Classes, see Natural Class. + Representation 45 +Phonological Features 47 +Phonological Rules 1, 69, 75 + +═════ PDF page 108 ═════ + +100 Parsing Conceptual Introduction + + Alpha Variables 79, 86 + Archiphonemes and 92 + Assimilation 79 + Constraining 88-89 + + Constrain by Category 88 + Constrain by Exception “Feature” 87, 89 + Constrain by Property 89 + Deletion Rule 78 + Examples 76, 77, 78, 79, 90 + Exception “Features” in 87, 88 + Feature Structures in 91 + Generative approach 69 + Iteration in Rule 78, 86 + Limitations 93 + Phoneme-Feature Matrix Examples 83, 90 + Phonemes and 91 + Rule Formula 75 + Rule Ordering 77, 78, 85 + Tips in using 91 + Tone 77 +Phrase 67 +Prefix 67 + Prefixing Interfix 67. See also Interfixes. +Proclitic, see Clitic. +Pronominal Hierarchy 44 +Reduplication 4, 51-54, 70 + Full Reduplication 51-52, 70, 85 + Partial Reduplication 52-54, 71 +Root 67 +Root and Pattern Morphology 4, 55, 74 +Root Variant, see Stem Allomorph Labels. +Segments, see Phonemes. +Slot, see Inflection, Inflectional Slot. +Stem 67 +Stem Allomorph, see Stem Allomorph Labels. +Stem Allomorph Labels 58 +Stem Variant, see Stem Allomorph Labels. +Subclass, see Inflection, Inflection Class, Subclass. +Suffix 67 + Suffixing Interfix 67. See also Interfixes. +Syllable 3, 4, 51, 52, 56 +Template, see Inflection, Inflectional Template. +Tone 47-48. See also Phonological Rules, Tone. +Vowel Coalescence, see Coalescence. +Word Boundary 45 +XAmple 53 +Zero Allomorph, see Null Allomorph. \ No newline at end of file diff --git a/Docs/ai-parser-help/workflow/sources/black-parser-workshop-2026-L02-fulltext.txt b/Docs/ai-parser-help/workflow/sources/black-parser-workshop-2026-L02-fulltext.txt new file mode 100644 index 0000000000..6e6bc70f3f --- /dev/null +++ b/Docs/ai-parser-help/workflow/sources/black-parser-workshop-2026-L02-fulltext.txt @@ -0,0 +1,658 @@ +2026 Parser Workshop +Foundations for Building a Parsing Project +Lizzie Poole +Ok, so earlier Kevin gave us an introduction to why we might want to set an automatic parser, and the two options we have in FLEx – Xample and Hermit Crab. + +You don’t need have memorised everything by the end of each lecture, that’s why we have plenty of time for you to try things out yourselves and meet with your consultant to help you think through the specific details for your project. + +And feel free to interrupt with questions and I can try and explain more clearly. + +Outline +High-level strategy +Starting project +Workflow for modeling a language +Preview of constraints +Running the parser +Linguistic terms from a FLEx perspective +Details of some key concepts +In this session we’re going to look at the general strategy for parsing, from the very first step of starting a project, to running the parser and testing that it works. + +We’re going to look at constraints – we’re not going to go into much detail, just an introduction, and more detail will be given in later lectures. + +Then we’ll see how to actually run the parser, and finally we’ll look at some terminology, and particularly clarify the ways certain terms are used in FLEx. + +High-level Strategy +Ok, so let’s start with an overview of how to go about setting up one of the parsers. + +Start with existing FLEx project, or new one? +Toy project +More freedom to experiment, make mistakes +Only the data needed—not distracting +Can intentionally choose what to add next +Can easily see progress +Existing project +Already has all your morphemes/roots in it +Won’t have to transfer setup later +The first question to ask is whether you want to use an existing project or set up a what we’re calling a toy project, a brand new project. + +We would lean towards recommending you start a toy project, and here are some reasons why: +more freedom to experiment and make mistakes without messing up data already in a project +you add only the data you need when you need it, so you’re not distracted by all the other data you have +once you’ve got the parser working for one particular paradigm you can intentionally choose what data to add next to for your next paradigm – so maybe you started with two nouns and two affixes, those are working so you choose to add one more noun and one more affix. It’s all very contained, whereas if you have your whole lexicon you have a lot to work with +you can easily see and keep track of the progress you are making + +The advantages of an existing project are that it already has morphemes and roots in it, and you won’t have to transfer things later from your toy project to the ‘real’ project. Some things we have tools that can help export and import, but other things need to be done manually. Depending on how far you’ve got with your real project, another option would be to export your lexicon and texts from your real project and import into the toy project so your toy project becomes your real project. + +The decision about which to do needs to be made on a project-by-project basis, but often starting a new project is a good recommendation because that simplifies a lot of things. And if you started a new project already to record your paradigms as you were preparing, then you don’t need to start another new project. If you’re unsure, show your consultant what you’ve got and discuss it with them. + +Any questions on that? + +Workflow for the whole endeavor +Make a rough list of what you want to cover +Word classes, paradigms, phonological phenomena +Start a toy FLEx project (or prep existing project) +Pick one paradigm from one word class +Once that is working, move to another +Iterate until you have good coverage +Start with short texts +So how does the whole workflow go? + +It’s a good idea to start off with a rough list of what you want to cover. You’re unlikely to get everything finished in this two weeks, but with the list you can then choose what to prioritise – probably pick something simpler to start with and while you get familiar with it what to do. Once you know how to do the basics, you could then leave other easy stuff that you know you can do by yourself, and move on to more complex stuff while here where there are people who can help. +what word classes? i.e. nouns, verbs, adjectives etc +what are the paradigms for each of these classes? You’ve been working on these in preparation for the workshop, but if you haven’t already got a straightforward list of your paradigms it’s probably a good idea to make one +what phonological phenomena do you need to cover? For example, assimilation, insertion, deletion, coalescence, metathesis etc. If you’ll be using Hermit Crab, then writing these down in the form of linear rules as far as possible will be a helpful head start + +Then, you’ll need to either start the new toy project to work in, or if using an existing project you’ll need to prepare it. In particular you’ll need to set up your phonemes, and Kevin will talk more about that tomorrow morning. + +Then choose one paradigm from one word class to work on. Choose something simpler initially – perhaps a word category where there’s only one type of inflection happening. This might be nouns, and choose one particular noun class or paradigm for singulars and plurals, or perhaps possessive forms, rather than verbs where you may have a lot more inflectional morphology such as person, number, gender, tense/aspect/mood, polarity etc. Again if you’re not sure, show your paradigms to your consultant and they can help you choose an appropriate place to start. + +The process of getting a paradigm to work correctly will be described on the next few slides, but in terms of the big picture of your workflow, once that paradigm is working correctly move to another, and so on, until you’ve covered a good selection of categories, paradigms and phonological phenomena from your original lists. + +At that point, you can start with short texts, perhaps just a sentence or even a clause, and as you add these you will probably come across more categories, paradigms and phonological phenomena that you need to address. But start slowly and build up. Don’t go wild and put a whole bible chapter in because that’s a sure way to get overwhelmed and frustrated! + +How do you “start with a paradigm”? +To model and test a given paradigm, you need: +Text with wordforms for the paradigm +Lexical entries: roots/stems, affixes +Category with template(s) + +Phonological constraints +Other constraints +Let’s look at what we mean by starting with a paradigm. + +In order to model and test a paradigm, you need to make sure your project includes all the relevant data, all the pieces. + +So firstly, you need a Text in Texts and Words which contains the wordforms of the paradigm. +You need to make sure that all relevant morphemes – roots/stems and affixes – are in the Lexicon. +You don’t actually need a template to get started right at the beginning, the parser will work without templates, but it is helpful to start thinking about them early on, so we’ll look at that tomorrow. + +The you need to think about the phonological issues – so if you have a root that starts with a vowel, does that make the prefixes change? Or maybe nasals do something different to non-nasals. So you want to put examples of different phonological environments that you know have an impact on how your affixes surface. + +There are other constraints as well, but we’ll leave those until next week. The ones mentioned so far are certainly enough for you to be starting with. + +Process: Text +Choose starting paradigm +In the Texts area, type or paste in the forms +Use Analysis Writing System for contrast (optional) + +So choose your starting paradigm, and in Texts and Words type or paste in the wordforms required. + +So here is an example from Mbugwe, a Bantu language in Tanzania. So I’ve started with just six words. I have chosen one verb stem, and one TAM form. And then with the six words I’ve just changed the subject, so we have 1sg, 2sg, 3sg, 1pl, 2pl and 3pl subjects. + +In the example here, the black text is the language I’m working on, and it uses a vernacular writing system. And then the labels for each paradigm are in blue and use an analysis writing system. Your consultant can help you font colours etc in your writing systems if you find that helpful. One other benefit of different writing systems if you include these kinds of labels in you Text, is that the parser won’t try to parse anything in an analysis writing system. + +So I have my text set up. + +Process: Lexicon +In Lexicon Edit, add the morphemes +Roots/stems for that paradigm +All affixes needed for that paradigm + +Next I need to make sure all the relevant morphemes are in the Lexicon. It’s best to do this in the Lexicon Edit area. + +Firstly I need the root/stem, shown on the left, and then also all the affixes used in paradigm, as you can see on the right. All the pieces need to be in the Lexicon so the parser can find them. + +Process: Lexicon +In Lexicon Edit, add the morphemes +Roots/stems for that paradigm +All affixes needed for that paradigm +Eventually add needed constraints +(Allomorphs, Environments, Inflection Classes, etc.) +Derivation +Does the form you are testing involve derivational affixes? +As you learn what constraints are needed, you’ll be able to add some of these in Lexicon Edit too, and others will be added elsewhere. +If the form you are testing involves derivational affixes you will need to include those as well, but typically you should start with inflectional paradigms. + +We’ll be talking much more about constraints and derivation vs inflection over the next few days. + +Process: Template +Grammar > Category Edit +Go to category, create a template +Add affixes to it +You will also need to create a template for the word category in the Grammar area. + +So you got to Grammar > Category Edit. You choose the relevant Category, add a template and then add affixes to the template. So this is what it might look like. + +You have a stem here and then you right-click to add a column before or after the stem, depending on whether you’re adding a prefix or a suffix. You’d give the column a name, so subject or TAM, for example. Then you can add affixes to the slot by right-clicking and choosing an affix from the list that comes up. Those are the steps, but Sara will talk more about templates tomorrow. + +Process: Testing +In Texts area, try parsing your paradigm +Parser > Parse Words in Text +In the Analyse tab of the Texts and Words area you then try parsing your paradigm. + +There is also another option, instead of Parse Words in Text you can also use the Run Tests facility. Kevin will talk about that more in another session. + +Process: Testing +You’ll hopefully get something that looks like this, meaning that the parser has found a possible analysis for your words. The tan background is important, it tells you that these analyses have been done by the parser. So tan is generally a good sign, if you wanted those words to parse! + + +Iterate +Debug until current paradigm parses completely +Move to next paradigm +Add forms to a Text +Build template +Add roots/affixes +Add or adjust constraints +Parse, debug, parse, until it works +Move to next phenomenon +If it doesn’t work, then you’ll need to debug, figure out what’s wrong, maybe something is missing in the Lexicon, maybe you haven’t accounted for phonological changes etc. So you’d make the corrections, and try again until it does parse.. + +Once it does work, you can move to the next paradigm. It’s an iterative process of adding wordforms, adding or expanding templates, entering morphemes into the Lexicon. The steps in the box don’t need to be done in any particular order, but all three need to be done before you can test whether the new wordforms parse or not. + +After the items in the box, you may find you need to add or adjust your constraints, and then try parsing, debug and try again until it works. + +After doing a set of related paradigms and confirming they all parse, you can move on to a different phenomenon. + +Working through patterns +Categories: Noun, Verb, Adj, Quant +Paradigm classes: AR verbs, ER verbs, IR verbs +Phonological variation: C-initial vs V-initial stems +Irregular patterns +You’ll need to work your way through all the patterns you’ve discovered as you’ve been preparing your paradigms. + +So you’ll need to work on different Categories, that is nouns, verbs, adjectives etc. +You may have different classes of paradigms. Here the term ‘class’ refers to sets within one category which may inflect differently. In Spanish for example, the person and tense affixes are different depending on whether the verb ends in ar, er or ir. We would call these the ar-class, the er-class and the ir-class. + +And you’d need to work with all the phonological variation possible with each paradigm + +And you may find you have some paradigms that don’t seem to fit with any others, using an irregular pattern, and you’ll need to account for those separately as well. These you would probably leave until last. + +There are many different ways you can go about this. + +Noun +First you want to model one paradigm for Nouns. + +Noun +Verb +Adj +Then you might do one for Verbs, and one for Adjectives. + +Noun +Verb +Adj + N2 +Then you decide to model a second pattern for Nouns. + +Noun +Verb +Adj + N2 + N3 + N4 +Then maybe you do more nouns, until you’ve covered all the possible noun patterns you’ve found so far. + +Noun +Verb +Adj +Adv + N2 + N3 + N4 +Then you decide to move on to Adverbs, + +Noun +Verb +Adj +Adv + N2 + Adv2 + N3 + N4 +and you model two different patterns for them. + +Noun +Verb +Adj +Adv + N2 + V-con + Adj2 + Adv2 + N3 + V-a + cons + N4 + V-i + a+ + V-u + u+ + V-irr1 + V-irr2 +Then you might decide to go really deep into Verbs and finish out all those patterns, and the same for Adjectives, including all the phonological variations. + +Noun +Verb +Adj +Adv +Quant +Prep +Det + N2 + V-con + Adj2 + Adv2 + Qnt2 + N3 + V-a + cons + N4 + V-i + a+ + V-u + u+ + V-irr1 + V-irr2 +Then maybe you decide to add a few more parts of speech. + +It doesn’t really matter what order you go in, but it helps to be systematic, and to keep track. + +Working through patterns +Categories: Noun, Verb, Adj, Quant +Paradigm classes: AR verbs, ER verbs, IR verbs +Phonological variation: cons-final stems vs vowel-final +Irregular patterns + +Pick a sequence that makes sense to you +It may help to have a checklist or table +As I said, you’ll need to work your way through all the categories, paradigms for each category, accounting for phonological variation and irregular forms. + +But it may help to have a checklist of all the phenomena you hope to model, or perhaps a table like we have just been looking at. You can then tick off the things that you’ve done, and keep track of what you still need to work on. + +Preview of parser constraints +We’re going to take a brief look at some of the ways you can constrain the parser, but more detail on each method/topic will be presented in lectures during the rest of the week. + +Process: Constraints +Detailed ways to describe the linguistic processes +Lexical constraints +Morphological constraints +Phonological constraints +Setting up these constraints in FLEx also documents how the language works + + + +Constraining the parser essentially means giving it the information it needs to know when it can and cannot use different morphemes or forms of morphemes. + +We can divide these into lexical constraints, morphological constraints and phonological constraints, and I’ll give examples of each of these in a moment. Setting up these constraints also documents how the language works, and it’s possible to print out a Grammar description containing all these constraints and maybe that’s the start of a paper, or you can share it with colleagues. + +Preview of Parser constraints +In the Lexicon +Morpheme type (affix vs root/stem vs clitic) +Affix type (derivational vs inflectional) +Category +Variants (fused forms) +Morphological constraints +Inflection template +Inflection class +Inflection features + + +Let’s start with Lexical constraints. + +You need to define the morpheme type – there are three broad categories – so is it an affix, a root/stem or clitic? And then within each of these there are subtypes. +If it’s an affix, is it derivational or inflectional? +What Category does the morpheme belong to or attach to? Category = part of speech / grammatical category +Next Monday we’ll also look at Variant forms + +Then Morphological constraints. + +You can use inflection templates to dictate the order of affixes in a word, inflection classes help you deal with situations where the same Category has different sets of words which make use of different forms of inflectional affixes, inflection features allow you to control which affixes can apply to which roots/stems, so perhaps nouns have a gender, and you want to make sure that only the masculine suffix can attach to the masculine nouns. + +Preview of Parser constraints (cont.) +Phonological constraints +Allomorphs and environments +Phonological rules (HC only) +Affix process rules (HC only) +Miscellaneous constraints +Parser parameters +Ad hoc rules +Compounds +Then Phonological constraints. + +Phonological constraints can tell the parser about allomorphs and their environments. This applies to both XAmple and Hermit Crab. + +Then specifically for the Hermit Crab parser, you can also use phonological rules and affix process rules to govern the processes that take place to turn the underlying morphemes into their surface representations. + +And final there are few other miscellaneous constraints such as the parameters set for the parser itself, ad hoc rules (ad hoc means ‘one-off’ in this context, these are rules you set up that are specific to particular morphemes or allomorphs). You can also set compounding rules. We’ll look at these next week. + +So those are broad categories of constraints, and we’ll look at them ni more detail over the next few days. + +Details about running the parser +We’ll now turn to looking at how to actually run the parser, and what different results look like. + +Navigate to your text +You need to be in Texts and Words, Interlinear Texts, choose the Text you’re working on and within that you need to be in the Analyse tab. + +Setting the parser to use +You need to choose which parser you will run, XAmple or Hermit Crab. + +You’d go to the Parser menu > Choose Parser. You can talk with your consultant about which one you will use. + +Interlinear text: in parsing development mode +During this workshop you want to use the Parser Development mode. Once the parser has been set up to correctly parse every word in the language, you can switch to Text Glossing for automatic interlinearisation. + +Running the parser +And then usually you’ll want to ‘Parse Words in Text’, especially at the beginning. If you’ve had a word fail to parse, have made a change that you hope will fix it and want to test just that word again you can choose to ‘Parse Current Word’, which runs the parser just on the word where the cursor is. That might be helpful as well as your Texts get longer, you may not want to wait for the whole text to parse. You also have the options of parsing or reparsing all words – that is every single wordform in every single text. That might end up taking a very long time so probably best to avoid that until you know you’ve got things set up correctly! + +Interlinear text: in parsing development mode +White background, tan outline: +This specific occurrence is user-approved and parses +White background, blue outline: +This specific occurrence is user-approved and may not parse +Boxed: Currently activated word. +In parsing development mode, the different coloured outlines and backgrounds that appear when a word is parsed give you information about the parser results. + +White background means this specific word in this specific text has been approved by the user. When there’s a tan outline, it means the parser is able to parse the word, or it did parse it at some point. If you’ve changed things and haven’t re-run the parser on the word then it will still have the tan outline, so you need to re-run the parser on previously successful words to check that they still parse after any changes you’ve made. + +White background with blue outline – the white background again is that this specific occurrence of this word has been approved by the user at some point but it may not parse – maybe the parser has never been run on it, or that it fails when parsing. So the user approved it, thinking it’s fine, but the parser doesn’t like it. + +The grey background with black outline is the currently activated word – so if you run the parser on ‘current word’ this is the one it will try. + + +Parser mechanics in FLEx: Interlinear text +Just stars, white background: +No analysis available +Blue background: +There was some approved analysis, somewhere, sometime and may not parse +Tan background: +This word has a parse done by the parser +Yellow baseline: More than one analysis (possibly none of them correct) +Yellow top line: More than one analysis (need to choose) +Just stars means there’s no analysis available, maybe it couldn’t parse, or if it’s monomorphemic then FLEx can’t find it in the Lexicon. Or it might mean it’s a new wordform and the parser hasn’t tried it yet, an the user hasn’t said anything about it yet. + +Blue background means this specific occurrence has not been approved, like the blue outline one, but it was approved somewhere else, perhaps elsewhere in the Text or even in a different Text altogether. So you approved it elsewhere and FLEx says, ‘ok you liked this before so you probably want it again and here’s blue to say it’s been approved previously’, but it may not parse because if it was parsing it would have a tan background. Tan overrides blue. + +Then ordinary tan – this word was done by the parser. It hasn’t been approved by the user yet, or it would have a white background, but it does parse. + +And then finally if the wordform has a yellow background it means more than one analysis is possible. You don’t know if those are by the parser or by the user, you just know that there’s more than one. So it means you need to investigate further – click on the down arrow to see what the possible analyses are. + + +Parser mechanics in FLEx: Interlinear text +And when you click on it, you’ll get something like this. So here there are two possible analyses to choose from the top is masc pl, the bottom is fem pl. And the check marks tell you whether it’s user-approved or parser-approved or both. Again blue means user-approved, and tan means the parser can parse it. So no check mark in the blue means the user hasn’t yet approved it; no check mark in the tan means the parser can’t or hasn’t yet tried to parse it. + +Does that make sense? Any questions on how that works? + +Basic color meanings +Tan = parser suggestion +Blue ≈ user + +Again tan means a parser suggestion, and blue almost always means the user. + +Blue can also mean that the wordform exists exactly as it is in the Lexicon, so that would work for monomorphemic words, but while we’re setting up the parser we’re going to be dealing with multimorphemic words, so generally you can think of tan as ‘parser’ and blue as ‘user’. + +Linguistic terms as used in FLEx +Finally we need to look at some linguistic terminology in the context of FLEx. + +FLEx and Linguistics +FLEx was created with linguistic principles in mind. +Sometimes we have to balance linguistic reality with the practical needs of parsing. +Hopefully they are not TOO far apart. +Sometimes the linguistic principles present two ways to go. In that case, we can decide what’s practical. +So FLEx was created by SIL with the cooperation of linguists and programmers, and was designed with linguistic principles in mind. + +But sometimes we have to balance the ideal linguistic way of doing something with what’s practical to get things parsing in FLEx. Hopefully those two things re not too far apart! + +Sometimes there a multiple ways of analysing something linguistically, and if you have that situation and one is better to model in the parser than the other, then you might want to go with the one that works better in the parser. + +Occasionally you might need to compromise a bit to get something working well in the parser. If you run into an issue where you feel like the compromise needed is crossing the line, then that might be a situation where you want to start a conversation on the FLEx parsing mailing list and see if others have encountered similar issues and found other ways to deal with it, and if it seems like it’s something that really needs to change or be added, then you might want to request that from the developers and they’ll take it into consideration for future versions of FLEx. + +Morpheme Types +root, stem, bound root, bound stem +prefix, suffix, infix, circumfix, etc. +clitic, proclitic, enclitic +Every morpheme has ONE of these specific values +Ok, so now onto the terminology. Firstly, morpheme types. + +When you add a word to the lexicon, by default it is labelled a ‘stem’ but you can choose from a long list of other morpheme types. + +There are three broad groupings, so along with stem there’s the option to select root, bound root or bound stem. +For affixes, you can have prefix, suffix, infix, circumfix etc. +And then there are clitics, proclitics and enclitics. + +Every morpheme must have one of these. So as I said it defaults to stem, and it you don’t want stem you need to pick on one of the other options. There’s no option to leave the morpheme type undefined. + +Morpheme Types +Fundamental constraint +Morpheme Type affects: +Information you can enter +Morphological processes +Display in interlinear texts (via token - = * ) +Sorting + + +And the morpheme type acts as a fundamental constraint both for the parser and in FLEx more generally. Morpheme type affects what other information you can enter, it affects morphological processes. + +Morpheme type also affects how the form is displayed, in the dictionary, so affixes have a hyphen, clitics have an =, and bound roots and stems have an asterisk. + +It also has some effect on sorting. + +Morpheme Type and Homographs in FLEx +Homographs are calculated separately for each Morpheme Type (more or less) + + + +And here’s an example of that. + +Here are 8 different ‘en’ forms, but they aren’t numbered 1-8. Two are ‘stem-like’ things, so they are numbered 1 and 2. Bound root and bound stem are then counted separately so restart at 1 and 2, and then similarly for prefixes and suffixes, they’re again each counted separately. + +Morpheme Types occur in “groups” +Specific Morpheme Types are clustered into three groups +Stem-type +Affix-type +Clitic-type +Each GROUP has certain properties and behaviors that apply to the whole group +If you change a morpheme’s type in FLEx, the information you can specify for it may change +These groupings are important in the FLEx data model. Anyone programming for FLExTools needs to understand this. But these labels are not exposed in the FLEx UI, though you can see the effects of them. + +Morpheme Types in the “stem-type” group +Root: most basic form +Stem: plus derivational affix +Treated the same in FLEx +Bound root/stem +Requires affixation +So linguistically the root is the most basic form, the stem is a root plus any derivational affixes. There could be no derivational affixes added though, and in that case the root ad stem look identical. + +Stems and roots are treated the same in FLEx, that is, FLEx doesn’t know the difference linguistically. So it’s up to you how you like to see it labelled, you can make the distinction or not and it makes no difference to the parser. + +On the other hand a bound root or bound stem requires affixation, so a bound root or bound stem will not successfully parse if there is no affix. + + +Notes about the stem-type Morpheme Types +Most projects: using stem is fine +If really needed: root, bound stem/root +Stems are fine in most projects. If you really need the distinction you can use them, but if you don’t feel like you need to then don’t go out of your way to make the distinction because it’s just extra work. + +If you do, be clear on how you are differentiating them, and think through what you gain – for example does it only affect the display or does it also affect the parser? + +Four different uses of the label “stem” +Linguistic definition: stem is “a root plus zero or more derivational affixes” +Morpheme Type: stem is one specific morpheme type +Short for ‘stem-type’, which covers stems, roots, bound stems and bound roots +Templates: stem base you build on with affix slots +any ‘stem-type’ morpheme can fill the stem slot of a template +So there are various ways in which the word ‘stem’ is used. + +Linguistically, a stem is a root plus zero or more derivational affixes. + +A stem is also one specific morpheme type, different to a bound root, a prefix, a clitic etc. + +‘stem’ is also sometimes a shorthand for ‘stem-type’, used as a cover term for stem, root, bound stem and bound root + +In templates, the stem is what you build on with affix slots – anything that is a ‘stem-type’ morpheme can fill the ‘stem’ slot in a template + +Clitics and parsing behavior in FLEx +Clitic (morpheme type in FLEx) +Is written as a separate word, orthographically +Proclitic/Enclitic (in FLEx) pro= =en +Attached to a word (like a prefix/suffix) +Attach outside of inflection +Can’t be in a template + +These are now FLEx-specific definitions. + +A clitic in FLEx is written as a separate orthographic word. Proclitics and enclitics in FLEx are written attached, like a prefix or suffix, but they attach outside of inflection and can’t appear in a template. + + +Affixes +Morpheme Types for affixes +prefix, suffix, circumfix, infix, and more +Affix type +Inflectional +Derivational +Unspecified (avoid) +There are lots of different morpheme types for affixes, so not all of them are listed on the slide, but these are probable the most common. + +And then affixes can be either inflectional or derivational. You can leave the affix type unspecified, but this isn’t recommended. The parser has so many different options when an affix is unspecified that it comes up with all sorts of crazy analyses and it mean the parser takes much longer to run. So picking one of the two and then finding you want to change the type is better than leaving it unspecified. + +Which terms go together? +Morpheme Type +Stem, bound root, prefix, circumfix, clitic, enclitic, …. +Affix Type +Inflectional vs Derivational +Category +Noun, Verb, Preposition, Particle, etc +So which terms go together? + +Category +Use linguistics to determine categories +Don’t have to use FLEx labels +For FLEx, ask, “Does it make a difference for parsing?” +For the parser, fewer categories is better. +Just because FLEx has a certain Category doesn’t mean you have to use it. And on the other hand, if FLEx doesn’t have one you need you can create your own. + +You don’t usually want to make them too fine-grained, for example having separate Categories or sub-Categories for masculine nouns, feminine nouns, masculine plural nouns etc. That’s too much dividing usually because it doesn’t really help you, and often actually makes things more difficult. Sometimes there are good reasons to have different categories. For example if in your language transitive and intransitive verbs have different morphological templates, then a different Category might be helpful. But if you don’t need that distinction, if transitive and intransitive verbs can make use of the same template, then creating the distinction just makes it more complicated. You can mark something like transitivity with inflection features instead. And so if you want finer distinctions in a dictionary there are other ways of doing it. + +The basic rule for parsing is to keep your Categories as few as possible. + +Category in FLEx related to Morpheme Types +Stems +Have a category via the sense +Affixes +Have a category they ATTACH TO +Clitics +Like stems, have a category +Like affixes, have a category they ATTACH TO + +So how do Categories relate to the different morpheme types? + +For a stem-type morpheme (ie stem, root, bound stem and bound root) the Category is defined via the sense. So a word could have two categories if it has two senses. +Affixes have categories that they attach to – does this affix attach to ouns or verbs etc? This is an automatic constraint to the parser – it stops verbal affixes attaching to nouns etc. +Clitics are in-between – they have their own category, like stems, but they also have a category they attach to, like affixes + +Inflection vs derivation +Not always clear cut! +Some view it as a continuum +In-between cases, we get to decide +Let’s think about inflection and derivation. + +This isn’t always clear-cut, and some would say that there’s a continuum. In FLEx however it’s a binary option and you have to decide which is most appropriate. If you need to make that decision, you might decide to go with whichever works best for the parser. + +Inflection +Usually part of a set of affixes +Applies to most roots of a category +Restrictive order of morphemes +Transparent meaning change +Typically a grammatical meaning, not semantic + +Examples +Person/number +Tense/aspect/mood + +Here are some general characteristics of inflection, and then we’ll look at the characteristics of derivation, and these can help you decide which you’re dealing with. + +Transparent meaning change – for example it’s pretty obvious that the plural affix means you’re talking about multiple things. + +Derivation +Less predictable +Applies to only some roots of a category +More significant meaning change +Often changes grammatical category or valence +Variation in where morphemes occur + +Examples +Nominalization/verbalization +Passive/causative/antipassive + +Derivation is generally less predictable, and you’ll see that this list is almost the opposite of the list for inflection. + +More significant meaning change – that’s deliberately quite a vague statement + +Inflection or Derivation? +-ly ADVLZR in English +Changes category +Applies to most adjectives +Predictable meaning change +Comparative +Bantu extensions (causative, passive, applicative) +Let’s look at some examples. + +Firstly the English adverbializer -ly, for example slow becomes slowly and bad becomes badly. +It changes the word’s category, and for most linguists, that is sufficient to call it derivational. However, since it applies ot most adjectives and has a transparent meaning change, some might say it has some characteristics of inflection. I think most would agree it’s derivational though, and I’m not disputing that, I just want to show that the lists we’ve just seen don’t always point in the same direction. + +The comparative –er is similar, for example big to bigger. It also has characteristics that make it seem sort of inflectional, especially that it doesn’t change the word category. But again most would consider it to be derivational. + +I think Bantuists often think of Bantu verb extensions as being inflectional, because we often include them in the verb template along with subject marking, TAM, and object marking etc, and that makes us think of them in the same way we think of those other inflectional affixes. They can also be added to most verbs. Before really thinking about it I definitely assumed they were inflectional. However, they are often valency-changing and the meaning change can be unpredictable, and having to think about which to use when parsing I’ve so far found that treating them as derivational is simpler. However, there’s a caveat to that which I’ll mention later. + +So you’ll need to decide which way to go, but again, your consultant can help you think it through and explain the impact each choice will have on the parser. + +How is it modeled in FLEx? +Inflection: affix template +“slots” for different sets of affixes +e.g., Person, Gender, Number +Slot order constrains morpheme order +Hard to define affixes (reduplication, infix) need a slot +So how are they modelled in FLEx? + +Inflection is modelled via an affix template, with slots for different affixes. So we saw the Mbugwe example earlier where I had a slot for subject, a slot for TAM, etc. +The order of the slots constrains the order of the morphemes. + +Even complicated affixes like reduplication, infixes and circumfixes get a slot and your consultant can help you figure out where to put the slot so the parser knows about it, and then if necessary other processes can move them to the right place. + +How is it modeled in FLEx? +Derivation +Category it attaches to? +Category it becomes? +No position restriction +Happens “before” or “inside” inflection +Once derived, template applies +For derivation, you need to define which Category the affix attaches to, and that constrains it quite a lot, and then you can define which Category it becomes. And these two can be the same, derivational affixes that don’t change Category are permitted. + +There is no way to restrict the position of a derivational affix, except that derivation always applies before inflection, so in your template a stem with a derivational affix uses the ‘stem’ slot, and inflectional affixes are then added outside of that. + +The Bantu extensions I mentioned do occur in a fixed order, so when there’s more than one it would be good to constrain the order, but this is something I only just of while preparing this lecture so I haven’t got an answer yet. I’ll be giving it some thought this week, and if some of you have similar phenomena, even if you’re not working on a Bantu language, then maybe later in the workshop I’ll have a more helpful suggestion! + +Deciding Inflectional vs Derivational (for affix) +Does it belong to a set of affixes, where one of the set is used? +Can I state clearly its position relative to other affixes? +Does it occur “outside” the less constrained affixes? +Is its meaning grammatical (not semantic or category change)? +If yes, then Inflectional. +If no, then Derivational. +So here are some questions you can ask when trying to decide between the two. + +Conclusion +The more you are reflecting linguistic reality, the easier it is supposed to be to model in FLEx. +But sometimes we have to make practical compromises. +Learn behaviors of different methods and use that to decide border cases. +So, to conclude... + +So start with the best linguistic approach that you can, and then see how it works in FLEx. As you go, you’ll learn the behaviours of different methods, and you can use that knowledge to help with the borderline cases. diff --git a/Src/Common/FwUtils/AiExportGate.cs b/Src/Common/FwUtils/AiExportGate.cs new file mode 100644 index 0000000000..087e046242 --- /dev/null +++ b/Src/Common/FwUtils/AiExportGate.cs @@ -0,0 +1,38 @@ +// Copyright (c) 2026 SIL International +// This software is licensed under the LGPL, version 2.1 or later +// (http://www.gnu.org/licenses/lgpl-2.1.html) + +using System; + +namespace SIL.FieldWorks.Common.FwUtils +{ + /// + /// The opt-in gate for exporting a project for AI analysis. Default is off: the export option is + /// absent from the export list until a user sets FLEX_AI_EXPORT. + /// + public static class AiExportGate + { + /// Environment variable a user sets to opt in to the AI-analysis export. + public const string EnabledVariable = "FLEX_AI_EXPORT"; + + /// Reads from the current process environment. + public static bool IsEnabled() => + IsEnabled(Environment.GetEnvironmentVariable(EnabledVariable)); + + /// + /// True for any except null, blank, "0", "false", and "off" + /// (case-insensitive), so the variable can be spelled "1", "true", or "yes" and still turn the + /// export on, while "0" reliably turns it back off. + /// + internal static bool IsEnabled(string variableValue) + { + if (string.IsNullOrWhiteSpace(variableValue)) + return false; + + var trimmed = variableValue.Trim(); + return !string.Equals(trimmed, "0", StringComparison.OrdinalIgnoreCase) && + !string.Equals(trimmed, "false", StringComparison.OrdinalIgnoreCase) && + !string.Equals(trimmed, "off", StringComparison.OrdinalIgnoreCase); + } + } +} diff --git a/Src/Common/FwUtils/EventConstants.cs b/Src/Common/FwUtils/EventConstants.cs index 85990748a8..ffeadca8d0 100644 --- a/Src/Common/FwUtils/EventConstants.cs +++ b/Src/Common/FwUtils/EventConstants.cs @@ -16,6 +16,7 @@ public static class EventConstants public const string DeleteRecord = "DeleteRecord"; public const string DialogInsertItemInVector = "DialogInsertItemInVector"; public const string DictionaryConfigured = "DictionaryConfigured"; + public const string ExportTextsAsFlexText = "ExportTextsAsFlexText"; public const string FilterListChanged = "FilterListChanged"; public const string FollowLink = "FollowLink"; public const string GetContentControlParameters = "GetContentControlParameters"; diff --git a/Src/Common/FwUtils/ExportTextsAsFlexTextRequest.cs b/Src/Common/FwUtils/ExportTextsAsFlexTextRequest.cs new file mode 100644 index 0000000000..b236a2e2ad --- /dev/null +++ b/Src/Common/FwUtils/ExportTextsAsFlexTextRequest.cs @@ -0,0 +1,33 @@ +// Copyright (c) 2026 SIL International +// This software is licensed under the LGPL, version 2.1 or later +// (http://www.gnu.org/licenses/lgpl-2.1.html) +using System.Collections.Generic; +using SIL.LCModel; + +namespace SIL.FieldWorks.Common.FwUtils +{ + /// + /// Published with EventConstants.ExportTextsAsFlexText and answered synchronously by a + /// globally-registered listener: it writes one .flextext file per text in + /// TextsToExport into OutputFolder, sets Handled to true, and appends a + /// "<name>: <message>" entry to Failures for any text it could not export. + /// + public sealed class ExportTextsAsFlexTextRequest + { + public ExportTextsAsFlexTextRequest(IEnumerable textsToExport, string outputFolder) + { + TextsToExport = textsToExport; + OutputFolder = outputFolder; + } + + public IEnumerable TextsToExport { get; } + + public string OutputFolder { get; } + + /// Set true by the subscriber that handled this request. + public bool Handled { get; set; } + + /// One entry per text that failed to export, formatted "<name>: <message>". + public List Failures { get; } = new List(); + } +} diff --git a/Src/Common/FwUtils/FwUtilsTests/AiExportGateTests.cs b/Src/Common/FwUtils/FwUtilsTests/AiExportGateTests.cs new file mode 100644 index 0000000000..54b00b189b --- /dev/null +++ b/Src/Common/FwUtils/FwUtilsTests/AiExportGateTests.cs @@ -0,0 +1,58 @@ +// Copyright (c) 2026 SIL International +// This software is licensed under the LGPL, version 2.1 or later +// (http://www.gnu.org/licenses/lgpl-2.1.html) + +using System; +using NUnit.Framework; + +namespace SIL.FieldWorks.Common.FwUtils +{ + /// + /// Covers which FLEX_AI_EXPORT values count as opting in to the AI-analysis export. + /// + [TestFixture] + public class AiExportGateTests + { + [TestCase("1")] + [TestCase("true")] + [TestCase("TRUE")] + [TestCase("yes")] + [TestCase("on")] + [TestCase(" 1 ")] + public void IsEnabled_TreatsAnyOtherValueAsOptedIn(string value) + { + Assert.That(AiExportGate.IsEnabled(value), Is.True); + } + + [TestCase(null)] + [TestCase("")] + [TestCase(" ")] + [TestCase("0")] + [TestCase("false")] + [TestCase("False")] + [TestCase("off")] + [TestCase(" off ")] + public void IsEnabled_FailsClosedForUnsetAndNegativeValues(string value) + { + Assert.That(AiExportGate.IsEnabled(value), Is.False); + } + + [Test] + public void IsEnabled_ReadsTheProcessEnvironmentAndDefaultsToOff() + { + var original = Environment.GetEnvironmentVariable(AiExportGate.EnabledVariable); + try + { + Environment.SetEnvironmentVariable(AiExportGate.EnabledVariable, "1"); + Assert.That(AiExportGate.IsEnabled(), Is.True); + + Environment.SetEnvironmentVariable(AiExportGate.EnabledVariable, null); + Assert.That(AiExportGate.IsEnabled(), Is.False); + } + finally + { + Environment.SetEnvironmentVariable(AiExportGate.EnabledVariable, original); + } + } + } +} diff --git a/Src/Common/FwUtils/FwUtilsTests/ExportTextsAsFlexTextRequestTests.cs b/Src/Common/FwUtils/FwUtilsTests/ExportTextsAsFlexTextRequestTests.cs new file mode 100644 index 0000000000..d183a8adb3 --- /dev/null +++ b/Src/Common/FwUtils/FwUtilsTests/ExportTextsAsFlexTextRequestTests.cs @@ -0,0 +1,37 @@ +// Copyright (c) 2026 SIL International +// This software is licensed under the LGPL, version 2.1 or later +// (http://www.gnu.org/licenses/lgpl-2.1.html) +using System.Collections.Generic; +using NUnit.Framework; +using SIL.LCModel; + +namespace SIL.FieldWorks.Common.FwUtils +{ + [TestFixture] + public class ExportTextsAsFlexTextRequestTests + { + [Test] + public void Constructor_SetsTextsAndFolder_LeavesHandledFalseAndFailuresEmpty() + { + var texts = new List(); + var request = new ExportTextsAsFlexTextRequest(texts, @"C:\some\folder"); + + Assert.That(request.TextsToExport, Is.SameAs(texts)); + Assert.That(request.OutputFolder, Is.EqualTo(@"C:\some\folder")); + Assert.That(request.Handled, Is.False); + Assert.That(request.Failures, Is.Empty); + } + + [Test] + public void Failures_CanBeAppendedByASubscriber() + { + var request = new ExportTextsAsFlexTextRequest(new List(), @"C:\folder"); + + request.Failures.Add("Some Text: disk full"); + request.Handled = true; + + Assert.That(request.Failures, Is.EqualTo(new[] { "Some Text: disk full" })); + Assert.That(request.Handled, Is.True); + } + } +} diff --git a/Src/LexText/Interlinear/FlexTextAIExportListener.cs b/Src/LexText/Interlinear/FlexTextAIExportListener.cs new file mode 100644 index 0000000000..276d9ade43 --- /dev/null +++ b/Src/LexText/Interlinear/FlexTextAIExportListener.cs @@ -0,0 +1,93 @@ +// Copyright (c) 2026 SIL International +// This software is licensed under the LGPL, version 2.1 or later +// (http://www.gnu.org/licenses/lgpl-2.1.html) +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Xml; +using SIL.LCModel; +using SIL.FieldWorks.Common.FwUtils; +using static SIL.FieldWorks.Common.FwUtils.FwUtils; +using SIL.FieldWorks.XWorks; +using XCore; + +namespace SIL.FieldWorks.IText +{ + /// + /// Answers EventConstants.ExportTextsAsFlexText: writes one .flextext file per + /// requested text into the given folder. Lives in ITextDll rather than xWorks + /// because InterlinVc/InterlinearExporter are only reachable from this project -- + /// a reference the other direction would be a build-breaking cycle. Registered + /// globally in Main.xml's <listeners> section, so it answers regardless of + /// which area is active. + /// + public class FlexTextAIExportListener : IxCoreColleague, IDisposable + { + private LcmCache m_cache; + private PropertyTable m_propertyTable; + private bool m_isDisposed; + + public void Init(Mediator mediator, PropertyTable propertyTable, XmlNode configurationParameters) + { + m_propertyTable = propertyTable; + m_cache = propertyTable.GetValue("cache"); + mediator.AddColleague(this); + Subscriber.Subscribe(EventConstants.ExportTextsAsFlexText, OnExportTextsAsFlexText, m_propertyTable.GetWindow()); + } + + public IxCoreColleague[] GetMessageTargets() + { + return new IxCoreColleague[] { this }; + } + + public bool ShouldNotCall => false; + + public int Priority => (int)ColleaguePriority.Medium; + + private void OnExportTextsAsFlexText(object parameterObj) + { + if (!(parameterObj is ExportTextsAsFlexTextRequest request)) + return; + ExportTextsAsFlexTextForTests(m_cache, request); + } + + /// The export logic itself, separate from the pub/sub handler. + internal void ExportTextsAsFlexTextForTests(LcmCache cache, ExportTextsAsFlexTextRequest request) + { + request.Handled = true; + var usedNames = new HashSet(StringComparer.OrdinalIgnoreCase) { "HCGrammar" }; + foreach (var stText in request.TextsToExport) + { + var name = GrammarTextsAIExportHelpers.GetTextDisplayName(stText); + try + { + var fileName = GrammarTextsAIExportHelpers.MakeSafeFileName(name, usedNames); + var filePath = Path.Combine(request.OutputFolder, fileName + ".flextext"); + var settings = new XmlWriterSettings { Encoding = Encoding.UTF8, Indent = true }; + using (var vc = new InterlinVc(cache)) + using (var writer = XmlWriter.Create(filePath, settings)) + { + vc.LineChoices = InterlinLineChoices.DefaultChoices(cache.LangProject, cache.DefaultVernWs, cache.DefaultAnalWs); + var exporter = InterlinearExporter.Create("xml", cache, writer, stText, vc.LineChoices, vc); + exporter.WriteBeginDocument(); + exporter.ExportDisplay(); + exporter.WriteEndDocument(); + } + } + catch (Exception e) + { + request.Failures.Add($"{name}: {e.Message}"); + } + } + } + + public void Dispose() + { + if (m_isDisposed) + return; + Subscriber.Unsubscribe(EventConstants.ExportTextsAsFlexText, OnExportTextsAsFlexText); + m_isDisposed = true; + } + } +} diff --git a/Src/LexText/Interlinear/ITextDllTests/FlexTextAIExportListenerTests.cs b/Src/LexText/Interlinear/ITextDllTests/FlexTextAIExportListenerTests.cs new file mode 100644 index 0000000000..05920f9f6a --- /dev/null +++ b/Src/LexText/Interlinear/ITextDllTests/FlexTextAIExportListenerTests.cs @@ -0,0 +1,48 @@ +// Copyright (c) 2026 SIL International +// This software is licensed under the LGPL, version 2.1 or later +// (http://www.gnu.org/licenses/lgpl-2.1.html) +using System.IO; +using NUnit.Framework; +using SIL.FieldWorks.Common.FwUtils; +using SIL.LCModel; +using SIL.LCModel.Core.Text; +using SIL.LCModel.DomainServices; + +namespace SIL.FieldWorks.IText +{ + [TestFixture] + public class FlexTextAIExportListenerTests : InterlinearTestBase + { + [Test] + public void ExportTextsAsFlexText_OneText_WritesOneFlexTextFileAndMarksHandled() + { + // Already runs inside an ambient undo task here, so this must not open its + // own -- LCM disallows nested undo tasks. + var text = Cache.ServiceLocator.GetInstance().Create(); + Cache.LangProject.Texts.Add(text); + text.ContentsOA = Cache.ServiceLocator.GetInstance().Create(); + text.Name.SetAnalysisDefaultWritingSystem("Listener Test Text"); + var para = Cache.ServiceLocator.GetInstance().Create(); + text.ContentsOA.ParagraphsOS.Add(para); + para.Contents = TsStringUtils.MakeString("hello world", Cache.DefaultVernWs); + + var tempFolder = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + Directory.CreateDirectory(tempFolder); + try + { + var listener = new FlexTextAIExportListener(); + var request = new ExportTextsAsFlexTextRequest(new[] { text.ContentsOA }, tempFolder); + + listener.ExportTextsAsFlexTextForTests(Cache, request); + + Assert.That(request.Handled, Is.True); + Assert.That(request.Failures, Is.Empty); + Assert.That(File.Exists(Path.Combine(tempFolder, "Listener Test Text.flextext")), Is.True); + } + finally + { + Directory.Delete(tempFolder, true); + } + } + } +} diff --git a/Src/xWorks/ExportDialog.cs b/Src/xWorks/ExportDialog.cs index 5f73d91a64..b5bf9a0a6f 100644 --- a/Src/xWorks/ExportDialog.cs +++ b/Src/xWorks/ExportDialog.cs @@ -17,6 +17,8 @@ using System.Xml.Xsl; using Microsoft.Win32; using SIL.Extensions; +using SIL.Machine.Morphology.HermitCrab; +using SIL.FieldWorks.WordWorks.Parser; using SIL.LCModel.Core.Text; using SIL.FieldWorks.Common.Controls; using SIL.FieldWorks.Common.Controls.FileDialog; @@ -89,7 +91,8 @@ protected internal enum FxtTypes kftWebonary, kftWordOpenXml, kftWordClassifiedDict, - kftPhonology + kftPhonology, + kftGrammarTextsAI } // ReSharper restore InconsistentNaming protected internal struct FxtType @@ -534,6 +537,7 @@ private Control EnsureViewInfo() private List m_translationWritingSystems; private List m_translatedLists; private bool m_allQuestions; // For semantic domains, export missing translations as English? + private List m_selectedTextsForAIExport; private void btnExport_Click(object sender, EventArgs e) { @@ -548,6 +552,8 @@ private void btnExport_Click(object sender, EventArgs e) if (!PrepareForExport()) return; bool fLiftExport = m_exportItems[0].SubItems[2].Text == "lift"; + bool fGrammarTextsAIExport = m_rgFxtTypes.Count > 0 + && m_rgFxtTypes[FxtIndex((string)m_exportItems[0].Tag)].m_ft == FxtTypes.kftGrammarTextsAI; string sFileName; string sDirectory; if (fLiftExport) @@ -592,6 +598,34 @@ private void btnExport_Click(object sender, EventArgs e) } } } + else if (fGrammarTextsAIExport) + { + var textList = InterestingTextsDecorator.GetInterestingTextList(m_mediator, m_propertyTable, m_cache.ServiceLocator).InterestingTexts; + using (var textDlg = new GrammarAndTextsAIExportSelectionDlg(m_cache, textList)) + { + var previousSelection = m_propertyTable.GetStringProperty("GrammarTextsAIExportSelection", null); + if (previousSelection != null) + textDlg.ApplyPreviousSelection(new HashSet(previousSelection.Split(','))); + if (textDlg.ShowDialog(this) != DialogResult.OK) + return; + m_selectedTextsForAIExport = textDlg.SelectedTexts.ToList(); + m_propertyTable.SetProperty("GrammarTextsAIExportSelection", + string.Join(",", m_selectedTextsForAIExport.Select(t => t.Guid.ToString())), true); + m_propertyTable.SetPropertyPersistence("GrammarTextsAIExportSelection", true); + } + using (var dlg = new FolderBrowserDialogAdapter()) + { + dlg.Description = xWorksStrings.ksChooseGrammarTextsAIExportFolder; + dlg.ShowNewFolderButton = true; + dlg.RootFolder = Environment.SpecialFolder.Desktop; + dlg.SelectedPath = m_propertyTable.GetStringProperty("ExportDir", + Environment.GetFolderPath(Environment.SpecialFolder.Personal)); + if (dlg.ShowDialog(this) != DialogResult.OK) + return; + sDirectory = dlg.SelectedPath; + } + sFileName = Path.Combine(sDirectory, "HCGrammar.xml"); + } else { FxtType ft; @@ -749,6 +783,12 @@ protected virtual bool PrepareForExport() return true; } + /// Sets the texts to include in a grammar+texts-for-AI export. + internal void SetSelectedTextsForAIExport(List texts) + { + m_selectedTextsForAIExport = texts; + } + /// /// This version is overridden by (currently) Interlinear and Discourse Chart exports. /// @@ -843,6 +883,19 @@ protected void DoExport(string outPath, bool fLiftOutput) progressDlg.Restartable = true; progressDlg.RunTask(true, ExportPhonology, outPath, ft.m_sDataType, ft.m_sXsltFiles); break; + case FxtTypes.kftGrammarTextsAI: + { + progressDlg.Minimum = 0; + progressDlg.Maximum = m_selectedTextsForAIExport.Count + 1; + progressDlg.AllowCancel = true; + var aiExportMessages = (List)progressDlg.RunTask(true, ExportGrammarAndTextsForAI, outPath); + if (aiExportMessages.Count > 0) + { + MessageBox.Show(this, string.Format(xWorksStrings.ksGrammarTextsAIExportSummary, + Environment.NewLine, string.Join(Environment.NewLine, aiExportMessages))); + } + break; + } case FxtTypes.kftWordOpenXml: case FxtTypes.kftWordClassifiedDict: progressDlg.Minimum = 0; @@ -1093,6 +1146,58 @@ private object ExportPhonology(IThreadedProgress progress, object[] args) return null; } + /// + /// Writes the HC grammar (HCGrammar.xml, at outPath), one .flextext file per + /// selected text, and the instructions file that tells an LLM how to read them, + /// all into the same folder. HCLoader already catches per-item + /// linguistic problems internally, so an exception escaping here aborts the + /// whole export; a per-text export failure only skips that one text. Runs on + /// the background task thread, so it returns the combined warning/failure list + /// rather than showing it -- the UI thread shows it once this returns. + /// + internal object ExportGrammarAndTextsForAI(IThreadedProgress progress, object[] args) + { + var outPath = (string)args[0]; + var outFolder = Path.GetDirectoryName(outPath); + var loadMessages = new List(); + var logger = new GrammarExportLoadLogger(loadMessages); + var language = HCLoader.Load(m_cache, logger); + XmlLanguageWriter.Save(language, outPath); + CopyAiExportInstructions(outFolder, loadMessages); + progress.Step(1); + + var texts = m_selectedTextsForAIExport ?? new List(); + var request = new ExportTextsAsFlexTextRequest(texts, outFolder); + Publisher.Publish(new PublisherParameterObject(EventConstants.ExportTextsAsFlexText, request, m_propertyTable.GetWindow())); + if (!request.Handled) + request.Failures.Add("FLExText export service was not available."); + foreach (var text in texts) + progress.Step(1); + + return loadMessages.Concat(request.Failures).ToList(); + } + + /// Name the exported instructions file gets in the export folder. + internal const string ksAiExportInstructionsFileName = "export-instructions.md"; + + /// + /// Copies the shipped AI-analysis instructions into , so that + /// dropping the whole folder into a chat carries the reading instructions and reference links + /// with it. A missing shipped file is reported in instead of + /// aborting, since the exported grammar and texts are still usable without it. + /// + private static void CopyAiExportInstructions(string outFolder, List messages) + { + var shippedFile = Path.Combine(FwDirectoryFinder.CodeDirectory, "Language Explorer", + "Export Templates", "AIExportInstructions.md"); + if (!File.Exists(shippedFile)) + { + messages.Add(string.Format(xWorksStrings.ksAIExportInstructionsMissing, shippedFile)); + return; + } + File.Copy(shippedFile, Path.Combine(outFolder, ksAiExportInstructionsFileName), true); + } + /// ------------------------------------------------------------------------------------ /// /// Exports as a LIFT file (possibly with one or more range files. @@ -1384,6 +1489,9 @@ protected void AddFxts(string[] filePaths) { continue; } + // The AI-analysis export is opt-in, so it stays out of the list entirely when unset. + if (IsAiExportTemplate(document) && !AiExportGate.IsEnabled()) + continue; XmlNode node = document.SelectSingleNode("//FxtDocumentDescription"); if (node == null) continue; @@ -1416,6 +1524,19 @@ protected void AddFxts(string[] filePaths) } + /// Value of the export template's type attribute that selects the AI-analysis export. + private const string ksAiExportTemplateType = "grammarTextsAI"; + + /// + /// True when the export template describes exporting a grammar and texts for AI analysis. + /// + internal static bool IsAiExportTemplate(XmlDocument document) + { + var templateRootNode = document.SelectSingleNode("//template"); + return templateRootNode != null + && XmlUtils.GetOptionalAttributeValue(templateRootNode, "type", "fxt") == ksAiExportTemplateType; + } + /// /// Store the attributes of the