From 3c35b85f6a162781af02f8d0205a246b43839a2b Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 11 Jul 2026 23:36:15 -0700 Subject: [PATCH 01/88] Add v2 Span, Role, and Token with construction validation Co-Authored-By: Claude Fable 5 --- nameparser/_types.py | 54 ++++++++++++++++++++++++++++++++++++++++++ tests/v2/__init__.py | 0 tests/v2/conftest.py | 16 +++++++++++++ tests/v2/test_types.py | 44 ++++++++++++++++++++++++++++++++++ 4 files changed, 114 insertions(+) create mode 100644 nameparser/_types.py create mode 100644 tests/v2/__init__.py create mode 100644 tests/v2/conftest.py create mode 100644 tests/v2/test_types.py diff --git a/nameparser/_types.py b/nameparser/_types.py new file mode 100644 index 0000000..cd75926 --- /dev/null +++ b/nameparser/_types.py @@ -0,0 +1,54 @@ +"""Core value types for the 2.0 API. + +Layering (enforced by tests/v2/test_layering.py): this module imports +nothing from nameparser -- it is the bottom of the dependency graph. +""" +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from typing import NamedTuple + + +class Role(Enum): + # Declaration order IS the canonical field order (conventions §3): + # every listing of the seven fields anywhere derives from this. + TITLE = "title" + GIVEN = "given" + MIDDLE = "middle" + FAMILY = "family" + SUFFIX = "suffix" + NICKNAME = "nickname" + MAIDEN = "maiden" + + +class Span(NamedTuple): + """Provenance range into ParsedName.original. end is exclusive.""" + + start: int + end: int + + +#: Stable, documented tag vocabulary (API). All other tags are +#: namespaced ("vocab:...", "patronymic:...") and unstable. +STABLE_TAGS = frozenset({"particle", "conjunction", "initial"}) + + +@dataclass(frozen=True, slots=True) +class Token: + text: str + span: Span | None # None = synthetic (from replace()) + role: Role + tags: frozenset[str] = frozenset() + + def __post_init__(self) -> None: + if not self.text: + raise ValueError("Token.text must be a non-empty string") + if self.span is not None: + start, end = self.span + if start < 0 or end < start: + raise ValueError( + f"invalid span ({start}, {end}): need 0 <= start <= end" + ) + object.__setattr__(self, "span", Span(start, end)) + object.__setattr__(self, "tags", frozenset(self.tags)) diff --git a/tests/v2/__init__.py b/tests/v2/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/v2/conftest.py b/tests/v2/conftest.py new file mode 100644 index 0000000..9bb5e18 --- /dev/null +++ b/tests/v2/conftest.py @@ -0,0 +1,16 @@ +"""Neutralize the v1 suite's autouse dual-run fixture for tests/v2. + +tests/conftest.py runs every test twice (empty_attribute_default '' and +None) and deep-copy-snapshots the shared CONSTANTS around each test. +v2 code never reads shared CONSTANTS, so both behaviors are pure +overhead here. Overriding the fixture by name in this conftest replaces +the parametrized parent version for this directory. +""" +from collections.abc import Iterator + +import pytest + + +@pytest.fixture(autouse=True) +def empty_attribute_default() -> Iterator[None]: + yield diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py new file mode 100644 index 0000000..80db2bb --- /dev/null +++ b/tests/v2/test_types.py @@ -0,0 +1,44 @@ +import pytest + +from nameparser._types import Role, Span, Token + + +def test_role_declaration_order_is_canonical_field_order(): + assert [r.value for r in Role] == [ + "title", "given", "middle", "family", "suffix", "nickname", "maiden", + ] + + +def test_token_construction_and_span_coercion(): + t = Token("Juan", (0, 4), Role.GIVEN) + assert t.span == Span(0, 4) + assert isinstance(t.span, Span) + assert t.span.start == 0 and t.span.end == 4 + assert t.tags == frozenset() + + +def test_synthetic_token_has_no_span(): + t = Token("Jane", None, Role.GIVEN) + assert t.span is None + + +def test_token_rejects_empty_text(): + with pytest.raises(ValueError, match="non-empty"): + Token("", (0, 0), Role.GIVEN) + + +def test_token_rejects_inverted_span(): + with pytest.raises(ValueError, match="start <= end"): + Token("x", (5, 2), Role.GIVEN) + + +def test_token_rejects_negative_span(): + with pytest.raises(ValueError, match="start <= end"): + Token("x", (-1, 1), Role.GIVEN) + + +def test_token_is_frozen_and_hashable(): + t = Token("Juan", (0, 4), Role.GIVEN) + with pytest.raises(AttributeError): + t.text = "X" # type: ignore[misc] + assert hash(t) == hash(Token("Juan", (0, 4), Role.GIVEN)) From 2f70ac76476d2dc6636db933dc729af367575307 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 11 Jul 2026 23:37:55 -0700 Subject: [PATCH 02/88] Raise the Python floor to 3.11 on the 2.0 branch (#257, partial) The v2 core design uses enum.StrEnum (3.11+). Full #257 (dropping typing_extensions, CI matrix) remains tracked on the issue. Co-Authored-By: Claude Fable 5 --- .python-version | 2 +- pyproject.toml | 3 +- uv.lock | 146 +++--------------------------------------------- 3 files changed, 10 insertions(+), 141 deletions(-) diff --git a/.python-version b/.python-version index c8cfe39..2c07333 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.10 +3.11 diff --git a/pyproject.toml b/pyproject.toml index 69bfe71..1e9ffef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ name = "nameparser" description = "A simple Python module for parsing human names into their individual components." readme = "README.rst" -requires-python = ">=3.10" +requires-python = ">=3.11" license = {text = "LGPL"} authors = [{name = "Derek Gulbranson", email = "derek73@gmail.com"}] dynamic = ["version"] @@ -13,7 +13,6 @@ classifiers = [ "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", diff --git a/uv.lock b/uv.lock index 64369b6..c9b8164 100644 --- a/uv.lock +++ b/uv.lock @@ -1,11 +1,10 @@ version = 1 revision = 3 -requires-python = ">=3.10" +requires-python = ">=3.11" resolution-markers = [ "python_full_version >= '3.15'", "python_full_version >= '3.12' and python_full_version < '3.15'", - "python_full_version == '3.11.*'", - "python_full_version < '3.11'", + "python_full_version < '3.12'", ] [[package]] @@ -106,22 +105,6 @@ version = "3.4.7" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/e7/a1/67fe25fac3c7642725500a3f6cfe5821ad557c3abb11c9d20d12c7008d3e/charset_normalizer-3.4.7.tar.gz", hash = "sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5", size = 144271, upload-time = "2026-04-02T09:28:39.342Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/26/08/0f303cb0b529e456bb116f2d50565a482694fbb94340bf56d44677e7ed03/charset_normalizer-3.4.7-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cdd68a1fb318e290a2077696b7eb7a21a49163c455979c639bf5a5dcdc46617d", size = 315182, upload-time = "2026-04-02T09:25:40.673Z" }, - { url = "https://files.pythonhosted.org/packages/24/47/b192933e94b546f1b1fe4df9cc1f84fcdbf2359f8d1081d46dd029b50207/charset_normalizer-3.4.7-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e17b8d5d6a8c47c85e68ca8379def1303fd360c3e22093a807cd34a71cd082b8", size = 209329, upload-time = "2026-04-02T09:25:42.354Z" }, - { url = "https://files.pythonhosted.org/packages/c2/b4/01fa81c5ca6141024d89a8fc15968002b71da7f825dd14113207113fabbd/charset_normalizer-3.4.7-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:511ef87c8aec0783e08ac18565a16d435372bc1ac25a91e6ac7f5ef2b0bff790", size = 231230, upload-time = "2026-04-02T09:25:44.281Z" }, - { url = "https://files.pythonhosted.org/packages/20/f7/7b991776844dfa058017e600e6e55ff01984a063290ca5622c0b63162f68/charset_normalizer-3.4.7-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:007d05ec7321d12a40227aae9e2bc6dca73f3cb21058999a1df9e193555a9dcc", size = 225890, upload-time = "2026-04-02T09:25:45.475Z" }, - { url = "https://files.pythonhosted.org/packages/20/e7/bed0024a0f4ab0c8a9c64d4445f39b30c99bd1acd228291959e3de664247/charset_normalizer-3.4.7-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf29836da5119f3c8a8a70667b0ef5fdca3bb12f80fd06487cfa575b3909b393", size = 216930, upload-time = "2026-04-02T09:25:46.58Z" }, - { url = "https://files.pythonhosted.org/packages/e2/ab/b18f0ab31cdd7b3ddb8bb76c4a414aeb8160c9810fdf1bc62f269a539d87/charset_normalizer-3.4.7-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:12d8baf840cc7889b37c7c770f478adea7adce3dcb3944d02ec87508e2dcf153", size = 202109, upload-time = "2026-04-02T09:25:48.031Z" }, - { url = "https://files.pythonhosted.org/packages/82/e5/7e9440768a06dfb3075936490cb82dbf0ee20a133bf0dd8551fa096914ec/charset_normalizer-3.4.7-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d560742f3c0d62afaccf9f41fe485ed69bd7661a241f86a3ef0f0fb8b1a397af", size = 214684, upload-time = "2026-04-02T09:25:49.245Z" }, - { url = "https://files.pythonhosted.org/packages/71/94/8c61d8da9f062fdf457c80acfa25060ec22bf1d34bbeaca4350f13bcfd07/charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b14b2d9dac08e28bb8046a1a0434b1750eb221c8f5b87a68f4fa11a6f97b5e34", size = 212785, upload-time = "2026-04-02T09:25:50.671Z" }, - { url = "https://files.pythonhosted.org/packages/66/cd/6e9889c648e72c0ab2e5967528bb83508f354d706637bc7097190c874e13/charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:bc17a677b21b3502a21f66a8cc64f5bfad4df8a0b8434d661666f8ce90ac3af1", size = 203055, upload-time = "2026-04-02T09:25:51.802Z" }, - { url = "https://files.pythonhosted.org/packages/92/2e/7a951d6a08aefb7eb8e1b54cdfb580b1365afdd9dd484dc4bee9e5d8f258/charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:750e02e074872a3fad7f233b47734166440af3cdea0add3e95163110816d6752", size = 232502, upload-time = "2026-04-02T09:25:53.388Z" }, - { url = "https://files.pythonhosted.org/packages/58/d5/abcf2d83bf8e0a1286df55cd0dc1d49af0da4282aa77e986df343e7de124/charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4e5163c14bffd570ef2affbfdd77bba66383890797df43dc8b4cc7d6f500bf53", size = 214295, upload-time = "2026-04-02T09:25:54.765Z" }, - { url = "https://files.pythonhosted.org/packages/47/3a/7d4cd7ed54be99973a0dc176032cba5cb1f258082c31fa6df35cff46acfc/charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6ed74185b2db44f41ef35fd1617c5888e59792da9bbc9190d6c7300617182616", size = 227145, upload-time = "2026-04-02T09:25:55.904Z" }, - { url = "https://files.pythonhosted.org/packages/1d/98/3a45bf8247889cf28262ebd3d0872edff11565b2a1e3064ccb132db3fbb0/charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:94e1885b270625a9a828c9793b4d52a64445299baa1fea5a173bf1d3dd9a1a5a", size = 218884, upload-time = "2026-04-02T09:25:57.074Z" }, - { url = "https://files.pythonhosted.org/packages/ad/80/2e8b7f8915ed5c9ef13aa828d82738e33888c485b65ebf744d615040c7ea/charset_normalizer-3.4.7-cp310-cp310-win32.whl", hash = "sha256:6785f414ae0f3c733c437e0f3929197934f526d19dfaa75e18fdb4f94c6fb374", size = 148343, upload-time = "2026-04-02T09:25:58.199Z" }, - { url = "https://files.pythonhosted.org/packages/35/1b/3b8c8c77184af465ee9ad88b5aea46ea6b2e1f7b9dc9502891e37af21e30/charset_normalizer-3.4.7-cp310-cp310-win_amd64.whl", hash = "sha256:6696b7688f54f5af4462118f0bfa7c1621eeb87154f77fa04b9295ce7a8f2943", size = 159174, upload-time = "2026-04-02T09:25:59.322Z" }, - { url = "https://files.pythonhosted.org/packages/be/c1/feb40dca40dbb21e0a908801782d9288c64fc8d8e562c2098e9994c8c21b/charset_normalizer-3.4.7-cp310-cp310-win_arm64.whl", hash = "sha256:66671f93accb62ed07da56613636f3641f1a12c13046ce91ffc923721f23c008", size = 147805, upload-time = "2026-04-02T09:26:00.756Z" }, { url = "https://files.pythonhosted.org/packages/c2/d7/b5b7020a0565c2e9fa8c09f4b5fa6232feb326b8c20081ccded47ea368fd/charset_normalizer-3.4.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7641bb8895e77f921102f72833904dcd9901df5d6d72a2ab8f31d04b7e51e4e7", size = 309705, upload-time = "2026-04-02T09:26:02.191Z" }, { url = "https://files.pythonhosted.org/packages/5a/53/58c29116c340e5456724ecd2fff4196d236b98f3da97b404bc5e51ac3493/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:202389074300232baeb53ae2569a60901f7efadd4245cf3a3bf0617d60b439d7", size = 206419, upload-time = "2026-04-02T09:26:03.583Z" }, { url = "https://files.pythonhosted.org/packages/b2/02/e8146dc6591a37a00e5144c63f29fb7c97a734ea8a111190783c0e60ab63/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:30b8d1d8c52a48c2c5690e152c169b673487a2a58de1ec7393196753063fcd5e", size = 227901, upload-time = "2026-04-02T09:26:04.738Z" }, @@ -220,20 +203,6 @@ version = "7.15.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/cc/8b/adeb62ea8951f13c4c7fef2e7a85e1a06b499c8d8237ea589d496029e53f/coverage-7.15.0.tar.gz", hash = "sha256:9ac3fe7a1435986463eaa8ee253ae2f2a268709ba4ae5c7dd1f52a05391ad78f", size = 925362, upload-time = "2026-07-02T13:10:50.535Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/97/c52dc440c390b6cfa87be9432b141a956e2d56d9b9f5fc8bd71c5f471722/coverage-7.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:50913d4bf5ddafa6ca3693da5e4dd833dd1b772e0283c99ca7f7d287db67331a", size = 220539, upload-time = "2026-07-02T13:08:19.252Z" }, - { url = "https://files.pythonhosted.org/packages/3f/26/602de8c2aec7e2e3e99ebfb8e04ba65598f746275396eea5f6794ff4673f/coverage-7.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:359e141ccd33893ce3f1ad5525f8b96083003677c82182e5907d62d4ea5799fc", size = 221058, upload-time = "2026-07-02T13:08:21.013Z" }, - { url = "https://files.pythonhosted.org/packages/fc/13/ebab0743138891c1d646d61e247ec29639afcbb6c4e1905e6a0f0c75291a/coverage-7.15.0-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3200b6204935f928c64b2ca1f923ab8c1acb7c9de45ec61569711b34d25cccaf", size = 247797, upload-time = "2026-07-02T13:08:22.474Z" }, - { url = "https://files.pythonhosted.org/packages/d3/b7/b6ffb9e042aa48dc4144a8a65529affaec8dca0685309353614a2a7386ad/coverage-7.15.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:be616bf61346883b2cfdc5178669647e03531d81ab761a7e378558b7e8bcb628", size = 249626, upload-time = "2026-07-02T13:08:23.803Z" }, - { url = "https://files.pythonhosted.org/packages/9c/06/243ff05b652333d8e3d060c11223efc2723b19cacf6605e433fa686ab5d4/coverage-7.15.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cc7bafc3fe1059463a8fdd97ca79972d6e2bf819d775c7d54991b5b1971201d6", size = 251493, upload-time = "2026-07-02T13:08:25.397Z" }, - { url = "https://files.pythonhosted.org/packages/d3/2b/867faa17030a806114dae388b32a3fa929d8cd4bf39226fbc11f6e6bb705/coverage-7.15.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b713aa7fcf325a01d4184d848acb46fd84f78fdb0978470c636b23a06a753d91", size = 253406, upload-time = "2026-07-02T13:08:26.842Z" }, - { url = "https://files.pythonhosted.org/packages/94/c0/d789ce18f6605afc4895db75723424be2ef494282f77f61d8e5832923183/coverage-7.15.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e38e6fba2d56652fdfaf0231f8f78aeb805234a912de25dc291ee5cce5b8faa4", size = 248512, upload-time = "2026-07-02T13:08:28.398Z" }, - { url = "https://files.pythonhosted.org/packages/c9/b6/b2673c30739f4a2e06649a0a38ad8b093c4d865462dc7bab0e9524a2c3b1/coverage-7.15.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:884499f42e382675be80770391983b90e0c0c774d87dbeeebf5f991cf6612b20", size = 249532, upload-time = "2026-07-02T13:08:29.731Z" }, - { url = "https://files.pythonhosted.org/packages/3c/2e/acd79e9a41beabee92b623afe4f30b549916f48566271475f2907e752828/coverage-7.15.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:840481b12e083dbcbafab14794a8781a958edf327c8d3d70b4eee42f9b8253aa", size = 247537, upload-time = "2026-07-02T13:08:31.173Z" }, - { url = "https://files.pythonhosted.org/packages/12/d4/2d301c4d1b3238d7c88b70ab9d13fd53ed9505662a7ff1b46ba1e2e4e3c3/coverage-7.15.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:276646e9481703d09f854f3b2f018f24e19fd7049ae670a92570043eb97203b1", size = 251348, upload-time = "2026-07-02T13:08:32.63Z" }, - { url = "https://files.pythonhosted.org/packages/35/bb/c67708b2bc00f32e12805ec23d5fa677a0a51652f449341a89f9d6b1b715/coverage-7.15.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4de4b4d3f5545aa6c60dc4efd9c63b5b5dcc3bf00fe83146b2bdfffb8f6613bd", size = 247806, upload-time = "2026-07-02T13:08:33.931Z" }, - { url = "https://files.pythonhosted.org/packages/eb/6c/57c4f653c47a6e917748f8938e389e72fbcae44e3643cd906664f0477a13/coverage-7.15.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5c504097b2a89b1e85bc6070d920df77daec701337e3aeef2c17775a5dd0ca90", size = 248410, upload-time = "2026-07-02T13:08:35.189Z" }, - { url = "https://files.pythonhosted.org/packages/6c/94/bb083041aef828903668f134273f319f2bd49224962875359c52faa5497f/coverage-7.15.0-cp310-cp310-win32.whl", hash = "sha256:f6e80ed91f98316e86b9c137206b04b2bcfbffccbdff49bd2eb09dddb1cf14e0", size = 222588, upload-time = "2026-07-02T13:08:36.486Z" }, - { url = "https://files.pythonhosted.org/packages/ef/94/a09d8ee618956f626741b0734854bac4425a00e10c0565f5abca64e7e751/coverage-7.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:b3b3e22030f3f6f5e01a5ce69936552a5c0f6992b7698777377b99041961031f", size = 223214, upload-time = "2026-07-02T13:08:37.885Z" }, { url = "https://files.pythonhosted.org/packages/ae/23/82e910835ef4b8391047025e1d53aa48d66029f444eb8b25373c849bf503/coverage-7.15.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:003fff99412ea848c0aaebcc78ed2b6ce7d8a1227ed17e68470672770b78a02a", size = 220662, upload-time = "2026-07-02T13:08:39.205Z" }, { url = "https://files.pythonhosted.org/packages/6d/0d/c7b213dde2f1579de5231062b386d8413f79c11667eb58c39319b25991da/coverage-7.15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5cbd804bf2784ce7b45114516050f346ecd50f960c4bb630a7ee9e1d78fa2118", size = 221168, upload-time = "2026-07-02T13:08:40.471Z" }, { url = "https://files.pythonhosted.org/packages/33/77/d000aeedfac085088337b3c7becdad328474b1f8a9e4c9368a0c99605d68/coverage-7.15.0-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8773e15c23305b58882a4611fb9b2755977eae0dc2e515366a1b6c98866cc4c2", size = 251587, upload-time = "2026-07-02T13:08:42.033Z" }, @@ -326,44 +295,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/77/dc8c558f7593132cf8fefec57c4f60c83b16941c574ac5f619abb3ae7933/dill-0.4.1-py3-none-any.whl", hash = "sha256:1e1ce33e978ae97fcfcff5638477032b801c46c7c65cf717f95fbc2248f79a9d", size = 120019, upload-time = "2026-01-19T02:36:55.663Z" }, ] -[[package]] -name = "docutils" -version = "0.21.2" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.11'", -] -sdist = { url = "https://files.pythonhosted.org/packages/ae/ed/aefcc8cd0ba62a0560c3c18c33925362d46c6075480bfa4df87b28e169a9/docutils-0.21.2.tar.gz", hash = "sha256:3a6b18732edf182daa3cd12775bbb338cf5691468f91eeeb109deff6ebfa986f", size = 2204444, upload-time = "2024-04-23T18:57:18.24Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8f/d7/9322c609343d929e75e7e5e6255e614fcc67572cfd083959cdef3b7aad79/docutils-0.21.2-py3-none-any.whl", hash = "sha256:dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2", size = 587408, upload-time = "2024-04-23T18:57:14.835Z" }, -] - [[package]] name = "docutils" version = "0.22.4" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.15'", - "python_full_version >= '3.12' and python_full_version < '3.15'", - "python_full_version == '3.11.*'", -] sdist = { url = "https://files.pythonhosted.org/packages/ae/b6/03bb70946330e88ffec97aefd3ea75ba575cb2e762061e0e62a213befee8/docutils-0.22.4.tar.gz", hash = "sha256:4db53b1fde9abecbb74d91230d32ab626d94f6badfc575d6db9194a49df29968", size = 2291750, upload-time = "2025-12-18T19:00:26.443Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl", hash = "sha256:d0013f540772d1420576855455d050a2180186c91c15779301ac2ccb3eeb68de", size = 633196, upload-time = "2025-12-18T19:00:18.077Z" }, ] -[[package]] -name = "exceptiongroup" -version = "1.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" }, -] - [[package]] name = "furo" version = "2025.12.19" @@ -372,8 +312,7 @@ dependencies = [ { name = "accessible-pygments" }, { name = "beautifulsoup4" }, { name = "pygments" }, - { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "sphinx-basic-ng" }, ] @@ -427,18 +366,6 @@ version = "0.11.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/40/08/9e7f6b5d2b5bed6ad055cdd5925f192bb403a51280f86b56554d9d0699a2/librt-0.11.0.tar.gz", hash = "sha256:075dc3ef4458a278e0195cbf6ac9d38808d9b906c5a6c7f7f79c3888276a3fb1", size = 200139, upload-time = "2026-05-10T18:17:25.138Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/83/10/37fd9e9ba96cb0bd742dfb20fc3d082e54bdbec759d7300df927f360ef07/librt-0.11.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6e94ebfcfa2d5e9926d6c3b9aa4617ffc42a845b4321fb84021b872358c82a0f", size = 141706, upload-time = "2026-05-10T18:15:16.129Z" }, - { url = "https://files.pythonhosted.org/packages/cf/72/1b1466f358e4a0b728051f69bc27e67b432c6eaa2e05b88db49d3785ae0d/librt-0.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ae627397a2f351560440d872d6f7c8dbb4072e57868e7b2fc5b8b430fe489d45", size = 142605, upload-time = "2026-05-10T18:15:18.148Z" }, - { url = "https://files.pythonhosted.org/packages/ca/85/ed26dd2f6bc9a0baf48306433e579e8d354d70b2bcb78134ed950a5d0e1e/librt-0.11.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc329359321b67d24efdf4bc69012b0597001649544db662c001db5a0184794c", size = 476555, upload-time = "2026-05-10T18:15:19.569Z" }, - { url = "https://files.pythonhosted.org/packages/66/fe/11891191c0e0a3fd617724e891f6e67a71a7658974a892b9a9a97fdb2977/librt-0.11.0-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:7e82e642ab0f7608ce2fe53d76ca2280a9ee33a1b06556142c7c6fe80a86fc33", size = 468434, upload-time = "2026-05-10T18:15:20.87Z" }, - { url = "https://files.pythonhosted.org/packages/6f/50/5ec949d7f9ce1a07af903aa3e13abb98b717923bdead6e719b2f824ccc07/librt-0.11.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:88145c15c67731d54283d135b03244028c750cc9edc334a96a4f5950ebdb2884", size = 496918, upload-time = "2026-05-10T18:15:22.616Z" }, - { url = "https://files.pythonhosted.org/packages/ea/c4/177336c7524e34875a38bf668e88b193a6723a4eb4045d07f74df6e1506c/librt-0.11.0-cp310-cp310-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9d36a51b3d93320b686588e27123f4995804dbf1bce81df78c02fc3c6eea9280", size = 490334, upload-time = "2026-05-10T18:15:24.2Z" }, - { url = "https://files.pythonhosted.org/packages/13/1f/da3112f7569eda3b49f9a2629bae1fe059812b6085df16c885f6454dff49/librt-0.11.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d00f3ac06a2a8b246327f11e186a53a100a4d5c7ed52346367e5ec751d51586c", size = 511287, upload-time = "2026-05-10T18:15:26.226Z" }, - { url = "https://files.pythonhosted.org/packages/fa/94/03fec301522e172d105581431223be56b27594ff46440ebfbb658a3735d5/librt-0.11.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:461bbceede621f1ffb8839755f8663e886087ee7af16294cab7fb4d782c62eeb", size = 517202, upload-time = "2026-05-10T18:15:27.965Z" }, - { url = "https://files.pythonhosted.org/packages/b7/6e/339f6e5a7b413ce014f1917a756dae630fe59cc99f34153205b1cb540901/librt-0.11.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:0cad8a4d6a8ff03c9b76f9414caccd78e7cfbc8a2e12fa334d8e1d9932753783", size = 497517, upload-time = "2026-05-10T18:15:29.614Z" }, - { url = "https://files.pythonhosted.org/packages/cd/43/acdd5ce317cb46e8253ca9bfbdb8b12e68a24d745949336a7f3d5fb79ba0/librt-0.11.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f37aa505b3cf60701562eddb32df74b12a9e380c207fd8b06dd157a943ac7ea0", size = 538878, upload-time = "2026-05-10T18:15:30.928Z" }, - { url = "https://files.pythonhosted.org/packages/29/b5/7a25bb12e3172839f647f196b3e988318b7bb1ca7501732a225c4dce2ec0/librt-0.11.0-cp310-cp310-win32.whl", hash = "sha256:94663a21534637f0e787ec2a2a756022df6e5b7b2335a5cdd7d8e33d68a2af89", size = 100070, upload-time = "2026-05-10T18:15:32.551Z" }, - { url = "https://files.pythonhosted.org/packages/c6/0d/ebbcf4d77999c02c937b05d2b90ff4cd4dcc7e9a365ba132329ac1fe7a0f/librt-0.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:dec7db73758c2b54953fd8b7fe348c45188fe26b39ee18446196edd08453a5d4", size = 117918, upload-time = "2026-05-10T18:15:33.678Z" }, { url = "https://files.pythonhosted.org/packages/fe/87/2bf31fe17587b29e3f93ec31421e2b1e1c3e349b8bf6c7c313dbad1d5340/librt-0.11.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:93d95bd45b7d58343d8b90d904450a545144eec19a002511163426f8ab1fae29", size = 141092, upload-time = "2026-05-10T18:15:34.795Z" }, { url = "https://files.pythonhosted.org/packages/cf/08/5c5bf772920b7ebac6e32bc91a643e0ab3870199c0b542356d3baa83970a/librt-0.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ee278c769a713638cdacd4c0436d72156e75df3ebc0166ab2b9dc43acc386c9", size = 142035, upload-time = "2026-05-10T18:15:36.242Z" }, { url = "https://files.pythonhosted.org/packages/06/20/662a03d254e5b000d838e8b345d83303ddb768c080fd488e40634c0fa66b/librt-0.11.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f230cb1cbc9faaa616f9a678f530ebcf186e414b6bcbd88b960e4ba1b92428d5", size = 475022, upload-time = "2026-05-10T18:15:37.56Z" }, @@ -512,17 +439,6 @@ version = "3.0.3" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e8/4b/3541d44f3937ba468b75da9eebcae497dcf67adb65caa16760b0a6807ebb/markupsafe-3.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559", size = 11631, upload-time = "2025-09-27T18:36:05.558Z" }, - { url = "https://files.pythonhosted.org/packages/98/1b/fbd8eed11021cabd9226c37342fa6ca4e8a98d8188a8d9b66740494960e4/markupsafe-3.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419", size = 12057, upload-time = "2025-09-27T18:36:07.165Z" }, - { url = "https://files.pythonhosted.org/packages/40/01/e560d658dc0bb8ab762670ece35281dec7b6c1b33f5fbc09ebb57a185519/markupsafe-3.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695", size = 22050, upload-time = "2025-09-27T18:36:08.005Z" }, - { url = "https://files.pythonhosted.org/packages/af/cd/ce6e848bbf2c32314c9b237839119c5a564a59725b53157c856e90937b7a/markupsafe-3.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591", size = 20681, upload-time = "2025-09-27T18:36:08.881Z" }, - { url = "https://files.pythonhosted.org/packages/c9/2a/b5c12c809f1c3045c4d580b035a743d12fcde53cf685dbc44660826308da/markupsafe-3.0.3-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c", size = 20705, upload-time = "2025-09-27T18:36:10.131Z" }, - { url = "https://files.pythonhosted.org/packages/cf/e3/9427a68c82728d0a88c50f890d0fc072a1484de2f3ac1ad0bfc1a7214fd5/markupsafe-3.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f", size = 21524, upload-time = "2025-09-27T18:36:11.324Z" }, - { url = "https://files.pythonhosted.org/packages/bc/36/23578f29e9e582a4d0278e009b38081dbe363c5e7165113fad546918a232/markupsafe-3.0.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6", size = 20282, upload-time = "2025-09-27T18:36:12.573Z" }, - { url = "https://files.pythonhosted.org/packages/56/21/dca11354e756ebd03e036bd8ad58d6d7168c80ce1fe5e75218e4945cbab7/markupsafe-3.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1", size = 20745, upload-time = "2025-09-27T18:36:13.504Z" }, - { url = "https://files.pythonhosted.org/packages/87/99/faba9369a7ad6e4d10b6a5fbf71fa2a188fe4a593b15f0963b73859a1bbd/markupsafe-3.0.3-cp310-cp310-win32.whl", hash = "sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa", size = 14571, upload-time = "2025-09-27T18:36:14.779Z" }, - { url = "https://files.pythonhosted.org/packages/d6/25/55dc3ab959917602c96985cb1253efaa4ff42f71194bddeb61eb7278b8be/markupsafe-3.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8", size = 15056, upload-time = "2025-09-27T18:36:16.125Z" }, - { url = "https://files.pythonhosted.org/packages/d0/9e/0a02226640c255d1da0b8d12e24ac2aa6734da68bff14c05dd53b94a0fc3/markupsafe-3.0.3-cp310-cp310-win_arm64.whl", hash = "sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1", size = 13932, upload-time = "2025-09-27T18:36:17.311Z" }, { url = "https://files.pythonhosted.org/packages/08/db/fefacb2136439fc8dd20e797950e749aa1f4997ed584c62cfb8ef7c2be0e/markupsafe-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad", size = 11631, upload-time = "2025-09-27T18:36:18.185Z" }, { url = "https://files.pythonhosted.org/packages/e1/2e/5898933336b61975ce9dc04decbc0a7f2fee78c30353c5efba7f2d6ff27a/markupsafe-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a", size = 12058, upload-time = "2025-09-27T18:36:19.444Z" }, { url = "https://files.pythonhosted.org/packages/1d/09/adf2df3699d87d1d8184038df46a9c80d78c0148492323f4693df54e17bb/markupsafe-3.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50", size = 24287, upload-time = "2025-09-27T18:36:20.768Z" }, @@ -600,18 +516,10 @@ dependencies = [ { name = "librt", marker = "platform_python_implementation != 'PyPy'" }, { name = "mypy-extensions" }, { name = "pathspec" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/82/15/cca9d88503549ed6fedeaa1d448cdddd542ee8a490232d732e278036fbf2/mypy-2.1.0.tar.gz", hash = "sha256:81e76ad12c2d804512e9b13240d1588316531bfba07558286078bfbce9613633", size = 3898359, upload-time = "2026-05-11T18:37:36.237Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a4/71/d351dca3e9b30da2328ee9d445c88b8388072808ebfbc49eb69d30b67749/mypy-2.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:11a6beb180257a805961aea9ec591bbd0bd17f1e18d35b8456d57aee5bedfedc", size = 14778792, upload-time = "2026-05-11T18:36:23.605Z" }, - { url = "https://files.pythonhosted.org/packages/2f/45/7d51594b644c17c0bcf74ed8cd5fc33b324276d708e8506f220b70dab9d9/mypy-2.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8ef78c1d306bbf9a8a12f526c44902c9c28dffd6c52c52bf6a72641ce18d3849", size = 13645739, upload-time = "2026-05-11T18:37:22.752Z" }, - { url = "https://files.pythonhosted.org/packages/65/01/455c31b170e9468265074840bf18863a8482a24103fdaabe4e199392aa5f/mypy-2.1.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c209a90853081ff01d01ee895cafe10f7db1474e0d95beaeef0f6c1db9119bbd", size = 14074199, upload-time = "2026-05-11T18:35:09.292Z" }, - { url = "https://files.pythonhosted.org/packages/41/5a/93093f0b29a9e982deafde698f740a2eb2e05886e79ccf0594c7fd5413a3/mypy-2.1.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:47cebf61abde7c088a4e27718a8b13a81655686b2e9c251f5c0915a802248166", size = 14953128, upload-time = "2026-05-11T18:31:57.678Z" }, - { url = "https://files.pythonhosted.org/packages/7f/2f/a196f5331d96170ad3d28f144d2aba690d4b2911381f68d51e489c7ab82a/mypy-2.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d57a90ae5e872138a425ec328edbc9b235d1934c4377881a33ec05b341acc9a8", size = 15249378, upload-time = "2026-05-11T18:33:00.101Z" }, - { url = "https://files.pythonhosted.org/packages/54/de/94d321cc12da9f71341ac0c270efbed5c725750c7b4c334d957de9a087d9/mypy-2.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:aea7f7a8a55b459c34275fc468ada6ca7c173a5e43a68f5dbe588a563d8a06b8", size = 11060994, upload-time = "2026-05-11T18:33:18.848Z" }, - { url = "https://files.pythonhosted.org/packages/e1/62/0c27ca55219a7c764a7fb88c7bb2b7b2f9780ade8bbf16bc8ed8400eef6b/mypy-2.1.0-cp310-cp310-win_arm64.whl", hash = "sha256:c989640253f0d76843e9c6c1bbf4bd48c5e85ada61bde4beb37cb3eca035685e", size = 9976743, upload-time = "2026-05-11T18:31:25.554Z" }, { url = "https://files.pythonhosted.org/packages/0a/a1/639f3024794a2a15899cb90707fe02e044c4412794c39c5769fd3df2e2ef/mypy-2.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a683016b16fe2f572dc04c72be7ee0504ac1605a265d0200f5cea695fb788f41", size = 14691685, upload-time = "2026-05-11T18:33:27.973Z" }, { url = "https://files.pythonhosted.org/packages/3b/08/9a585dea4325f20d8b80dc78623fa50d1fd2173b710f6237afd6ba6ab39b/mypy-2.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1a293c534adb55271fef24a26da04b855540a8c13cc07bc5917b9fd2c394f2ca", size = 13555165, upload-time = "2026-05-11T18:32:16.107Z" }, { url = "https://files.pythonhosted.org/packages/81/dc/7c42cc9c6cb01e8eb09961f1f738741d3e9c7e9d5c5b30ec69222625cd5f/mypy-2.1.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7406f4d048e71e576f5356d317e5b0a9e666dfd966bd99f9d14ca06e1a341538", size = 13994376, upload-time = "2026-05-11T18:32:39.256Z" }, @@ -662,9 +570,6 @@ wheels = [ [[package]] name = "nameparser" source = { editable = "." } -dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, -] [package.dev-dependencies] dev = [ @@ -675,8 +580,7 @@ dev = [ { name = "pytest-cov" }, { name = "pytest-timeout" }, { name = "ruff" }, - { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] @@ -737,12 +641,10 @@ version = "9.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, { name = "iniconfig" }, { name = "packaging" }, { name = "pluggy" }, { name = "pygments" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/84/0e/b5858858d74958632c49b72cb25a3976ff9f632397626715be71c89d3971/pytest-9.1.0.tar.gz", hash = "sha256:41dd9148c08072446394cefd3d79701701335a9f4cae69ba92e39f6c7f5c061c", size = 1634181, upload-time = "2026-06-13T18:52:45.983Z" } wheels = [ @@ -842,49 +744,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5e/f5/0c41cb68dcae6b7de4fac4188a3a9589e21fb31df21ea3a2e888db95e6c9/soupsieve-2.8.4-py3-none-any.whl", hash = "sha256:e7e6b0769c8f51ed59acab6e994b00621096cfb1c640a7509295987388fbaf65", size = 37304, upload-time = "2026-05-24T13:55:55.406Z" }, ] -[[package]] -name = "sphinx" -version = "8.1.3" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.11'", -] -dependencies = [ - { name = "alabaster" }, - { name = "babel" }, - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "docutils", version = "0.21.2", source = { registry = "https://pypi.org/simple" } }, - { name = "imagesize" }, - { name = "jinja2" }, - { name = "packaging" }, - { name = "pygments" }, - { name = "requests" }, - { name = "snowballstemmer" }, - { name = "sphinxcontrib-applehelp" }, - { name = "sphinxcontrib-devhelp" }, - { name = "sphinxcontrib-htmlhelp" }, - { name = "sphinxcontrib-jsmath" }, - { name = "sphinxcontrib-qthelp" }, - { name = "sphinxcontrib-serializinghtml" }, - { name = "tomli" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/be0b61178fe2cdcb67e2a92fc9ebb488e3c51c4f74a36a7824c0adf23425/sphinx-8.1.3.tar.gz", hash = "sha256:43c1911eecb0d3e161ad78611bc905d1ad0e523e4ddc202a58a821773dc4c927", size = 8184611, upload-time = "2024-10-13T20:27:13.93Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/26/60/1ddff83a56d33aaf6f10ec8ce84b4c007d9368b21008876fceda7e7381ef/sphinx-8.1.3-py3-none-any.whl", hash = "sha256:09719015511837b76bf6e03e42eb7595ac8c2e41eeb9c29c5b755c6b677992a2", size = 3487125, upload-time = "2024-10-13T20:27:10.448Z" }, -] - [[package]] name = "sphinx" version = "9.0.4" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.11.*'", + "python_full_version < '3.12'", ] dependencies = [ { name = "alabaster" }, { name = "babel" }, { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "docutils", version = "0.22.4", source = { registry = "https://pypi.org/simple" } }, + { name = "docutils" }, { name = "imagesize" }, { name = "jinja2" }, { name = "packaging" }, @@ -916,7 +787,7 @@ dependencies = [ { name = "alabaster" }, { name = "babel" }, { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "docutils", version = "0.22.4", source = { registry = "https://pypi.org/simple" } }, + { name = "docutils" }, { name = "imagesize" }, { name = "jinja2" }, { name = "packaging" }, @@ -941,8 +812,7 @@ name = "sphinx-basic-ng" version = "1.0.0b2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/98/0b/a866924ded68efec7a1759587a4e478aec7559d8165fac8b2ad1c0e774d6/sphinx_basic_ng-1.0.0b2.tar.gz", hash = "sha256:9ec55a47c90c8c002b5960c57492ec3021f5193cb26cebc2dc4ea226848651c9", size = 20736, upload-time = "2023-07-08T18:40:54.166Z" } From 0a416ed8585da4bafdf320ab8366f5625e167e62 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 11 Jul 2026 23:44:22 -0700 Subject: [PATCH 03/88] Fail loud on malformed Token spans and non-string text Co-Authored-By: Claude Fable 5 --- nameparser/_types.py | 15 +++++++++++++-- tests/v2/test_types.py | 12 ++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/nameparser/_types.py b/nameparser/_types.py index cd75926..77b7b89 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -42,9 +42,20 @@ class Token: tags: frozenset[str] = frozenset() def __post_init__(self) -> None: - if not self.text: - raise ValueError("Token.text must be a non-empty string") + if not isinstance(self.text, str) or not self.text: + raise ValueError( + f"Token.text must be a non-empty string, got {self.text!r}" + ) if self.span is not None: + if not ( + isinstance(self.span, tuple) + and len(self.span) == 2 + and all(isinstance(v, int) for v in self.span) + ): + raise ValueError( + f"invalid span {self.span!r}: expected a (start, end) " + "pair of ints or None" + ) start, end = self.span if start < 0 or end < start: raise ValueError( diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 80db2bb..e1ba6bf 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -37,6 +37,18 @@ def test_token_rejects_negative_span(): Token("x", (-1, 1), Role.GIVEN) +def test_token_rejects_malformed_span_shapes(): + with pytest.raises(ValueError, match="expected a \\(start, end\\) pair"): + Token("x", (0, 4, 9), Role.GIVEN) # type: ignore[arg-type] + with pytest.raises(ValueError, match="expected a \\(start, end\\) pair"): + Token("x", 5, Role.GIVEN) # type: ignore[arg-type] + + +def test_token_rejects_non_string_text(): + with pytest.raises(ValueError, match="got None"): + Token(None, None, Role.GIVEN) # type: ignore[arg-type] + + def test_token_is_frozen_and_hashable(): t = Token("Juan", (0, 4), Role.GIVEN) with pytest.raises(AttributeError): From 3f3d4488a3136b5fc4040f2d4b7160cc2b5588fb Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 11 Jul 2026 23:47:16 -0700 Subject: [PATCH 04/88] Add v2 AmbiguityKind StrEnum and Ambiguity type Co-Authored-By: Claude Fable 5 --- nameparser/_types.py | 30 +++++++++++++++++++++++++++++- tests/v2/test_types.py | 20 ++++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/nameparser/_types.py b/nameparser/_types.py index 77b7b89..43bc470 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -6,7 +6,7 @@ from __future__ import annotations from dataclasses import dataclass -from enum import Enum +from enum import Enum, StrEnum from typing import NamedTuple @@ -63,3 +63,31 @@ def __post_init__(self) -> None: ) object.__setattr__(self, "span", Span(start, end)) object.__setattr__(self, "tags", frozenset(self.tags)) + + +class AmbiguityKind(StrEnum): + """Stable identifiers (API); members ARE their string values.""" + + ORDER = "order" + SUFFIX_OR_NICKNAME = "suffix-or-nickname" + PARTICLE_OR_GIVEN = "particle-or-given" + UNBALANCED_DELIMITER = "unbalanced-delimiter" + COMMA_STRUCTURE = "comma-structure" + + +@dataclass(frozen=True, slots=True) +class Ambiguity: + kind: AmbiguityKind + detail: str + tokens: tuple[Token, ...] + + def __post_init__(self) -> None: + if not isinstance(self.kind, AmbiguityKind): + try: + object.__setattr__(self, "kind", AmbiguityKind(self.kind)) + except ValueError: + valid = ", ".join(k.value for k in AmbiguityKind) + raise ValueError( + f"unknown AmbiguityKind {self.kind!r}; valid kinds: {valid}" + ) from None + object.__setattr__(self, "tokens", tuple(self.tokens)) diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index e1ba6bf..0389bad 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -54,3 +54,23 @@ def test_token_is_frozen_and_hashable(): with pytest.raises(AttributeError): t.text = "X" # type: ignore[misc] assert hash(t) == hash(Token("Juan", (0, 4), Role.GIVEN)) + + +from nameparser._types import Ambiguity, AmbiguityKind + + +def test_ambiguity_kind_members_are_their_string_values(): + assert AmbiguityKind.PARTICLE_OR_GIVEN == "particle-or-given" + assert AmbiguityKind("order") is AmbiguityKind.ORDER + + +def test_ambiguity_construction_coerces_kind_string(): + t = Token("Van", (0, 3), Role.GIVEN, frozenset({"particle"})) + a = Ambiguity("particle-or-given", "leading 'van' may be a particle", (t,)) + assert a.kind is AmbiguityKind.PARTICLE_OR_GIVEN + assert a.tokens == (t,) + + +def test_ambiguity_rejects_unknown_kind(): + with pytest.raises(ValueError, match="particle-or-given"): + Ambiguity("no-such-kind", "detail", ()) From de78ebf7f648354fd5d81cbbe9e718495fc9151e Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 11 Jul 2026 23:50:10 -0700 Subject: [PATCH 05/88] Hoist v2 test imports to the top-of-file block Co-Authored-By: Claude Fable 5 --- tests/v2/test_types.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 0389bad..42424a9 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -1,6 +1,6 @@ import pytest -from nameparser._types import Role, Span, Token +from nameparser._types import Ambiguity, AmbiguityKind, Role, Span, Token def test_role_declaration_order_is_canonical_field_order(): @@ -56,9 +56,6 @@ def test_token_is_frozen_and_hashable(): assert hash(t) == hash(Token("Juan", (0, 4), Role.GIVEN)) -from nameparser._types import Ambiguity, AmbiguityKind - - def test_ambiguity_kind_members_are_their_string_values(): assert AmbiguityKind.PARTICLE_OR_GIVEN == "particle-or-given" assert AmbiguityKind("order") is AmbiguityKind.ORDER From 072e2da1f50a05f152b00baaef2651c084501b03 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 11 Jul 2026 23:53:52 -0700 Subject: [PATCH 06/88] Validate Ambiguity detail and token element types Co-Authored-By: Claude Fable 5 --- nameparser/_types.py | 13 ++++++++++++- tests/v2/test_types.py | 10 ++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/nameparser/_types.py b/nameparser/_types.py index 43bc470..7081069 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -90,4 +90,15 @@ def __post_init__(self) -> None: raise ValueError( f"unknown AmbiguityKind {self.kind!r}; valid kinds: {valid}" ) from None - object.__setattr__(self, "tokens", tuple(self.tokens)) + if not isinstance(self.detail, str) or not self.detail: + raise ValueError( + f"Ambiguity.detail must be a non-empty string, got {self.detail!r}" + ) + toks = tuple(self.tokens) + for tok in toks: + if not isinstance(tok, Token): + raise ValueError( + f"Ambiguity.tokens must contain only Token instances, " + f"got {tok!r}" + ) + object.__setattr__(self, "tokens", toks) diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 42424a9..573e1c4 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -71,3 +71,13 @@ def test_ambiguity_construction_coerces_kind_string(): def test_ambiguity_rejects_unknown_kind(): with pytest.raises(ValueError, match="particle-or-given"): Ambiguity("no-such-kind", "detail", ()) + + +def test_ambiguity_rejects_non_token_elements(): + with pytest.raises(ValueError, match="only Token instances"): + Ambiguity("order", "detail", ("not-a-token",)) # type: ignore[arg-type] + + +def test_ambiguity_rejects_empty_detail(): + with pytest.raises(ValueError, match="non-empty string"): + Ambiguity("order", "", ()) From ef6d2b833de95b13cc8790f235a43d71d6edd3c8 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 11 Jul 2026 23:54:31 -0700 Subject: [PATCH 07/88] Add v2 ParsedName with constructor-enforced span invariants Co-Authored-By: Claude Fable 5 --- nameparser/_types.py | 44 +++++++++++++++++++++++++++ tests/v2/test_types.py | 67 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 110 insertions(+), 1 deletion(-) diff --git a/nameparser/_types.py b/nameparser/_types.py index 7081069..2d9bdbc 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -102,3 +102,47 @@ def __post_init__(self) -> None: f"got {tok!r}" ) object.__setattr__(self, "tokens", toks) + + +@dataclass(frozen=True, slots=True) +class ParsedName: + """Immutable result of a parse. Constructor-enforced invariants: + spans ascending, non-overlapping, in bounds of `original`; every + Ambiguity's tokens are a subset of `tokens`. Provenance semantics + (text == original[span] for parser-produced names) are documented, + not enforced -- transforms like replace() legitimately break them. + """ + + original: str + tokens: tuple[Token, ...] + ambiguities: tuple[Ambiguity, ...] = () + + def __post_init__(self) -> None: + object.__setattr__(self, "tokens", tuple(self.tokens)) + object.__setattr__(self, "ambiguities", tuple(self.ambiguities)) + prev_end = 0 + for tok in self.tokens: + if tok.span is None: + continue + if tok.span.end > len(self.original): + raise ValueError( + f"token {tok.text!r} span {tuple(tok.span)} is out of " + f"bounds for original of length {len(self.original)}" + ) + if tok.span.start < prev_end: + raise ValueError( + f"token spans must be ascending and non-overlapping; " + f"token {tok.text!r} at {tuple(tok.span)} begins before " + f"offset {prev_end}" + ) + prev_end = tok.span.end + for amb in self.ambiguities: + for tok in amb.tokens: + if tok not in self.tokens: + raise ValueError( + f"Ambiguity token {tok.text!r} is not a subset of " + f"this ParsedName's tokens" + ) + + def __bool__(self) -> bool: + return bool(self.tokens) diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 573e1c4..5dcd79e 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -1,6 +1,6 @@ import pytest -from nameparser._types import Ambiguity, AmbiguityKind, Role, Span, Token +from nameparser._types import Ambiguity, AmbiguityKind, ParsedName, Role, Span, Token def test_role_declaration_order_is_canonical_field_order(): @@ -81,3 +81,68 @@ def test_ambiguity_rejects_non_token_elements(): def test_ambiguity_rejects_empty_detail(): with pytest.raises(ValueError, match="non-empty string"): Ambiguity("order", "", ()) + + +def _pn(original, tokens, ambiguities=()): + return ParsedName(original=original, tokens=tuple(tokens), + ambiguities=tuple(ambiguities)) + + +def test_parsedname_accepts_valid_spans_and_is_truthy(): + pn = _pn("John Smith", [ + Token("John", (0, 4), Role.GIVEN), + Token("Smith", (5, 10), Role.FAMILY), + ]) + assert bool(pn) is True + + +def test_empty_parse_is_falsy(): + assert bool(_pn("", [])) is False + assert bool(_pn(" ", [])) is False + + +def test_parsedname_rejects_out_of_bounds_span(): + with pytest.raises(ValueError, match="out of bounds"): + _pn("John", [Token("Johnny", (0, 6), Role.GIVEN)]) + + +def test_parsedname_rejects_overlapping_spans(): + with pytest.raises(ValueError, match="ascending"): + _pn("John Smith", [ + Token("John", (0, 4), Role.GIVEN), + Token("ohn S", (1, 6), Role.FAMILY), + ]) + + +def test_parsedname_rejects_descending_spans(): + with pytest.raises(ValueError, match="ascending"): + _pn("John Smith", [ + Token("Smith", (5, 10), Role.FAMILY), + Token("John", (0, 4), Role.GIVEN), + ]) + + +def test_synthetic_tokens_skip_span_checks(): + pn = _pn("John Smith", [ + Token("John", (0, 4), Role.GIVEN), + Token("Qux", None, Role.MIDDLE), + Token("Smith", (5, 10), Role.FAMILY), + ]) + assert len(pn.tokens) == 3 + + +def test_ambiguity_tokens_must_be_subset_of_parse_tokens(): + inside = Token("Van", (0, 3), Role.GIVEN) + outside = Token("Zzz", None, Role.GIVEN) + with pytest.raises(ValueError, match="subset"): + _pn("Van Johnson", + [inside, Token("Johnson", (4, 11), Role.FAMILY)], + [Ambiguity(AmbiguityKind.PARTICLE_OR_GIVEN, "d", (outside,))]) + + +def test_parsedname_equality_is_strict_structural(): + a = _pn("John", [Token("John", (0, 4), Role.GIVEN)]) + b = _pn("John", [Token("John", (0, 4), Role.GIVEN)]) + c = _pn("John ", [Token("John", (0, 4), Role.GIVEN)]) + assert a == b and hash(a) == hash(b) + assert a != c # different original: not interchangeable From 4a79080a171d872e89808d3da441abde968a61b9 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 11 Jul 2026 23:59:49 -0700 Subject: [PATCH 08/88] Validate ParsedName element types and original Co-Authored-By: Claude Fable 5 --- nameparser/_types.py | 16 ++++++++++++++++ tests/v2/test_types.py | 12 ++++++++++++ 2 files changed, 28 insertions(+) diff --git a/nameparser/_types.py b/nameparser/_types.py index 2d9bdbc..efede7e 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -118,8 +118,24 @@ class ParsedName: ambiguities: tuple[Ambiguity, ...] = () def __post_init__(self) -> None: + if not isinstance(self.original, str): + raise ValueError( + f"ParsedName.original must be a str, got {self.original!r}" + ) object.__setattr__(self, "tokens", tuple(self.tokens)) object.__setattr__(self, "ambiguities", tuple(self.ambiguities)) + for tok in self.tokens: + if not isinstance(tok, Token): + raise ValueError( + f"ParsedName.tokens must contain only Token instances, " + f"got {tok!r}" + ) + for amb in self.ambiguities: + if not isinstance(amb, Ambiguity): + raise ValueError( + f"ParsedName.ambiguities must contain only Ambiguity " + f"instances, got {amb!r}" + ) prev_end = 0 for tok in self.tokens: if tok.span is None: diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 5dcd79e..b14f9f9 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -146,3 +146,15 @@ def test_parsedname_equality_is_strict_structural(): c = _pn("John ", [Token("John", (0, 4), Role.GIVEN)]) assert a == b and hash(a) == hash(b) assert a != c # different original: not interchangeable + + +def test_parsedname_rejects_non_str_original(): + with pytest.raises(ValueError, match="must be a str"): + _pn(None, []) # type: ignore[arg-type] + + +def test_parsedname_rejects_non_token_and_non_ambiguity_elements(): + with pytest.raises(ValueError, match="only Token instances"): + _pn("x", ["not-a-token"]) # type: ignore[list-item] + with pytest.raises(ValueError, match="only Ambiguity instances"): + _pn("John", [Token("John", (0, 4), Role.GIVEN)], ["nope"]) # type: ignore[list-item] From 8e8160b870e892637ccf345cf4c928a116a0d960 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 00:00:21 -0700 Subject: [PATCH 09/88] Add v2 ParsedName string properties, derived views, and as_dict Co-Authored-By: Claude Fable 5 --- nameparser/_types.py | 74 ++++++++++++++++++++++++++++++++++++++++++ tests/v2/test_types.py | 58 +++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+) diff --git a/nameparser/_types.py b/nameparser/_types.py index efede7e..9d31a79 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -162,3 +162,77 @@ def __post_init__(self) -> None: def __bool__(self) -> bool: return bool(self.tokens) + + # -- string views (canonical order = Role declaration order) -------- + + def _text_for(self, *roles: Role, tag: str | None = None, + without_tag: str | None = None) -> str: + joiner = ", " if roles == (Role.SUFFIX,) else " " + parts = [] + for tok in self.tokens: + if tok.role not in roles: + continue + if tag is not None and tag not in tok.tags: + continue + if without_tag is not None and without_tag in tok.tags: + continue + parts.append(tok.text) + return joiner.join(parts) + + @property + def title(self) -> str: + return self._text_for(Role.TITLE) + + @property + def given(self) -> str: + return self._text_for(Role.GIVEN) + + @property + def middle(self) -> str: + return self._text_for(Role.MIDDLE) + + @property + def family(self) -> str: + return self._text_for(Role.FAMILY) + + @property + def suffix(self) -> str: + return self._text_for(Role.SUFFIX) + + @property + def nickname(self) -> str: + return self._text_for(Role.NICKNAME) + + @property + def maiden(self) -> str: + return self._text_for(Role.MAIDEN) + + # -- derived views (filters over roles + STABLE tags only) ---------- + + @property + def family_particles(self) -> str: + return self._text_for(Role.FAMILY, tag="particle") + + @property + def family_base(self) -> str: + return self._text_for(Role.FAMILY, without_tag="particle") + + @property + def surnames(self) -> str: + return self._text_for(Role.MIDDLE, Role.FAMILY) + + @property + def given_names(self) -> str: + return self._text_for(Role.GIVEN, Role.MIDDLE) + + # -- structured access ---------------------------------------------- + + def tokens_for(self, role: Role) -> tuple[Token, ...]: + return tuple(t for t in self.tokens if t.role is role) + + def as_dict(self, include_empty: bool = True) -> dict[str, str]: + # _text_for handles the suffix ", "-join (single-role SUFFIX call) + d = {role.value: self._text_for(role) for role in Role} + if not include_empty: + d = {k: v for k, v in d.items() if v} + return d diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index b14f9f9..ba5241b 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -158,3 +158,61 @@ def test_parsedname_rejects_non_token_and_non_ambiguity_elements(): _pn("x", ["not-a-token"]) # type: ignore[list-item] with pytest.raises(ValueError, match="only Ambiguity instances"): _pn("John", [Token("John", (0, 4), Role.GIVEN)], ["nope"]) # type: ignore[list-item] + + +def _delavega(): + # "Dr. Juan de la Vega III" -- hand-built, spans verified by hand + # 0123456789012345678901234 + return _pn("Dr. Juan de la Vega III", [ + Token("Dr.", (0, 3), Role.TITLE), + Token("Juan", (4, 8), Role.GIVEN), + Token("de", (9, 11), Role.FAMILY, frozenset({"particle"})), + Token("la", (12, 14), Role.FAMILY, frozenset({"particle"})), + Token("Vega", (15, 19), Role.FAMILY), + Token("III", (20, 23), Role.SUFFIX), + ]) + + +def test_string_properties_join_by_role(): + pn = _delavega() + assert pn.title == "Dr." + assert pn.given == "Juan" + assert pn.middle == "" + assert pn.family == "de la Vega" + assert pn.suffix == "III" + assert pn.nickname == "" + assert pn.maiden == "" + + +def test_suffix_joins_with_comma_space(): + pn = _pn("John Smith PhD MD", [ + Token("John", (0, 4), Role.GIVEN), + Token("Smith", (5, 10), Role.FAMILY), + Token("PhD", (11, 14), Role.SUFFIX), + Token("MD", (15, 17), Role.SUFFIX), + ]) + assert pn.suffix == "PhD, MD" + + +def test_derived_views_filter_on_stable_particle_tag(): + pn = _delavega() + assert pn.family_particles == "de la" + assert pn.family_base == "Vega" + assert pn.surnames == "de la Vega" # middle + family + assert pn.given_names == "Juan" # given + middle + + +def test_tokens_for_preserves_order(): + pn = _delavega() + assert [t.text for t in pn.tokens_for(Role.FAMILY)] == ["de", "la", "Vega"] + assert pn.tokens_for(Role.NICKNAME) == () + + +def test_as_dict_canonical_order_and_empty_filtering(): + pn = _delavega() + d = pn.as_dict() + assert list(d) == ["title", "given", "middle", "family", + "suffix", "nickname", "maiden"] + assert d["family"] == "de la Vega" and d["middle"] == "" + d2 = pn.as_dict(include_empty=False) + assert list(d2) == ["title", "given", "family", "suffix"] From 3190abd3c34eb07ca3d9b9aead8146aa879f1737 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 00:05:29 -0700 Subject: [PATCH 10/88] Add v2 ParsedName.replace() with positional synthetic tokens Co-Authored-By: Claude Fable 5 --- nameparser/_types.py | 37 +++++++++++++++++++++++++++++++++++++ tests/v2/test_types.py | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/nameparser/_types.py b/nameparser/_types.py index 9d31a79..25fa4ef 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -236,3 +236,40 @@ def as_dict(self, include_empty: bool = True) -> dict[str, str]: if not include_empty: d = {k: v for k, v in d.items() if v} return d + + # -- editing ---------------------------------------------------------- + + def replace(self, **fields: str) -> "ParsedName": + """Return a new ParsedName with the named fields re-tokenized as + synthetic tokens (span=None). Whitespace-splits each value; an + empty value clears the field. original is unchanged (provenance). + """ + by_value = {role.value: role for role in Role} + for key in fields: + if key not in by_value: + raise TypeError( + f"unknown field {key!r}; expected one of " + f"{', '.join(by_value)}" + ) + + def synthetic(value: str, role: Role) -> list[Token]: + return [Token(word, None, role) for word in value.split()] + + replaced = {by_value[k]: v for k, v in fields.items()} + new_tokens: list[Token] = [] + emitted: set[Role] = set() + for tok in self.tokens: + if tok.role in replaced: + if tok.role not in emitted: + new_tokens.extend(synthetic(replaced[tok.role], tok.role)) + emitted.add(tok.role) + continue + new_tokens.append(tok) + for role, value in replaced.items(): + if role not in emitted: + new_tokens.extend(synthetic(value, role)) + kept = tuple( + amb for amb in self.ambiguities + if all(t in new_tokens for t in amb.tokens) + ) + return ParsedName(self.original, tuple(new_tokens), kept) diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index ba5241b..7889ead 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -216,3 +216,44 @@ def test_as_dict_canonical_order_and_empty_filtering(): assert d["family"] == "de la Vega" and d["middle"] == "" d2 = pn.as_dict(include_empty=False) assert list(d2) == ["title", "given", "family", "suffix"] + + +def test_replace_swaps_field_with_synthetic_tokens_in_place(): + pn = _delavega() + pn2 = pn.replace(given="Jean Paul") + assert pn2.given == "Jean Paul" + assert pn2.family == "de la Vega" # untouched + assert pn2.original == pn.original # provenance unchanged + assert all(t.span is None for t in pn2.tokens_for(Role.GIVEN)) + assert pn.given == "Juan" # source object unchanged + # positional: synthetic given tokens sit where the old ones were + assert [t.role for t in pn2.tokens][:3] == [Role.TITLE, Role.GIVEN, Role.GIVEN] + + +def test_replace_adds_missing_field_at_end(): + pn = _pn("John Smith", [ + Token("John", (0, 4), Role.GIVEN), + Token("Smith", (5, 10), Role.FAMILY), + ]) + pn2 = pn.replace(suffix="Jr") + assert pn2.suffix == "Jr" + assert pn2.tokens[-1].role is Role.SUFFIX + + +def test_replace_with_empty_string_clears_field(): + pn = _delavega() + assert pn.replace(title="").title == "" + + +def test_replace_rejects_unknown_field(): + with pytest.raises(TypeError, match="given"): + _delavega().replace(firstname="X") + + +def test_replace_drops_ambiguities_referencing_removed_tokens(): + van = Token("Van", (0, 3), Role.GIVEN) + pn = _pn("Van Johnson", + [van, Token("Johnson", (4, 11), Role.FAMILY)], + [Ambiguity(AmbiguityKind.PARTICLE_OR_GIVEN, "d", (van,))]) + assert pn.replace(given="Bob").ambiguities == () + assert pn.replace(family="Smith").ambiguities != () From 06a7e221aa680c6399e90c7bcd0ab8f6f66555fd Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 00:10:32 -0700 Subject: [PATCH 11/88] Fail loud on non-str replace values; canonical append order - Validate that all replace() field values are str (user error if None) - Append missing-role synthetics in canonical Role order, not kwargs order - Unquote return type annotation (postponed annotations in effect) Co-Authored-By: Claude Fable 5 --- nameparser/_types.py | 14 +++++++++----- tests/v2/test_types.py | 11 +++++++++++ 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/nameparser/_types.py b/nameparser/_types.py index 25fa4ef..515eeb3 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -239,18 +239,22 @@ def as_dict(self, include_empty: bool = True) -> dict[str, str]: # -- editing ---------------------------------------------------------- - def replace(self, **fields: str) -> "ParsedName": + def replace(self, **fields: str) -> ParsedName: """Return a new ParsedName with the named fields re-tokenized as synthetic tokens (span=None). Whitespace-splits each value; an empty value clears the field. original is unchanged (provenance). """ by_value = {role.value: role for role in Role} - for key in fields: + for key, value in fields.items(): if key not in by_value: raise TypeError( f"unknown field {key!r}; expected one of " f"{', '.join(by_value)}" ) + if not isinstance(value, str): + raise TypeError( + f"field {key!r} must be a str, got {value!r}" + ) def synthetic(value: str, role: Role) -> list[Token]: return [Token(word, None, role) for word in value.split()] @@ -265,9 +269,9 @@ def synthetic(value: str, role: Role) -> list[Token]: emitted.add(tok.role) continue new_tokens.append(tok) - for role, value in replaced.items(): - if role not in emitted: - new_tokens.extend(synthetic(value, role)) + for role in Role: + if role in replaced and role not in emitted: + new_tokens.extend(synthetic(replaced[role], role)) kept = tuple( amb for amb in self.ambiguities if all(t in new_tokens for t in amb.tokens) diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 7889ead..3d620fa 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -257,3 +257,14 @@ def test_replace_drops_ambiguities_referencing_removed_tokens(): [Ambiguity(AmbiguityKind.PARTICLE_OR_GIVEN, "d", (van,))]) assert pn.replace(given="Bob").ambiguities == () assert pn.replace(family="Smith").ambiguities != () + + +def test_replace_rejects_non_str_value(): + with pytest.raises(TypeError, match="must be a str"): + _delavega().replace(given=None) # type: ignore[arg-type] + + +def test_replace_appends_missing_roles_in_canonical_order(): + pn = _pn("John", [Token("John", (0, 4), Role.GIVEN)]) + pn2 = pn.replace(maiden="X", suffix="Y") + assert [t.role for t in pn2.tokens] == [Role.GIVEN, Role.SUFFIX, Role.MAIDEN] From aeefe51156beb8becaaf1b8f7e7fdb82795b59d9 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 00:11:02 -0700 Subject: [PATCH 12/88] Add v2 ParsedName.comparison_key() Casefolded seven-component tuple in canonical Role order for dedup, dict keys, and sorting. Semantic layer comparison; __eq__ remains strict. Co-Authored-By: Claude Fable 5 --- nameparser/_types.py | 8 ++++++++ tests/v2/test_types.py | 20 ++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/nameparser/_types.py b/nameparser/_types.py index 515eeb3..d0cc802 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -277,3 +277,11 @@ def synthetic(value: str, role: Role) -> list[Token]: if all(t in new_tokens for t in amb.tokens) ) return ParsedName(self.original, tuple(new_tokens), kept) + + # -- comparison ------------------------------------------------------- + + def comparison_key(self) -> tuple[str, ...]: + """Casefolded seven components in canonical order, for dedup, + dict keys, and sorting. The semantic layer; __eq__ stays strict. + """ + return tuple(self._text_for(role).casefold() for role in Role) diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 3d620fa..4ddfab0 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -268,3 +268,23 @@ def test_replace_appends_missing_roles_in_canonical_order(): pn = _pn("John", [Token("John", (0, 4), Role.GIVEN)]) pn2 = pn.replace(maiden="X", suffix="Y") assert [t.role for t in pn2.tokens] == [Role.GIVEN, Role.SUFFIX, Role.MAIDEN] + + +def test_comparison_key_is_casefolded_canonical_seven_tuple(): + pn = _delavega() + assert pn.comparison_key() == ( + "dr.", "juan", "", "de la vega", "iii", "", "", + ) + upper = _pn("JUAN DE LA VEGA", [ + Token("JUAN", (0, 4), Role.GIVEN), + Token("DE", (5, 7), Role.FAMILY, frozenset({"particle"})), + Token("LA", (8, 10), Role.FAMILY, frozenset({"particle"})), + Token("VEGA", (11, 15), Role.FAMILY), + ]) + lower = _pn("juan de la vega", [ + Token("juan", (0, 4), Role.GIVEN), + Token("de", (5, 7), Role.FAMILY, frozenset({"particle"})), + Token("la", (8, 10), Role.FAMILY, frozenset({"particle"})), + Token("vega", (11, 15), Role.FAMILY), + ]) + assert upper.comparison_key() == lower.comparison_key() From 9d781c79d605022639220de5a4eb799682521e0f Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 00:16:54 -0700 Subject: [PATCH 13/88] Add v2 Lexicon with canonical storage and v1-sourced default Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 118 +++++++++++++++++++++++++++++++++++++++ tests/v2/test_lexicon.py | 48 ++++++++++++++++ 2 files changed, 166 insertions(+) create mode 100644 nameparser/_lexicon.py create mode 100644 tests/v2/test_lexicon.py diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py new file mode 100644 index 0000000..545147c --- /dev/null +++ b/nameparser/_lexicon.py @@ -0,0 +1,118 @@ +"""Immutable vocabulary configuration for the 2.0 API. + +Layering: may import nameparser.config DATA modules (titles, suffixes, +prefixes, conjunctions, capitalization, bound_first_names) as the single +source of vocabulary during 2.x -- never nameparser.config itself, never +nameparser.parser. Enforced by tests/v2/test_layering.py. +""" +from __future__ import annotations + +import functools +from collections.abc import Iterable, Mapping +from dataclasses import dataclass, field +from types import MappingProxyType + +#: Vocabulary set fields, in declaration order. add()/remove()/__or__ +#: (a later task) operate on exactly these; capitalization_exceptions is +#: deliberately excluded (its entries are pairs -- use dataclasses.replace). +_VOCAB_FIELDS = ( + "titles", "given_name_titles", "suffix_acronyms", "suffix_words", + "suffix_acronyms_ambiguous", "particles", "particles_ambiguous", + "conjunctions", "bound_given_names", "maiden_markers", +) + + +def _normalize(word: str) -> str: + """Casefold and strip periods -- v1's lc(). Membership tests never + re-normalize because construction already did.""" + return word.casefold().replace(".", "").strip() + + +def _normset(entries: Iterable[str]) -> frozenset[str]: + result = frozenset(_normalize(w) for w in entries) + return frozenset(w for w in result if w) + + +@dataclass(frozen=True, slots=True) +class Lexicon: + titles: frozenset[str] = frozenset() + given_name_titles: frozenset[str] = frozenset() + suffix_acronyms: frozenset[str] = frozenset() + suffix_words: frozenset[str] = frozenset() + suffix_acronyms_ambiguous: frozenset[str] = frozenset() + particles: frozenset[str] = frozenset() + particles_ambiguous: frozenset[str] = frozenset() + conjunctions: frozenset[str] = frozenset() + bound_given_names: frozenset[str] = frozenset() + maiden_markers: frozenset[str] = frozenset() + # Canonical storage: sorted tuple of (key, value) pairs. The + # constructor tolerates any Mapping (or pair iterable) at runtime and + # canonicalizes here; this closes the caller-aliasing hole and keeps + # Lexicon hashable. Read via capitalization_exceptions_map. + capitalization_exceptions: tuple[tuple[str, str], ...] = () + _cap_map: Mapping[str, str] = field( + init=False, repr=False, compare=False, hash=False, + default_factory=lambda: MappingProxyType({})) + + def __post_init__(self) -> None: + for name in _VOCAB_FIELDS: + object.__setattr__(self, name, _normset(getattr(self, name))) + raw = self.capitalization_exceptions + pairs = raw.items() if isinstance(raw, Mapping) else raw + canonical = tuple(sorted((_normalize(k), v) for k, v in pairs)) + object.__setattr__(self, "capitalization_exceptions", canonical) + object.__setattr__(self, "_cap_map", MappingProxyType(dict(canonical))) + if not self.particles_ambiguous <= self.particles: + extra = ", ".join(sorted(self.particles_ambiguous - self.particles)) + raise ValueError( + f"particles_ambiguous must be a subset of particles; " + f"not in particles: {extra}" + ) + + @property + def capitalization_exceptions_map(self) -> Mapping[str, str]: + return self._cap_map + + # -- constructors ---------------------------------------------------- + + @classmethod + def empty(cls) -> Lexicon: + return cls() + + @classmethod + def default(cls) -> Lexicon: + return _default_lexicon() + + +@functools.cache +def _default_lexicon() -> Lexicon: + # v1 data modules are the single source of vocabulary through 2.x. + from nameparser.config.bound_first_names import BOUND_FIRST_NAMES + from nameparser.config.capitalization import CAPITALIZATION_EXCEPTIONS + from nameparser.config.conjunctions import CONJUNCTIONS + from nameparser.config.prefixes import NON_FIRST_NAME_PREFIXES, PREFIXES + from nameparser.config.suffixes import ( + SUFFIX_ACRONYMS, SUFFIX_ACRONYMS_AMBIGUOUS, SUFFIX_NOT_ACRONYMS, + ) + from nameparser.config.titles import FIRST_NAME_TITLES, TITLES + + # v1 data modules export plain `set[str]`; wrap each at this call site + # so the strictly-typed frozenset[str] fields never see a bare set. + return Lexicon( + titles=frozenset(TITLES), + given_name_titles=frozenset(FIRST_NAME_TITLES), + suffix_acronyms=frozenset(SUFFIX_ACRONYMS), + suffix_words=frozenset(SUFFIX_NOT_ACRONYMS), + suffix_acronyms_ambiguous=frozenset(SUFFIX_ACRONYMS_AMBIGUOUS), + particles=frozenset(PREFIXES), + # FLIPPED from v1: v1 marks the never-given subset; v2 marks the + # may-be-given subset (migration: complement translation). + particles_ambiguous=frozenset(PREFIXES - NON_FIRST_NAME_PREFIXES), + conjunctions=frozenset(CONJUNCTIONS), + bound_given_names=frozenset(BOUND_FIRST_NAMES), + maiden_markers=frozenset({"née", "nee", "geb"}), + # pass canonical pair-tuples so this strictly-typed call site never + # feeds a Mapping to the tuple-annotated field; __post_init__ + # still tolerates a Mapping at runtime for interactive use + capitalization_exceptions=tuple(sorted(CAPITALIZATION_EXCEPTIONS.items())), + ) diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py new file mode 100644 index 0000000..74440cb --- /dev/null +++ b/tests/v2/test_lexicon.py @@ -0,0 +1,48 @@ +import dataclasses + +import pytest + +from nameparser._lexicon import Lexicon + + +def test_entries_are_normalized_at_construction(): + lex = Lexicon(titles={"Dr.", "MR"}) + assert lex.titles == frozenset({"dr", "mr"}) + + +def test_default_sources_v1_vocabulary(): + lex = Lexicon.default() + assert "dr" in lex.titles + assert "van" in lex.particles + assert "phd" in lex.suffix_acronyms + # flipped model: 'dos' is never-given in v1, so NOT ambiguous here + assert "dos" in lex.particles and "dos" not in lex.particles_ambiguous + assert "van" in lex.particles_ambiguous + # v1's CAPITALIZATION_EXCEPTIONS maps 'phd' -> 'Ph.D.' (verbatim, not + # normalized -- only keys are casefolded/period-stripped at + # construction, values pass through unchanged). + assert lex.capitalization_exceptions_map["phd"] == "Ph.D." + + +def test_default_is_cached_single_instance(): + assert Lexicon.default() is Lexicon.default() + + +def test_particles_ambiguous_must_be_subset_of_particles(): + with pytest.raises(ValueError, match="subset"): + Lexicon(particles_ambiguous={"van"}) + + +def test_capitalization_exceptions_canonical_and_no_aliasing(): + exceptions = {"phd": "PhD", "ii": "II"} + lex = Lexicon.empty() + lex2 = dataclasses.replace(lex, capitalization_exceptions=exceptions) + exceptions["iii"] = "III" # mutate caller's dict afterwards + assert "iii" not in lex2.capitalization_exceptions_map + # canonical: insertion order does not affect equality/hash + lex3 = dataclasses.replace(lex, capitalization_exceptions={"ii": "II", "phd": "PhD"}) + assert lex2 == lex3 and hash(lex2) == hash(lex3) + + +def test_lexicon_is_hashable(): + assert isinstance(hash(Lexicon.default()), int) From 61362bd2a50aa98fcb969d245ba58403188c1513 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 00:24:27 -0700 Subject: [PATCH 14/88] Fail loud on malformed Lexicon input; dedupe exception keys Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 34 ++++++++++++++++++++++++++++++---- tests/v2/test_lexicon.py | 22 ++++++++++++++++++++++ 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 545147c..ffe75ed 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -28,8 +28,22 @@ def _normalize(word: str) -> str: return word.casefold().replace(".", "").strip() -def _normset(entries: Iterable[str]) -> frozenset[str]: - result = frozenset(_normalize(w) for w in entries) +def _normset(entries: Iterable[str], field_name: str) -> frozenset[str]: + # Reject a bare str before iterating: iterating "dr" would silently + # yield the single characters {'d', 'r'} -- the set(str) footgun on + # the primary customization surface. + if isinstance(entries, str): + raise ValueError( + f"Lexicon.{field_name} must be an iterable of strings, " + f"not a bare string" + ) + items = tuple(entries) # materialize once; entries may be a generator + for w in items: + if not isinstance(w, str): + raise ValueError( + f"Lexicon.{field_name} entries must be strings, got {w!r}" + ) + result = frozenset(_normalize(w) for w in items) return frozenset(w for w in result if w) @@ -56,10 +70,22 @@ class Lexicon: def __post_init__(self) -> None: for name in _VOCAB_FIELDS: - object.__setattr__(self, name, _normset(getattr(self, name))) + object.__setattr__(self, name, _normset(getattr(self, name), name)) raw = self.capitalization_exceptions pairs = raw.items() if isinstance(raw, Mapping) else raw - canonical = tuple(sorted((_normalize(k), v) for k, v in pairs)) + # Dedupe on the NORMALIZED key before storing so the tuple and the + # map always agree ("Ph.D." and "phd" collide after normalization). + # Last occurrence wins, matching dict semantics and the right-bias + # rule used elsewhere. + deduped: dict[str, str] = {} + for k, v in pairs: + if not isinstance(k, str) or not isinstance(v, str): + raise ValueError( + f"capitalization_exceptions entries must be " + f"str -> str, got {k!r}: {v!r}" + ) + deduped[_normalize(k)] = v + canonical = tuple(sorted(deduped.items())) object.__setattr__(self, "capitalization_exceptions", canonical) object.__setattr__(self, "_cap_map", MappingProxyType(dict(canonical))) if not self.particles_ambiguous <= self.particles: diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index 74440cb..8072371 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -46,3 +46,25 @@ def test_capitalization_exceptions_canonical_and_no_aliasing(): def test_lexicon_is_hashable(): assert isinstance(hash(Lexicon.default()), int) + + +def test_lexicon_rejects_bare_string_vocab(): + with pytest.raises(ValueError, match="bare string"): + Lexicon(titles="dr") # type: ignore[arg-type] + + +def test_lexicon_rejects_non_str_vocab_entries(): + with pytest.raises(ValueError, match="entries must be strings"): + Lexicon(titles={"Dr.", 42}) # type: ignore[arg-type] + + +def test_colliding_exception_keys_dedupe_last_wins(): + lex = Lexicon(capitalization_exceptions={"Ph.D.": "A", "phd": "B"}) + assert lex.capitalization_exceptions == (("phd", "B"),) + rebuilt = Lexicon(capitalization_exceptions=lex.capitalization_exceptions_map) + assert rebuilt == lex and hash(rebuilt) == hash(lex) + + +def test_lexicon_rejects_non_str_exception_values(): + with pytest.raises(ValueError, match="str -> str"): + Lexicon(capitalization_exceptions={"phd": 42}) # type: ignore[dict-item] From f216e5fce24cd2e32452997b461702ea9faeae5b Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 00:28:23 -0700 Subject: [PATCH 15/88] Add v2 Lexicon composition: add/remove and field-wise union Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 48 ++++++++++++++++++++++++++++++++++++++++ tests/v2/test_lexicon.py | 37 +++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index ffe75ed..2499cc0 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -7,6 +7,7 @@ """ from __future__ import annotations +import dataclasses import functools from collections.abc import Iterable, Mapping from dataclasses import dataclass, field @@ -109,6 +110,53 @@ def empty(cls) -> Lexicon: def default(cls) -> Lexicon: return _default_lexicon() + # -- composition ------------------------------------------------------ + + def _edit(self, op: str, entries: Mapping[str, Iterable[str]]) -> Lexicon: + updates: dict[str, frozenset[str]] = {} + for name, words in entries.items(): + if name == "capitalization_exceptions": + raise TypeError( + "capitalization_exceptions holds key->value pairs; " + "use dataclasses.replace(lexicon, " + "capitalization_exceptions={...}) instead of " + f"{op}()" + ) + if name not in _VOCAB_FIELDS: + raise TypeError( + f"unknown Lexicon field {name!r}; valid fields: " + f"{', '.join(_VOCAB_FIELDS)}" + ) + current: frozenset[str] = getattr(self, name) + normalized = _normset(words, name) + updates[name] = (current | normalized if op == "add" + else current - normalized) + # mypy's dataclasses.replace() typing checks a **dict's single + # value type against every field's type (it can't see which keys + # are actually present behind the unpack), so a homogeneous + # frozenset[str] dict is flagged against the tuple/Mapping-typed + # capitalization_exceptions/_cap_map fields even though this dict + # never contains those keys (guarded above). + return dataclasses.replace(self, **updates) # type: ignore[arg-type] + + def add(self, **entries: Iterable[str]) -> Lexicon: + return self._edit("add", entries) + + def remove(self, **entries: Iterable[str]) -> Lexicon: + return self._edit("remove", entries) + + def __or__(self, other: Lexicon) -> Lexicon: + if not isinstance(other, Lexicon): + return NotImplemented + updates: dict[str, object] = { + name: getattr(self, name) | getattr(other, name) + for name in _VOCAB_FIELDS + } + # right-biased on key conflicts, mirroring later-wins for scalars + merged = dict(self._cap_map) | dict(other._cap_map) + updates["capitalization_exceptions"] = tuple(sorted(merged.items())) + return dataclasses.replace(self, **updates) # type: ignore[arg-type] + @functools.cache def _default_lexicon() -> Lexicon: diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index 8072371..a952a92 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -68,3 +68,40 @@ def test_colliding_exception_keys_dedupe_last_wins(): def test_lexicon_rejects_non_str_exception_values(): with pytest.raises(ValueError, match="str -> str"): Lexicon(capitalization_exceptions={"phd": 42}) # type: ignore[dict-item] + + +def test_add_and_remove_return_new_lexicons(): + # "zqtitle" is a synthetic word absent from v1's TITLES data (unlike + # e.g. "dra", the feminine "dr." abbreviation, which is already there). + base = Lexicon.default() + lex = base.add(titles={"zqtitle"}).remove(suffix_words={"bishop"}) + assert "zqtitle" in lex.titles and "zqtitle" not in base.titles + assert "bishop" not in lex.suffix_words + + +def test_add_unknown_field_raises_with_valid_names(): + with pytest.raises(TypeError, match="prefixes"): + Lexicon.default().add(prefixes={"van"}) # v1 name: helpful error + + +def test_add_capitalization_exceptions_raises_pointing_at_replace(): + with pytest.raises(TypeError, match="dataclasses.replace"): + Lexicon.default().add(capitalization_exceptions={"x": "X"}) + + +def test_union_is_fieldwise_and_right_biased_for_exceptions(): + a = dataclasses.replace(Lexicon.empty(), + capitalization_exceptions={"phd": "PhD"}) + a = a.add(titles={"dr"}) + b = dataclasses.replace(Lexicon.empty(), + capitalization_exceptions={"phd": "Ph.D."}) + b = b.add(titles={"mr"}) + u = a | b + assert u.titles == frozenset({"dr", "mr"}) + assert u.capitalization_exceptions_map["phd"] == "Ph.D." # right wins + + +def test_remove_breaking_subset_invariant_raises(): + lex = Lexicon(particles={"van"}, particles_ambiguous={"van"}) + with pytest.raises(ValueError, match="subset"): + lex.remove(particles={"van"}) # would orphan particles_ambiguous From 0be9e5323aea265182676e050fe18cdd014a7242 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 00:33:31 -0700 Subject: [PATCH 16/88] Add v2 Policy with order-spec validation and PatronymicRule Co-Authored-By: Claude Fable 5 --- nameparser/_policy.py | 72 +++++++++++++++++++++++++++++++++++++++++ tests/v2/test_policy.py | 49 ++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 nameparser/_policy.py create mode 100644 tests/v2/test_policy.py diff --git a/nameparser/_policy.py b/nameparser/_policy.py new file mode 100644 index 0000000..4067e30 --- /dev/null +++ b/nameparser/_policy.py @@ -0,0 +1,72 @@ +"""Immutable behavior configuration for the 2.0 API. + +Layering: imports nameparser._types only (tests/v2/test_layering.py, +added in a later task, enforces this). +""" +from __future__ import annotations + +from dataclasses import dataclass +from enum import StrEnum + +from nameparser._types import Role + + +class PatronymicRule(StrEnum): + """Stable rule names (API); implementations live in the pipeline.""" + + EAST_SLAVIC = "east-slavic" + TURKIC = "turkic" + + +# Order-spec constants (#270). Each reads as its contents because roles +# are named given/family, not first/last. +GIVEN_FIRST = (Role.GIVEN, Role.MIDDLE, Role.FAMILY) +FAMILY_FIRST = (Role.FAMILY, Role.GIVEN, Role.MIDDLE) +FAMILY_FIRST_GIVEN_LAST = (Role.FAMILY, Role.MIDDLE, Role.GIVEN) + +_NAME_ROLES = frozenset({Role.GIVEN, Role.MIDDLE, Role.FAMILY}) + + +@dataclass(frozen=True, slots=True) +class Policy: + name_order: tuple[Role, Role, Role] = GIVEN_FIRST + patronymic_rules: frozenset[PatronymicRule] = frozenset() + middle_as_family: bool = False # v1's middle_name_as_last + # v1 default delimiter set (#273) + nickname_delimiters: frozenset[tuple[str, str]] = frozenset( + {("'", "'"), ('"', '"'), ("(", ")")} + ) + # empty by default (v1 parity); route ("(", ")") here to send + # parenthesized content to maiden instead of nickname (#274) + maiden_delimiters: frozenset[tuple[str, str]] = frozenset() + extra_suffix_delimiters: frozenset[str] = frozenset() + lenient_comma_suffixes: bool = True + strip_emoji: bool = True + strip_bidi: bool = True # replaces v1's CONSTANTS.regexes.bidi = False + + def __post_init__(self) -> None: + order = tuple(self.name_order) + if len(order) != 3 or set(order) != _NAME_ROLES: + raise ValueError( + f"name_order must be a permutation of (Role.GIVEN, " + f"Role.MIDDLE, Role.FAMILY), got {order!r}; use " + f"GIVEN_FIRST, FAMILY_FIRST, or FAMILY_FIRST_GIVEN_LAST" + ) + object.__setattr__(self, "name_order", order) + try: + rules = frozenset(PatronymicRule(r) for r in self.patronymic_rules) + except ValueError: + valid = ", ".join(r.value for r in PatronymicRule) + raise ValueError( + f"unknown patronymic rule in {set(self.patronymic_rules)!r}; " + f"valid rules: {valid}" + ) from None + object.__setattr__(self, "patronymic_rules", rules) + for pairs_name in ("nickname_delimiters", "maiden_delimiters"): + for pair in getattr(self, pairs_name): + if (len(pair) != 2 or not all( + isinstance(s, str) and s for s in pair)): + raise ValueError( + f"{pairs_name} entries must be pairs of non-empty " + f"strings, got {pair!r}" + ) diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py new file mode 100644 index 0000000..430cfec --- /dev/null +++ b/tests/v2/test_policy.py @@ -0,0 +1,49 @@ +import dataclasses + +import pytest + +from nameparser._policy import ( + FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, GIVEN_FIRST, + PatronymicRule, Policy, +) +from nameparser._types import Role + + +def test_order_constants_read_as_their_contents(): + assert GIVEN_FIRST == (Role.GIVEN, Role.MIDDLE, Role.FAMILY) + assert FAMILY_FIRST == (Role.FAMILY, Role.GIVEN, Role.MIDDLE) + assert FAMILY_FIRST_GIVEN_LAST == (Role.FAMILY, Role.MIDDLE, Role.GIVEN) + + +def test_policy_defaults(): + p = Policy() + assert p.name_order == GIVEN_FIRST + assert p.patronymic_rules == frozenset() + assert ("(", ")") in p.nickname_delimiters + assert p.maiden_delimiters == frozenset() + assert p.strip_emoji and p.strip_bidi and p.lenient_comma_suffixes + + +def test_policy_is_hashable_and_replaceable(): + p = dataclasses.replace(Policy(), name_order=FAMILY_FIRST) + assert p.name_order == FAMILY_FIRST + assert isinstance(hash(p), int) + + +def test_name_order_must_be_permutation_and_error_names_constants(): + with pytest.raises(ValueError, match="FAMILY_FIRST_GIVEN_LAST"): + Policy(name_order=(Role.TITLE, Role.GIVEN, Role.FAMILY)) + with pytest.raises(ValueError, match="GIVEN_FIRST"): + Policy(name_order=(Role.GIVEN, Role.GIVEN, Role.FAMILY)) + + +def test_patronymic_rules_coerce_and_reject(): + p = Policy(patronymic_rules=frozenset({"east-slavic"})) + assert p.patronymic_rules == frozenset({PatronymicRule.EAST_SLAVIC}) + with pytest.raises(ValueError, match="east-slavic, turkic"): + Policy(patronymic_rules=frozenset({"klingon"})) + + +def test_delimiter_pairs_must_be_nonempty_string_pairs(): + with pytest.raises(ValueError, match="non-empty"): + Policy(nickname_delimiters=frozenset({("", ")")})) From 21c817b7f9405f989b7686f4c2b8816eebc6d5ec Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 00:38:56 -0700 Subject: [PATCH 17/88] Reject malformed Policy delimiter pairs and patronymic rules Co-Authored-By: Claude Fable 5 --- nameparser/_policy.py | 17 +++++++++++------ tests/v2/test_policy.py | 12 ++++++++++++ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 4067e30..e8095b3 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -53,20 +53,25 @@ def __post_init__(self) -> None: f"GIVEN_FIRST, FAMILY_FIRST, or FAMILY_FIRST_GIVEN_LAST" ) object.__setattr__(self, "name_order", order) + if isinstance(self.patronymic_rules, str): + raise ValueError( + f"patronymic_rules must be an iterable of rule names, " + f"not a bare string: {self.patronymic_rules!r}" + ) try: rules = frozenset(PatronymicRule(r) for r in self.patronymic_rules) - except ValueError: + except (TypeError, ValueError): valid = ", ".join(r.value for r in PatronymicRule) raise ValueError( - f"unknown patronymic rule in {set(self.patronymic_rules)!r}; " + f"unknown patronymic rule in {self.patronymic_rules!r}; " f"valid rules: {valid}" ) from None object.__setattr__(self, "patronymic_rules", rules) for pairs_name in ("nickname_delimiters", "maiden_delimiters"): for pair in getattr(self, pairs_name): - if (len(pair) != 2 or not all( - isinstance(s, str) and s for s in pair)): + if (not isinstance(pair, tuple) or len(pair) != 2 + or not all(isinstance(s, str) and s for s in pair)): raise ValueError( - f"{pairs_name} entries must be pairs of non-empty " - f"strings, got {pair!r}" + f"{pairs_name} entries must be (open, close) tuples " + f"of non-empty strings, got {pair!r}" ) diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index 430cfec..cef3f4f 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -47,3 +47,15 @@ def test_patronymic_rules_coerce_and_reject(): def test_delimiter_pairs_must_be_nonempty_string_pairs(): with pytest.raises(ValueError, match="non-empty"): Policy(nickname_delimiters=frozenset({("", ")")})) + + +def test_delimiter_pair_rejects_two_char_string(): + with pytest.raises(ValueError, match="tuples"): + Policy(nickname_delimiters=frozenset({"()"})) # type: ignore[arg-type] + + +def test_patronymic_rules_rejects_bare_string_and_non_iterable(): + with pytest.raises(ValueError, match="bare string"): + Policy(patronymic_rules="east-slavic") # type: ignore[arg-type] + with pytest.raises(ValueError, match="valid rules"): + Policy(patronymic_rules=5) # type: ignore[arg-type] From 719fee6d418ebcb80c22ff693546081d39ee8805 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 00:42:39 -0700 Subject: [PATCH 18/88] Coerce and validate all Policy collection fields Co-Authored-By: Claude Fable 5 --- nameparser/_policy.py | 19 ++++++++++++++++++- tests/v2/test_policy.py | 24 ++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/nameparser/_policy.py b/nameparser/_policy.py index e8095b3..1e6534b 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -68,10 +68,27 @@ def __post_init__(self) -> None: ) from None object.__setattr__(self, "patronymic_rules", rules) for pairs_name in ("nickname_delimiters", "maiden_delimiters"): - for pair in getattr(self, pairs_name): + pairs = tuple(getattr(self, pairs_name)) + for pair in pairs: if (not isinstance(pair, tuple) or len(pair) != 2 or not all(isinstance(s, str) and s for s in pair)): raise ValueError( f"{pairs_name} entries must be (open, close) tuples " f"of non-empty strings, got {pair!r}" ) + object.__setattr__(self, pairs_name, frozenset(pairs)) + if isinstance(self.extra_suffix_delimiters, str): + raise ValueError( + f"extra_suffix_delimiters must be an iterable of strings, " + f"not a bare string: {self.extra_suffix_delimiters!r}" + ) + delimiters = tuple(self.extra_suffix_delimiters) + for d in delimiters: + if not isinstance(d, str) or not d: + raise ValueError( + f"extra_suffix_delimiters entries must be non-empty " + f"strings, got {d!r}" + ) + object.__setattr__( + self, "extra_suffix_delimiters", frozenset(delimiters) + ) diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index cef3f4f..1a242e9 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -59,3 +59,27 @@ def test_patronymic_rules_rejects_bare_string_and_non_iterable(): Policy(patronymic_rules="east-slavic") # type: ignore[arg-type] with pytest.raises(ValueError, match="valid rules"): Policy(patronymic_rules=5) # type: ignore[arg-type] + + +def test_policy_delimiters_coerce_to_frozensets(): + p = Policy(nickname_delimiters=[("(", ")")]) # type: ignore[arg-type] + assert isinstance(p.nickname_delimiters, frozenset) + assert isinstance(hash(p), int) + assert p == Policy(nickname_delimiters=frozenset({("(", ")")})) + + +def test_policy_delimiters_do_not_alias_caller_containers(): + source = {("(", ")")} + p = Policy(nickname_delimiters=source) # type: ignore[arg-type] + source.add(("'", "'")) + assert ("'", "'") not in p.nickname_delimiters + + +def test_extra_suffix_delimiters_validated_and_coerced(): + with pytest.raises(ValueError, match="bare string"): + Policy(extra_suffix_delimiters="ab") # type: ignore[arg-type] + with pytest.raises(ValueError, match="non-empty strings"): + Policy(extra_suffix_delimiters={""}) + p = Policy(extra_suffix_delimiters=["-"]) # type: ignore[arg-type] + assert p.extra_suffix_delimiters == frozenset({"-"}) + assert isinstance(hash(p), int) From 0a47671a7692c6f3b4969db6e13fff9f618d1b25 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 00:45:38 -0700 Subject: [PATCH 19/88] Add v2 PolicyPatch with UNSET sentinel and declared composition Co-Authored-By: Claude Fable 5 --- nameparser/_policy.py | 55 +++++++++++++++++++++++++++++++++++++++-- tests/v2/test_policy.py | 32 +++++++++++++++++++++++- 2 files changed, 84 insertions(+), 3 deletions(-) diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 1e6534b..adee223 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -5,8 +5,9 @@ """ from __future__ import annotations -from dataclasses import dataclass -from enum import StrEnum +import dataclasses +from dataclasses import dataclass, field +from enum import Enum, StrEnum, auto from nameparser._types import Role @@ -92,3 +93,53 @@ def __post_init__(self) -> None: object.__setattr__( self, "extra_suffix_delimiters", frozenset(delimiters) ) + + +class _Unset(Enum): + UNSET = auto() + + +#: Sentinel for "this patch does not set this field" (picklable enum +#: member, distinguishable from every real value including None/False). +UNSET = _Unset.UNSET + +_UNION = {"compose": "union"} # field metadata: set-valued -> union + + +@dataclass(frozen=True, slots=True) +class PolicyPatch: + """A partial Policy: one field per Policy field, all defaulting to + UNSET. Composition per field is DECLARED via metadata -- set-valued + fields union, scalars override (later wins). Kept in lockstep with + Policy by the parity test in tests/v2/test_policy.py. + """ + + name_order: tuple[Role, Role, Role] | _Unset = UNSET + patronymic_rules: frozenset[PatronymicRule] | _Unset = field( + default=UNSET, metadata=_UNION) + middle_as_family: bool | _Unset = UNSET + nickname_delimiters: frozenset[tuple[str, str]] | _Unset = field( + default=UNSET, metadata=_UNION) + maiden_delimiters: frozenset[tuple[str, str]] | _Unset = field( + default=UNSET, metadata=_UNION) + extra_suffix_delimiters: frozenset[str] | _Unset = field( + default=UNSET, metadata=_UNION) + lenient_comma_suffixes: bool | _Unset = UNSET + strip_emoji: bool | _Unset = UNSET + strip_bidi: bool | _Unset = UNSET + + +def apply_patch(policy: Policy, patch: PolicyPatch) -> Policy: + """Fold a PolicyPatch onto a Policy. Policy.__post_init__ re-runs via + dataclasses.replace, so patched values are revalidated for free.""" + updates: dict[str, object] = {} + for f in dataclasses.fields(PolicyPatch): + value = getattr(patch, f.name) + if value is UNSET: + continue + if f.metadata.get("compose") == "union": + value = getattr(policy, f.name) | value + updates[f.name] = value + if not updates: + return policy + return dataclasses.replace(policy, **updates) # type: ignore[arg-type] diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index 1a242e9..f52bf1a 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -4,7 +4,7 @@ from nameparser._policy import ( FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, GIVEN_FIRST, - PatronymicRule, Policy, + PatronymicRule, Policy, PolicyPatch, UNSET, apply_patch, ) from nameparser._types import Role @@ -83,3 +83,33 @@ def test_extra_suffix_delimiters_validated_and_coerced(): p = Policy(extra_suffix_delimiters=["-"]) # type: ignore[arg-type] assert p.extra_suffix_delimiters == frozenset({"-"}) assert isinstance(hash(p), int) + + +def test_policy_patch_mirrors_policy_fields_and_types(): + policy_fields = {f.name for f in dataclasses.fields(Policy)} + patch_fields = {f.name for f in dataclasses.fields(PolicyPatch)} + assert policy_fields == patch_fields + + +def test_apply_patch_overrides_scalars_and_unions_sets(): + base = Policy(patronymic_rules=frozenset({PatronymicRule.EAST_SLAVIC})) + patch = PolicyPatch( + name_order=FAMILY_FIRST, + patronymic_rules=frozenset({PatronymicRule.TURKIC}), + ) + out = apply_patch(base, patch) + assert out.name_order == FAMILY_FIRST # override + assert out.patronymic_rules == frozenset( # union + {PatronymicRule.EAST_SLAVIC, PatronymicRule.TURKIC}) + assert out.strip_emoji is True # untouched + + +def test_apply_patch_with_empty_patch_returns_same_policy(): + base = Policy() + assert apply_patch(base, PolicyPatch()) is base + + +def test_unset_fields_are_distinguishable_from_defaults(): + patch = PolicyPatch(strip_emoji=True) # explicitly set to the default value + assert patch.strip_emoji is True + assert PolicyPatch().strip_emoji is UNSET From cd46673aede740fe54c51941dbde43142dc18c9d Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 00:48:45 -0700 Subject: [PATCH 20/88] Document the replace type-ignore; rename parity test Co-Authored-By: Claude Fable 5 --- nameparser/_policy.py | 2 ++ tests/v2/test_policy.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/nameparser/_policy.py b/nameparser/_policy.py index adee223..629ced9 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -142,4 +142,6 @@ def apply_patch(policy: Policy, patch: PolicyPatch) -> Policy: updates[f.name] = value if not updates: return policy + # Known mypy limitation with **dict-unpacked replace; see the full + # explanation at Lexicon._edit in _lexicon.py. return dataclasses.replace(policy, **updates) # type: ignore[arg-type] diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index f52bf1a..f5572d1 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -85,7 +85,7 @@ def test_extra_suffix_delimiters_validated_and_coerced(): assert isinstance(hash(p), int) -def test_policy_patch_mirrors_policy_fields_and_types(): +def test_policy_patch_mirrors_policy_field_names(): policy_fields = {f.name for f in dataclasses.fields(Policy)} patch_fields = {f.name for f in dataclasses.fields(PolicyPatch)} assert policy_fields == patch_fields From 29f603fe76991ad98e50b7925baa3965d4c84ad3 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 00:53:34 -0700 Subject: [PATCH 21/88] Canonicalize PolicyPatch union fields; lock field types in parity test Co-Authored-By: Claude Fable 5 --- nameparser/_policy.py | 20 ++++++++++++++++++++ tests/v2/test_policy.py | 19 +++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 629ced9..e6267d0 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -112,6 +112,9 @@ class PolicyPatch: UNSET. Composition per field is DECLARED via metadata -- set-valued fields union, scalars override (later wins). Kept in lockstep with Policy by the parity test in tests/v2/test_policy.py. + + Values are validated when the patch is applied (Policy's constructor + re-runs), not at patch construction. """ name_order: tuple[Role, Role, Role] | _Unset = UNSET @@ -128,6 +131,23 @@ class PolicyPatch: strip_emoji: bool | _Unset = UNSET strip_bidi: bool | _Unset = UNSET + def __post_init__(self) -> None: + # Canonicalize (but do NOT validate) union-composed fields so a + # patch built from a set/list literal is hashable and unions + # cleanly in apply_patch. + for f in dataclasses.fields(self): + if f.metadata.get("compose") != "union": + continue + value = getattr(self, f.name) + if value is UNSET: + continue + if isinstance(value, str): + raise ValueError( + f"{f.name} must be an iterable, " + f"not a bare string: {value!r}" + ) + object.__setattr__(self, f.name, frozenset(value)) + def apply_patch(policy: Policy, patch: PolicyPatch) -> Policy: """Fold a PolicyPatch onto a Policy. Policy.__post_init__ re-runs via diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index f5572d1..ac0293d 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -91,6 +91,25 @@ def test_policy_patch_mirrors_policy_field_names(): assert policy_fields == patch_fields +def test_policy_patch_mirrors_policy_field_types(): + for f in dataclasses.fields(Policy): + patch_annotation = PolicyPatch.__dataclass_fields__[f.name].type + assert patch_annotation == f"{f.type} | _Unset" + + +def test_policy_patch_canonicalizes_union_fields(): + p = PolicyPatch(extra_suffix_delimiters={"-"}) + assert isinstance(p.extra_suffix_delimiters, frozenset) + assert isinstance(hash(p), int) + out = apply_patch(Policy(), PolicyPatch(extra_suffix_delimiters=["-"])) # type: ignore[arg-type] + assert out.extra_suffix_delimiters == frozenset({"-"}) + + +def test_policy_patch_rejects_bare_string_union_fields(): + with pytest.raises(ValueError, match="bare string"): + PolicyPatch(extra_suffix_delimiters="ab") # type: ignore[arg-type] + + def test_apply_patch_overrides_scalars_and_unions_sets(): base = Policy(patronymic_rules=frozenset({PatronymicRule.EAST_SLAVIC})) patch = PolicyPatch( From 2d3dd044f3426612cc0b2d04babb5760a9808e7d Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 00:57:22 -0700 Subject: [PATCH 22/88] Add v2 Locale type Co-Authored-By: Claude Fable 5 --- nameparser/_locale.py | 40 ++++++++++++++++++++++++++++++++++++++++ tests/v2/test_locale.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 nameparser/_locale.py create mode 100644 tests/v2/test_locale.py diff --git a/nameparser/_locale.py b/nameparser/_locale.py new file mode 100644 index 0000000..8a6707d --- /dev/null +++ b/nameparser/_locale.py @@ -0,0 +1,40 @@ +"""The Locale type: a named delta over (Lexicon, Policy). + +A Locale dissolves at parser construction (parser_for, a later plan): +lexicon fragments union onto the base, the PolicyPatch folds via +apply_patch. Packs are pure data; they have no privileged capabilities. + +Layering: imports _lexicon and _policy only (tests/v2/test_layering.py, +added in a later task, enforces this). +""" +from __future__ import annotations + +from dataclasses import dataclass + +from nameparser._lexicon import Lexicon +from nameparser._policy import PolicyPatch + + +@dataclass(frozen=True, slots=True) +class Locale: + code: str + lexicon: Lexicon + policy: PolicyPatch = PolicyPatch() + + def __post_init__(self) -> None: + if not isinstance(self.code, str) or not self.code.strip(): + raise ValueError( + f"Locale.code must be a non-empty string, got {self.code!r}" + ) + if self.code != self.code.lower(): + raise ValueError( + f"Locale.code must be lowercase, got {self.code!r}" + ) + if not isinstance(self.lexicon, Lexicon): + raise ValueError( + f"Locale.lexicon must be a Lexicon, got {self.lexicon!r}" + ) + if not isinstance(self.policy, PolicyPatch): + raise ValueError( + f"Locale.policy must be a PolicyPatch, got {self.policy!r}" + ) diff --git a/tests/v2/test_locale.py b/tests/v2/test_locale.py new file mode 100644 index 0000000..6617708 --- /dev/null +++ b/tests/v2/test_locale.py @@ -0,0 +1,40 @@ +import pytest + +from nameparser._lexicon import Lexicon +from nameparser._locale import Locale +from nameparser._policy import PatronymicRule, PolicyPatch + + +def test_locale_holds_code_lexicon_fragment_and_patch(): + ru = Locale( + code="ru", + lexicon=Lexicon.empty(), + policy=PolicyPatch( + patronymic_rules=frozenset({PatronymicRule.EAST_SLAVIC})), + ) + assert ru.code == "ru" + assert ru.policy.patronymic_rules == frozenset( + {PatronymicRule.EAST_SLAVIC}) + + +def test_locale_defaults_to_empty_patch(): + assert Locale(code="xx", lexicon=Lexicon.empty()).policy == PolicyPatch() + + +def test_locale_code_must_be_nonempty_lowercase(): + with pytest.raises(ValueError, match="lowercase"): + Locale(code="RU", lexicon=Lexicon.empty()) + with pytest.raises(ValueError, match="non-empty"): + Locale(code="", lexicon=Lexicon.empty()) + + +def test_locale_is_hashable(): + loc = Locale(code="ru", lexicon=Lexicon.empty()) + assert isinstance(hash(loc), int) + + +def test_locale_validates_component_types(): + with pytest.raises(ValueError, match="Lexicon"): + Locale(code="ru", lexicon={"titles": set()}) # type: ignore[arg-type] + with pytest.raises(ValueError, match="PolicyPatch"): + Locale(code="ru", lexicon=Lexicon.empty(), policy={"name_order": None}) # type: ignore[arg-type] From 91845fdfaa26a70397c1f2afa9d73d8a0cd8ebf1 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 01:01:59 -0700 Subject: [PATCH 23/88] Reject whitespace in Locale codes Co-Authored-By: Claude Fable 5 --- nameparser/_locale.py | 4 ++++ tests/v2/test_locale.py | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/nameparser/_locale.py b/nameparser/_locale.py index 8a6707d..d483d34 100644 --- a/nameparser/_locale.py +++ b/nameparser/_locale.py @@ -30,6 +30,10 @@ def __post_init__(self) -> None: raise ValueError( f"Locale.code must be lowercase, got {self.code!r}" ) + if any(c.isspace() for c in self.code): + raise ValueError( + f"Locale.code must not contain whitespace, got {self.code!r}" + ) if not isinstance(self.lexicon, Lexicon): raise ValueError( f"Locale.lexicon must be a Lexicon, got {self.lexicon!r}" diff --git a/tests/v2/test_locale.py b/tests/v2/test_locale.py index 6617708..b3a8f54 100644 --- a/tests/v2/test_locale.py +++ b/tests/v2/test_locale.py @@ -28,6 +28,12 @@ def test_locale_code_must_be_nonempty_lowercase(): Locale(code="", lexicon=Lexicon.empty()) +def test_locale_code_rejects_whitespace(): + for bad in ("ru ", " ru", "ru\n", "r u"): + with pytest.raises(ValueError, match="whitespace"): + Locale(code=bad, lexicon=Lexicon.empty()) + + def test_locale_is_hashable(): loc = Locale(code="ru", lexicon=Lexicon.empty()) assert isinstance(hash(loc), int) From b88d3ac95934251184c843d792ded6587b46dd25 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 01:06:40 -0700 Subject: [PATCH 24/88] Drop exception keys that normalize to empty Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 5 ++++- tests/v2/test_lexicon.py | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 2499cc0..f7394b6 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -85,7 +85,10 @@ def __post_init__(self) -> None: f"capitalization_exceptions entries must be " f"str -> str, got {k!r}: {v!r}" ) - deduped[_normalize(k)] = v + normalized_key = _normalize(k) + if not normalized_key: + continue # mirror _normset's drop-empty rule + deduped[normalized_key] = v canonical = tuple(sorted(deduped.items())) object.__setattr__(self, "capitalization_exceptions", canonical) object.__setattr__(self, "_cap_map", MappingProxyType(dict(canonical))) diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index a952a92..944c4fb 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -58,6 +58,11 @@ def test_lexicon_rejects_non_str_vocab_entries(): Lexicon(titles={"Dr.", 42}) # type: ignore[arg-type] +def test_exception_keys_normalizing_to_empty_are_dropped(): + lex = Lexicon(capitalization_exceptions={"...": "X", "phd": "PhD"}) + assert lex.capitalization_exceptions == (("phd", "PhD"),) + + def test_colliding_exception_keys_dedupe_last_wins(): lex = Lexicon(capitalization_exceptions={"Ph.D.": "A", "phd": "B"}) assert lex.capitalization_exceptions == (("phd", "B"),) From 0838e9878d158b31c0dbc590cf132bb75cb0038f Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 01:08:36 -0700 Subject: [PATCH 25/88] Add bounded deviation-only reprs to all v2 types Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 19 +++++++++++++ nameparser/_locale.py | 12 ++++++++- nameparser/_policy.py | 24 +++++++++++++++++ nameparser/_types.py | 29 ++++++++++++++++++++ tests/v2/test_reprs.py | 61 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 144 insertions(+), 1 deletion(-) create mode 100644 tests/v2/test_reprs.py diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index f7394b6..9394899 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -99,6 +99,25 @@ def __post_init__(self) -> None: f"not in particles: {extra}" ) + def __repr__(self) -> str: + # Bounded: renders only which fields deviate from default() and by + # how many entries -- never the entries themselves (design rule, + # see nameparser._types module docstring). + default = Lexicon.default() + if self == default: + return "Lexicon(default)" + deltas = [] + for name in _VOCAB_FIELDS + ("capitalization_exceptions",): + mine = set(getattr(self, name)) + theirs = set(getattr(default, name)) + added, removed = len(mine - theirs), len(theirs - mine) + if added or removed: + delta = "".join( + part for part, n in ((f"+{added}", added), (f"-{removed}", removed)) if n + ) + deltas.append(f"{name}: {delta}") + return f"Lexicon(default + {', '.join(deltas)})" + @property def capitalization_exceptions_map(self) -> Mapping[str, str]: return self._cap_map diff --git a/nameparser/_locale.py b/nameparser/_locale.py index d483d34..2494b3e 100644 --- a/nameparser/_locale.py +++ b/nameparser/_locale.py @@ -9,10 +9,11 @@ """ from __future__ import annotations +import dataclasses from dataclasses import dataclass from nameparser._lexicon import Lexicon -from nameparser._policy import PolicyPatch +from nameparser._policy import UNSET, PolicyPatch @dataclass(frozen=True, slots=True) @@ -42,3 +43,12 @@ def __post_init__(self) -> None: raise ValueError( f"Locale.policy must be a PolicyPatch, got {self.policy!r}" ) + + def __repr__(self) -> str: + # Bounded: shows the code and which Policy fields the patch sets, + # never the Lexicon contents or the patched values themselves + # (design rule, see nameparser._types module docstring). + patched = [f.name for f in dataclasses.fields(self.policy) + if getattr(self.policy, f.name) is not UNSET] + suffix = f": {', '.join(patched)}" if patched else "" + return f"Locale({self.code!r}{suffix})" diff --git a/nameparser/_policy.py b/nameparser/_policy.py index e6267d0..7427f16 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -94,6 +94,30 @@ def __post_init__(self) -> None: self, "extra_suffix_delimiters", frozenset(delimiters) ) + def __repr__(self) -> str: + # Bounded: only fields that deviate from the default are shown + # (design rule, see nameparser._types module docstring). + constant_names = { + GIVEN_FIRST: "GIVEN_FIRST", + FAMILY_FIRST: "FAMILY_FIRST", + FAMILY_FIRST_GIVEN_LAST: "FAMILY_FIRST_GIVEN_LAST", + } + parts = [] + for f in dataclasses.fields(self): + value = getattr(self, f.name) + if value == f.default: + continue + if f.name == "name_order": + # Only 3 of the 6 possible role permutations have named + # constants; fall back to a compact role-name tuple for + # the rest so this can't KeyError on an unnamed order. + order_repr = constant_names.get( + value, "(" + ", ".join(r.name for r in value) + ")") + parts.append(f"name_order={order_repr}") + else: + parts.append(f"{f.name}={value!r}") + return f"Policy({', '.join(parts)})" + class _Unset(Enum): UNSET = auto() diff --git a/nameparser/_types.py b/nameparser/_types.py index d0cc802..97dd941 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -2,6 +2,11 @@ Layering (enforced by tests/v2/test_layering.py): this module imports nothing from nameparser -- it is the bottom of the dependency graph. + +Repr policy (applies to every v2 type's __repr__, across this module and +_lexicon.py/_policy.py/_locale.py): bounded output only. No repr may scale +with vocabulary size -- collections render as counts or deltas, never +contents. """ from __future__ import annotations @@ -64,6 +69,14 @@ def __post_init__(self) -> None: object.__setattr__(self, "span", Span(start, end)) object.__setattr__(self, "tags", frozenset(self.tags)) + def __repr__(self) -> str: + # Bounded output: a single token's text/span/role/tags, never + # scales with vocabulary size (design rule -- see module docstring). + where = (f"@{self.span.start}:{self.span.end}" + if self.span is not None else "@synthetic") + tags = f" {{{', '.join(sorted(self.tags))}}}" if self.tags else "" + return f"Token({self.text!r} {where} {self.role.name}{tags})" + class AmbiguityKind(StrEnum): """Stable identifiers (API); members ARE their string values.""" @@ -103,6 +116,10 @@ def __post_init__(self) -> None: ) object.__setattr__(self, "tokens", toks) + def __repr__(self) -> str: + texts = "/".join(repr(t.text) for t in self.tokens) + return f"Ambiguity({self.kind.value!r}: {texts})" + @dataclass(frozen=True, slots=True) class ParsedName: @@ -163,6 +180,18 @@ def __post_init__(self) -> None: def __bool__(self) -> bool: return bool(self.tokens) + def __repr__(self) -> str: + lines = [] + for role in Role: + text = self._text_for(role) + if text: + lines.append(f"\t{role.value}: {text!r}") + if self.ambiguities: + kinds = [a.kind.value for a in self.ambiguities] + lines.append(f"\tambiguities: {kinds!r}") + body = "\n".join(lines) + return f"" if lines else "" + # -- string views (canonical order = Role declaration order) -------- def _text_for(self, *roles: Role, tag: str | None = None, diff --git a/tests/v2/test_reprs.py b/tests/v2/test_reprs.py new file mode 100644 index 0000000..a5c421d --- /dev/null +++ b/tests/v2/test_reprs.py @@ -0,0 +1,61 @@ +import dataclasses + +from nameparser._lexicon import Lexicon +from nameparser._locale import Locale +from nameparser._policy import FAMILY_FIRST, PatronymicRule, Policy, PolicyPatch +from nameparser._types import Ambiguity, AmbiguityKind, ParsedName, Role, Token + + +def test_token_repr_is_compact(): + t = Token("de", (9, 11), Role.FAMILY, frozenset({"particle"})) + assert repr(t) == "Token('de' @9:11 FAMILY {particle})" + assert repr(Token("Jane", None, Role.GIVEN)) == "Token('Jane' @synthetic GIVEN)" + + +def test_ambiguity_repr_shows_kind_and_token_texts(): + van = Token("Van", (0, 3), Role.GIVEN) + a = Ambiguity(AmbiguityKind.PARTICLE_OR_GIVEN, "detail", (van,)) + assert repr(a) == "Ambiguity('particle-or-given': 'Van')" + + +def test_parsedname_repr_lists_nonempty_fields_in_canonical_order(): + pn = ParsedName("John Smith", ( + Token("John", (0, 4), Role.GIVEN), + Token("Smith", (5, 10), Role.FAMILY), + )) + assert repr(pn) == ( + "" + ) + + +def test_parsedname_repr_includes_ambiguities_line_when_present(): + van = Token("Van", (0, 3), Role.GIVEN) + pn = ParsedName("Van Johnson", + (van, Token("Johnson", (4, 11), Role.FAMILY)), + (Ambiguity(AmbiguityKind.PARTICLE_OR_GIVEN, "d", (van,)),)) + assert "ambiguities: ['particle-or-given']" in repr(pn) + + +def test_empty_parsedname_repr(): + assert repr(ParsedName("", ())) == "" + + +def test_policy_repr_shows_only_nondefault_fields(): + assert repr(Policy()) == "Policy()" + p = Policy(name_order=FAMILY_FIRST, strip_bidi=False) + assert repr(p) == "Policy(name_order=FAMILY_FIRST, strip_bidi=False)" + + +def test_lexicon_repr_is_bounded(): + assert repr(Lexicon.default()) == "Lexicon(default)" + lex = Lexicon.default().add(titles={"zqx", "zqy"}) + assert repr(lex) == "Lexicon(default + titles: +2)" + assert "zqx" not in repr(lex) # never dump contents + + +def test_locale_repr_shows_code_and_patched_fields(): + ru = Locale("ru", Lexicon.empty(), + PolicyPatch(patronymic_rules=frozenset( + {PatronymicRule.EAST_SLAVIC}))) + assert repr(ru) == "Locale('ru': patronymic_rules)" + assert repr(Locale("xx", Lexicon.empty())) == "Locale('xx')" From af6a01ad787cc704d5e97cadb684866231a948c8 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 01:20:54 -0700 Subject: [PATCH 26/88] Export v2 core types; enforce layering and mypy strictness Adds tests/v2/test_layering.py to mechanically enforce the conventions doc's import layering and the public v2 export surface. Appends the v2 core types (Span, Role, Token, Ambiguity, AmbiguityKind, ParsedName, Lexicon, Policy, PolicyPatch, PatronymicRule, UNSET, GIVEN_FIRST, FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, Locale) to nameparser/__init__.py alongside the existing v1 HumanName export. Turns on component-flag mypy strictness for the four v2 modules and check_untyped_defs for tests/v2, then fixes the resulting test-only type mismatches by using the precise constructed types (Span(...), frozenset({...}), tuple-of-pairs) where the literal type didn't matter to the test, and adding narrow # type: ignore[arg-type] comments only where a test deliberately exercises runtime coercion/validation of an intentionally mismatched static type (e.g. Token span/Ambiguity kind coercion tests, PatronymicRule string coercion, dict aliasing test). Co-Authored-By: Claude Fable 5 --- nameparser/__init__.py | 29 ++++++++++++ pyproject.toml | 17 +++++++ tests/v2/test_layering.py | 52 +++++++++++++++++++++ tests/v2/test_lexicon.py | 22 ++++----- tests/v2/test_policy.py | 8 ++-- tests/v2/test_reprs.py | 14 +++--- tests/v2/test_types.py | 96 +++++++++++++++++++-------------------- 7 files changed, 168 insertions(+), 70 deletions(-) create mode 100644 tests/v2/test_layering.py diff --git a/nameparser/__init__.py b/nameparser/__init__.py index c82d800..5e3adcb 100644 --- a/nameparser/__init__.py +++ b/nameparser/__init__.py @@ -5,3 +5,32 @@ __author_email__ = 'derek73@gmail.com' __license__ = "LGPL" __url__ = "https://github.com/derek73/python-nameparser" + +from nameparser._lexicon import Lexicon +from nameparser._locale import Locale +from nameparser._policy import ( + FAMILY_FIRST, + FAMILY_FIRST_GIVEN_LAST, + GIVEN_FIRST, + UNSET, + PatronymicRule, + Policy, + PolicyPatch, +) +from nameparser._types import ( + Ambiguity, + AmbiguityKind, + ParsedName, + Role, + Span, + Token, +) + +__all__ = [ + # v1 (compatibility layer) + "HumanName", + # v2 core + "Span", "Role", "Token", "Ambiguity", "AmbiguityKind", "ParsedName", + "Lexicon", "Policy", "PolicyPatch", "PatronymicRule", "UNSET", + "GIVEN_FIRST", "FAMILY_FIRST", "FAMILY_FIRST_GIVEN_LAST", "Locale", +] diff --git a/pyproject.toml b/pyproject.toml index 1e9ffef..1beee8f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,23 @@ module = [ ] ignore_missing_imports = true +[[tool.mypy.overrides]] +module = [ + "nameparser._types", + "nameparser._lexicon", + "nameparser._policy", + "nameparser._locale", +] +disallow_untyped_defs = true +disallow_incomplete_defs = true +disallow_any_generics = true +warn_return_any = true +strict_equality = true + +[[tool.mypy.overrides]] +module = ["tests.v2.*"] +check_untyped_defs = true + [tool.pytest.ini_options] testpaths = ["tests", "nameparser"] addopts = "--doctest-modules" diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py new file mode 100644 index 0000000..99ae45c --- /dev/null +++ b/tests/v2/test_layering.py @@ -0,0 +1,52 @@ +"""Enforce the conventions doc's import layering mechanically.""" +import ast +import pathlib + +import nameparser + +PKG = pathlib.Path(nameparser.__file__).parent + +# module -> prefixes it may import from within nameparser +ALLOWED = { + "_types.py": (), + "_lexicon.py": ("nameparser.config.",), # DATA modules only, during 2.x + "_policy.py": ("nameparser._types",), + "_locale.py": ("nameparser._lexicon", "nameparser._policy"), +} + + +def _nameparser_imports(path: pathlib.Path) -> list[str]: + tree = ast.parse(path.read_text()) + found = [] + for node in ast.walk(tree): + if isinstance(node, ast.Import): + found += [a.name for a in node.names] + elif isinstance(node, ast.ImportFrom) and node.module: + found.append(node.module) + return [m for m in found if m.startswith("nameparser")] + + +def test_layering_contract(): + for mod, allowed in ALLOWED.items(): + for imported in _nameparser_imports(PKG / mod): + assert imported.startswith(allowed), ( + f"{mod} imports {imported}, which the layering contract " + f"forbids (allowed prefixes: {allowed or 'none'})" + ) + + +def test_lexicon_never_imports_config_package_root_or_parser(): + for imported in _nameparser_imports(PKG / "_lexicon.py"): + assert imported != "nameparser.config" + assert not imported.startswith("nameparser.parser") + + +def test_public_exports(): + expected = { + "Span", "Role", "Token", "Ambiguity", "AmbiguityKind", "ParsedName", + "Lexicon", "Policy", "PolicyPatch", "PatronymicRule", "UNSET", + "GIVEN_FIRST", "FAMILY_FIRST", "FAMILY_FIRST_GIVEN_LAST", "Locale", + } + assert expected <= set(nameparser.__all__) + for name in expected: + assert getattr(nameparser, name) is not None diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index 944c4fb..4102a27 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -6,7 +6,7 @@ def test_entries_are_normalized_at_construction(): - lex = Lexicon(titles={"Dr.", "MR"}) + lex = Lexicon(titles=frozenset({"Dr.", "MR"})) assert lex.titles == frozenset({"dr", "mr"}) @@ -30,17 +30,17 @@ def test_default_is_cached_single_instance(): def test_particles_ambiguous_must_be_subset_of_particles(): with pytest.raises(ValueError, match="subset"): - Lexicon(particles_ambiguous={"van"}) + Lexicon(particles_ambiguous=frozenset({"van"})) def test_capitalization_exceptions_canonical_and_no_aliasing(): exceptions = {"phd": "PhD", "ii": "II"} lex = Lexicon.empty() - lex2 = dataclasses.replace(lex, capitalization_exceptions=exceptions) + lex2 = dataclasses.replace(lex, capitalization_exceptions=exceptions) # type: ignore[arg-type] exceptions["iii"] = "III" # mutate caller's dict afterwards assert "iii" not in lex2.capitalization_exceptions_map # canonical: insertion order does not affect equality/hash - lex3 = dataclasses.replace(lex, capitalization_exceptions={"ii": "II", "phd": "PhD"}) + lex3 = dataclasses.replace(lex, capitalization_exceptions=(("ii", "II"), ("phd", "PhD"))) assert lex2 == lex3 and hash(lex2) == hash(lex3) @@ -59,20 +59,20 @@ def test_lexicon_rejects_non_str_vocab_entries(): def test_exception_keys_normalizing_to_empty_are_dropped(): - lex = Lexicon(capitalization_exceptions={"...": "X", "phd": "PhD"}) + lex = Lexicon(capitalization_exceptions=(("...", "X"), ("phd", "PhD"))) assert lex.capitalization_exceptions == (("phd", "PhD"),) def test_colliding_exception_keys_dedupe_last_wins(): - lex = Lexicon(capitalization_exceptions={"Ph.D.": "A", "phd": "B"}) + lex = Lexicon(capitalization_exceptions=(("Ph.D.", "A"), ("phd", "B"))) assert lex.capitalization_exceptions == (("phd", "B"),) - rebuilt = Lexicon(capitalization_exceptions=lex.capitalization_exceptions_map) + rebuilt = Lexicon(capitalization_exceptions=lex.capitalization_exceptions_map) # type: ignore[arg-type] assert rebuilt == lex and hash(rebuilt) == hash(lex) def test_lexicon_rejects_non_str_exception_values(): with pytest.raises(ValueError, match="str -> str"): - Lexicon(capitalization_exceptions={"phd": 42}) # type: ignore[dict-item] + Lexicon(capitalization_exceptions={"phd": 42}) # type: ignore[dict-item, arg-type] def test_add_and_remove_return_new_lexicons(): @@ -96,10 +96,10 @@ def test_add_capitalization_exceptions_raises_pointing_at_replace(): def test_union_is_fieldwise_and_right_biased_for_exceptions(): a = dataclasses.replace(Lexicon.empty(), - capitalization_exceptions={"phd": "PhD"}) + capitalization_exceptions=(("phd", "PhD"),)) a = a.add(titles={"dr"}) b = dataclasses.replace(Lexicon.empty(), - capitalization_exceptions={"phd": "Ph.D."}) + capitalization_exceptions=(("phd", "Ph.D."),)) b = b.add(titles={"mr"}) u = a | b assert u.titles == frozenset({"dr", "mr"}) @@ -107,6 +107,6 @@ def test_union_is_fieldwise_and_right_biased_for_exceptions(): def test_remove_breaking_subset_invariant_raises(): - lex = Lexicon(particles={"van"}, particles_ambiguous={"van"}) + lex = Lexicon(particles=frozenset({"van"}), particles_ambiguous=frozenset({"van"})) with pytest.raises(ValueError, match="subset"): lex.remove(particles={"van"}) # would orphan particles_ambiguous diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index ac0293d..83e33bf 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -38,10 +38,10 @@ def test_name_order_must_be_permutation_and_error_names_constants(): def test_patronymic_rules_coerce_and_reject(): - p = Policy(patronymic_rules=frozenset({"east-slavic"})) + p = Policy(patronymic_rules=frozenset({"east-slavic"})) # type: ignore[arg-type] assert p.patronymic_rules == frozenset({PatronymicRule.EAST_SLAVIC}) with pytest.raises(ValueError, match="east-slavic, turkic"): - Policy(patronymic_rules=frozenset({"klingon"})) + Policy(patronymic_rules=frozenset({"klingon"})) # type: ignore[arg-type] def test_delimiter_pairs_must_be_nonempty_string_pairs(): @@ -79,7 +79,7 @@ def test_extra_suffix_delimiters_validated_and_coerced(): with pytest.raises(ValueError, match="bare string"): Policy(extra_suffix_delimiters="ab") # type: ignore[arg-type] with pytest.raises(ValueError, match="non-empty strings"): - Policy(extra_suffix_delimiters={""}) + Policy(extra_suffix_delimiters=frozenset({""})) p = Policy(extra_suffix_delimiters=["-"]) # type: ignore[arg-type] assert p.extra_suffix_delimiters == frozenset({"-"}) assert isinstance(hash(p), int) @@ -98,7 +98,7 @@ def test_policy_patch_mirrors_policy_field_types(): def test_policy_patch_canonicalizes_union_fields(): - p = PolicyPatch(extra_suffix_delimiters={"-"}) + p = PolicyPatch(extra_suffix_delimiters=frozenset({"-"})) assert isinstance(p.extra_suffix_delimiters, frozenset) assert isinstance(hash(p), int) out = apply_patch(Policy(), PolicyPatch(extra_suffix_delimiters=["-"])) # type: ignore[arg-type] diff --git a/tests/v2/test_reprs.py b/tests/v2/test_reprs.py index a5c421d..4f6672b 100644 --- a/tests/v2/test_reprs.py +++ b/tests/v2/test_reprs.py @@ -3,25 +3,25 @@ from nameparser._lexicon import Lexicon from nameparser._locale import Locale from nameparser._policy import FAMILY_FIRST, PatronymicRule, Policy, PolicyPatch -from nameparser._types import Ambiguity, AmbiguityKind, ParsedName, Role, Token +from nameparser._types import Ambiguity, AmbiguityKind, ParsedName, Role, Span, Token def test_token_repr_is_compact(): - t = Token("de", (9, 11), Role.FAMILY, frozenset({"particle"})) + t = Token("de", Span(9, 11), Role.FAMILY, frozenset({"particle"})) assert repr(t) == "Token('de' @9:11 FAMILY {particle})" assert repr(Token("Jane", None, Role.GIVEN)) == "Token('Jane' @synthetic GIVEN)" def test_ambiguity_repr_shows_kind_and_token_texts(): - van = Token("Van", (0, 3), Role.GIVEN) + van = Token("Van", Span(0, 3), Role.GIVEN) a = Ambiguity(AmbiguityKind.PARTICLE_OR_GIVEN, "detail", (van,)) assert repr(a) == "Ambiguity('particle-or-given': 'Van')" def test_parsedname_repr_lists_nonempty_fields_in_canonical_order(): pn = ParsedName("John Smith", ( - Token("John", (0, 4), Role.GIVEN), - Token("Smith", (5, 10), Role.FAMILY), + Token("John", Span(0, 4), Role.GIVEN), + Token("Smith", Span(5, 10), Role.FAMILY), )) assert repr(pn) == ( "" @@ -29,9 +29,9 @@ def test_parsedname_repr_lists_nonempty_fields_in_canonical_order(): def test_parsedname_repr_includes_ambiguities_line_when_present(): - van = Token("Van", (0, 3), Role.GIVEN) + van = Token("Van", Span(0, 3), Role.GIVEN) pn = ParsedName("Van Johnson", - (van, Token("Johnson", (4, 11), Role.FAMILY)), + (van, Token("Johnson", Span(4, 11), Role.FAMILY)), (Ambiguity(AmbiguityKind.PARTICLE_OR_GIVEN, "d", (van,)),)) assert "ambiguities: ['particle-or-given']" in repr(pn) diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 4ddfab0..21cedd8 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -10,7 +10,7 @@ def test_role_declaration_order_is_canonical_field_order(): def test_token_construction_and_span_coercion(): - t = Token("Juan", (0, 4), Role.GIVEN) + t = Token("Juan", (0, 4), Role.GIVEN) # type: ignore[arg-type] assert t.span == Span(0, 4) assert isinstance(t.span, Span) assert t.span.start == 0 and t.span.end == 4 @@ -24,17 +24,17 @@ def test_synthetic_token_has_no_span(): def test_token_rejects_empty_text(): with pytest.raises(ValueError, match="non-empty"): - Token("", (0, 0), Role.GIVEN) + Token("", Span(0, 0), Role.GIVEN) def test_token_rejects_inverted_span(): with pytest.raises(ValueError, match="start <= end"): - Token("x", (5, 2), Role.GIVEN) + Token("x", Span(5, 2), Role.GIVEN) def test_token_rejects_negative_span(): with pytest.raises(ValueError, match="start <= end"): - Token("x", (-1, 1), Role.GIVEN) + Token("x", Span(-1, 1), Role.GIVEN) def test_token_rejects_malformed_span_shapes(): @@ -50,10 +50,10 @@ def test_token_rejects_non_string_text(): def test_token_is_frozen_and_hashable(): - t = Token("Juan", (0, 4), Role.GIVEN) + t = Token("Juan", Span(0, 4), Role.GIVEN) with pytest.raises(AttributeError): t.text = "X" # type: ignore[misc] - assert hash(t) == hash(Token("Juan", (0, 4), Role.GIVEN)) + assert hash(t) == hash(Token("Juan", Span(0, 4), Role.GIVEN)) def test_ambiguity_kind_members_are_their_string_values(): @@ -62,15 +62,15 @@ def test_ambiguity_kind_members_are_their_string_values(): def test_ambiguity_construction_coerces_kind_string(): - t = Token("Van", (0, 3), Role.GIVEN, frozenset({"particle"})) - a = Ambiguity("particle-or-given", "leading 'van' may be a particle", (t,)) + t = Token("Van", Span(0, 3), Role.GIVEN, frozenset({"particle"})) + a = Ambiguity("particle-or-given", "leading 'van' may be a particle", (t,)) # type: ignore[arg-type] assert a.kind is AmbiguityKind.PARTICLE_OR_GIVEN assert a.tokens == (t,) def test_ambiguity_rejects_unknown_kind(): with pytest.raises(ValueError, match="particle-or-given"): - Ambiguity("no-such-kind", "detail", ()) + Ambiguity("no-such-kind", "detail", ()) # type: ignore[arg-type] def test_ambiguity_rejects_non_token_elements(): @@ -80,7 +80,7 @@ def test_ambiguity_rejects_non_token_elements(): def test_ambiguity_rejects_empty_detail(): with pytest.raises(ValueError, match="non-empty string"): - Ambiguity("order", "", ()) + Ambiguity(AmbiguityKind.ORDER, "", ()) def _pn(original, tokens, ambiguities=()): @@ -90,8 +90,8 @@ def _pn(original, tokens, ambiguities=()): def test_parsedname_accepts_valid_spans_and_is_truthy(): pn = _pn("John Smith", [ - Token("John", (0, 4), Role.GIVEN), - Token("Smith", (5, 10), Role.FAMILY), + Token("John", Span(0, 4), Role.GIVEN), + Token("Smith", Span(5, 10), Role.FAMILY), ]) assert bool(pn) is True @@ -103,47 +103,47 @@ def test_empty_parse_is_falsy(): def test_parsedname_rejects_out_of_bounds_span(): with pytest.raises(ValueError, match="out of bounds"): - _pn("John", [Token("Johnny", (0, 6), Role.GIVEN)]) + _pn("John", [Token("Johnny", Span(0, 6), Role.GIVEN)]) def test_parsedname_rejects_overlapping_spans(): with pytest.raises(ValueError, match="ascending"): _pn("John Smith", [ - Token("John", (0, 4), Role.GIVEN), - Token("ohn S", (1, 6), Role.FAMILY), + Token("John", Span(0, 4), Role.GIVEN), + Token("ohn S", Span(1, 6), Role.FAMILY), ]) def test_parsedname_rejects_descending_spans(): with pytest.raises(ValueError, match="ascending"): _pn("John Smith", [ - Token("Smith", (5, 10), Role.FAMILY), - Token("John", (0, 4), Role.GIVEN), + Token("Smith", Span(5, 10), Role.FAMILY), + Token("John", Span(0, 4), Role.GIVEN), ]) def test_synthetic_tokens_skip_span_checks(): pn = _pn("John Smith", [ - Token("John", (0, 4), Role.GIVEN), + Token("John", Span(0, 4), Role.GIVEN), Token("Qux", None, Role.MIDDLE), - Token("Smith", (5, 10), Role.FAMILY), + Token("Smith", Span(5, 10), Role.FAMILY), ]) assert len(pn.tokens) == 3 def test_ambiguity_tokens_must_be_subset_of_parse_tokens(): - inside = Token("Van", (0, 3), Role.GIVEN) + inside = Token("Van", Span(0, 3), Role.GIVEN) outside = Token("Zzz", None, Role.GIVEN) with pytest.raises(ValueError, match="subset"): _pn("Van Johnson", - [inside, Token("Johnson", (4, 11), Role.FAMILY)], + [inside, Token("Johnson", Span(4, 11), Role.FAMILY)], [Ambiguity(AmbiguityKind.PARTICLE_OR_GIVEN, "d", (outside,))]) def test_parsedname_equality_is_strict_structural(): - a = _pn("John", [Token("John", (0, 4), Role.GIVEN)]) - b = _pn("John", [Token("John", (0, 4), Role.GIVEN)]) - c = _pn("John ", [Token("John", (0, 4), Role.GIVEN)]) + a = _pn("John", [Token("John", Span(0, 4), Role.GIVEN)]) + b = _pn("John", [Token("John", Span(0, 4), Role.GIVEN)]) + c = _pn("John ", [Token("John", Span(0, 4), Role.GIVEN)]) assert a == b and hash(a) == hash(b) assert a != c # different original: not interchangeable @@ -157,19 +157,19 @@ def test_parsedname_rejects_non_token_and_non_ambiguity_elements(): with pytest.raises(ValueError, match="only Token instances"): _pn("x", ["not-a-token"]) # type: ignore[list-item] with pytest.raises(ValueError, match="only Ambiguity instances"): - _pn("John", [Token("John", (0, 4), Role.GIVEN)], ["nope"]) # type: ignore[list-item] + _pn("John", [Token("John", Span(0, 4), Role.GIVEN)], ["nope"]) # type: ignore[list-item] def _delavega(): # "Dr. Juan de la Vega III" -- hand-built, spans verified by hand # 0123456789012345678901234 return _pn("Dr. Juan de la Vega III", [ - Token("Dr.", (0, 3), Role.TITLE), - Token("Juan", (4, 8), Role.GIVEN), - Token("de", (9, 11), Role.FAMILY, frozenset({"particle"})), - Token("la", (12, 14), Role.FAMILY, frozenset({"particle"})), - Token("Vega", (15, 19), Role.FAMILY), - Token("III", (20, 23), Role.SUFFIX), + Token("Dr.", Span(0, 3), Role.TITLE), + Token("Juan", Span(4, 8), Role.GIVEN), + Token("de", Span(9, 11), Role.FAMILY, frozenset({"particle"})), + Token("la", Span(12, 14), Role.FAMILY, frozenset({"particle"})), + Token("Vega", Span(15, 19), Role.FAMILY), + Token("III", Span(20, 23), Role.SUFFIX), ]) @@ -186,10 +186,10 @@ def test_string_properties_join_by_role(): def test_suffix_joins_with_comma_space(): pn = _pn("John Smith PhD MD", [ - Token("John", (0, 4), Role.GIVEN), - Token("Smith", (5, 10), Role.FAMILY), - Token("PhD", (11, 14), Role.SUFFIX), - Token("MD", (15, 17), Role.SUFFIX), + Token("John", Span(0, 4), Role.GIVEN), + Token("Smith", Span(5, 10), Role.FAMILY), + Token("PhD", Span(11, 14), Role.SUFFIX), + Token("MD", Span(15, 17), Role.SUFFIX), ]) assert pn.suffix == "PhD, MD" @@ -232,8 +232,8 @@ def test_replace_swaps_field_with_synthetic_tokens_in_place(): def test_replace_adds_missing_field_at_end(): pn = _pn("John Smith", [ - Token("John", (0, 4), Role.GIVEN), - Token("Smith", (5, 10), Role.FAMILY), + Token("John", Span(0, 4), Role.GIVEN), + Token("Smith", Span(5, 10), Role.FAMILY), ]) pn2 = pn.replace(suffix="Jr") assert pn2.suffix == "Jr" @@ -251,9 +251,9 @@ def test_replace_rejects_unknown_field(): def test_replace_drops_ambiguities_referencing_removed_tokens(): - van = Token("Van", (0, 3), Role.GIVEN) + van = Token("Van", Span(0, 3), Role.GIVEN) pn = _pn("Van Johnson", - [van, Token("Johnson", (4, 11), Role.FAMILY)], + [van, Token("Johnson", Span(4, 11), Role.FAMILY)], [Ambiguity(AmbiguityKind.PARTICLE_OR_GIVEN, "d", (van,))]) assert pn.replace(given="Bob").ambiguities == () assert pn.replace(family="Smith").ambiguities != () @@ -265,7 +265,7 @@ def test_replace_rejects_non_str_value(): def test_replace_appends_missing_roles_in_canonical_order(): - pn = _pn("John", [Token("John", (0, 4), Role.GIVEN)]) + pn = _pn("John", [Token("John", Span(0, 4), Role.GIVEN)]) pn2 = pn.replace(maiden="X", suffix="Y") assert [t.role for t in pn2.tokens] == [Role.GIVEN, Role.SUFFIX, Role.MAIDEN] @@ -276,15 +276,15 @@ def test_comparison_key_is_casefolded_canonical_seven_tuple(): "dr.", "juan", "", "de la vega", "iii", "", "", ) upper = _pn("JUAN DE LA VEGA", [ - Token("JUAN", (0, 4), Role.GIVEN), - Token("DE", (5, 7), Role.FAMILY, frozenset({"particle"})), - Token("LA", (8, 10), Role.FAMILY, frozenset({"particle"})), - Token("VEGA", (11, 15), Role.FAMILY), + Token("JUAN", Span(0, 4), Role.GIVEN), + Token("DE", Span(5, 7), Role.FAMILY, frozenset({"particle"})), + Token("LA", Span(8, 10), Role.FAMILY, frozenset({"particle"})), + Token("VEGA", Span(11, 15), Role.FAMILY), ]) lower = _pn("juan de la vega", [ - Token("juan", (0, 4), Role.GIVEN), - Token("de", (5, 7), Role.FAMILY, frozenset({"particle"})), - Token("la", (8, 10), Role.FAMILY, frozenset({"particle"})), - Token("vega", (11, 15), Role.FAMILY), + Token("juan", Span(0, 4), Role.GIVEN), + Token("de", Span(5, 7), Role.FAMILY, frozenset({"particle"})), + Token("la", Span(8, 10), Role.FAMILY, frozenset({"particle"})), + Token("vega", Span(11, 15), Role.FAMILY), ]) assert upper.comparison_key() == lower.comparison_key() From 30ef7630c1007e94a5bba666c3b924d1283170c3 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 04:54:37 -0700 Subject: [PATCH 27/88] Make Lexicon picklable despite its mappingproxy slot pickle.dumps(Lexicon.default()) raised TypeError because the default slots-dataclass pickle path serializes the _cap_map MappingProxyType. Ship every other slot and rebuild the proxy from the canonical capitalization_exceptions tuple on load. Parser (Plan 3) is picklable by construction per the core spec, and a Parser holds a Lexicon. Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 13 +++++++++++++ tests/v2/test_lexicon.py | 15 +++++++++++++++ tests/v2/test_locale.py | 7 +++++++ 3 files changed, 35 insertions(+) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 9394899..b9e20ba 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -118,6 +118,19 @@ def __repr__(self) -> str: deltas.append(f"{name}: {delta}") return f"Lexicon(default + {', '.join(deltas)})" + def __getstate__(self) -> dict[str, object]: + # _cap_map is a MappingProxyType, which pickle rejects; ship every + # other slot and rebuild the proxy from the canonical tuple on load. + return {f.name: getattr(self, f.name) + for f in dataclasses.fields(self) if f.name != "_cap_map"} + + def __setstate__(self, state: dict[str, object]) -> None: + for name, value in state.items(): + object.__setattr__(self, name, value) + object.__setattr__( + self, "_cap_map", + MappingProxyType(dict(self.capitalization_exceptions))) + @property def capitalization_exceptions_map(self) -> Mapping[str, str]: return self._cap_map diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index 4102a27..5b1e272 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -110,3 +110,18 @@ def test_remove_breaking_subset_invariant_raises(): lex = Lexicon(particles=frozenset({"van"}), particles_ambiguous=frozenset({"van"})) with pytest.raises(ValueError, match="subset"): lex.remove(particles={"van"}) # would orphan particles_ambiguous + + +def test_pickle_round_trip_preserves_equality_and_cap_map(): + # _cap_map holds a MappingProxyType, which pickle rejects; Lexicon + # must round-trip anyway because Parser is picklable by construction + # (spec: 2026-07-11-v2-core-api-design.md) and a Parser holds a Lexicon. + import pickle + + for lex in (Lexicon.default(), + Lexicon.empty().add(titles={"Dr."})): + loaded = pickle.loads(pickle.dumps(lex)) + assert loaded == lex + assert hash(loaded) == hash(lex) + assert (loaded.capitalization_exceptions_map + == lex.capitalization_exceptions_map) diff --git a/tests/v2/test_locale.py b/tests/v2/test_locale.py index b3a8f54..b5829ab 100644 --- a/tests/v2/test_locale.py +++ b/tests/v2/test_locale.py @@ -44,3 +44,10 @@ def test_locale_validates_component_types(): Locale(code="ru", lexicon={"titles": set()}) # type: ignore[arg-type] with pytest.raises(ValueError, match="PolicyPatch"): Locale(code="ru", lexicon=Lexicon.empty(), policy={"name_order": None}) # type: ignore[arg-type] + + +def test_locale_with_lexicon_pickles_round_trip(): + import pickle + + loc = Locale(code="ru", lexicon=Lexicon.empty().add(titles={"Dr."})) + assert pickle.loads(pickle.dumps(loc)) == loc From 401535cae216bd1daf182c1b0a99ae7b3d21f976 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 04:55:25 -0700 Subject: [PATCH 28/88] Drop Python 3.10 from the CI matrix (#257, partial) requires-python is >=3.11 on this branch, so the 3.10 job can no longer install the package (uv sync either fails or silently substitutes a managed 3.11). The rest of #257 (typing_extensions, classifiers) stays tracked on the issue. Co-Authored-By: Claude Fable 5 --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 290ee0a..072f44c 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -20,7 +20,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v7 From f2b42c8bb68946b32f5fc80351ebd8ea1d841de7 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 04:58:10 -0700 Subject: [PATCH 29/88] Drop the pre-3.11 typing_extensions shim (#257) With requires-python at >=3.11, Self imports directly from typing and the conditional dependency can never activate; ruff (UP035/UP036) flags the dead version block once the floor is raised. Co-Authored-By: Claude Fable 5 --- nameparser/config/__init__.py | 7 +------ pyproject.toml | 4 +--- tests/v2/test_layering.py | 15 ++++++++++++++- tests/v2/test_types.py | 7 ++++++- uv.lock | 1 - 5 files changed, 22 insertions(+), 12 deletions(-) diff --git a/nameparser/config/__init__.py b/nameparser/config/__init__.py index 6f1e798..a0bbdce 100644 --- a/nameparser/config/__init__.py +++ b/nameparser/config/__init__.py @@ -45,12 +45,7 @@ import sys import warnings from collections.abc import Callable, Iterable, Iterator, Mapping, Set -from typing import Any, TypeVar, overload - -if sys.version_info >= (3, 11): - from typing import Self -else: - from typing_extensions import Self +from typing import Any, Self, TypeVar, overload from nameparser.util import lc from nameparser.config.prefixes import PREFIXES, NON_FIRST_NAME_PREFIXES diff --git a/pyproject.toml b/pyproject.toml index 1beee8f..afbfeca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,9 +22,7 @@ classifiers = [ "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: Text Processing :: Linguistic", ] -dependencies = [ - "typing_extensions (>=4.5.0); python_version < '3.11'" -] +dependencies = [] [build-system] requires = ["setuptools (>=82.0.1)"] diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index 99ae45c..67f4a49 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -26,10 +26,23 @@ def _nameparser_imports(path: pathlib.Path) -> list[str]: return [m for m in found if m.startswith("nameparser")] +def _permitted(imported: str, allowed: tuple[str, ...]) -> bool: + # An entry ending in "." is a pure prefix (subpackage contents only); + # any other entry means that exact module or its submodules -- a bare + # startswith would also admit siblings like nameparser._types_helpers. + for entry in allowed: + if entry.endswith("."): + if imported.startswith(entry): + return True + elif imported == entry or imported.startswith(entry + "."): + return True + return False + + def test_layering_contract(): for mod, allowed in ALLOWED.items(): for imported in _nameparser_imports(PKG / mod): - assert imported.startswith(allowed), ( + assert _permitted(imported, allowed), ( f"{mod} imports {imported}, which the layering contract " f"forbids (allowed prefixes: {allowed or 'none'})" ) diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 21cedd8..95bef68 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -1,6 +1,8 @@ import pytest -from nameparser._types import Ambiguity, AmbiguityKind, ParsedName, Role, Span, Token +from nameparser._types import ( + STABLE_TAGS, Ambiguity, AmbiguityKind, ParsedName, Role, Span, Token, +) def test_role_declaration_order_is_canonical_field_order(): @@ -195,6 +197,9 @@ def test_suffix_joins_with_comma_space(): def test_derived_views_filter_on_stable_particle_tag(): + # Pin the hard-coded "particle" string in _text_for to the published + # contract until Plan 3's tag-emission contract tests land. + assert "particle" in STABLE_TAGS pn = _delavega() assert pn.family_particles == "de la" assert pn.family_base == "Vega" diff --git a/uv.lock b/uv.lock index c9b8164..fe84787 100644 --- a/uv.lock +++ b/uv.lock @@ -585,7 +585,6 @@ dev = [ ] [package.metadata] -requires-dist = [{ name = "typing-extensions", marker = "python_full_version < '3.11'", specifier = ">=4.5.0" }] [package.metadata.requires-dev] dev = [ From 4c171fc06a6bb0ee4b5b9b24ab8fc4a9972f4bc2 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 04:59:11 -0700 Subject: [PATCH 30/88] Annotate v2 test returns for ruff ANN compliance The v1 suite is fully annotated (since #250 put tests/ under mypy), so pyproject carries no per-file ANN ignores; the new tests/v2 modules must be annotated too or 'ruff check' fails in CI. Also drops an unused import ruff flagged (F401). Co-Authored-By: Claude Fable 5 --- tests/v2/test_lexicon.py | 34 +++++++++++++++++----------------- tests/v2/test_locale.py | 14 +++++++------- tests/v2/test_policy.py | 36 ++++++++++++++++++------------------ tests/v2/test_reprs.py | 17 ++++++++--------- 4 files changed, 50 insertions(+), 51 deletions(-) diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index 5b1e272..d2cfe5f 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -5,12 +5,12 @@ from nameparser._lexicon import Lexicon -def test_entries_are_normalized_at_construction(): +def test_entries_are_normalized_at_construction() -> None: lex = Lexicon(titles=frozenset({"Dr.", "MR"})) assert lex.titles == frozenset({"dr", "mr"}) -def test_default_sources_v1_vocabulary(): +def test_default_sources_v1_vocabulary() -> None: lex = Lexicon.default() assert "dr" in lex.titles assert "van" in lex.particles @@ -24,16 +24,16 @@ def test_default_sources_v1_vocabulary(): assert lex.capitalization_exceptions_map["phd"] == "Ph.D." -def test_default_is_cached_single_instance(): +def test_default_is_cached_single_instance() -> None: assert Lexicon.default() is Lexicon.default() -def test_particles_ambiguous_must_be_subset_of_particles(): +def test_particles_ambiguous_must_be_subset_of_particles() -> None: with pytest.raises(ValueError, match="subset"): Lexicon(particles_ambiguous=frozenset({"van"})) -def test_capitalization_exceptions_canonical_and_no_aliasing(): +def test_capitalization_exceptions_canonical_and_no_aliasing() -> None: exceptions = {"phd": "PhD", "ii": "II"} lex = Lexicon.empty() lex2 = dataclasses.replace(lex, capitalization_exceptions=exceptions) # type: ignore[arg-type] @@ -44,38 +44,38 @@ def test_capitalization_exceptions_canonical_and_no_aliasing(): assert lex2 == lex3 and hash(lex2) == hash(lex3) -def test_lexicon_is_hashable(): +def test_lexicon_is_hashable() -> None: assert isinstance(hash(Lexicon.default()), int) -def test_lexicon_rejects_bare_string_vocab(): +def test_lexicon_rejects_bare_string_vocab() -> None: with pytest.raises(ValueError, match="bare string"): Lexicon(titles="dr") # type: ignore[arg-type] -def test_lexicon_rejects_non_str_vocab_entries(): +def test_lexicon_rejects_non_str_vocab_entries() -> None: with pytest.raises(ValueError, match="entries must be strings"): Lexicon(titles={"Dr.", 42}) # type: ignore[arg-type] -def test_exception_keys_normalizing_to_empty_are_dropped(): +def test_exception_keys_normalizing_to_empty_are_dropped() -> None: lex = Lexicon(capitalization_exceptions=(("...", "X"), ("phd", "PhD"))) assert lex.capitalization_exceptions == (("phd", "PhD"),) -def test_colliding_exception_keys_dedupe_last_wins(): +def test_colliding_exception_keys_dedupe_last_wins() -> None: lex = Lexicon(capitalization_exceptions=(("Ph.D.", "A"), ("phd", "B"))) assert lex.capitalization_exceptions == (("phd", "B"),) rebuilt = Lexicon(capitalization_exceptions=lex.capitalization_exceptions_map) # type: ignore[arg-type] assert rebuilt == lex and hash(rebuilt) == hash(lex) -def test_lexicon_rejects_non_str_exception_values(): +def test_lexicon_rejects_non_str_exception_values() -> None: with pytest.raises(ValueError, match="str -> str"): Lexicon(capitalization_exceptions={"phd": 42}) # type: ignore[dict-item, arg-type] -def test_add_and_remove_return_new_lexicons(): +def test_add_and_remove_return_new_lexicons() -> None: # "zqtitle" is a synthetic word absent from v1's TITLES data (unlike # e.g. "dra", the feminine "dr." abbreviation, which is already there). base = Lexicon.default() @@ -84,17 +84,17 @@ def test_add_and_remove_return_new_lexicons(): assert "bishop" not in lex.suffix_words -def test_add_unknown_field_raises_with_valid_names(): +def test_add_unknown_field_raises_with_valid_names() -> None: with pytest.raises(TypeError, match="prefixes"): Lexicon.default().add(prefixes={"van"}) # v1 name: helpful error -def test_add_capitalization_exceptions_raises_pointing_at_replace(): +def test_add_capitalization_exceptions_raises_pointing_at_replace() -> None: with pytest.raises(TypeError, match="dataclasses.replace"): Lexicon.default().add(capitalization_exceptions={"x": "X"}) -def test_union_is_fieldwise_and_right_biased_for_exceptions(): +def test_union_is_fieldwise_and_right_biased_for_exceptions() -> None: a = dataclasses.replace(Lexicon.empty(), capitalization_exceptions=(("phd", "PhD"),)) a = a.add(titles={"dr"}) @@ -106,13 +106,13 @@ def test_union_is_fieldwise_and_right_biased_for_exceptions(): assert u.capitalization_exceptions_map["phd"] == "Ph.D." # right wins -def test_remove_breaking_subset_invariant_raises(): +def test_remove_breaking_subset_invariant_raises() -> None: lex = Lexicon(particles=frozenset({"van"}), particles_ambiguous=frozenset({"van"})) with pytest.raises(ValueError, match="subset"): lex.remove(particles={"van"}) # would orphan particles_ambiguous -def test_pickle_round_trip_preserves_equality_and_cap_map(): +def test_pickle_round_trip_preserves_equality_and_cap_map() -> None: # _cap_map holds a MappingProxyType, which pickle rejects; Lexicon # must round-trip anyway because Parser is picklable by construction # (spec: 2026-07-11-v2-core-api-design.md) and a Parser holds a Lexicon. diff --git a/tests/v2/test_locale.py b/tests/v2/test_locale.py index b5829ab..0af17fe 100644 --- a/tests/v2/test_locale.py +++ b/tests/v2/test_locale.py @@ -5,7 +5,7 @@ from nameparser._policy import PatronymicRule, PolicyPatch -def test_locale_holds_code_lexicon_fragment_and_patch(): +def test_locale_holds_code_lexicon_fragment_and_patch() -> None: ru = Locale( code="ru", lexicon=Lexicon.empty(), @@ -17,36 +17,36 @@ def test_locale_holds_code_lexicon_fragment_and_patch(): {PatronymicRule.EAST_SLAVIC}) -def test_locale_defaults_to_empty_patch(): +def test_locale_defaults_to_empty_patch() -> None: assert Locale(code="xx", lexicon=Lexicon.empty()).policy == PolicyPatch() -def test_locale_code_must_be_nonempty_lowercase(): +def test_locale_code_must_be_nonempty_lowercase() -> None: with pytest.raises(ValueError, match="lowercase"): Locale(code="RU", lexicon=Lexicon.empty()) with pytest.raises(ValueError, match="non-empty"): Locale(code="", lexicon=Lexicon.empty()) -def test_locale_code_rejects_whitespace(): +def test_locale_code_rejects_whitespace() -> None: for bad in ("ru ", " ru", "ru\n", "r u"): with pytest.raises(ValueError, match="whitespace"): Locale(code=bad, lexicon=Lexicon.empty()) -def test_locale_is_hashable(): +def test_locale_is_hashable() -> None: loc = Locale(code="ru", lexicon=Lexicon.empty()) assert isinstance(hash(loc), int) -def test_locale_validates_component_types(): +def test_locale_validates_component_types() -> None: with pytest.raises(ValueError, match="Lexicon"): Locale(code="ru", lexicon={"titles": set()}) # type: ignore[arg-type] with pytest.raises(ValueError, match="PolicyPatch"): Locale(code="ru", lexicon=Lexicon.empty(), policy={"name_order": None}) # type: ignore[arg-type] -def test_locale_with_lexicon_pickles_round_trip(): +def test_locale_with_lexicon_pickles_round_trip() -> None: import pickle loc = Locale(code="ru", lexicon=Lexicon.empty().add(titles={"Dr."})) diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index 83e33bf..0c9b610 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -9,13 +9,13 @@ from nameparser._types import Role -def test_order_constants_read_as_their_contents(): +def test_order_constants_read_as_their_contents() -> None: assert GIVEN_FIRST == (Role.GIVEN, Role.MIDDLE, Role.FAMILY) assert FAMILY_FIRST == (Role.FAMILY, Role.GIVEN, Role.MIDDLE) assert FAMILY_FIRST_GIVEN_LAST == (Role.FAMILY, Role.MIDDLE, Role.GIVEN) -def test_policy_defaults(): +def test_policy_defaults() -> None: p = Policy() assert p.name_order == GIVEN_FIRST assert p.patronymic_rules == frozenset() @@ -24,58 +24,58 @@ def test_policy_defaults(): assert p.strip_emoji and p.strip_bidi and p.lenient_comma_suffixes -def test_policy_is_hashable_and_replaceable(): +def test_policy_is_hashable_and_replaceable() -> None: p = dataclasses.replace(Policy(), name_order=FAMILY_FIRST) assert p.name_order == FAMILY_FIRST assert isinstance(hash(p), int) -def test_name_order_must_be_permutation_and_error_names_constants(): +def test_name_order_must_be_permutation_and_error_names_constants() -> None: with pytest.raises(ValueError, match="FAMILY_FIRST_GIVEN_LAST"): Policy(name_order=(Role.TITLE, Role.GIVEN, Role.FAMILY)) with pytest.raises(ValueError, match="GIVEN_FIRST"): Policy(name_order=(Role.GIVEN, Role.GIVEN, Role.FAMILY)) -def test_patronymic_rules_coerce_and_reject(): +def test_patronymic_rules_coerce_and_reject() -> None: p = Policy(patronymic_rules=frozenset({"east-slavic"})) # type: ignore[arg-type] assert p.patronymic_rules == frozenset({PatronymicRule.EAST_SLAVIC}) with pytest.raises(ValueError, match="east-slavic, turkic"): Policy(patronymic_rules=frozenset({"klingon"})) # type: ignore[arg-type] -def test_delimiter_pairs_must_be_nonempty_string_pairs(): +def test_delimiter_pairs_must_be_nonempty_string_pairs() -> None: with pytest.raises(ValueError, match="non-empty"): Policy(nickname_delimiters=frozenset({("", ")")})) -def test_delimiter_pair_rejects_two_char_string(): +def test_delimiter_pair_rejects_two_char_string() -> None: with pytest.raises(ValueError, match="tuples"): Policy(nickname_delimiters=frozenset({"()"})) # type: ignore[arg-type] -def test_patronymic_rules_rejects_bare_string_and_non_iterable(): +def test_patronymic_rules_rejects_bare_string_and_non_iterable() -> None: with pytest.raises(ValueError, match="bare string"): Policy(patronymic_rules="east-slavic") # type: ignore[arg-type] with pytest.raises(ValueError, match="valid rules"): Policy(patronymic_rules=5) # type: ignore[arg-type] -def test_policy_delimiters_coerce_to_frozensets(): +def test_policy_delimiters_coerce_to_frozensets() -> None: p = Policy(nickname_delimiters=[("(", ")")]) # type: ignore[arg-type] assert isinstance(p.nickname_delimiters, frozenset) assert isinstance(hash(p), int) assert p == Policy(nickname_delimiters=frozenset({("(", ")")})) -def test_policy_delimiters_do_not_alias_caller_containers(): +def test_policy_delimiters_do_not_alias_caller_containers() -> None: source = {("(", ")")} p = Policy(nickname_delimiters=source) # type: ignore[arg-type] source.add(("'", "'")) assert ("'", "'") not in p.nickname_delimiters -def test_extra_suffix_delimiters_validated_and_coerced(): +def test_extra_suffix_delimiters_validated_and_coerced() -> None: with pytest.raises(ValueError, match="bare string"): Policy(extra_suffix_delimiters="ab") # type: ignore[arg-type] with pytest.raises(ValueError, match="non-empty strings"): @@ -85,19 +85,19 @@ def test_extra_suffix_delimiters_validated_and_coerced(): assert isinstance(hash(p), int) -def test_policy_patch_mirrors_policy_field_names(): +def test_policy_patch_mirrors_policy_field_names() -> None: policy_fields = {f.name for f in dataclasses.fields(Policy)} patch_fields = {f.name for f in dataclasses.fields(PolicyPatch)} assert policy_fields == patch_fields -def test_policy_patch_mirrors_policy_field_types(): +def test_policy_patch_mirrors_policy_field_types() -> None: for f in dataclasses.fields(Policy): patch_annotation = PolicyPatch.__dataclass_fields__[f.name].type assert patch_annotation == f"{f.type} | _Unset" -def test_policy_patch_canonicalizes_union_fields(): +def test_policy_patch_canonicalizes_union_fields() -> None: p = PolicyPatch(extra_suffix_delimiters=frozenset({"-"})) assert isinstance(p.extra_suffix_delimiters, frozenset) assert isinstance(hash(p), int) @@ -105,12 +105,12 @@ def test_policy_patch_canonicalizes_union_fields(): assert out.extra_suffix_delimiters == frozenset({"-"}) -def test_policy_patch_rejects_bare_string_union_fields(): +def test_policy_patch_rejects_bare_string_union_fields() -> None: with pytest.raises(ValueError, match="bare string"): PolicyPatch(extra_suffix_delimiters="ab") # type: ignore[arg-type] -def test_apply_patch_overrides_scalars_and_unions_sets(): +def test_apply_patch_overrides_scalars_and_unions_sets() -> None: base = Policy(patronymic_rules=frozenset({PatronymicRule.EAST_SLAVIC})) patch = PolicyPatch( name_order=FAMILY_FIRST, @@ -123,12 +123,12 @@ def test_apply_patch_overrides_scalars_and_unions_sets(): assert out.strip_emoji is True # untouched -def test_apply_patch_with_empty_patch_returns_same_policy(): +def test_apply_patch_with_empty_patch_returns_same_policy() -> None: base = Policy() assert apply_patch(base, PolicyPatch()) is base -def test_unset_fields_are_distinguishable_from_defaults(): +def test_unset_fields_are_distinguishable_from_defaults() -> None: patch = PolicyPatch(strip_emoji=True) # explicitly set to the default value assert patch.strip_emoji is True assert PolicyPatch().strip_emoji is UNSET diff --git a/tests/v2/test_reprs.py b/tests/v2/test_reprs.py index 4f6672b..cacfdbc 100644 --- a/tests/v2/test_reprs.py +++ b/tests/v2/test_reprs.py @@ -1,4 +1,3 @@ -import dataclasses from nameparser._lexicon import Lexicon from nameparser._locale import Locale @@ -6,19 +5,19 @@ from nameparser._types import Ambiguity, AmbiguityKind, ParsedName, Role, Span, Token -def test_token_repr_is_compact(): +def test_token_repr_is_compact() -> None: t = Token("de", Span(9, 11), Role.FAMILY, frozenset({"particle"})) assert repr(t) == "Token('de' @9:11 FAMILY {particle})" assert repr(Token("Jane", None, Role.GIVEN)) == "Token('Jane' @synthetic GIVEN)" -def test_ambiguity_repr_shows_kind_and_token_texts(): +def test_ambiguity_repr_shows_kind_and_token_texts() -> None: van = Token("Van", Span(0, 3), Role.GIVEN) a = Ambiguity(AmbiguityKind.PARTICLE_OR_GIVEN, "detail", (van,)) assert repr(a) == "Ambiguity('particle-or-given': 'Van')" -def test_parsedname_repr_lists_nonempty_fields_in_canonical_order(): +def test_parsedname_repr_lists_nonempty_fields_in_canonical_order() -> None: pn = ParsedName("John Smith", ( Token("John", Span(0, 4), Role.GIVEN), Token("Smith", Span(5, 10), Role.FAMILY), @@ -28,7 +27,7 @@ def test_parsedname_repr_lists_nonempty_fields_in_canonical_order(): ) -def test_parsedname_repr_includes_ambiguities_line_when_present(): +def test_parsedname_repr_includes_ambiguities_line_when_present() -> None: van = Token("Van", Span(0, 3), Role.GIVEN) pn = ParsedName("Van Johnson", (van, Token("Johnson", Span(4, 11), Role.FAMILY)), @@ -36,24 +35,24 @@ def test_parsedname_repr_includes_ambiguities_line_when_present(): assert "ambiguities: ['particle-or-given']" in repr(pn) -def test_empty_parsedname_repr(): +def test_empty_parsedname_repr() -> None: assert repr(ParsedName("", ())) == "" -def test_policy_repr_shows_only_nondefault_fields(): +def test_policy_repr_shows_only_nondefault_fields() -> None: assert repr(Policy()) == "Policy()" p = Policy(name_order=FAMILY_FIRST, strip_bidi=False) assert repr(p) == "Policy(name_order=FAMILY_FIRST, strip_bidi=False)" -def test_lexicon_repr_is_bounded(): +def test_lexicon_repr_is_bounded() -> None: assert repr(Lexicon.default()) == "Lexicon(default)" lex = Lexicon.default().add(titles={"zqx", "zqy"}) assert repr(lex) == "Lexicon(default + titles: +2)" assert "zqx" not in repr(lex) # never dump contents -def test_locale_repr_shows_code_and_patched_fields(): +def test_locale_repr_shows_code_and_patched_fields() -> None: ru = Locale("ru", Lexicon.empty(), PolicyPatch(patronymic_rules=frozenset( {PatronymicRule.EAST_SLAVIC}))) From 353d10aceb5cc038c5076c7de6bcb9fd136e83a8 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 04:59:11 -0700 Subject: [PATCH 31/88] Tighten the layering test to exact-module matching A bare startswith('nameparser._types') would also admit a future sibling like nameparser._types_helpers. Entries ending in '.' stay pure prefixes (subpackage contents); anything else now means that exact module or its submodules. Also carries this file's ANN annotations. Co-Authored-By: Claude Fable 5 --- tests/v2/test_layering.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index 67f4a49..f9d9605 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -39,7 +39,7 @@ def _permitted(imported: str, allowed: tuple[str, ...]) -> bool: return False -def test_layering_contract(): +def test_layering_contract() -> None: for mod, allowed in ALLOWED.items(): for imported in _nameparser_imports(PKG / mod): assert _permitted(imported, allowed), ( @@ -48,13 +48,13 @@ def test_layering_contract(): ) -def test_lexicon_never_imports_config_package_root_or_parser(): +def test_lexicon_never_imports_config_package_root_or_parser() -> None: for imported in _nameparser_imports(PKG / "_lexicon.py"): assert imported != "nameparser.config" assert not imported.startswith("nameparser.parser") -def test_public_exports(): +def test_public_exports() -> None: expected = { "Span", "Role", "Token", "Ambiguity", "AmbiguityKind", "ParsedName", "Lexicon", "Policy", "PolicyPatch", "PatronymicRule", "UNSET", From 09cc85d6105d0b2d036bde78ad5e41df7761e16c Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 04:59:11 -0700 Subject: [PATCH 32/88] Pin the derived views' particle tag to STABLE_TAGS Nothing tied the 'particle' string hard-coded in _text_for callers to the published STABLE_TAGS constant; assert the linkage in the derived-view test until Plan 3's tag-emission contract tests land. Also carries this file's ANN annotations. Co-Authored-By: Claude Fable 5 --- tests/v2/test_types.py | 81 ++++++++++++++++++++++-------------------- 1 file changed, 42 insertions(+), 39 deletions(-) diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 95bef68..7446cbd 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -1,3 +1,5 @@ +from collections.abc import Iterable + import pytest from nameparser._types import ( @@ -5,13 +7,13 @@ ) -def test_role_declaration_order_is_canonical_field_order(): +def test_role_declaration_order_is_canonical_field_order() -> None: assert [r.value for r in Role] == [ "title", "given", "middle", "family", "suffix", "nickname", "maiden", ] -def test_token_construction_and_span_coercion(): +def test_token_construction_and_span_coercion() -> None: t = Token("Juan", (0, 4), Role.GIVEN) # type: ignore[arg-type] assert t.span == Span(0, 4) assert isinstance(t.span, Span) @@ -19,78 +21,79 @@ def test_token_construction_and_span_coercion(): assert t.tags == frozenset() -def test_synthetic_token_has_no_span(): +def test_synthetic_token_has_no_span() -> None: t = Token("Jane", None, Role.GIVEN) assert t.span is None -def test_token_rejects_empty_text(): +def test_token_rejects_empty_text() -> None: with pytest.raises(ValueError, match="non-empty"): Token("", Span(0, 0), Role.GIVEN) -def test_token_rejects_inverted_span(): +def test_token_rejects_inverted_span() -> None: with pytest.raises(ValueError, match="start <= end"): Token("x", Span(5, 2), Role.GIVEN) -def test_token_rejects_negative_span(): +def test_token_rejects_negative_span() -> None: with pytest.raises(ValueError, match="start <= end"): Token("x", Span(-1, 1), Role.GIVEN) -def test_token_rejects_malformed_span_shapes(): +def test_token_rejects_malformed_span_shapes() -> None: with pytest.raises(ValueError, match="expected a \\(start, end\\) pair"): Token("x", (0, 4, 9), Role.GIVEN) # type: ignore[arg-type] with pytest.raises(ValueError, match="expected a \\(start, end\\) pair"): Token("x", 5, Role.GIVEN) # type: ignore[arg-type] -def test_token_rejects_non_string_text(): +def test_token_rejects_non_string_text() -> None: with pytest.raises(ValueError, match="got None"): Token(None, None, Role.GIVEN) # type: ignore[arg-type] -def test_token_is_frozen_and_hashable(): +def test_token_is_frozen_and_hashable() -> None: t = Token("Juan", Span(0, 4), Role.GIVEN) with pytest.raises(AttributeError): t.text = "X" # type: ignore[misc] assert hash(t) == hash(Token("Juan", Span(0, 4), Role.GIVEN)) -def test_ambiguity_kind_members_are_their_string_values(): +def test_ambiguity_kind_members_are_their_string_values() -> None: assert AmbiguityKind.PARTICLE_OR_GIVEN == "particle-or-given" assert AmbiguityKind("order") is AmbiguityKind.ORDER -def test_ambiguity_construction_coerces_kind_string(): +def test_ambiguity_construction_coerces_kind_string() -> None: t = Token("Van", Span(0, 3), Role.GIVEN, frozenset({"particle"})) a = Ambiguity("particle-or-given", "leading 'van' may be a particle", (t,)) # type: ignore[arg-type] assert a.kind is AmbiguityKind.PARTICLE_OR_GIVEN assert a.tokens == (t,) -def test_ambiguity_rejects_unknown_kind(): +def test_ambiguity_rejects_unknown_kind() -> None: with pytest.raises(ValueError, match="particle-or-given"): Ambiguity("no-such-kind", "detail", ()) # type: ignore[arg-type] -def test_ambiguity_rejects_non_token_elements(): +def test_ambiguity_rejects_non_token_elements() -> None: with pytest.raises(ValueError, match="only Token instances"): Ambiguity("order", "detail", ("not-a-token",)) # type: ignore[arg-type] -def test_ambiguity_rejects_empty_detail(): +def test_ambiguity_rejects_empty_detail() -> None: with pytest.raises(ValueError, match="non-empty string"): Ambiguity(AmbiguityKind.ORDER, "", ()) -def _pn(original, tokens, ambiguities=()): +def _pn(original: str, tokens: Iterable[Token], + ambiguities: Iterable[Ambiguity] = ()) -> ParsedName: return ParsedName(original=original, tokens=tuple(tokens), ambiguities=tuple(ambiguities)) -def test_parsedname_accepts_valid_spans_and_is_truthy(): +def test_parsedname_accepts_valid_spans_and_is_truthy() -> None: pn = _pn("John Smith", [ Token("John", Span(0, 4), Role.GIVEN), Token("Smith", Span(5, 10), Role.FAMILY), @@ -98,17 +101,17 @@ def test_parsedname_accepts_valid_spans_and_is_truthy(): assert bool(pn) is True -def test_empty_parse_is_falsy(): +def test_empty_parse_is_falsy() -> None: assert bool(_pn("", [])) is False assert bool(_pn(" ", [])) is False -def test_parsedname_rejects_out_of_bounds_span(): +def test_parsedname_rejects_out_of_bounds_span() -> None: with pytest.raises(ValueError, match="out of bounds"): _pn("John", [Token("Johnny", Span(0, 6), Role.GIVEN)]) -def test_parsedname_rejects_overlapping_spans(): +def test_parsedname_rejects_overlapping_spans() -> None: with pytest.raises(ValueError, match="ascending"): _pn("John Smith", [ Token("John", Span(0, 4), Role.GIVEN), @@ -116,7 +119,7 @@ def test_parsedname_rejects_overlapping_spans(): ]) -def test_parsedname_rejects_descending_spans(): +def test_parsedname_rejects_descending_spans() -> None: with pytest.raises(ValueError, match="ascending"): _pn("John Smith", [ Token("Smith", Span(5, 10), Role.FAMILY), @@ -124,7 +127,7 @@ def test_parsedname_rejects_descending_spans(): ]) -def test_synthetic_tokens_skip_span_checks(): +def test_synthetic_tokens_skip_span_checks() -> None: pn = _pn("John Smith", [ Token("John", Span(0, 4), Role.GIVEN), Token("Qux", None, Role.MIDDLE), @@ -133,7 +136,7 @@ def test_synthetic_tokens_skip_span_checks(): assert len(pn.tokens) == 3 -def test_ambiguity_tokens_must_be_subset_of_parse_tokens(): +def test_ambiguity_tokens_must_be_subset_of_parse_tokens() -> None: inside = Token("Van", Span(0, 3), Role.GIVEN) outside = Token("Zzz", None, Role.GIVEN) with pytest.raises(ValueError, match="subset"): @@ -142,7 +145,7 @@ def test_ambiguity_tokens_must_be_subset_of_parse_tokens(): [Ambiguity(AmbiguityKind.PARTICLE_OR_GIVEN, "d", (outside,))]) -def test_parsedname_equality_is_strict_structural(): +def test_parsedname_equality_is_strict_structural() -> None: a = _pn("John", [Token("John", Span(0, 4), Role.GIVEN)]) b = _pn("John", [Token("John", Span(0, 4), Role.GIVEN)]) c = _pn("John ", [Token("John", Span(0, 4), Role.GIVEN)]) @@ -150,19 +153,19 @@ def test_parsedname_equality_is_strict_structural(): assert a != c # different original: not interchangeable -def test_parsedname_rejects_non_str_original(): +def test_parsedname_rejects_non_str_original() -> None: with pytest.raises(ValueError, match="must be a str"): _pn(None, []) # type: ignore[arg-type] -def test_parsedname_rejects_non_token_and_non_ambiguity_elements(): +def test_parsedname_rejects_non_token_and_non_ambiguity_elements() -> None: with pytest.raises(ValueError, match="only Token instances"): _pn("x", ["not-a-token"]) # type: ignore[list-item] with pytest.raises(ValueError, match="only Ambiguity instances"): _pn("John", [Token("John", Span(0, 4), Role.GIVEN)], ["nope"]) # type: ignore[list-item] -def _delavega(): +def _delavega() -> ParsedName: # "Dr. Juan de la Vega III" -- hand-built, spans verified by hand # 0123456789012345678901234 return _pn("Dr. Juan de la Vega III", [ @@ -175,7 +178,7 @@ def _delavega(): ]) -def test_string_properties_join_by_role(): +def test_string_properties_join_by_role() -> None: pn = _delavega() assert pn.title == "Dr." assert pn.given == "Juan" @@ -186,7 +189,7 @@ def test_string_properties_join_by_role(): assert pn.maiden == "" -def test_suffix_joins_with_comma_space(): +def test_suffix_joins_with_comma_space() -> None: pn = _pn("John Smith PhD MD", [ Token("John", Span(0, 4), Role.GIVEN), Token("Smith", Span(5, 10), Role.FAMILY), @@ -196,7 +199,7 @@ def test_suffix_joins_with_comma_space(): assert pn.suffix == "PhD, MD" -def test_derived_views_filter_on_stable_particle_tag(): +def test_derived_views_filter_on_stable_particle_tag() -> None: # Pin the hard-coded "particle" string in _text_for to the published # contract until Plan 3's tag-emission contract tests land. assert "particle" in STABLE_TAGS @@ -207,13 +210,13 @@ def test_derived_views_filter_on_stable_particle_tag(): assert pn.given_names == "Juan" # given + middle -def test_tokens_for_preserves_order(): +def test_tokens_for_preserves_order() -> None: pn = _delavega() assert [t.text for t in pn.tokens_for(Role.FAMILY)] == ["de", "la", "Vega"] assert pn.tokens_for(Role.NICKNAME) == () -def test_as_dict_canonical_order_and_empty_filtering(): +def test_as_dict_canonical_order_and_empty_filtering() -> None: pn = _delavega() d = pn.as_dict() assert list(d) == ["title", "given", "middle", "family", @@ -223,7 +226,7 @@ def test_as_dict_canonical_order_and_empty_filtering(): assert list(d2) == ["title", "given", "family", "suffix"] -def test_replace_swaps_field_with_synthetic_tokens_in_place(): +def test_replace_swaps_field_with_synthetic_tokens_in_place() -> None: pn = _delavega() pn2 = pn.replace(given="Jean Paul") assert pn2.given == "Jean Paul" @@ -235,7 +238,7 @@ def test_replace_swaps_field_with_synthetic_tokens_in_place(): assert [t.role for t in pn2.tokens][:3] == [Role.TITLE, Role.GIVEN, Role.GIVEN] -def test_replace_adds_missing_field_at_end(): +def test_replace_adds_missing_field_at_end() -> None: pn = _pn("John Smith", [ Token("John", Span(0, 4), Role.GIVEN), Token("Smith", Span(5, 10), Role.FAMILY), @@ -245,17 +248,17 @@ def test_replace_adds_missing_field_at_end(): assert pn2.tokens[-1].role is Role.SUFFIX -def test_replace_with_empty_string_clears_field(): +def test_replace_with_empty_string_clears_field() -> None: pn = _delavega() assert pn.replace(title="").title == "" -def test_replace_rejects_unknown_field(): +def test_replace_rejects_unknown_field() -> None: with pytest.raises(TypeError, match="given"): _delavega().replace(firstname="X") -def test_replace_drops_ambiguities_referencing_removed_tokens(): +def test_replace_drops_ambiguities_referencing_removed_tokens() -> None: van = Token("Van", Span(0, 3), Role.GIVEN) pn = _pn("Van Johnson", [van, Token("Johnson", Span(4, 11), Role.FAMILY)], @@ -264,18 +267,18 @@ def test_replace_drops_ambiguities_referencing_removed_tokens(): assert pn.replace(family="Smith").ambiguities != () -def test_replace_rejects_non_str_value(): +def test_replace_rejects_non_str_value() -> None: with pytest.raises(TypeError, match="must be a str"): _delavega().replace(given=None) # type: ignore[arg-type] -def test_replace_appends_missing_roles_in_canonical_order(): +def test_replace_appends_missing_roles_in_canonical_order() -> None: pn = _pn("John", [Token("John", Span(0, 4), Role.GIVEN)]) pn2 = pn.replace(maiden="X", suffix="Y") assert [t.role for t in pn2.tokens] == [Role.GIVEN, Role.SUFFIX, Role.MAIDEN] -def test_comparison_key_is_casefolded_canonical_seven_tuple(): +def test_comparison_key_is_casefolded_canonical_seven_tuple() -> None: pn = _delavega() assert pn.comparison_key() == ( "dr.", "juan", "", "de la vega", "iii", "", "", From 473c0b96292109b8a713fe9aa504ec15b27bfa90 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 04:59:46 -0700 Subject: [PATCH 33/88] Reject mappings passed to plain Lexicon vocab fields Lexicon(titles={'Dr.': 'Doctor'}) silently kept only the keys -- the lone quiet coercion on an otherwise fail-loud surface, and a dict here almost always means the field was confused with capitalization_exceptions. Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 8 ++++++++ tests/v2/test_lexicon.py | 10 ++++++++++ 2 files changed, 18 insertions(+) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index b9e20ba..1608696 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -38,6 +38,14 @@ def _normset(entries: Iterable[str], field_name: str) -> frozenset[str]: f"Lexicon.{field_name} must be an iterable of strings, " f"not a bare string" ) + # A Mapping would silently contribute only its keys; a dict here + # almost always means the caller confused this field with + # capitalization_exceptions. + if isinstance(entries, Mapping): + raise ValueError( + f"Lexicon.{field_name} must be an iterable of strings, not a " + f"mapping (only capitalization_exceptions holds key->value pairs)" + ) items = tuple(entries) # materialize once; entries may be a generator for w in items: if not isinstance(w, str): diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index d2cfe5f..d200440 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -125,3 +125,13 @@ def test_pickle_round_trip_preserves_equality_and_cap_map() -> None: assert hash(loaded) == hash(lex) assert (loaded.capitalization_exceptions_map == lex.capitalization_exceptions_map) + + +def test_lexicon_rejects_mapping_for_plain_vocab_field() -> None: + # A dict here almost always means the caller confused the field with + # capitalization_exceptions; silently keeping just the keys would be + # the lone quiet coercion on an otherwise fail-loud surface. + with pytest.raises(ValueError, match="mapping"): + Lexicon(titles={"Dr.": "Doctor"}) # type: ignore[arg-type] + with pytest.raises(ValueError, match="mapping"): + Lexicon.empty().add(titles={"Dr.": "Doctor"}) From ce7b36bc4ac9eddced6de3c4f5ee1c08fa8c7a72 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 05:00:32 -0700 Subject: [PATCH 34/88] Diff Lexicon reprs against the nearer named baseline repr(Lexicon.empty().add(titles={'zqx'})) rendered as default() plus ten '-N' deltas -- technically bounded, but the wrong story for the documented build-from-empty() power-user path. Compare against both named constructors and render the smaller diff. Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 38 +++++++++++++++++++++++++------------- tests/v2/test_reprs.py | 8 ++++++++ 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 1608696..7087222 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -107,24 +107,36 @@ def __post_init__(self) -> None: f"not in particles: {extra}" ) - def __repr__(self) -> str: - # Bounded: renders only which fields deviate from default() and by - # how many entries -- never the entries themselves (design rule, - # see nameparser._types module docstring). - default = Lexicon.default() - if self == default: - return "Lexicon(default)" + def _deltas_from(self, baseline: Lexicon) -> list[tuple[str, int, int]]: deltas = [] for name in _VOCAB_FIELDS + ("capitalization_exceptions",): mine = set(getattr(self, name)) - theirs = set(getattr(default, name)) + theirs = set(getattr(baseline, name)) added, removed = len(mine - theirs), len(theirs - mine) if added or removed: - delta = "".join( - part for part, n in ((f"+{added}", added), (f"-{removed}", removed)) if n - ) - deltas.append(f"{name}: {delta}") - return f"Lexicon(default + {', '.join(deltas)})" + deltas.append((name, added, removed)) + return deltas + + def __repr__(self) -> str: + # Bounded: renders only which fields deviate from the nearer of + # the two named constructors and by how many entries -- never the + # entries themselves (design rule, see nameparser._types module + # docstring). Diffing empty()-built lexicons against default() + # would tell the wrong story ("default minus ~700 entries"). + if self == Lexicon.default(): + return "Lexicon(default)" + if self == Lexicon.empty(): + return "Lexicon(empty)" + candidates = [(label, self._deltas_from(baseline)) + for label, baseline in (("default", Lexicon.default()), + ("empty", Lexicon.empty()))] + label, deltas = min( + candidates, key=lambda c: sum(a + r for _, a, r in c[1])) + rendered = ", ".join( + name + ": " + "".join( + part for part, n in ((f"+{a}", a), (f"-{r}", r)) if n) + for name, a, r in deltas) + return f"Lexicon({label} + {rendered})" def __getstate__(self) -> dict[str, object]: # _cap_map is a MappingProxyType, which pickle rejects; ship every diff --git a/tests/v2/test_reprs.py b/tests/v2/test_reprs.py index cacfdbc..61c31e1 100644 --- a/tests/v2/test_reprs.py +++ b/tests/v2/test_reprs.py @@ -58,3 +58,11 @@ def test_locale_repr_shows_code_and_patched_fields() -> None: {PatronymicRule.EAST_SLAVIC}))) assert repr(ru) == "Locale('ru': patronymic_rules)" assert repr(Locale("xx", Lexicon.empty())) == "Locale('xx')" + + +def test_lexicon_repr_diffs_against_nearer_baseline() -> None: + # An empty()-built Lexicon must not render as default() minus ~700 + # entries -- diff against whichever named constructor is nearer. + assert repr(Lexicon.empty()) == "Lexicon(empty)" + lex = Lexicon.empty().add(titles={"zqx"}) + assert repr(lex) == "Lexicon(empty + titles: +1)" From 334c617ff954f6d7a482067e42dbf55902ae2c77 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 05:01:46 -0700 Subject: [PATCH 35/88] =?UTF-8?q?Order=20Lexicon=20sections=20per=20conven?= =?UTF-8?q?tions=20doc=20=C2=A74?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pure move: constructors ahead of dunders, __or__ grouped with the dunders, then properties, then the editing methods (with _edit at the head of its section per the documented exception). No behavior change. Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 50 +++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 7087222..296c18e 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -107,6 +107,18 @@ def __post_init__(self) -> None: f"not in particles: {extra}" ) + # -- constructors ---------------------------------------------------- + + @classmethod + def empty(cls) -> Lexicon: + return cls() + + @classmethod + def default(cls) -> Lexicon: + return _default_lexicon() + + # -- dunders ---------------------------------------------------------- + def _deltas_from(self, baseline: Lexicon) -> list[tuple[str, int, int]]: deltas = [] for name in _VOCAB_FIELDS + ("capitalization_exceptions",): @@ -151,21 +163,25 @@ def __setstate__(self, state: dict[str, object]) -> None: self, "_cap_map", MappingProxyType(dict(self.capitalization_exceptions))) + def __or__(self, other: Lexicon) -> Lexicon: + if not isinstance(other, Lexicon): + return NotImplemented + updates: dict[str, object] = { + name: getattr(self, name) | getattr(other, name) + for name in _VOCAB_FIELDS + } + # right-biased on key conflicts, mirroring later-wins for scalars + merged = dict(self._cap_map) | dict(other._cap_map) + updates["capitalization_exceptions"] = tuple(sorted(merged.items())) + return dataclasses.replace(self, **updates) # type: ignore[arg-type] + + # -- properties ------------------------------------------------------- + @property def capitalization_exceptions_map(self) -> Mapping[str, str]: return self._cap_map - # -- constructors ---------------------------------------------------- - - @classmethod - def empty(cls) -> Lexicon: - return cls() - - @classmethod - def default(cls) -> Lexicon: - return _default_lexicon() - - # -- composition ------------------------------------------------------ + # -- editing ---------------------------------------------------------- def _edit(self, op: str, entries: Mapping[str, Iterable[str]]) -> Lexicon: updates: dict[str, frozenset[str]] = {} @@ -200,18 +216,6 @@ def add(self, **entries: Iterable[str]) -> Lexicon: def remove(self, **entries: Iterable[str]) -> Lexicon: return self._edit("remove", entries) - def __or__(self, other: Lexicon) -> Lexicon: - if not isinstance(other, Lexicon): - return NotImplemented - updates: dict[str, object] = { - name: getattr(self, name) | getattr(other, name) - for name in _VOCAB_FIELDS - } - # right-biased on key conflicts, mirroring later-wins for scalars - merged = dict(self._cap_map) | dict(other._cap_map) - updates["capitalization_exceptions"] = tuple(sorted(merged.items())) - return dataclasses.replace(self, **updates) # type: ignore[arg-type] - @functools.cache def _default_lexicon() -> Lexicon: From 5a5bf801cf9a3e70683ffa9165bad38064087a6d Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 11:23:50 -0700 Subject: [PATCH 36/88] Adopt the stdlib exception taxonomy across v2 validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wrong type (including wrong element type in a collection, bare str for an iterable-of-strings field, Mapping for a plain iterable) raises TypeError; well-typed but unacceptable values (empty required strings, inverted spans, unknown enum values, subset violations) raise ValueError. Matches the boundary the generated dataclass __init__ already draws -- unknown kwargs are TypeError natively -- so the constructors can never present an all-ValueError contract anyway. Mixed checks (Token.text, Ambiguity.detail, Locale.code, delimiter pairs, extra_suffix_delimiters entries) split into a type check and a value check. Enum lookups stay ValueError for any element, matching stdlib EnumType.__call__; non-iterable patronymic_rules now surfaces its natural TypeError instead of being converted. Rule recorded in the conventions doc §6. Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 8 ++++---- nameparser/_locale.py | 10 +++++++--- nameparser/_policy.py | 31 +++++++++++++++++++++---------- nameparser/_types.py | 26 +++++++++++++++----------- tests/v2/test_lexicon.py | 10 +++++----- tests/v2/test_locale.py | 6 ++++-- tests/v2/test_policy.py | 12 +++++++----- tests/v2/test_types.py | 19 ++++++++++++------- 8 files changed, 75 insertions(+), 47 deletions(-) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 296c18e..61e356d 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -34,7 +34,7 @@ def _normset(entries: Iterable[str], field_name: str) -> frozenset[str]: # yield the single characters {'d', 'r'} -- the set(str) footgun on # the primary customization surface. if isinstance(entries, str): - raise ValueError( + raise TypeError( f"Lexicon.{field_name} must be an iterable of strings, " f"not a bare string" ) @@ -42,14 +42,14 @@ def _normset(entries: Iterable[str], field_name: str) -> frozenset[str]: # almost always means the caller confused this field with # capitalization_exceptions. if isinstance(entries, Mapping): - raise ValueError( + raise TypeError( f"Lexicon.{field_name} must be an iterable of strings, not a " f"mapping (only capitalization_exceptions holds key->value pairs)" ) items = tuple(entries) # materialize once; entries may be a generator for w in items: if not isinstance(w, str): - raise ValueError( + raise TypeError( f"Lexicon.{field_name} entries must be strings, got {w!r}" ) result = frozenset(_normalize(w) for w in items) @@ -89,7 +89,7 @@ def __post_init__(self) -> None: deduped: dict[str, str] = {} for k, v in pairs: if not isinstance(k, str) or not isinstance(v, str): - raise ValueError( + raise TypeError( f"capitalization_exceptions entries must be " f"str -> str, got {k!r}: {v!r}" ) diff --git a/nameparser/_locale.py b/nameparser/_locale.py index 2494b3e..5e06d17 100644 --- a/nameparser/_locale.py +++ b/nameparser/_locale.py @@ -23,7 +23,11 @@ class Locale: policy: PolicyPatch = PolicyPatch() def __post_init__(self) -> None: - if not isinstance(self.code, str) or not self.code.strip(): + if not isinstance(self.code, str): + raise TypeError( + f"Locale.code must be a str, got {self.code!r}" + ) + if not self.code.strip(): raise ValueError( f"Locale.code must be a non-empty string, got {self.code!r}" ) @@ -36,11 +40,11 @@ def __post_init__(self) -> None: f"Locale.code must not contain whitespace, got {self.code!r}" ) if not isinstance(self.lexicon, Lexicon): - raise ValueError( + raise TypeError( f"Locale.lexicon must be a Lexicon, got {self.lexicon!r}" ) if not isinstance(self.policy, PolicyPatch): - raise ValueError( + raise TypeError( f"Locale.policy must be a PolicyPatch, got {self.policy!r}" ) diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 7427f16..53ab123 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -55,13 +55,15 @@ def __post_init__(self) -> None: ) object.__setattr__(self, "name_order", order) if isinstance(self.patronymic_rules, str): - raise ValueError( + raise TypeError( f"patronymic_rules must be an iterable of rule names, " f"not a bare string: {self.patronymic_rules!r}" ) + # A non-iterable raises its natural TypeError from the frozenset + # call; only failed enum lookups get the enriched message. try: rules = frozenset(PatronymicRule(r) for r in self.patronymic_rules) - except (TypeError, ValueError): + except ValueError: valid = ", ".join(r.value for r in PatronymicRule) raise ValueError( f"unknown patronymic rule in {self.patronymic_rules!r}; " @@ -72,23 +74,32 @@ def __post_init__(self) -> None: pairs = tuple(getattr(self, pairs_name)) for pair in pairs: if (not isinstance(pair, tuple) or len(pair) != 2 - or not all(isinstance(s, str) and s for s in pair)): - raise ValueError( + or not all(isinstance(s, str) for s in pair)): + raise TypeError( f"{pairs_name} entries must be (open, close) tuples " - f"of non-empty strings, got {pair!r}" + f"of strings, got {pair!r}" + ) + if not all(pair): + raise ValueError( + f"{pairs_name} entries must be pairs of non-empty " + f"strings, got {pair!r}" ) object.__setattr__(self, pairs_name, frozenset(pairs)) if isinstance(self.extra_suffix_delimiters, str): - raise ValueError( + raise TypeError( f"extra_suffix_delimiters must be an iterable of strings, " f"not a bare string: {self.extra_suffix_delimiters!r}" ) delimiters = tuple(self.extra_suffix_delimiters) for d in delimiters: - if not isinstance(d, str) or not d: + if not isinstance(d, str): + raise TypeError( + f"extra_suffix_delimiters entries must be strings, " + f"got {d!r}" + ) + if not d: raise ValueError( - f"extra_suffix_delimiters entries must be non-empty " - f"strings, got {d!r}" + "extra_suffix_delimiters entries must be non-empty strings" ) object.__setattr__( self, "extra_suffix_delimiters", frozenset(delimiters) @@ -166,7 +177,7 @@ def __post_init__(self) -> None: if value is UNSET: continue if isinstance(value, str): - raise ValueError( + raise TypeError( f"{f.name} must be an iterable, " f"not a bare string: {value!r}" ) diff --git a/nameparser/_types.py b/nameparser/_types.py index 97dd941..5ec3f19 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -47,17 +47,19 @@ class Token: tags: frozenset[str] = frozenset() def __post_init__(self) -> None: - if not isinstance(self.text, str) or not self.text: - raise ValueError( - f"Token.text must be a non-empty string, got {self.text!r}" + if not isinstance(self.text, str): + raise TypeError( + f"Token.text must be a str, got {self.text!r}" ) + if not self.text: + raise ValueError("Token.text must be a non-empty string") if self.span is not None: if not ( isinstance(self.span, tuple) and len(self.span) == 2 and all(isinstance(v, int) for v in self.span) ): - raise ValueError( + raise TypeError( f"invalid span {self.span!r}: expected a (start, end) " "pair of ints or None" ) @@ -103,14 +105,16 @@ def __post_init__(self) -> None: raise ValueError( f"unknown AmbiguityKind {self.kind!r}; valid kinds: {valid}" ) from None - if not isinstance(self.detail, str) or not self.detail: - raise ValueError( - f"Ambiguity.detail must be a non-empty string, got {self.detail!r}" + if not isinstance(self.detail, str): + raise TypeError( + f"Ambiguity.detail must be a str, got {self.detail!r}" ) + if not self.detail: + raise ValueError("Ambiguity.detail must be a non-empty string") toks = tuple(self.tokens) for tok in toks: if not isinstance(tok, Token): - raise ValueError( + raise TypeError( f"Ambiguity.tokens must contain only Token instances, " f"got {tok!r}" ) @@ -136,20 +140,20 @@ class ParsedName: def __post_init__(self) -> None: if not isinstance(self.original, str): - raise ValueError( + raise TypeError( f"ParsedName.original must be a str, got {self.original!r}" ) object.__setattr__(self, "tokens", tuple(self.tokens)) object.__setattr__(self, "ambiguities", tuple(self.ambiguities)) for tok in self.tokens: if not isinstance(tok, Token): - raise ValueError( + raise TypeError( f"ParsedName.tokens must contain only Token instances, " f"got {tok!r}" ) for amb in self.ambiguities: if not isinstance(amb, Ambiguity): - raise ValueError( + raise TypeError( f"ParsedName.ambiguities must contain only Ambiguity " f"instances, got {amb!r}" ) diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index d200440..210da29 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -49,12 +49,12 @@ def test_lexicon_is_hashable() -> None: def test_lexicon_rejects_bare_string_vocab() -> None: - with pytest.raises(ValueError, match="bare string"): + with pytest.raises(TypeError, match="bare string"): Lexicon(titles="dr") # type: ignore[arg-type] def test_lexicon_rejects_non_str_vocab_entries() -> None: - with pytest.raises(ValueError, match="entries must be strings"): + with pytest.raises(TypeError, match="entries must be strings"): Lexicon(titles={"Dr.", 42}) # type: ignore[arg-type] @@ -71,7 +71,7 @@ def test_colliding_exception_keys_dedupe_last_wins() -> None: def test_lexicon_rejects_non_str_exception_values() -> None: - with pytest.raises(ValueError, match="str -> str"): + with pytest.raises(TypeError, match="str -> str"): Lexicon(capitalization_exceptions={"phd": 42}) # type: ignore[dict-item, arg-type] @@ -131,7 +131,7 @@ def test_lexicon_rejects_mapping_for_plain_vocab_field() -> None: # A dict here almost always means the caller confused the field with # capitalization_exceptions; silently keeping just the keys would be # the lone quiet coercion on an otherwise fail-loud surface. - with pytest.raises(ValueError, match="mapping"): + with pytest.raises(TypeError, match="mapping"): Lexicon(titles={"Dr.": "Doctor"}) # type: ignore[arg-type] - with pytest.raises(ValueError, match="mapping"): + with pytest.raises(TypeError, match="mapping"): Lexicon.empty().add(titles={"Dr.": "Doctor"}) diff --git a/tests/v2/test_locale.py b/tests/v2/test_locale.py index 0af17fe..bc25eda 100644 --- a/tests/v2/test_locale.py +++ b/tests/v2/test_locale.py @@ -40,10 +40,12 @@ def test_locale_is_hashable() -> None: def test_locale_validates_component_types() -> None: - with pytest.raises(ValueError, match="Lexicon"): + with pytest.raises(TypeError, match="Lexicon"): Locale(code="ru", lexicon={"titles": set()}) # type: ignore[arg-type] - with pytest.raises(ValueError, match="PolicyPatch"): + with pytest.raises(TypeError, match="PolicyPatch"): Locale(code="ru", lexicon=Lexicon.empty(), policy={"name_order": None}) # type: ignore[arg-type] + with pytest.raises(TypeError, match="must be a str"): + Locale(code=5, lexicon=Lexicon.empty()) # type: ignore[arg-type] def test_locale_with_lexicon_pickles_round_trip() -> None: diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index 0c9b610..06cef02 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -50,14 +50,14 @@ def test_delimiter_pairs_must_be_nonempty_string_pairs() -> None: def test_delimiter_pair_rejects_two_char_string() -> None: - with pytest.raises(ValueError, match="tuples"): + with pytest.raises(TypeError, match="tuples"): Policy(nickname_delimiters=frozenset({"()"})) # type: ignore[arg-type] def test_patronymic_rules_rejects_bare_string_and_non_iterable() -> None: - with pytest.raises(ValueError, match="bare string"): + with pytest.raises(TypeError, match="bare string"): Policy(patronymic_rules="east-slavic") # type: ignore[arg-type] - with pytest.raises(ValueError, match="valid rules"): + with pytest.raises(TypeError, match="iterable"): Policy(patronymic_rules=5) # type: ignore[arg-type] @@ -76,8 +76,10 @@ def test_policy_delimiters_do_not_alias_caller_containers() -> None: def test_extra_suffix_delimiters_validated_and_coerced() -> None: - with pytest.raises(ValueError, match="bare string"): + with pytest.raises(TypeError, match="bare string"): Policy(extra_suffix_delimiters="ab") # type: ignore[arg-type] + with pytest.raises(TypeError, match="must be strings"): + Policy(extra_suffix_delimiters=frozenset({5})) # type: ignore[arg-type] with pytest.raises(ValueError, match="non-empty strings"): Policy(extra_suffix_delimiters=frozenset({""})) p = Policy(extra_suffix_delimiters=["-"]) # type: ignore[arg-type] @@ -106,7 +108,7 @@ def test_policy_patch_canonicalizes_union_fields() -> None: def test_policy_patch_rejects_bare_string_union_fields() -> None: - with pytest.raises(ValueError, match="bare string"): + with pytest.raises(TypeError, match="bare string"): PolicyPatch(extra_suffix_delimiters="ab") # type: ignore[arg-type] diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 7446cbd..973021c 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -42,14 +42,14 @@ def test_token_rejects_negative_span() -> None: def test_token_rejects_malformed_span_shapes() -> None: - with pytest.raises(ValueError, match="expected a \\(start, end\\) pair"): + with pytest.raises(TypeError, match="expected a \\(start, end\\) pair"): Token("x", (0, 4, 9), Role.GIVEN) # type: ignore[arg-type] - with pytest.raises(ValueError, match="expected a \\(start, end\\) pair"): + with pytest.raises(TypeError, match="expected a \\(start, end\\) pair"): Token("x", 5, Role.GIVEN) # type: ignore[arg-type] def test_token_rejects_non_string_text() -> None: - with pytest.raises(ValueError, match="got None"): + with pytest.raises(TypeError, match="got None"): Token(None, None, Role.GIVEN) # type: ignore[arg-type] @@ -78,7 +78,7 @@ def test_ambiguity_rejects_unknown_kind() -> None: def test_ambiguity_rejects_non_token_elements() -> None: - with pytest.raises(ValueError, match="only Token instances"): + with pytest.raises(TypeError, match="only Token instances"): Ambiguity("order", "detail", ("not-a-token",)) # type: ignore[arg-type] @@ -87,6 +87,11 @@ def test_ambiguity_rejects_empty_detail() -> None: Ambiguity(AmbiguityKind.ORDER, "", ()) +def test_ambiguity_rejects_non_str_detail() -> None: + with pytest.raises(TypeError, match="must be a str"): + Ambiguity(AmbiguityKind.ORDER, None, ()) # type: ignore[arg-type] + + def _pn(original: str, tokens: Iterable[Token], ambiguities: Iterable[Ambiguity] = ()) -> ParsedName: return ParsedName(original=original, tokens=tuple(tokens), @@ -154,14 +159,14 @@ def test_parsedname_equality_is_strict_structural() -> None: def test_parsedname_rejects_non_str_original() -> None: - with pytest.raises(ValueError, match="must be a str"): + with pytest.raises(TypeError, match="must be a str"): _pn(None, []) # type: ignore[arg-type] def test_parsedname_rejects_non_token_and_non_ambiguity_elements() -> None: - with pytest.raises(ValueError, match="only Token instances"): + with pytest.raises(TypeError, match="only Token instances"): _pn("x", ["not-a-token"]) # type: ignore[list-item] - with pytest.raises(ValueError, match="only Ambiguity instances"): + with pytest.raises(TypeError, match="only Ambiguity instances"): _pn("John", [Token("John", Span(0, 4), Role.GIVEN)], ["nope"]) # type: ignore[list-item] From 974aec2218f0f5fbf0867b480fceaeb0cea55300 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 12:14:55 -0700 Subject: [PATCH 37/88] Source maiden markers from a config data module (#274) maiden_markers was the only Lexicon.default() field fed by an inline literal instead of a nameparser/config data module. Add config/maiden_markers.py with the #274 candidate set: French, German, Dutch, Czech/Slovak, and Russian markers, both genders and both e/yo spellings (casefold does not fold them). Non-colliding Cyrillic entries belong in the default lexicon per the locales design's sorting rule. Deliberately absent: 'z domu' (two-token marker, pending the pipeline's multi-token matching decision) and Scandinavian 'f.' (collides with the initial F). The 1.x parser does not read the module. Co-Authored-By: Claude Fable 5 --- docs/modules.rst | 2 ++ nameparser/_lexicon.py | 3 ++- nameparser/config/maiden_markers.py | 30 +++++++++++++++++++++++++++++ tests/v2/test_lexicon.py | 6 ++++++ 4 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 nameparser/config/maiden_markers.py diff --git a/docs/modules.rst b/docs/modules.rst index bdc6b52..8f3b199 100644 --- a/docs/modules.rst +++ b/docs/modules.rst @@ -33,6 +33,8 @@ HumanName.config Defaults :members: .. automodule:: nameparser.config.conjunctions :members: +.. automodule:: nameparser.config.maiden_markers + :members: .. automodule:: nameparser.config.capitalization :members: .. automodule:: nameparser.config.regexes diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 61e356d..a43e9e7 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -223,6 +223,7 @@ def _default_lexicon() -> Lexicon: from nameparser.config.bound_first_names import BOUND_FIRST_NAMES from nameparser.config.capitalization import CAPITALIZATION_EXCEPTIONS from nameparser.config.conjunctions import CONJUNCTIONS + from nameparser.config.maiden_markers import MAIDEN_MARKERS from nameparser.config.prefixes import NON_FIRST_NAME_PREFIXES, PREFIXES from nameparser.config.suffixes import ( SUFFIX_ACRONYMS, SUFFIX_ACRONYMS_AMBIGUOUS, SUFFIX_NOT_ACRONYMS, @@ -243,7 +244,7 @@ def _default_lexicon() -> Lexicon: particles_ambiguous=frozenset(PREFIXES - NON_FIRST_NAME_PREFIXES), conjunctions=frozenset(CONJUNCTIONS), bound_given_names=frozenset(BOUND_FIRST_NAMES), - maiden_markers=frozenset({"née", "nee", "geb"}), + maiden_markers=frozenset(MAIDEN_MARKERS), # pass canonical pair-tuples so this strictly-typed call site never # feeds a Mapping to the tuple-annotated field; __post_init__ # still tolerates a Mapping at runtime for interactive use diff --git a/nameparser/config/maiden_markers.py b/nameparser/config/maiden_markers.py new file mode 100644 index 0000000..47910bf --- /dev/null +++ b/nameparser/config/maiden_markers.py @@ -0,0 +1,30 @@ +MAIDEN_MARKERS = { + 'née', + 'né', + 'nee', + 'geb', + 'geborene', + 'geboren', + 'roz', + 'rozená', + 'урожд', + 'урождённая', + 'урожденная', + 'урождённый', + 'урожденный', +} +""" +Marker words that introduce a birth surname, e.g. "Jane Smith née Jones" +(#274). French née/né/nee, German geb./geborene, Dutch geboren, +Czech/Slovak roz./rozená, Russian урожд./урождённая (both ё and е +spellings — ``str.casefold()`` does not fold them, and running text +routinely writes е). Entries are stored normalized: lowercase, no +periods. + +Consumed by the 2.0 parser's default lexicon. The 1.x parser does not +read this module. + +Deliberately absent: Polish "z domu" (a two-token marker; pending the +2.0 pipeline's multi-token matching decision) and Scandinavian "f." +(født/född — collides with the initial "F."). +""" diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index 210da29..6945615 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -22,6 +22,12 @@ def test_default_sources_v1_vocabulary() -> None: # normalized -- only keys are casefolded/period-stripped at # construction, values pass through unchanged). assert lex.capitalization_exceptions_map["phd"] == "Ph.D." + # maiden markers source from the same data-module pattern (#274); + # non-colliding Cyrillic entries live in the default per the locales + # design's sorting rule, and both ё/е spellings are listed because + # casefold() does not fold them. + assert "geborene" in lex.maiden_markers + assert "урожденная" in lex.maiden_markers and "урождённая" in lex.maiden_markers def test_default_is_cached_single_instance() -> None: From 5d7893375f1ae2e94872b6b46a23ec8f030d7813 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 12:36:34 -0700 Subject: [PATCH 38/88] Add Scandinavian participle maiden markers (#274) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The issue's veto covered only the abbreviation 'f.' (collides with the initial F); the full participles født (da/nb), fødd (nn), and född (sv) cannot appear as name tokens and are the standard convention in running text. No ASCII variants -- dropping the diacritic is not standard practice in Scandinavian text, unlike nee in English. Co-Authored-By: Claude Fable 5 --- nameparser/config/maiden_markers.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/nameparser/config/maiden_markers.py b/nameparser/config/maiden_markers.py index 47910bf..0c36c5e 100644 --- a/nameparser/config/maiden_markers.py +++ b/nameparser/config/maiden_markers.py @@ -7,6 +7,9 @@ 'geboren', 'roz', 'rozená', + 'født', + 'fødd', + 'född', 'урожд', 'урождённая', 'урожденная', @@ -16,15 +19,16 @@ """ Marker words that introduce a birth surname, e.g. "Jane Smith née Jones" (#274). French née/né/nee, German geb./geborene, Dutch geboren, -Czech/Slovak roz./rozená, Russian урожд./урождённая (both ё and е -spellings — ``str.casefold()`` does not fold them, and running text -routinely writes е). Entries are stored normalized: lowercase, no -periods. +Czech/Slovak roz./rozená, Danish/Norwegian født (Nynorsk fødd), Swedish +född, Russian урожд./урождённая (both ё and е spellings — +``str.casefold()`` does not fold them, and running text routinely +writes е). Entries are stored normalized: lowercase, no periods. Consumed by the 2.0 parser's default lexicon. The 1.x parser does not read this module. Deliberately absent: Polish "z domu" (a two-token marker; pending the -2.0 pipeline's multi-token matching decision) and Scandinavian "f." -(født/född — collides with the initial "F."). +2.0 pipeline's multi-token matching decision) and the Scandinavian +abbreviation "f." (collides with the initial "F." — only the full +participles are safe). """ From a16364edf539ca92e6e6924f4a393939fb5b0edc Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 12:48:17 -0700 Subject: [PATCH 39/88] Distill 2.0 implementation conventions into AGENTS.md The enforceable subset of the (untracked) conventions doc -- module layout, layering, canonical field order, method organization, exception taxonomy, bounded reprs, typing/doctest posture, pickling, the one-global rule, and tests/v2 conventions -- now has a tracked home the v1 sections don't cover, including this week's two amendments (section-head helpers, TypeError/ValueError split). Establishes the same-commit rule: convention changes update this section in the commit that makes them. Co-Authored-By: Claude Fable 5 --- AGENTS.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 46115f4..6aefe8a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -107,6 +107,21 @@ Parse flow: Each named attribute (`title`, `first`, etc.) is a `@property` that joins its corresponding `_list`. Setters call `_set_list()` which runs the value through `parse_pieces()`, so assigning `hn.last = "de la Vega"` correctly re-parses prefix tokens. +## 2.0 API modules (`nameparser/_*.py`, `tests/v2/`) + +The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These conventions apply to all new-API code and are stricter than the v1 sections above. The full design record (rationale, settled-decision logs, dated amendments) lives in untracked `docs/superpowers/specs/`; this section is the enforceable subset. **A commit that establishes or amends one of these conventions must update this section in the same commit** — grep-driven staleness sweeps miss paraphrased prose (see Workflow above), so write-time maintenance is the mechanism, audits are the backstop. + +- **Module layout**: every new module is underscore-private (`_types.py`, `_lexicon.py`, `_policy.py`, `_locale.py`; later `_pipeline/`, `_parser.py`, `_render.py`). The public import surface is exactly `nameparser` (and later `nameparser.locales`); `nameparser/__init__.py` holds re-exports and `__all__` only — no logic. Old paths (`nameparser.parser`, `nameparser.config`) stay v1-shaped. +- **Layering is enforced by `tests/v2/test_layering.py`** (exact-module matching): `_types` imports nothing internal; `_lexicon` and `_policy` sit above it independently; `_locale` on top. `_lexicon` may import `nameparser.config.*` DATA modules only — vocabulary is single-sourced from the v1 data modules through 2.x (`config/maiden_markers.py` is 2.0-only data that lives there for the same reason). Extend the test's `ALLOWED` table when adding a module. +- **Canonical field order** — `title, given, middle, family, suffix, nickname, maiden` — is defined once by `Role` enum declaration order and derived everywhere (properties, `as_dict`, reprs, `comparison_key`). Never restate the order literally. +- **Method organization**, fixed section order in every class: fields + `__post_init__` validation → alternative constructors → dunders (construction/equality → protocol → operators) → properties → public methods by concern (access → editing → comparison → rendering delegates) → private helpers last, except a helper serving exactly one section may sit at that section's head. +- **Validation is eager and fail-loud**: every `raise` states the offending value, the expected form, and the fix. Exception taxonomy: wrong type — including wrong element type inside a collection, bare `str` where an iterable of strings is expected, or a `Mapping` where a plain iterable is expected — raises `TypeError`; well-typed but unacceptable values raise `ValueError`; failed enum lookups stay `ValueError` for any input (stdlib `EnumType` precedent). +- **Reprs are bounded**: render which fields deviate from a named baseline and by how much, never contents (`Lexicon(default + titles: +2)`). +- **Typing/docs**: `from __future__ import annotations`; `frozen=True, slots=True` on every public dataclass; mypy strict via per-module overrides in pyproject. Docstrings state contracts in prose with **no doctest blocks** — `--doctest-modules` makes every example a test; behavior examples go to unit tests per the lean-docs rule. +- **Pickling**: v2 types must round-trip (`Parser` will be picklable by construction, and it holds a `Lexicon`). `Lexicon` needs its custom `__getstate__`/`__setstate__` because of its `mappingproxy` slot; a new unpicklable slot type needs the same treatment plus a round-trip test. +- **One sanctioned global**: the (future) cached default `Parser`. Any second piece of module-level mutable state requires amending the conventions doc, on purpose, in review. +- **Tests**: all v2 tests live in the `tests/v2/` package (its `conftest.py` overrides the v1 dual-run fixture — v2 code never reads shared `CONSTANTS`), one test module per source module, names stating behavior. Never assert `Lexicon.default()` contents; the narrow sourcing spot-checks in `test_default_sources_v1_vocabulary` that pin the v1→v2 migration contract (e.g. the flipped `particles_ambiguous` model) are the sanctioned exception. + ## Extension Patterns **Adding a scalar `Constants` attribute + `HumanName` kwarg** (e.g. `initials_separator`, `suffix_delimiter`): From 339a3edb53592a87c6c571081e6b1e359531c912 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 13:08:30 -0700 Subject: [PATCH 40/88] Guard Token.tags and coerce Token.role tags was the one collection input on the v2 surface without the bare-string/mapping/element-type guards: tags='particle' silently became its character set and every STABLE_TAGS-based derived view misclassified with no error. role was the one enum field without coercion; a raw 'given' string broke 'role is Role.GIVEN' identity checks silently. role now mirrors Ambiguity.kind: coerce the string form, enriched ValueError naming valid roles on any failed lookup. Co-Authored-By: Claude Fable 5 --- nameparser/_types.py | 29 ++++++++++++++++++++++++++++- tests/v2/test_types.py | 23 +++++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/nameparser/_types.py b/nameparser/_types.py index 5ec3f19..c73d8c8 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -10,6 +10,7 @@ """ from __future__ import annotations +from collections.abc import Mapping from dataclasses import dataclass from enum import Enum, StrEnum from typing import NamedTuple @@ -53,6 +54,14 @@ def __post_init__(self) -> None: ) if not self.text: raise ValueError("Token.text must be a non-empty string") + if not isinstance(self.role, Role): + try: + object.__setattr__(self, "role", Role(self.role)) + except ValueError: + valid = ", ".join(r.value for r in Role) + raise ValueError( + f"unknown Role {self.role!r}; valid roles: {valid}" + ) from None if self.span is not None: if not ( isinstance(self.span, tuple) @@ -69,7 +78,25 @@ def __post_init__(self) -> None: f"invalid span ({start}, {end}): need 0 <= start <= end" ) object.__setattr__(self, "span", Span(start, end)) - object.__setattr__(self, "tags", frozenset(self.tags)) + # The same guards _normset applies to Lexicon vocabulary: a bare + # string would become its character set, a mapping would silently + # contribute only its keys. + if isinstance(self.tags, str): + raise TypeError( + "Token.tags must be an iterable of strings, " + "not a bare string" + ) + if isinstance(self.tags, Mapping): + raise TypeError( + "Token.tags must be an iterable of strings, not a mapping" + ) + tags = frozenset(self.tags) + for tag in tags: + if not isinstance(tag, str): + raise TypeError( + f"Token.tags must contain only strings, got {tag!r}" + ) + object.__setattr__(self, "tags", tags) def __repr__(self) -> str: # Bounded output: a single token's text/span/role/tags, never diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 973021c..921ff39 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -301,3 +301,26 @@ def test_comparison_key_is_casefolded_canonical_seven_tuple() -> None: Token("vega", Span(11, 15), Role.FAMILY), ]) assert upper.comparison_key() == lower.comparison_key() + + +def test_token_rejects_bare_string_and_mapping_tags() -> None: + # frozenset("particle") is the set(str) footgun: eight single chars. + with pytest.raises(TypeError, match="bare string"): + Token("Van", None, Role.GIVEN, tags="particle") # type: ignore[arg-type] + with pytest.raises(TypeError, match="mapping"): + Token("Van", None, Role.GIVEN, tags={"particle": 1}) # type: ignore[arg-type] + + +def test_token_rejects_non_str_tags() -> None: + with pytest.raises(TypeError, match="tags must contain only strings"): + Token("Van", None, Role.GIVEN, tags=frozenset({1})) # type: ignore[arg-type] + + +def test_token_coerces_role_string_and_rejects_unknown() -> None: + # mirror Ambiguity.kind: coerce the string form, ValueError for any + # failed enum lookup (stdlib EnumType precedent). + assert Token("Juan", None, "given").role is Role.GIVEN # type: ignore[arg-type] + with pytest.raises(ValueError, match="title, given"): + Token("Juan", None, "chief") # type: ignore[arg-type] + with pytest.raises(ValueError, match="title, given"): + Token("Juan", None, 5) # type: ignore[arg-type] From 9d7493c5fcb8db1e61866ddd3f7a7860f6978b4c Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 13:09:26 -0700 Subject: [PATCH 41/88] Close the fail-loud gaps in Policy.__post_init__ The four bool flags were the only unvalidated Policy fields: truthy strings like 'no' stored fine and behaved as True, silently inverting the caller's intent. And the patronymic-rule try wrapped the whole iteration, so a ValueError raised inside a caller's generator was rewritten as 'unknown patronymic rule' with the real traceback erased by 'from None'. Materialize first (the _normset pattern), convert per-element, and name the offending entry in the error. Co-Authored-By: Claude Fable 5 --- nameparser/_policy.py | 37 ++++++++++++++++++++++++++----------- tests/v2/test_policy.py | 25 +++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 11 deletions(-) diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 53ab123..80c6c69 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -59,17 +59,22 @@ def __post_init__(self) -> None: f"patronymic_rules must be an iterable of rule names, " f"not a bare string: {self.patronymic_rules!r}" ) - # A non-iterable raises its natural TypeError from the frozenset - # call; only failed enum lookups get the enriched message. - try: - rules = frozenset(PatronymicRule(r) for r in self.patronymic_rules) - except ValueError: - valid = ", ".join(r.value for r in PatronymicRule) - raise ValueError( - f"unknown patronymic rule in {self.patronymic_rules!r}; " - f"valid rules: {valid}" - ) from None - object.__setattr__(self, "patronymic_rules", rules) + # Materialize before converting (the _normset pattern): a + # non-iterable raises its natural TypeError here, and an exception + # raised inside a caller's generator propagates untouched instead + # of being rewritten as an unknown-rule error. Only the enum + # lookup itself gets the enriched message, naming the offender. + items = tuple(self.patronymic_rules) + rules = set() + for r in items: + try: + rules.add(PatronymicRule(r)) + except ValueError: + valid = ", ".join(v.value for v in PatronymicRule) + raise ValueError( + f"unknown patronymic rule {r!r}; valid rules: {valid}" + ) from None + object.__setattr__(self, "patronymic_rules", frozenset(rules)) for pairs_name in ("nickname_delimiters", "maiden_delimiters"): pairs = tuple(getattr(self, pairs_name)) for pair in pairs: @@ -104,6 +109,16 @@ def __post_init__(self) -> None: object.__setattr__( self, "extra_suffix_delimiters", frozenset(delimiters) ) + # Truthy strings ("no", "false") would silently invert the + # caller's intent downstream; bools are the one field kind the + # coercing checks above can't cover. + for flag in ("middle_as_family", "lenient_comma_suffixes", + "strip_emoji", "strip_bidi"): + value = getattr(self, flag) + if not isinstance(value, bool): + raise TypeError( + f"{flag} must be a bool, got {value!r}" + ) def __repr__(self) -> str: # Bounded: only fields that deviate from the default are shown diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index 06cef02..a067418 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -134,3 +134,28 @@ def test_unset_fields_are_distinguishable_from_defaults() -> None: patch = PolicyPatch(strip_emoji=True) # explicitly set to the default value assert patch.strip_emoji is True assert PolicyPatch().strip_emoji is UNSET + + +def test_policy_rejects_non_bool_flags() -> None: + # "no" and "false" are truthy: storing them would silently invert + # the caller's intent downstream. + for flag in ("middle_as_family", "lenient_comma_suffixes", + "strip_emoji", "strip_bidi"): + with pytest.raises(TypeError, match="must be a bool"): + Policy(**{flag: "no"}) # type: ignore[arg-type] + + +def test_patronymic_rules_generator_errors_propagate_untouched() -> None: + # A ValueError raised inside the caller's own generator must not be + # rewritten as "unknown patronymic rule" with the traceback erased. + def bad_loader(): # noqa: ANN202 + yield "east-slavic" + raise ValueError("config line 7: bad int") + + with pytest.raises(ValueError, match="config line 7"): + Policy(patronymic_rules=bad_loader()) # type: ignore[arg-type] + + +def test_unknown_patronymic_rule_error_names_the_offender() -> None: + with pytest.raises(ValueError, match="klingon"): + Policy(patronymic_rules=iter(["east-slavic", "klingon"])) # type: ignore[arg-type] From cff23fff154feb28dc716e948b16fbe6db8cc8f2 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 13:10:07 -0700 Subject: [PATCH 42/88] Canonicalize PolicyPatch's scalar name_order to a tuple Union fields were already canonicalized at patch construction, but a list name_order stored as-is -- making the patch and any Locale holding it unhashable, with the failure surfacing only when a locale became a dict key. Policy re-coerces at apply time, so the patch was the one container in the chain that could silently carry the list. Co-Authored-By: Claude Fable 5 --- nameparser/_policy.py | 10 +++++++--- tests/v2/test_policy.py | 8 ++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 80c6c69..df27793 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -182,9 +182,13 @@ class PolicyPatch: strip_bidi: bool | _Unset = UNSET def __post_init__(self) -> None: - # Canonicalize (but do NOT validate) union-composed fields so a - # patch built from a set/list literal is hashable and unions - # cleanly in apply_patch. + # Canonicalize (but do NOT validate) collection fields so a patch + # built from a set/list literal is hashable and unions cleanly in + # apply_patch. name_order needs the same treatment: Policy would + # coerce a list at apply time, but the patch itself (and any + # Locale holding it) must already be hashable. + if self.name_order is not UNSET: + object.__setattr__(self, "name_order", tuple(self.name_order)) for f in dataclasses.fields(self): if f.metadata.get("compose") != "union": continue diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index a067418..abf21af 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -159,3 +159,11 @@ def bad_loader(): # noqa: ANN202 def test_unknown_patronymic_rule_error_names_the_offender() -> None: with pytest.raises(ValueError, match="klingon"): Policy(patronymic_rules=iter(["east-slavic", "klingon"])) # type: ignore[arg-type] + + +def test_policy_patch_canonicalizes_scalar_name_order() -> None: + # A list name_order stored as-is made the patch -- and any Locale + # holding it -- unhashable, failing far from the construction site. + p = PolicyPatch(name_order=[Role.FAMILY, Role.GIVEN, Role.MIDDLE]) # type: ignore[arg-type] + assert p.name_order == (Role.FAMILY, Role.GIVEN, Role.MIDDLE) + assert isinstance(hash(p), int) From d4cb1b234ab6cbc5155038a62c3925975efec991 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 13:11:03 -0700 Subject: [PATCH 43/88] Validate capitalization_exceptions entry shapes A bare string or a mis-shaped entry leaked raw unpack errors ('not enough values to unpack') naming neither the field nor the fix -- and a 2-char string entry like 'ab' unpacked silently into {'a': 'b'}. All three now raise the taxonomy's TypeError with the offending value and expected form. Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 22 +++++++++++++++++++++- tests/v2/test_lexicon.py | 11 +++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index a43e9e7..620f2a8 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -81,13 +81,33 @@ def __post_init__(self) -> None: for name in _VOCAB_FIELDS: object.__setattr__(self, name, _normset(getattr(self, name), name)) raw = self.capitalization_exceptions + if isinstance(raw, str): + raise TypeError( + "capitalization_exceptions must be a mapping or an " + "iterable of (key, value) pairs, not a bare string" + ) pairs = raw.items() if isinstance(raw, Mapping) else raw # Dedupe on the NORMALIZED key before storing so the tuple and the # map always agree ("Ph.D." and "phd" collide after normalization). # Last occurrence wins, matching dict semantics and the right-bias # rule used elsewhere. deduped: dict[str, str] = {} - for k, v in pairs: + for entry in pairs: + # A 2-char str entry would unpack "ab" into ("a", "b") + # silently, so reject str outright; other mis-shapes would + # otherwise surface as bare unpack errors. + if isinstance(entry, str): + raise TypeError( + f"capitalization_exceptions entries must be " + f"(key, value) pairs, got {entry!r}" + ) + try: + k, v = entry + except (TypeError, ValueError): + raise TypeError( + f"capitalization_exceptions entries must be " + f"(key, value) pairs, got {entry!r}" + ) from None if not isinstance(k, str) or not isinstance(v, str): raise TypeError( f"capitalization_exceptions entries must be " diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index 6945615..ef8e114 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -141,3 +141,14 @@ def test_lexicon_rejects_mapping_for_plain_vocab_field() -> None: Lexicon(titles={"Dr.": "Doctor"}) # type: ignore[arg-type] with pytest.raises(TypeError, match="mapping"): Lexicon.empty().add(titles={"Dr.": "Doctor"}) + + +def test_capitalization_exceptions_rejects_malformed_shapes() -> None: + with pytest.raises(TypeError, match="bare string"): + Lexicon(capitalization_exceptions="ab") # type: ignore[arg-type] + with pytest.raises(TypeError, match="pairs"): + Lexicon(capitalization_exceptions=(("a", "b", "c"),)) # type: ignore[arg-type] + with pytest.raises(TypeError, match="pairs"): + # a 2-char string entry would unpack into two chars and silently + # store {"a": "b"} -- reject str entries outright + Lexicon(capitalization_exceptions=("ab",)) # type: ignore[arg-type] From af3aad02bf9b3340a22e9e1922382095e5d51a8b Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 13:11:37 -0700 Subject: [PATCH 44/88] Fail at unpickle time on Lexicon field-layout skew __setstate__ accepted any state dict: a pickle from an older or newer Lexicon (field added, renamed) loaded fine and failed at the first attribute read, far from the unpickle site with no hint the cause was a stale pickle. Check the field-name set at load and name the missing/unexpected fields. Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 11 +++++++++++ tests/v2/test_lexicon.py | 16 ++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 620f2a8..b4a5307 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -177,6 +177,17 @@ def __getstate__(self) -> dict[str, object]: for f in dataclasses.fields(self) if f.name != "_cap_map"} def __setstate__(self, state: dict[str, object]) -> None: + # Fail at the unpickle site if the state comes from a different + # Lexicon field layout (version skew) -- silently loading it + # would defer the failure to some distant attribute read. + expected = {f.name for f in dataclasses.fields(Lexicon)} - {"_cap_map"} + if set(state) != expected: + missing = ", ".join(sorted(expected - set(state))) or "none" + unexpected = ", ".join(sorted(set(state) - expected)) or "none" + raise ValueError( + f"incompatible Lexicon pickle: missing fields: {missing}; " + f"unexpected fields: {unexpected}" + ) for name, value in state.items(): object.__setattr__(self, name, value) object.__setattr__( diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index ef8e114..ebdea70 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -152,3 +152,19 @@ def test_capitalization_exceptions_rejects_malformed_shapes() -> None: # a 2-char string entry would unpack into two chars and silently # store {"a": "b"} -- reject str entries outright Lexicon(capitalization_exceptions=("ab",)) # type: ignore[arg-type] + + +def test_setstate_rejects_mismatched_field_layout() -> None: + # A pickle from a different Lexicon version (field added/renamed) + # must fail at load time with a message naming the mismatch, not at + # some later attribute read far from the unpickle site. + lex = Lexicon.empty() + good_state = lex.__getstate__() + missing = dict(good_state) + del missing["titles"] + with pytest.raises(ValueError, match="titles"): + Lexicon.__new__(Lexicon).__setstate__(missing) + extra = dict(good_state) + extra["zq_future_field"] = frozenset() + with pytest.raises(ValueError, match="zq_future_field"): + Lexicon.__new__(Lexicon).__setstate__(extra) From c5090d1f2c37c069ddd77ce47600d7e3b3b60e87 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 13:12:18 -0700 Subject: [PATCH 45/88] Fix vacuous removal assertion in the add/remove test 'bishop' is a v1 TITLE, not a suffix word, so removing it from suffix_words asserted nothing -- the test kept passing even if remove() were a no-op. Remove an entry that is actually present and assert the precondition first. Co-Authored-By: Claude Fable 5 --- tests/v2/test_lexicon.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index ebdea70..9eb4d19 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -85,9 +85,12 @@ def test_add_and_remove_return_new_lexicons() -> None: # "zqtitle" is a synthetic word absent from v1's TITLES data (unlike # e.g. "dra", the feminine "dr." abbreviation, which is already there). base = Lexicon.default() - lex = base.add(titles={"zqtitle"}).remove(suffix_words={"bishop"}) + lex = base.add(titles={"zqtitle"}).remove(suffix_words={"esquire"}) assert "zqtitle" in lex.titles and "zqtitle" not in base.titles - assert "bishop" not in lex.suffix_words + # precondition, or the removal assertion passes vacuously (this is a + # guard for the operation under test, not a vocabulary content pin) + assert "esquire" in base.suffix_words + assert "esquire" not in lex.suffix_words def test_add_unknown_field_raises_with_valid_names() -> None: From 2310fb1c790d0aedb21c85adceb509940c0336a7 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 13:12:56 -0700 Subject: [PATCH 46/88] Pin four stated-but-untested v2 contracts - apply_patch revalidates deferred patch values (PolicyPatch documents lazy validation; nothing asserted the apply-time failure) - all four set-valued PolicyPatch fields declare union composition (apply_patch is metadata-driven; dropping one silently flips locale layering from union to override) - pickle round-trips for Token/Ambiguity/ParsedName/Policy/PolicyPatch, pinning that UNSET survives unpickling BY IDENTITY -- apply_patch gates on 'is UNSET', which only works because Enum members unpickle to the same object - _normalize is casefold + all-periods stripping, stricter than v1's lc() (lower + edge periods only) Co-Authored-By: Claude Fable 5 --- tests/v2/test_lexicon.py | 8 ++++++++ tests/v2/test_policy.py | 39 +++++++++++++++++++++++++++++++++++++++ tests/v2/test_types.py | 11 +++++++++++ 3 files changed, 58 insertions(+) diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index 9eb4d19..cdb8f24 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -171,3 +171,11 @@ def test_setstate_rejects_mismatched_field_layout() -> None: extra["zq_future_field"] = frozenset() with pytest.raises(ValueError, match="zq_future_field"): Lexicon.__new__(Lexicon).__setstate__(extra) + + +def test_normalization_casefolds_and_strips_interior_periods() -> None: + # Stricter than v1's lc(), which lower()s and trims only EDGE + # periods: casefold handles ß, and interior periods are removed too. + # A "simplify to .lower()/.strip('.')" regression must fail here. + lex = Lexicon(titles=frozenset({"STRAßE", "Ph.D"})) + assert lex.titles == frozenset({"strasse", "phd"}) diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index abf21af..045ac6e 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -167,3 +167,42 @@ def test_policy_patch_canonicalizes_scalar_name_order() -> None: p = PolicyPatch(name_order=[Role.FAMILY, Role.GIVEN, Role.MIDDLE]) # type: ignore[arg-type] assert p.name_order == (Role.FAMILY, Role.GIVEN, Role.MIDDLE) assert isinstance(hash(p), int) + + +def test_apply_patch_revalidates_deferred_values() -> None: + # PolicyPatch documents lazy validation: invalid values sit latent in + # the patch and must fail when applied, not silently flow into Policy. + bad_order = PolicyPatch(name_order=(Role.TITLE, Role.GIVEN, Role.FAMILY)) + with pytest.raises(ValueError, match="permutation"): + apply_patch(Policy(), bad_order) + bad_rules = PolicyPatch(patronymic_rules=frozenset({"klingon"})) # type: ignore[arg-type] + with pytest.raises(ValueError, match="valid rules"): + apply_patch(Policy(), bad_rules) + + +def test_all_set_valued_patch_fields_declare_union_composition() -> None: + # apply_patch is driven by this metadata; dropping it from one field + # would silently flip locale layering from union to override. + union_fields = { + f.name for f in dataclasses.fields(PolicyPatch) + if f.metadata.get("compose") == "union" + } + assert union_fields == { + "patronymic_rules", "nickname_delimiters", + "maiden_delimiters", "extra_suffix_delimiters", + } + + +def test_policy_and_patch_pickle_round_trip_preserves_unset_identity() -> None: + import pickle + + p = Policy(patronymic_rules=frozenset({PatronymicRule.TURKIC})) + assert pickle.loads(pickle.dumps(p)) == p + patch = PolicyPatch(strip_emoji=False) + loaded = pickle.loads(pickle.dumps(patch)) + assert loaded == patch + # apply_patch gates on 'value is UNSET'; an unpickled patch is only + # correct because Enum members round-trip BY IDENTITY. A plain + # object() sentinel would break this silently. + assert loaded.name_order is UNSET + assert loaded.strip_emoji is False diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 921ff39..c956456 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -324,3 +324,14 @@ def test_token_coerces_role_string_and_rejects_unknown() -> None: Token("Juan", None, "chief") # type: ignore[arg-type] with pytest.raises(ValueError, match="title, given"): Token("Juan", None, 5) # type: ignore[arg-type] + + +def test_types_pickle_round_trip() -> None: + import pickle + + pn = _delavega() + assert pickle.loads(pickle.dumps(pn)) == pn + amb = Ambiguity(AmbiguityKind.ORDER, "two-comma structure", ()) + assert pickle.loads(pickle.dumps(amb)) == amb + tok = Token("de", Span(9, 11), Role.FAMILY, frozenset({"particle"})) + assert pickle.loads(pickle.dumps(tok)) == tok From cbd8cdff837c2c43f5db520ef6eeafdf37a4f4fb Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 13:15:31 -0700 Subject: [PATCH 47/88] Fix stale and inaccurate comments found in review Three 'added in a later task' notes were false at HEAD; the _VOCAB_FIELDS comment claimed __or__ excludes capitalization_exceptions (it merges them right-biased); the _lexicon module docstring's data- module list omitted maiden_markers (now points at _default_lexicon's imports, which cannot rot); _normalize claimed equivalence with v1's lc() (lc lowers and trims only edge periods); the maiden_markers docstring omitted the masculine Russian forms it ships; replace() did not document that it drops ambiguities referencing replaced tokens; and AGENTS.md overstated 'mypy strict' (per-module strict is not valid mypy config) and the one-test-module-per-source rule. Also notes the layering table's config prefix enforces package granularity only. Co-Authored-By: Claude Fable 5 --- AGENTS.md | 6 +++--- nameparser/_lexicon.py | 19 +++++++++++-------- nameparser/_locale.py | 4 ++-- nameparser/_policy.py | 6 +++--- nameparser/_types.py | 6 ++++-- nameparser/config/maiden_markers.py | 7 +++++-- tests/v2/test_layering.py | 4 +++- tests/v2/test_types.py | 2 +- 8 files changed, 32 insertions(+), 22 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6aefe8a..4e52806 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -113,14 +113,14 @@ The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These - **Module layout**: every new module is underscore-private (`_types.py`, `_lexicon.py`, `_policy.py`, `_locale.py`; later `_pipeline/`, `_parser.py`, `_render.py`). The public import surface is exactly `nameparser` (and later `nameparser.locales`); `nameparser/__init__.py` holds re-exports and `__all__` only — no logic. Old paths (`nameparser.parser`, `nameparser.config`) stay v1-shaped. - **Layering is enforced by `tests/v2/test_layering.py`** (exact-module matching): `_types` imports nothing internal; `_lexicon` and `_policy` sit above it independently; `_locale` on top. `_lexicon` may import `nameparser.config.*` DATA modules only — vocabulary is single-sourced from the v1 data modules through 2.x (`config/maiden_markers.py` is 2.0-only data that lives there for the same reason). Extend the test's `ALLOWED` table when adding a module. -- **Canonical field order** — `title, given, middle, family, suffix, nickname, maiden` — is defined once by `Role` enum declaration order and derived everywhere (properties, `as_dict`, reprs, `comparison_key`). Never restate the order literally. +- **Canonical field order** — the seven roles in `Role` enum declaration order, defined once and derived everywhere (properties, `as_dict`, reprs, `comparison_key`). Never restate the order literally. - **Method organization**, fixed section order in every class: fields + `__post_init__` validation → alternative constructors → dunders (construction/equality → protocol → operators) → properties → public methods by concern (access → editing → comparison → rendering delegates) → private helpers last, except a helper serving exactly one section may sit at that section's head. - **Validation is eager and fail-loud**: every `raise` states the offending value, the expected form, and the fix. Exception taxonomy: wrong type — including wrong element type inside a collection, bare `str` where an iterable of strings is expected, or a `Mapping` where a plain iterable is expected — raises `TypeError`; well-typed but unacceptable values raise `ValueError`; failed enum lookups stay `ValueError` for any input (stdlib `EnumType` precedent). - **Reprs are bounded**: render which fields deviate from a named baseline and by how much, never contents (`Lexicon(default + titles: +2)`). -- **Typing/docs**: `from __future__ import annotations`; `frozen=True, slots=True` on every public dataclass; mypy strict via per-module overrides in pyproject. Docstrings state contracts in prose with **no doctest blocks** — `--doctest-modules` makes every example a test; behavior examples go to unit tests per the lean-docs rule. +- **Typing/docs**: `from __future__ import annotations`; `frozen=True, slots=True` on every public dataclass; strict-profile mypy flags via per-module overrides in pyproject (`strict = true` itself is not valid per-module). Docstrings state contracts in prose with **no doctest blocks** — `--doctest-modules` makes every example a test; behavior examples go to unit tests per the lean-docs rule. - **Pickling**: v2 types must round-trip (`Parser` will be picklable by construction, and it holds a `Lexicon`). `Lexicon` needs its custom `__getstate__`/`__setstate__` because of its `mappingproxy` slot; a new unpicklable slot type needs the same treatment plus a round-trip test. - **One sanctioned global**: the (future) cached default `Parser`. Any second piece of module-level mutable state requires amending the conventions doc, on purpose, in review. -- **Tests**: all v2 tests live in the `tests/v2/` package (its `conftest.py` overrides the v1 dual-run fixture — v2 code never reads shared `CONSTANTS`), one test module per source module, names stating behavior. Never assert `Lexicon.default()` contents; the narrow sourcing spot-checks in `test_default_sources_v1_vocabulary` that pin the v1→v2 migration contract (e.g. the flipped `particles_ambiguous` model) are the sanctioned exception. +- **Tests**: all v2 tests live in the `tests/v2/` package (its `conftest.py` overrides the v1 dual-run fixture — v2 code never reads shared `CONSTANTS`), one test module per source module plus cross-cutting `test_reprs.py` and `test_layering.py`, names stating behavior. Never assert `Lexicon.default()` contents; the narrow sourcing spot-checks in `test_default_sources_v1_vocabulary` that pin the v1→v2 migration contract (e.g. the flipped `particles_ambiguous` model) are the sanctioned exception. ## Extension Patterns diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index b4a5307..e1d07a4 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -1,8 +1,8 @@ """Immutable vocabulary configuration for the 2.0 API. -Layering: may import nameparser.config DATA modules (titles, suffixes, -prefixes, conjunctions, capitalization, bound_first_names) as the single -source of vocabulary during 2.x -- never nameparser.config itself, never +Layering: may import nameparser.config DATA modules (the imports in +_default_lexicon() are the authoritative list) as the single source of +vocabulary during 2.x -- never nameparser.config itself, never nameparser.parser. Enforced by tests/v2/test_layering.py. """ from __future__ import annotations @@ -13,9 +13,10 @@ from dataclasses import dataclass, field from types import MappingProxyType -#: Vocabulary set fields, in declaration order. add()/remove()/__or__ -#: (a later task) operate on exactly these; capitalization_exceptions is -#: deliberately excluded (its entries are pairs -- use dataclasses.replace). +#: Vocabulary set fields, in declaration order. add()/remove() operate +#: on exactly these and reject capitalization_exceptions (its entries +#: are pairs -- use dataclasses.replace); __or__ unions these AND merges +#: capitalization_exceptions right-biased. _VOCAB_FIELDS = ( "titles", "given_name_titles", "suffix_acronyms", "suffix_words", "suffix_acronyms_ambiguous", "particles", "particles_ambiguous", @@ -24,8 +25,10 @@ def _normalize(word: str) -> str: - """Casefold and strip periods -- v1's lc(). Membership tests never - re-normalize because construction already did.""" + """Casefold, remove ALL periods, strip whitespace -- v2's stricter + analogue of v1's lc(), which lower()s and trims only edge periods. + Membership tests never re-normalize because construction already + did.""" return word.casefold().replace(".", "").strip() diff --git a/nameparser/_locale.py b/nameparser/_locale.py index 5e06d17..346fb0d 100644 --- a/nameparser/_locale.py +++ b/nameparser/_locale.py @@ -4,8 +4,8 @@ lexicon fragments union onto the base, the PolicyPatch folds via apply_patch. Packs are pure data; they have no privileged capabilities. -Layering: imports _lexicon and _policy only (tests/v2/test_layering.py, -added in a later task, enforces this). +Layering: imports _lexicon and _policy only (enforced by +tests/v2/test_layering.py). """ from __future__ import annotations diff --git a/nameparser/_policy.py b/nameparser/_policy.py index df27793..9f12fad 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -1,7 +1,7 @@ """Immutable behavior configuration for the 2.0 API. -Layering: imports nameparser._types only (tests/v2/test_layering.py, -added in a later task, enforces this). +Layering: imports nameparser._types only (enforced by +tests/v2/test_layering.py). """ from __future__ import annotations @@ -43,7 +43,7 @@ class Policy: extra_suffix_delimiters: frozenset[str] = frozenset() lenient_comma_suffixes: bool = True strip_emoji: bool = True - strip_bidi: bool = True # replaces v1's CONSTANTS.regexes.bidi = False + strip_bidi: bool = True # =False replaces v1's opt-out CONSTANTS.regexes.bidi = False def __post_init__(self) -> None: order = tuple(self.name_order) diff --git a/nameparser/_types.py b/nameparser/_types.py index c73d8c8..8420f3f 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -303,6 +303,7 @@ def replace(self, **fields: str) -> ParsedName: """Return a new ParsedName with the named fields re-tokenized as synthetic tokens (span=None). Whitespace-splits each value; an empty value clears the field. original is unchanged (provenance). + Ambiguities referencing replaced tokens are dropped. """ by_value = {role.value: role for role in Role} for key, value in fields.items(): @@ -341,7 +342,8 @@ def synthetic(value: str, role: Role) -> list[Token]: # -- comparison ------------------------------------------------------- def comparison_key(self) -> tuple[str, ...]: - """Casefolded seven components in canonical order, for dedup, - dict keys, and sorting. The semantic layer; __eq__ stays strict. + """One casefolded component per Role, in canonical order, for + dedup, dict keys, and sorting. The semantic layer; __eq__ stays + strict. """ return tuple(self._text_for(role).casefold() for role in Role) diff --git a/nameparser/config/maiden_markers.py b/nameparser/config/maiden_markers.py index 0c36c5e..22399e3 100644 --- a/nameparser/config/maiden_markers.py +++ b/nameparser/config/maiden_markers.py @@ -20,9 +20,12 @@ Marker words that introduce a birth surname, e.g. "Jane Smith née Jones" (#274). French née/né/nee, German geb./geborene, Dutch geboren, Czech/Slovak roz./rozená, Danish/Norwegian født (Nynorsk fødd), Swedish -född, Russian урожд./урождённая (both ё and е spellings — +född, Russian урожд./урождённая/урождённый (both ё and е spellings — ``str.casefold()`` does not fold them, and running text routinely -writes е). Entries are stored normalized: lowercase, no periods. +writes е). Both grammatical genders are listed where #274 or review +attested them (née/né, урождённая/урождённый); Czech masculine rozený +awaits the same vetting. Entries are stored normalized: lowercase, no +periods. Consumed by the 2.0 parser's default lexicon. The 1.x parser does not read this module. diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index f9d9605..cfbd63b 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -9,7 +9,9 @@ # module -> prefixes it may import from within nameparser ALLOWED = { "_types.py": (), - "_lexicon.py": ("nameparser.config.",), # DATA modules only, during 2.x + # intent: DATA modules only, during 2.x -- mechanically this admits + # any config submodule; "data only" holds by convention/review + "_lexicon.py": ("nameparser.config.",), "_policy.py": ("nameparser._types",), "_locale.py": ("nameparser._lexicon", "nameparser._policy"), } diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index c956456..339e4e0 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -206,7 +206,7 @@ def test_suffix_joins_with_comma_space() -> None: def test_derived_views_filter_on_stable_particle_tag() -> None: # Pin the hard-coded "particle" string in _text_for to the published - # contract until Plan 3's tag-emission contract tests land. + # contract until parser tag-emission contract tests land. assert "particle" in STABLE_TAGS pn = _delavega() assert pn.family_particles == "de la" From 358ce8074e6bf54b2fca4fb900301963f2b336d4 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 13:49:00 -0700 Subject: [PATCH 48/88] Raise on entries that normalize to empty '.', '', or whitespace is a data bug (stray split artifact, empty CSV cell) on the primary customization surface; silently dropping it also meant a future data-module typo would vanish instead of failing CI. One rule for vocab fields AND capitalization-exception keys -- the previous behavior dropped both silently (keys deliberately, vocab undocumented). Raising now and relaxing later is compatible; the reverse is a break. Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 20 +++++++++++++++++--- tests/v2/test_lexicon.py | 13 ++++++++++--- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index e1d07a4..dea001a 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -50,13 +50,23 @@ def _normset(entries: Iterable[str], field_name: str) -> frozenset[str]: f"mapping (only capitalization_exceptions holds key->value pairs)" ) items = tuple(entries) # materialize once; entries may be a generator + normalized = set() for w in items: if not isinstance(w, str): raise TypeError( f"Lexicon.{field_name} entries must be strings, got {w!r}" ) - result = frozenset(_normalize(w) for w in items) - return frozenset(w for w in result if w) + n = _normalize(w) + # "." or "" is a data bug (stray split artifact, empty CSV + # cell); dropping it silently would also let a data-module typo + # vanish instead of failing CI. + if not n: + raise ValueError( + f"Lexicon.{field_name} entry {w!r} normalizes to empty " + f"(casefold + strip periods/whitespace leaves nothing)" + ) + normalized.add(n) + return frozenset(normalized) @dataclass(frozen=True, slots=True) @@ -118,7 +128,11 @@ def __post_init__(self) -> None: ) normalized_key = _normalize(k) if not normalized_key: - continue # mirror _normset's drop-empty rule + raise ValueError( + f"capitalization_exceptions key {k!r} normalizes to " + f"empty (casefold + strip periods/whitespace leaves " + f"nothing)" + ) deduped[normalized_key] = v canonical = tuple(sorted(deduped.items())) object.__setattr__(self, "capitalization_exceptions", canonical) diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index cdb8f24..99bf095 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -64,9 +64,16 @@ def test_lexicon_rejects_non_str_vocab_entries() -> None: Lexicon(titles={"Dr.", 42}) # type: ignore[arg-type] -def test_exception_keys_normalizing_to_empty_are_dropped() -> None: - lex = Lexicon(capitalization_exceptions=(("...", "X"), ("phd", "PhD"))) - assert lex.capitalization_exceptions == (("phd", "PhD"),) +def test_entries_normalizing_to_empty_raise() -> None: + # "." or "" is a data bug (stray split artifact, empty CSV cell); + # dropping it silently would also let a future data-module typo + # vanish instead of failing CI. One rule for vocab AND exception keys. + with pytest.raises(ValueError, match="normalizes to empty"): + Lexicon(titles=frozenset({"Dr.", "."})) + with pytest.raises(ValueError, match="normalizes to empty"): + Lexicon.empty().add(titles={" "}) + with pytest.raises(ValueError, match="normalizes to empty"): + Lexicon(capitalization_exceptions=(("...", "X"), ("phd", "PhD"))) def test_colliding_exception_keys_dedupe_last_wins() -> None: From 22fe2eaf478438c538546d53ea7463bc20dc6c5e Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 13:50:04 -0700 Subject: [PATCH 49/88] Block Span concatenation and bool span coordinates Span(0,2) + Span(3,4) inherited tuple concatenation, producing a 4-tuple -- the natural but wrong spelling of 'covering span' that the pipeline's join stage will tempt. A real cover() operation ships with that consumer; blocking + now is compatible, blocking it after 2.0.0 would be a break. Also reject bools in span coordinates (bool is an int subclass; (False, True) is a leaked comparison result, not a span). Co-Authored-By: Claude Fable 5 --- nameparser/_types.py | 17 +++++++++++++++-- tests/v2/test_types.py | 15 +++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/nameparser/_types.py b/nameparser/_types.py index 8420f3f..497d7d5 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -13,7 +13,7 @@ from collections.abc import Mapping from dataclasses import dataclass from enum import Enum, StrEnum -from typing import NamedTuple +from typing import NamedTuple, NoReturn class Role(Enum): @@ -34,6 +34,16 @@ class Span(NamedTuple): start: int end: int + def __add__(self, other: object) -> NoReturn: # type: ignore[override] + # Inherited tuple + would concatenate two spans into a 4-tuple -- + # the natural but wrong spelling of "covering span". The real + # covering operation ships with its consumer, the pipeline's + # join stage. + raise TypeError( + "Span does not support +; tuple concatenation is not a " + "covering span" + ) + #: Stable, documented tag vocabulary (API). All other tags are #: namespaced ("vocab:...", "patronymic:...") and unstable. @@ -66,7 +76,10 @@ def __post_init__(self) -> None: if not ( isinstance(self.span, tuple) and len(self.span) == 2 - and all(isinstance(v, int) for v in self.span) + # bool is an int subclass: (False, True) is a comparison + # result leaking into a coordinate slot, not a span + and all(isinstance(v, int) and not isinstance(v, bool) + for v in self.span) ): raise TypeError( f"invalid span {self.span!r}: expected a (start, end) " diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 339e4e0..5211730 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -335,3 +335,18 @@ def test_types_pickle_round_trip() -> None: assert pickle.loads(pickle.dumps(amb)) == amb tok = Token("de", Span(9, 11), Role.FAMILY, frozenset({"particle"})) assert pickle.loads(pickle.dumps(tok)) == tok + + +def test_span_add_is_blocked() -> None: + # NamedTuple + would concatenate into a 4-tuple, the natural but + # wrong spelling of "covering span" (a real cover() arrives with the + # pipeline's join stage, its consumer). + with pytest.raises(TypeError, match="covering span"): + Span(0, 2) + Span(3, 4) # type: ignore[operator] + + +def test_token_rejects_bool_span_coordinates() -> None: + # bool is an int subclass; (False, True) is a comparison result + # leaking into a coordinate slot, not a span. + with pytest.raises(TypeError, match="pair of ints"): + Token("x", (False, True), Role.GIVEN) # type: ignore[arg-type] From f07082b369b94d2644770356a7834f006ffece61 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 13:50:57 -0700 Subject: [PATCH 50/88] Pin Locale.code to the registry-key charset [a-z0-9_]+ Codes become registry keys the moment parser_for and third-party packs exist; every character accepted today is supported forever. The pin matches the shipped ru/tr_az convention and deliberately allows one separator only -- accepting '-' as well would make tr-az and tr_az distinct keys. Relaxing later is compatible; tightening later breaks published packs. Co-Authored-By: Claude Fable 5 --- nameparser/_locale.py | 9 +++++++++ tests/v2/test_locale.py | 11 +++++++++++ 2 files changed, 20 insertions(+) diff --git a/nameparser/_locale.py b/nameparser/_locale.py index 346fb0d..14784c3 100644 --- a/nameparser/_locale.py +++ b/nameparser/_locale.py @@ -10,6 +10,7 @@ from __future__ import annotations import dataclasses +import re from dataclasses import dataclass from nameparser._lexicon import Lexicon @@ -39,6 +40,14 @@ def __post_init__(self) -> None: raise ValueError( f"Locale.code must not contain whitespace, got {self.code!r}" ) + # Codes are registry keys (parser_for, third-party packs): every + # accepted character is supported forever, so pin the charset + # while relaxing later is still compatible. One separator only -- + # allowing '-' too would make tr-az and tr_az distinct keys. + if not re.fullmatch(r"[a-z0-9_]+", self.code): + raise ValueError( + f"Locale.code must match [a-z0-9_]+, got {self.code!r}" + ) if not isinstance(self.lexicon, Lexicon): raise TypeError( f"Locale.lexicon must be a Lexicon, got {self.lexicon!r}" diff --git a/tests/v2/test_locale.py b/tests/v2/test_locale.py index bc25eda..f84728f 100644 --- a/tests/v2/test_locale.py +++ b/tests/v2/test_locale.py @@ -53,3 +53,14 @@ def test_locale_with_lexicon_pickles_round_trip() -> None: loc = Locale(code="ru", lexicon=Lexicon.empty().add(titles={"Dr."})) assert pickle.loads(pickle.dumps(loc)) == loc + + +def test_locale_code_is_pinned_to_registry_charset() -> None: + # Codes become registry keys the moment parser_for and third-party + # packs exist: every accepted character is supported forever, so pin + # [a-z0-9_]+ now (matches ru/tr_az). One separator only -- allowing + # both '-' and '_' would make tr-az and tr_az distinct keys. + for bad in ("ru!", "tr-az", "тр", "zh/tw"): + with pytest.raises(ValueError, match="a-z0-9_"): + Locale(code=bad, lexicon=Lexicon.empty()) + assert Locale(code="tr_az", lexicon=Lexicon.empty()).code == "tr_az" From 22ea503dc27efda001406eec39eab652ab81a878 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 14:05:06 -0700 Subject: [PATCH 51/88] Extract _coerce_enum for the duplicated enum coercion blocks Token.role and Ambiguity.kind carried byte-parallel coerce-or-raise blocks that had to be kept in sync by hand; both message shapes are pinned by tests. One parametrized module-private helper, two one-line call sites, identical messages. Co-Authored-By: Claude Fable 5 --- nameparser/_types.py | 40 +++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/nameparser/_types.py b/nameparser/_types.py index 497d7d5..f147909 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -13,7 +13,7 @@ from collections.abc import Mapping from dataclasses import dataclass from enum import Enum, StrEnum -from typing import NamedTuple, NoReturn +from typing import NamedTuple, NoReturn, TypeVar class Role(Enum): @@ -49,6 +49,23 @@ def __add__(self, other: object) -> NoReturn: # type: ignore[override] #: namespaced ("vocab:...", "patronymic:...") and unstable. STABLE_TAGS = frozenset({"particle", "conjunction", "initial"}) +_E = TypeVar("_E", bound=Enum) + + +def _coerce_enum(value: object, enum_cls: type[_E], noun: str, plural: str) -> _E: + """Coerce value to enum_cls, or raise the enriched ValueError listing + every valid member (enum lookups stay ValueError for any input -- + stdlib EnumType precedent, see AGENTS.md's taxonomy rule).""" + if isinstance(value, enum_cls): + return value + try: + return enum_cls(value) + except ValueError: + valid = ", ".join(str(m.value) for m in enum_cls) + raise ValueError( + f"unknown {noun} {value!r}; valid {plural}: {valid}" + ) from None + @dataclass(frozen=True, slots=True) class Token: @@ -64,14 +81,8 @@ def __post_init__(self) -> None: ) if not self.text: raise ValueError("Token.text must be a non-empty string") - if not isinstance(self.role, Role): - try: - object.__setattr__(self, "role", Role(self.role)) - except ValueError: - valid = ", ".join(r.value for r in Role) - raise ValueError( - f"unknown Role {self.role!r}; valid roles: {valid}" - ) from None + object.__setattr__( + self, "role", _coerce_enum(self.role, Role, "Role", "roles")) if self.span is not None: if not ( isinstance(self.span, tuple) @@ -137,14 +148,9 @@ class Ambiguity: tokens: tuple[Token, ...] def __post_init__(self) -> None: - if not isinstance(self.kind, AmbiguityKind): - try: - object.__setattr__(self, "kind", AmbiguityKind(self.kind)) - except ValueError: - valid = ", ".join(k.value for k in AmbiguityKind) - raise ValueError( - f"unknown AmbiguityKind {self.kind!r}; valid kinds: {valid}" - ) from None + object.__setattr__( + self, "kind", + _coerce_enum(self.kind, AmbiguityKind, "AmbiguityKind", "kinds")) if not isinstance(self.detail, str): raise TypeError( f"Ambiguity.detail must be a str, got {self.detail!r}" From eaba92f260adfdd8d3fa4657c291db83d3f7ab19 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 14:05:47 -0700 Subject: [PATCH 52/88] Extract _normpairs as _normset's sibling The module's convention is that collection normalization lives in a module-level _norm* helper the constructor calls; capitalization exceptions were the one field whose ~40-line validation body sat inlined in __post_init__, mixing orchestration with one field's rules. Pure move -- messages and behavior unchanged. Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 90 ++++++++++++++++++++++-------------------- 1 file changed, 48 insertions(+), 42 deletions(-) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index dea001a..e4f08e9 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -69,6 +69,53 @@ def _normset(entries: Iterable[str], field_name: str) -> frozenset[str]: return frozenset(normalized) +def _normpairs( + raw: Mapping[str, str] | Iterable[tuple[str, str]], +) -> tuple[tuple[str, str], ...]: + """Canonicalize capitalization_exceptions input: _normset's sibling + for the one pair-valued field. Dedupes on the NORMALIZED key so the + tuple and the derived map always agree ("Ph.D." and "phd" collide + after normalization); last occurrence wins, matching dict semantics + and the right-bias rule used elsewhere.""" + if isinstance(raw, str): + raise TypeError( + "capitalization_exceptions must be a mapping or an " + "iterable of (key, value) pairs, not a bare string" + ) + pairs = raw.items() if isinstance(raw, Mapping) else raw + deduped: dict[str, str] = {} + for entry in pairs: + # A 2-char str entry would unpack "ab" into ("a", "b") + # silently, so reject str outright; other mis-shapes would + # otherwise surface as bare unpack errors. + if isinstance(entry, str): + raise TypeError( + f"capitalization_exceptions entries must be " + f"(key, value) pairs, got {entry!r}" + ) + try: + k, v = entry + except (TypeError, ValueError): + raise TypeError( + f"capitalization_exceptions entries must be " + f"(key, value) pairs, got {entry!r}" + ) from None + if not isinstance(k, str) or not isinstance(v, str): + raise TypeError( + f"capitalization_exceptions entries must be " + f"str -> str, got {k!r}: {v!r}" + ) + normalized_key = _normalize(k) + if not normalized_key: + raise ValueError( + f"capitalization_exceptions key {k!r} normalizes to " + f"empty (casefold + strip periods/whitespace leaves " + f"nothing)" + ) + deduped[normalized_key] = v + return tuple(sorted(deduped.items())) + + @dataclass(frozen=True, slots=True) class Lexicon: titles: frozenset[str] = frozenset() @@ -93,48 +140,7 @@ class Lexicon: def __post_init__(self) -> None: for name in _VOCAB_FIELDS: object.__setattr__(self, name, _normset(getattr(self, name), name)) - raw = self.capitalization_exceptions - if isinstance(raw, str): - raise TypeError( - "capitalization_exceptions must be a mapping or an " - "iterable of (key, value) pairs, not a bare string" - ) - pairs = raw.items() if isinstance(raw, Mapping) else raw - # Dedupe on the NORMALIZED key before storing so the tuple and the - # map always agree ("Ph.D." and "phd" collide after normalization). - # Last occurrence wins, matching dict semantics and the right-bias - # rule used elsewhere. - deduped: dict[str, str] = {} - for entry in pairs: - # A 2-char str entry would unpack "ab" into ("a", "b") - # silently, so reject str outright; other mis-shapes would - # otherwise surface as bare unpack errors. - if isinstance(entry, str): - raise TypeError( - f"capitalization_exceptions entries must be " - f"(key, value) pairs, got {entry!r}" - ) - try: - k, v = entry - except (TypeError, ValueError): - raise TypeError( - f"capitalization_exceptions entries must be " - f"(key, value) pairs, got {entry!r}" - ) from None - if not isinstance(k, str) or not isinstance(v, str): - raise TypeError( - f"capitalization_exceptions entries must be " - f"str -> str, got {k!r}: {v!r}" - ) - normalized_key = _normalize(k) - if not normalized_key: - raise ValueError( - f"capitalization_exceptions key {k!r} normalizes to " - f"empty (casefold + strip periods/whitespace leaves " - f"nothing)" - ) - deduped[normalized_key] = v - canonical = tuple(sorted(deduped.items())) + canonical = _normpairs(self.capitalization_exceptions) object.__setattr__(self, "capitalization_exceptions", canonical) object.__setattr__(self, "_cap_map", MappingProxyType(dict(canonical))) if not self.particles_ambiguous <= self.particles: From cac42311899e79282f964e0ffe74a7e1f01d417e Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 14:06:14 -0700 Subject: [PATCH 53/88] Drop the whitespace check the charset pin subsumed The [a-z0-9_]+ fullmatch landed after the whitespace check and fully absorbs it: all-whitespace input already fails the strip() check, and interior whitespace fails the charset with an accurate message. The empty and lowercase pre-checks stay -- they guard the common mistakes with materially friendlier messages. Co-Authored-By: Claude Fable 5 --- nameparser/_locale.py | 4 ---- tests/v2/test_locale.py | 4 +++- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/nameparser/_locale.py b/nameparser/_locale.py index 14784c3..640a6a6 100644 --- a/nameparser/_locale.py +++ b/nameparser/_locale.py @@ -36,10 +36,6 @@ def __post_init__(self) -> None: raise ValueError( f"Locale.code must be lowercase, got {self.code!r}" ) - if any(c.isspace() for c in self.code): - raise ValueError( - f"Locale.code must not contain whitespace, got {self.code!r}" - ) # Codes are registry keys (parser_for, third-party packs): every # accepted character is supported forever, so pin the charset # while relaxing later is still compatible. One separator only -- diff --git a/tests/v2/test_locale.py b/tests/v2/test_locale.py index f84728f..c900ff5 100644 --- a/tests/v2/test_locale.py +++ b/tests/v2/test_locale.py @@ -29,8 +29,10 @@ def test_locale_code_must_be_nonempty_lowercase() -> None: def test_locale_code_rejects_whitespace() -> None: + # caught by the [a-z0-9_]+ charset pin (a dedicated whitespace check + # would be pure redundancy; the charset message is accurate) for bad in ("ru ", " ru", "ru\n", "r u"): - with pytest.raises(ValueError, match="whitespace"): + with pytest.raises(ValueError, match="a-z0-9_"): Locale(code=bad, lexicon=Lexicon.empty()) From a2d88eaa61fbf558c8e0863095bb1171a37b1cc7 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 14:28:09 -0700 Subject: [PATCH 54/88] Test the real matrix interpreter, and add 3.15 pre-release (#259) Every matrix job was silently testing CPython 3.11: uv sync honors .python-version (pinned 3.11 by the floor bump) over setup-python's interpreter, so 'build (3.14)' ran the suite on 3.11.15 (verified in the PR #288 run logs). UV_PYTHON makes the matrix real. With that fixed, add 3.15 (final 2026-10-01) as a non-blocking pre-release job so interpreter breakage surfaces while 2.0 is being built; flip it to blocking when 3.15 goes final. Locally green today on 3.15.0b3: pytest, mypy, ruff. Co-Authored-By: Claude Fable 5 --- .github/workflows/python-package.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 072f44c..aacd72a 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -20,7 +20,14 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.11", "3.12", "3.13", "3.14"] + python-version: ["3.11", "3.12", "3.13", "3.14", "3.15"] + # 3.15 is pre-release until 2026-10-01 (#259): surface breakage + # without blocking the branch; flip to blocking when it goes final. + continue-on-error: ${{ matrix.python-version == '3.15' }} + env: + # Without this, uv sync honors .python-version (3.11) over the + # matrix interpreter and every job silently tests 3.11. + UV_PYTHON: ${{ matrix.python-version }} steps: - uses: actions/checkout@v7 @@ -28,6 +35,7 @@ jobs: uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} + allow-prereleases: true - name: Install uv uses: astral-sh/setup-uv@v6 with: From 73e816eed8dae48085f9e83d055a99770a886013 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 15:37:10 -0700 Subject: [PATCH 55/88] Add _render with the #254-normative collapse; extend layering for it Co-Authored-By: Claude Fable 5 --- AGENTS.md | 2 +- nameparser/_render.py | 30 +++++++++++++++++++++++ pyproject.toml | 1 + tests/v2/test_layering.py | 50 +++++++++++++++++++++++++++++++++++---- tests/v2/test_render.py | 20 ++++++++++++++++ 5 files changed, 97 insertions(+), 6 deletions(-) create mode 100644 nameparser/_render.py create mode 100644 tests/v2/test_render.py diff --git a/AGENTS.md b/AGENTS.md index 4e52806..0166c0e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -112,7 +112,7 @@ Each named attribute (`title`, `first`, etc.) is a `@property` that joins its co The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These conventions apply to all new-API code and are stricter than the v1 sections above. The full design record (rationale, settled-decision logs, dated amendments) lives in untracked `docs/superpowers/specs/`; this section is the enforceable subset. **A commit that establishes or amends one of these conventions must update this section in the same commit** — grep-driven staleness sweeps miss paraphrased prose (see Workflow above), so write-time maintenance is the mechanism, audits are the backstop. - **Module layout**: every new module is underscore-private (`_types.py`, `_lexicon.py`, `_policy.py`, `_locale.py`; later `_pipeline/`, `_parser.py`, `_render.py`). The public import surface is exactly `nameparser` (and later `nameparser.locales`); `nameparser/__init__.py` holds re-exports and `__all__` only — no logic. Old paths (`nameparser.parser`, `nameparser.config`) stay v1-shaped. -- **Layering is enforced by `tests/v2/test_layering.py`** (exact-module matching): `_types` imports nothing internal; `_lexicon` and `_policy` sit above it independently; `_locale` on top. `_lexicon` may import `nameparser.config.*` DATA modules only — vocabulary is single-sourced from the v1 data modules through 2.x (`config/maiden_markers.py` is 2.0-only data that lives there for the same reason). Extend the test's `ALLOWED` table when adding a module. +- **Layering is enforced by `tests/v2/test_layering.py`** (exact-module matching; `if TYPE_CHECKING:` imports don't count): `_types` imports nothing internal at module level — its rendering delegates import `_render` at call time; `_lexicon` and `_policy` sit above `_types` independently (`_lexicon` may import `nameparser.config.*` DATA modules only — vocabulary is single-sourced from the v1 data modules through 2.x, e.g. `config/maiden_markers.py`); `_locale` sits on `_lexicon`+`_policy`; `_render` imports `_types` and `_lexicon` (for `Lexicon.default()`). Extend the test's `ALLOWED` table when adding a module. - **Canonical field order** — the seven roles in `Role` enum declaration order, defined once and derived everywhere (properties, `as_dict`, reprs, `comparison_key`). Never restate the order literally. - **Method organization**, fixed section order in every class: fields + `__post_init__` validation → alternative constructors → dunders (construction/equality → protocol → operators) → properties → public methods by concern (access → editing → comparison → rendering delegates) → private helpers last, except a helper serving exactly one section may sit at that section's head. - **Validation is eager and fail-loud**: every `raise` states the offending value, the expected form, and the fix. Exception taxonomy: wrong type — including wrong element type inside a collection, bare `str` where an iterable of strings is expected, or a `Mapping` where a plain iterable is expected — raises `TypeError`; well-typed but unacceptable values raise `ValueError`; failed enum lookups stay `ValueError` for any input (stdlib `EnumType` precedent). diff --git a/nameparser/_render.py b/nameparser/_render.py new file mode 100644 index 0000000..fa67303 --- /dev/null +++ b/nameparser/_render.py @@ -0,0 +1,30 @@ +"""Rendering for the 2.0 API: ParsedName -> display strings. + +Layering: imports nameparser._types, and nameparser._lexicon only for +Lexicon.default() when capitalized() receives lexicon=None (enforced by +tests/v2/test_layering.py). Parsing code never imports this module; +ParsedName's rendering methods delegate here via call-time imports. +""" +from __future__ import annotations + +import re + +_SPACES = re.compile(r"\s+") +_SPACE_BEFORE_COMMA = re.compile(r"\s+,") +_COMMA_CHAR = re.compile(r"[,،，]") # ASCII, Arabic, fullwidth + + +def _collapse(rendered: str) -> str: + """The #254 collapse, normative (core spec §5b): empty fields + substitute '' and every artifact of that is removed -- dangling + empty-nickname wrappers, space runs, space-before-comma, one + trailing comma character (any script), leading/trailing ', ' + debris.""" + rendered = (rendered.replace(" ()", "") + .replace(" ''", "") + .replace(' ""', "")) + rendered = _SPACE_BEFORE_COMMA.sub(",", rendered) + rendered = _SPACES.sub(" ", rendered.strip()) + if rendered and _COMMA_CHAR.fullmatch(rendered[-1]): + rendered = rendered[:-1] + return rendered.strip(", ") diff --git a/pyproject.toml b/pyproject.toml index afbfeca..bba07a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,7 @@ module = [ "nameparser._types", "nameparser._lexicon", "nameparser._policy", + "nameparser._render", "nameparser._locale", ] disallow_untyped_defs = true diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index cfbd63b..cb7c7c3 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -8,23 +8,47 @@ # module -> prefixes it may import from within nameparser ALLOWED = { - "_types.py": (), + # call-time imports only (inside the rendering-delegate method + # bodies); module level stays import-free. TYPE_CHECKING imports + # are skipped by _nameparser_imports and need no entry. + "_types.py": ("nameparser._render",), # intent: DATA modules only, during 2.x -- mechanically this admits # any config submodule; "data only" holds by convention/review "_lexicon.py": ("nameparser.config.",), "_policy.py": ("nameparser._types",), "_locale.py": ("nameparser._lexicon", "nameparser._policy"), + # _lexicon is needed at runtime: capitalized(lexicon=None) resolves + # to Lexicon.default() + "_render.py": ("nameparser._types", "nameparser._lexicon"), } +def _is_type_checking(test: ast.expr) -> bool: + return (isinstance(test, ast.Name) and test.id == "TYPE_CHECKING") or ( + isinstance(test, ast.Attribute) and test.attr == "TYPE_CHECKING") + + def _nameparser_imports(path: pathlib.Path) -> list[str]: - tree = ast.parse(path.read_text()) - found = [] - for node in ast.walk(tree): + """All nameparser-internal imports in the module, at any nesting + depth, EXCEPT those under `if TYPE_CHECKING:` -- annotation-only + imports are not runtime dependencies.""" + found: list[str] = [] + + def _record(node: ast.AST) -> None: + # guard checked on EVERY node uniformly, so `elif TYPE_CHECKING:` + # (a nested If in orelse) is skipped exactly like the plain form + if isinstance(node, ast.If) and _is_type_checking(node.test): + for stmt in node.orelse: + _record(stmt) + return if isinstance(node, ast.Import): - found += [a.name for a in node.names] + found.extend(a.name for a in node.names) elif isinstance(node, ast.ImportFrom) and node.module: found.append(node.module) + for child in ast.iter_child_nodes(node): + _record(child) + + _record(ast.parse(path.read_text())) return [m for m in found if m.startswith("nameparser")] @@ -65,3 +89,19 @@ def test_public_exports() -> None: assert expected <= set(nameparser.__all__) for name in expected: assert getattr(nameparser, name) is not None + + +def test_type_checking_imports_do_not_count(tmp_path: pathlib.Path) -> None: + mod = tmp_path / "snippet.py" + mod.write_text( + "from typing import TYPE_CHECKING\n" + "if TYPE_CHECKING:\n" + " from nameparser._lexicon import Lexicon\n" + "elif TYPE_CHECKING:\n" + " import nameparser.never_runtime\n" + "else:\n" + " import nameparser.util\n" + "def f():\n" + " from nameparser import _render\n" + ) + assert _nameparser_imports(mod) == ["nameparser.util", "nameparser"] diff --git a/tests/v2/test_render.py b/tests/v2/test_render.py new file mode 100644 index 0000000..7cfa87a --- /dev/null +++ b/tests/v2/test_render.py @@ -0,0 +1,20 @@ +# NOTE: the import block grows task by task -- importing names before +# their task lands would fail ruff F401. Task 1 needs only _collapse. +from nameparser._render import _collapse + + +def test_collapse_is_the_254_algorithm() -> None: + # normative: leading/trailing whitespace, doubled spaces, + # space-before-comma, one trailing comma char (incl. Arabic/CJK), + # leading/trailing ', ' debris, and empty-wrapper artifacts from + # empty fields are removed + assert _collapse(" John Smith ") == "John Smith" + assert _collapse("Smith , John") == "Smith, John" + assert _collapse("John Smith ,") == "John Smith" + assert _collapse("John Smith،") == "John Smith" # Arabic comma + assert _collapse("John Smith，") == "John Smith" # fullwidth comma + assert _collapse(", John Smith, ") == "John Smith" + assert _collapse("John Smith ()") == "John Smith" + assert _collapse("John Smith ''") == "John Smith" + assert _collapse('John Smith ""') == "John Smith" + assert _collapse("") == "" From a6750dc3a95378cedc989216ecea943ff5efcf26 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 15:57:59 -0700 Subject: [PATCH 56/88] Add render(): spec formatting over role fields and derived views Co-Authored-By: Claude Fable 5 --- nameparser/_render.py | 22 ++++++++++++++++ tests/v2/test_render.py | 57 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 77 insertions(+), 2 deletions(-) diff --git a/nameparser/_render.py b/nameparser/_render.py index fa67303..14b9b68 100644 --- a/nameparser/_render.py +++ b/nameparser/_render.py @@ -9,10 +9,17 @@ import re +from nameparser._types import ParsedName, Role + _SPACES = re.compile(r"\s+") _SPACE_BEFORE_COMMA = re.compile(r"\s+,") _COMMA_CHAR = re.compile(r"[,،，]") # ASCII, Arabic, fullwidth +#: str.format keys render() accepts: the seven role fields in canonical +#: order (derived from Role -- never restated) plus the derived views. +_DERIVED_VIEWS = ("family_base", "family_particles", "surnames", "given_names") +_RENDER_KEYS = tuple(r.value for r in Role) + _DERIVED_VIEWS + def _collapse(rendered: str) -> str: """The #254 collapse, normative (core spec §5b): empty fields @@ -28,3 +35,18 @@ def _collapse(rendered: str) -> str: if rendered and _COMMA_CHAR.fullmatch(rendered[-1]): rendered = rendered[:-1] return rendered.strip(", ") + + +def render(name: ParsedName, spec: str) -> str: + """Fill the str.format spec from the seven role fields and the + derived views (empty fields substitute ''), then apply the #254 + collapse. Unknown keys raise KeyError naming the valid fields.""" + values = {key: getattr(name, key) for key in _RENDER_KEYS} + try: + rendered = spec.format(**values) + except KeyError as exc: + raise KeyError( + f"unknown render field {exc.args[0]!r}; valid fields: " + f"{', '.join(_RENDER_KEYS)}" + ) from None + return _collapse(rendered) diff --git a/tests/v2/test_render.py b/tests/v2/test_render.py index 7cfa87a..142707b 100644 --- a/tests/v2/test_render.py +++ b/tests/v2/test_render.py @@ -1,6 +1,7 @@ -# NOTE: the import block grows task by task -- importing names before -# their task lands would fail ruff F401. Task 1 needs only _collapse. +import pytest + from nameparser._render import _collapse +from nameparser._types import ParsedName, Role, Span, Token def test_collapse_is_the_254_algorithm() -> None: @@ -18,3 +19,55 @@ def test_collapse_is_the_254_algorithm() -> None: assert _collapse("John Smith ''") == "John Smith" assert _collapse('John Smith ""') == "John Smith" assert _collapse("") == "" + + +def _pn(original: str, tokens: list[Token]) -> ParsedName: + return ParsedName(original=original, tokens=tuple(tokens)) + + +def _delavega() -> ParsedName: + # "Dr. Juan de la Vega III" -- spans verified by hand + # 0123456789012345678901234 + return _pn("Dr. Juan de la Vega III", [ + Token("Dr.", Span(0, 3), Role.TITLE), + Token("Juan", Span(4, 8), Role.GIVEN), + Token("de", Span(9, 11), Role.FAMILY, frozenset({"particle"})), + Token("la", Span(12, 14), Role.FAMILY, frozenset({"particle"})), + Token("Vega", Span(15, 19), Role.FAMILY), + Token("III", Span(20, 23), Role.SUFFIX), + ]) + + +def test_render_fills_fields_and_collapses() -> None: + from nameparser._render import render + pn = _delavega() + assert render(pn, "{title} {given} {middle} {family} {suffix}") \ + == "Dr. Juan de la Vega III" + # empty middle collapses; comma survives correctly + assert render(pn, "{family}, {given} {middle}") == "de la Vega, Juan" + + +def test_render_accepts_derived_view_keys() -> None: + from nameparser._render import render + assert render(_delavega(), "{family_base}, {given} {family_particles}") \ + == "Vega, Juan de la" + assert render(_delavega(), "{surnames}") == "de la Vega" + assert render(_delavega(), "{given_names}") == "Juan" + + +def test_render_every_role_key_is_valid() -> None: + from nameparser._render import render + pn = _delavega() + for role in Role: + render(pn, f"{{{role.value}}}") # must not raise + + +def test_render_unknown_key_raises_enriched_keyerror() -> None: + from nameparser._render import render + with pytest.raises(KeyError, match="valid fields"): + render(_delavega(), "{first}") # v1 spelling: redirected loudly + + +def test_render_empty_parse_is_empty_string() -> None: + from nameparser._render import render + assert render(_pn("", []), "{title} {given} {middle} {family} {suffix}") == "" From bbca3653209747dac1dc29ba8dc2c964fb6ad0c9 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 16:03:58 -0700 Subject: [PATCH 57/88] Add ParsedName.render() and __str__ via call-time _render delegation Co-Authored-By: Claude Fable 5 --- nameparser/_types.py | 15 +++++++++++++++ tests/v2/test_render.py | 8 ++++++++ 2 files changed, 23 insertions(+) diff --git a/nameparser/_types.py b/nameparser/_types.py index f147909..7d6410e 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -230,6 +230,9 @@ def __post_init__(self) -> None: def __bool__(self) -> bool: return bool(self.tokens) + def __str__(self) -> str: + return self.render() + def __repr__(self) -> str: lines = [] for role in Role: @@ -366,3 +369,15 @@ def comparison_key(self) -> tuple[str, ...]: strict. """ return tuple(self._text_for(role).casefold() for role in Role) + + # -- rendering delegates ---------------------------------------------- + # One-line delegation to nameparser._render (core spec §5b): parsing + # code physically cannot import formatting logic, so these import at + # call time -- module level stays internal-import-free. + + def render(self, spec: str = "{title} {given} {middle} {family} {suffix}") -> str: + """Fill the str.format spec from the seven role fields and the + derived views; empty fields collapse (#254). Unknown keys raise + KeyError naming the valid fields.""" + import nameparser._render as _render + return _render.render(self, spec) diff --git a/tests/v2/test_render.py b/tests/v2/test_render.py index 142707b..064f74c 100644 --- a/tests/v2/test_render.py +++ b/tests/v2/test_render.py @@ -71,3 +71,11 @@ def test_render_unknown_key_raises_enriched_keyerror() -> None: def test_render_empty_parse_is_empty_string() -> None: from nameparser._render import render assert render(_pn("", []), "{title} {given} {middle} {family} {suffix}") == "" + + +def test_parsedname_render_and_str_delegate() -> None: + pn = _delavega() + assert pn.render() == "Dr. Juan de la Vega III" + assert pn.render("{family}, {given}") == "de la Vega, Juan" + assert str(pn) == pn.render() + assert str(_pn("", [])) == "" From 445571d138a4007e60775574026d35677fa8c4b8 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 16:07:50 -0700 Subject: [PATCH 58/88] Add initials() with call-site delimiter/separator arguments Co-Authored-By: Claude Fable 5 --- nameparser/_render.py | 34 ++++++++++++++++++++++ nameparser/_types.py | 8 ++++++ tests/v2/test_render.py | 62 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 104 insertions(+) diff --git a/nameparser/_render.py b/nameparser/_render.py index 14b9b68..2f8ae5a 100644 --- a/nameparser/_render.py +++ b/nameparser/_render.py @@ -20,6 +20,13 @@ _DERIVED_VIEWS = ("family_base", "family_particles", "surnames", "given_names") _RENDER_KEYS = tuple(r.value for r in Role) + _DERIVED_VIEWS +#: str.format keys initials() accepts: the three name-bearing roles. +_INITIALS_KEYS = (Role.GIVEN.value, Role.MIDDLE.value, Role.FAMILY.value) + +#: Tags whose tokens contribute no initial outside the given group. +#: Not STABLE_TAGS -- that also contains "initial", which must contribute. +_SKIP_TAGS = frozenset({"particle", "conjunction"}) + def _collapse(rendered: str) -> str: """The #254 collapse, normative (core spec §5b): empty fields @@ -50,3 +57,30 @@ def render(name: ParsedName, spec: str) -> str: f"{', '.join(_RENDER_KEYS)}" ) from None return _collapse(rendered) + + +def initials(name: ParsedName, spec: str, delimiter: str, separator: str) -> str: + """First letter of each contributing token per group, v1 semantics: + delimiter follows each initial, separator sits between initials + within a group. Tokens tagged particle/conjunction contribute no + initial in middle/family (given-name tokens always contribute); + tags come from the pipeline -- hand-built untagged tokens all + contribute. Valid spec keys: given, middle, family.""" + values: dict[str, str] = {} + for key in _INITIALS_KEYS: + role = Role(key) + tokens = name.tokens_for(role) + if role is not Role.GIVEN: + tokens = tuple(t for t in tokens + if not (_SKIP_TAGS & t.tags)) + letters = [t.text[0] for t in tokens] + values[key] = ((delimiter + separator).join(letters) + delimiter + if letters else "") + try: + rendered = spec.format(**values) + except KeyError as exc: + raise KeyError( + f"unknown initials field {exc.args[0]!r}; valid fields: " + f"{', '.join(_INITIALS_KEYS)}" + ) from None + return _collapse(rendered) diff --git a/nameparser/_types.py b/nameparser/_types.py index 7d6410e..a700bd1 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -381,3 +381,11 @@ def render(self, spec: str = "{title} {given} {middle} {family} {suffix}") -> st KeyError naming the valid fields.""" import nameparser._render as _render return _render.render(self, spec) + + def initials(self, spec: str = "{given} {middle} {family}", + delimiter: str = ".", separator: str = " ") -> str: + """Initials per group; v1's initials_format/_delimiter/_separator + become call-site arguments (core spec §5b). Valid spec keys: + given, middle, family.""" + import nameparser._render as _render + return _render.initials(self, spec, delimiter, separator) diff --git a/tests/v2/test_render.py b/tests/v2/test_render.py index 064f74c..15e0861 100644 --- a/tests/v2/test_render.py +++ b/tests/v2/test_render.py @@ -79,3 +79,65 @@ def test_parsedname_render_and_str_delegate() -> None: assert pn.render("{family}, {given}") == "de la Vega, Juan" assert str(pn) == pn.render() assert str(_pn("", [])) == "" + + +def _bobdole() -> ParsedName: + # "Sir Bob Andrew Dole" + # 01234567890123456789 + return _pn("Sir Bob Andrew Dole", [ + Token("Sir", Span(0, 3), Role.TITLE), + Token("Bob", Span(4, 7), Role.GIVEN), + Token("Andrew", Span(8, 14), Role.MIDDLE), + Token("Dole", Span(15, 19), Role.FAMILY), + ]) + + +def test_initials_default_spec() -> None: + assert _bobdole().initials() == "B. A. D." + + +def test_initials_skips_tagged_particles_outside_given() -> None: + # family "de la Vega" with particle tags -> only V contributes; + # a given-name token always contributes even if tagged + assert _delavega().initials() == "J. V." + # conjunction tag skips too + pn = _pn("Mr. and Mrs. Smith", [ + Token("Mr.", Span(0, 3), Role.TITLE), + Token("and", Span(4, 7), Role.FAMILY, frozenset({"conjunction"})), + Token("Smith", Span(13, 18), Role.FAMILY), + ]) + assert pn.initials("{family}") == "S." + + +def test_initials_custom_delimiter_and_separator() -> None: + assert _bobdole().initials(delimiter="", separator="") == "B A D" + + +def test_initials_multiword_group_joins_within_group() -> None: + pn = _pn("Mary Jane Watson", [ + Token("Mary", Span(0, 4), Role.GIVEN), + Token("Jane", Span(5, 9), Role.GIVEN), + Token("Watson", Span(10, 16), Role.FAMILY), + ]) + assert pn.initials() == "M. J. W." + + +def test_initials_custom_spec_and_unknown_key() -> None: + assert _bobdole().initials("{given} {middle}") == "B. A." + with pytest.raises(KeyError, match="valid fields"): + _bobdole().initials("{title}") + + +def test_initials_already_initial_words() -> None: + pn = _pn("J. Doe", [ + Token("J.", Span(0, 2), Role.GIVEN), + Token("Doe", Span(3, 6), Role.FAMILY), + ]) + assert pn.initials() == "J. D." + + +def test_initials_empty_group_renders_empty() -> None: + # v2 returns "" for an empty result -- no v1-style + # empty_attribute_default fallback + assert _bobdole().initials("{middle}") == "A." + assert _pn("Cher", [Token("Cher", Span(0, 4), Role.GIVEN)]).initials("{middle}") == "" From da94eb5f18c18d90a8d6e8414a826c8fd91f335c Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 16:14:53 -0700 Subject: [PATCH 59/88] Add capitalized(): lexicon-driven case fixing, same spans Co-Authored-By: Claude Fable 5 --- nameparser/_render.py | 56 +++++++++++++++++++++++++++- nameparser/_types.py | 14 ++++++- tests/v2/test_render.py | 82 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 149 insertions(+), 3 deletions(-) diff --git a/nameparser/_render.py b/nameparser/_render.py index 2f8ae5a..6a4dd0a 100644 --- a/nameparser/_render.py +++ b/nameparser/_render.py @@ -9,11 +9,14 @@ import re -from nameparser._types import ParsedName, Role +from nameparser._lexicon import Lexicon, _normalize +from nameparser._types import Ambiguity, ParsedName, Role, Token _SPACES = re.compile(r"\s+") _SPACE_BEFORE_COMMA = re.compile(r"\s+,") _COMMA_CHAR = re.compile(r"[,،，]") # ASCII, Arabic, fullwidth +_MAC = re.compile(r"^(ma?c)(\w{2,})", re.IGNORECASE) +_WORD = re.compile(r"(\w|\.)+") #: str.format keys render() accepts: the seven role fields in canonical #: order (derived from Role -- never restated) plus the derived views. @@ -84,3 +87,54 @@ def initials(name: ParsedName, spec: str, delimiter: str, separator: str) -> str f"{', '.join(_INITIALS_KEYS)}" ) from None return _collapse(rendered) + + +def _cap_word(word: str, role: Role, lex: Lexicon) -> str: + # v1 cap_word order: particle/conjunction rule first, then the + # exceptions map, then Mac/Mc, then str.capitalize + normalized = _normalize(word) + if (normalized in lex.particles + and role in (Role.MIDDLE, Role.FAMILY)) \ + or normalized in lex.conjunctions: + return word.lower() + exception = lex.capitalization_exceptions_map.get(normalized) + if exception is not None: + return exception + if _MAC.match(word): + return _MAC.sub( + lambda m: m.group(1).capitalize() + m.group(2).capitalize(), + word) + return word.capitalize() + + +def _cap_text(text: str, role: Role, lex: Lexicon) -> str: + # word-by-word within the token text: hyphenated names capitalize + # both sides ("macdole-eisenhower" -> "MacDole-Eisenhower") + return _WORD.sub(lambda m: _cap_word(m.group(0), role, lex), text) + + +def capitalized(name: ParsedName, lexicon: Lexicon | None, *, + force: bool) -> ParsedName: + """Case-fixing transform -> new ParsedName, same spans, new token + texts (core spec §5b). Gate (v1 parity): only single-case input is + touched unless force=True; the gate reads the joined token texts. + Idempotent: without force, a capitalized result is mixed-case and + the gate returns it unchanged; with force, every _cap_word rule is + a fixpoint on its own output.""" + lex = Lexicon.default() if lexicon is None else lexicon + joined = " ".join(t.text for t in name.tokens) + if not force and joined not in (joined.upper(), joined.lower()): + return name + new_tokens = tuple( + Token(_cap_text(t.text, t.role, lex), t.span, t.role, t.tags) + for t in name.tokens) + # equal tokens (possible only for synthetic span=None duplicates) + # collapse to one mapping entry -- benign: the rebuilt ambiguity + # references an equal token, so the subset invariant still holds + replacement = dict(zip(name.tokens, new_tokens)) + new_ambiguities = tuple( + Ambiguity(a.kind, a.detail, + tuple(replacement[t] for t in a.tokens)) + for a in name.ambiguities) + return ParsedName(original=name.original, tokens=new_tokens, + ambiguities=new_ambiguities) diff --git a/nameparser/_types.py b/nameparser/_types.py index a700bd1..fee45e4 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -13,7 +13,10 @@ from collections.abc import Mapping from dataclasses import dataclass from enum import Enum, StrEnum -from typing import NamedTuple, NoReturn, TypeVar +from typing import TYPE_CHECKING, NamedTuple, NoReturn, TypeVar + +if TYPE_CHECKING: + from nameparser._lexicon import Lexicon class Role(Enum): @@ -389,3 +392,12 @@ def initials(self, spec: str = "{given} {middle} {family}", given, middle, family.""" import nameparser._render as _render return _render.initials(self, spec, delimiter, separator) + + def capitalized(self, lexicon: Lexicon | None = None, *, + force: bool = False) -> ParsedName: + """Case-fixing transform -> new ParsedName, same spans, new + token texts. Needs a lexicon for capitalization_exceptions and + particle rules; None uses the default lexicon. force=False + preserves mixed-case input (v1 parity). Idempotent.""" + import nameparser._render as _render + return _render.capitalized(self, lexicon, force=force) diff --git a/tests/v2/test_render.py b/tests/v2/test_render.py index 15e0861..b79db4f 100644 --- a/tests/v2/test_render.py +++ b/tests/v2/test_render.py @@ -1,7 +1,8 @@ import pytest +from nameparser._lexicon import Lexicon from nameparser._render import _collapse -from nameparser._types import ParsedName, Role, Span, Token +from nameparser._types import Ambiguity, AmbiguityKind, ParsedName, Role, Span, Token def test_collapse_is_the_254_algorithm() -> None: @@ -141,3 +142,82 @@ def test_initials_empty_group_renders_empty() -> None: # empty_attribute_default fallback assert _bobdole().initials("{middle}") == "A." assert _pn("Cher", [Token("Cher", Span(0, 4), Role.GIVEN)]).initials("{middle}") == "" + + +def _lowercase_mac() -> ParsedName: + # v1 capitalize() doctest input: 'bob v. de la macdole-eisenhower phd' + # 0123456789012345678901234567890123456 + return _pn("bob v. de la macdole-eisenhower phd", [ + Token("bob", Span(0, 3), Role.GIVEN), + Token("v.", Span(4, 6), Role.MIDDLE), + Token("de", Span(7, 9), Role.FAMILY), + Token("la", Span(10, 12), Role.FAMILY), + Token("macdole-eisenhower", Span(13, 31), Role.FAMILY), + Token("phd", Span(32, 35), Role.SUFFIX), + ]) + + +def test_capitalized_all_lower_input_v1_parity() -> None: + out = _lowercase_mac().capitalized() + assert out.given == "Bob" + assert out.middle == "V." + assert out.family == "de la MacDole-Eisenhower" # particles stay lower + assert out.suffix == "Ph.D." # exceptions map, verbatim + # same spans, new texts (provenance is a documented non-invariant) + assert [t.span for t in out.tokens] == [t.span for t in _lowercase_mac().tokens] + + +def test_capitalized_all_upper_input() -> None: + pn = _pn("JOHN SMITH", [ + Token("JOHN", Span(0, 4), Role.GIVEN), + Token("SMITH", Span(5, 10), Role.FAMILY), + ]) + assert str(pn.capitalized()) == "John Smith" + + +def test_capitalized_preserves_mixed_case_unless_forced() -> None: + pn = _pn("Shirley Maclaine", [ + Token("Shirley", Span(0, 7), Role.GIVEN), + Token("Maclaine", Span(8, 16), Role.FAMILY), + ]) + assert pn.capitalized() == pn # untouched + assert pn.capitalized(force=True).family == "MacLaine" + + +def test_capitalized_is_idempotent() -> None: + once = _lowercase_mac().capitalized() + assert once.capitalized() == once + assert once.capitalized(force=True) == once + + +def test_capitalized_with_explicit_lexicon() -> None: + # empty lexicon: no particle rule, no exceptions -> plain capitalize + out = _lowercase_mac().capitalized(Lexicon.empty()) + assert out.family == "De La MacDole-Eisenhower" + assert out.suffix == "Phd" + + +def test_capitalized_lowers_conjunctions() -> None: + # 01234567890123456789 + pn = _pn("juan ortega Y gasset", [ + Token("juan", Span(0, 4), Role.GIVEN), + Token("ortega", Span(5, 11), Role.FAMILY), + Token("Y", Span(12, 13), Role.FAMILY, frozenset({"conjunction"})), + Token("gasset", Span(14, 20), Role.FAMILY), + ]) + out = pn.capitalized(force=True) + assert out.family == "Ortega y Gasset" + + +def test_capitalized_rebuilds_ambiguity_tokens() -> None: + tok = Token("van", Span(0, 3), Role.GIVEN, frozenset({"particle"})) + pn = ParsedName( + original="van johnson", + tokens=(tok, Token("johnson", Span(4, 11), Role.FAMILY)), + ambiguities=(Ambiguity(AmbiguityKind.PARTICLE_OR_GIVEN, + "leading 'van' may be a particle", (tok,)),), + ) + out = pn.capitalized() + # the ambiguity references the NEW capitalized token (subset invariant) + assert out.ambiguities[0].tokens[0] is out.tokens[0] + assert out.ambiguities[0].tokens[0].text == "Van" From 936d3476b29a3d8513d193510a67fc6895c5a431 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 16:27:28 -0700 Subject: [PATCH 60/88] True up module docstrings to the rendering delegation _types still claimed to import nothing from nameparser -- now false in the unqualified form (call-time _render imports, TYPE_CHECKING Lexicon) and contradicting the layering ALLOWED entry; AGENTS.md had the qualified wording but the docstring a reader actually opens did not. _render's docstring understated its _lexicon dependency (_normalize is used unconditionally) and now notes that only unknown KEYS get the enriched KeyError. Co-Authored-By: Claude Fable 5 --- nameparser/_render.py | 13 +++++++++---- nameparser/_types.py | 5 ++++- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/nameparser/_render.py b/nameparser/_render.py index 6a4dd0a..d3c7ea7 100644 --- a/nameparser/_render.py +++ b/nameparser/_render.py @@ -1,9 +1,14 @@ """Rendering for the 2.0 API: ParsedName -> display strings. -Layering: imports nameparser._types, and nameparser._lexicon only for -Lexicon.default() when capitalized() receives lexicon=None (enforced by -tests/v2/test_layering.py). Parsing code never imports this module; -ParsedName's rendering methods delegate here via call-time imports. +Layering: imports nameparser._types, and nameparser._lexicon for +Lexicon.default() (capitalized() with lexicon=None) and _normalize +(enforced by tests/v2/test_layering.py). Parsing code never imports +this module; ParsedName's rendering methods delegate here via +call-time imports. + +Malformed str.format specs beyond unknown keys (positional fields, +bad conversions) surface the raw str.format error; only unknown KEYS +get the enriched KeyError. """ from __future__ import annotations diff --git a/nameparser/_types.py b/nameparser/_types.py index fee45e4..195c7e6 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -1,7 +1,10 @@ """Core value types for the 2.0 API. Layering (enforced by tests/v2/test_layering.py): this module imports -nothing from nameparser -- it is the bottom of the dependency graph. +nothing from nameparser at module level -- it is the bottom of the +module-import dependency graph. The rendering delegates import _render +at call time, and a TYPE_CHECKING-only _lexicon import supplies the +Lexicon annotation. Repr policy (applies to every v2 type's __repr__, across this module and _lexicon.py/_policy.py/_locale.py): bounded output only. No repr may scale From 27616dcdc1240e3086cda99b52322037cbb78129 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 16:59:36 -0700 Subject: [PATCH 61/88] Guard all frozen types against pickle layout skew Lexicon failed at the load site on field-layout mismatch; its five frozen siblings (Token, Ambiguity, ParsedName, Policy, PolicyPatch, Locale) restored slots silently and failed at a distant attribute read. Shared _guarded_getstate/_guarded_setstate functions in _types are ASSIGNED IN EACH CLASS BODY -- @dataclass(slots=True) regenerates the class and installs its own pickle methods unless the names are in the class's own __dict__, which is why a mixin cannot work. Lexicon keeps its own copy (layering forbids _lexicon importing _types) plus its mappingproxy rebuild. Policy, stated in code and AGENTS.md: unpickle validates field LAYOUT, not values -- pickle is not a security boundary, and canonical state comes only from a validated instance. _locale's ALLOWED entry gains the downward _types import. Co-Authored-By: Claude Fable 5 --- AGENTS.md | 4 ++-- nameparser/_locale.py | 5 +++++ nameparser/_policy.py | 10 ++++++++- nameparser/_types.py | 46 +++++++++++++++++++++++++++++++++++++++ tests/v2/test_layering.py | 5 ++++- tests/v2/test_locale.py | 8 +++++++ tests/v2/test_policy.py | 7 ++++++ tests/v2/test_types.py | 18 +++++++++++++++ 8 files changed, 99 insertions(+), 4 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 0166c0e..8b7ed4e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -112,13 +112,13 @@ Each named attribute (`title`, `first`, etc.) is a `@property` that joins its co The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These conventions apply to all new-API code and are stricter than the v1 sections above. The full design record (rationale, settled-decision logs, dated amendments) lives in untracked `docs/superpowers/specs/`; this section is the enforceable subset. **A commit that establishes or amends one of these conventions must update this section in the same commit** — grep-driven staleness sweeps miss paraphrased prose (see Workflow above), so write-time maintenance is the mechanism, audits are the backstop. - **Module layout**: every new module is underscore-private (`_types.py`, `_lexicon.py`, `_policy.py`, `_locale.py`; later `_pipeline/`, `_parser.py`, `_render.py`). The public import surface is exactly `nameparser` (and later `nameparser.locales`); `nameparser/__init__.py` holds re-exports and `__all__` only — no logic. Old paths (`nameparser.parser`, `nameparser.config`) stay v1-shaped. -- **Layering is enforced by `tests/v2/test_layering.py`** (exact-module matching; `if TYPE_CHECKING:` imports don't count): `_types` imports nothing internal at module level — its rendering delegates import `_render` at call time; `_lexicon` and `_policy` sit above `_types` independently (`_lexicon` may import `nameparser.config.*` DATA modules only — vocabulary is single-sourced from the v1 data modules through 2.x, e.g. `config/maiden_markers.py`); `_locale` sits on `_lexicon`+`_policy`; `_render` imports `_types` and `_lexicon` (for `Lexicon.default()`). Extend the test's `ALLOWED` table when adding a module. +- **Layering is enforced by `tests/v2/test_layering.py`** (exact-module matching; `if TYPE_CHECKING:` imports don't count): `_types` imports nothing internal at module level — its rendering delegates import `_render` at call time; `_lexicon` and `_policy` sit above `_types` independently (`_lexicon` may import `nameparser.config.*` DATA modules only — vocabulary is single-sourced from the v1 data modules through 2.x, e.g. `config/maiden_markers.py`); `_locale` sits on `_lexicon`+`_policy` (plus `_types` for the shared pickle mixin); `_render` imports `_types` and `_lexicon` (for `Lexicon.default()` and `_normalize`). Extend the test's `ALLOWED` table when adding a module. - **Canonical field order** — the seven roles in `Role` enum declaration order, defined once and derived everywhere (properties, `as_dict`, reprs, `comparison_key`). Never restate the order literally. - **Method organization**, fixed section order in every class: fields + `__post_init__` validation → alternative constructors → dunders (construction/equality → protocol → operators) → properties → public methods by concern (access → editing → comparison → rendering delegates) → private helpers last, except a helper serving exactly one section may sit at that section's head. - **Validation is eager and fail-loud**: every `raise` states the offending value, the expected form, and the fix. Exception taxonomy: wrong type — including wrong element type inside a collection, bare `str` where an iterable of strings is expected, or a `Mapping` where a plain iterable is expected — raises `TypeError`; well-typed but unacceptable values raise `ValueError`; failed enum lookups stay `ValueError` for any input (stdlib `EnumType` precedent). - **Reprs are bounded**: render which fields deviate from a named baseline and by how much, never contents (`Lexicon(default + titles: +2)`). - **Typing/docs**: `from __future__ import annotations`; `frozen=True, slots=True` on every public dataclass; strict-profile mypy flags via per-module overrides in pyproject (`strict = true` itself is not valid per-module). Docstrings state contracts in prose with **no doctest blocks** — `--doctest-modules` makes every example a test; behavior examples go to unit tests per the lean-docs rule. -- **Pickling**: v2 types must round-trip (`Parser` will be picklable by construction, and it holds a `Lexicon`). `Lexicon` needs its custom `__getstate__`/`__setstate__` because of its `mappingproxy` slot; a new unpicklable slot type needs the same treatment plus a round-trip test. +- **Pickling**: v2 types must round-trip (`Parser` will be picklable by construction, and it holds a `Lexicon`). Every frozen type assigns `_guarded_getstate`/`_guarded_setstate` (`_types.py`) in its class body (`@dataclass(slots=True)` would override inherited pickle methods) — unpickling fails at the LOAD site on field-layout skew, and values are deliberately NOT re-validated (pickle is not a security boundary; canonical state comes from a validated instance). `Lexicon` keeps its own copy of the guard (layering) plus the `mappingproxy` slot rebuild; a new unpicklable slot type needs the same treatment plus a round-trip test. - **One sanctioned global**: the (future) cached default `Parser`. Any second piece of module-level mutable state requires amending the conventions doc, on purpose, in review. - **Tests**: all v2 tests live in the `tests/v2/` package (its `conftest.py` overrides the v1 dual-run fixture — v2 code never reads shared `CONSTANTS`), one test module per source module plus cross-cutting `test_reprs.py` and `test_layering.py`, names stating behavior. Never assert `Lexicon.default()` contents; the narrow sourcing spot-checks in `test_default_sources_v1_vocabulary` that pin the v1→v2 migration contract (e.g. the flipped `particles_ambiguous` model) are the sanctioned exception. diff --git a/nameparser/_locale.py b/nameparser/_locale.py index 640a6a6..d50cd6e 100644 --- a/nameparser/_locale.py +++ b/nameparser/_locale.py @@ -15,6 +15,7 @@ from nameparser._lexicon import Lexicon from nameparser._policy import UNSET, PolicyPatch +from nameparser._types import _guarded_getstate, _guarded_setstate @dataclass(frozen=True, slots=True) @@ -23,6 +24,10 @@ class Locale: lexicon: Lexicon policy: PolicyPatch = PolicyPatch() + # in the class body so @dataclass(slots=True) keeps them + __getstate__ = _guarded_getstate + __setstate__ = _guarded_setstate + def __post_init__(self) -> None: if not isinstance(self.code, str): raise TypeError( diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 9f12fad..f0c375a 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -9,7 +9,7 @@ from dataclasses import dataclass, field from enum import Enum, StrEnum, auto -from nameparser._types import Role +from nameparser._types import Role, _guarded_getstate, _guarded_setstate class PatronymicRule(StrEnum): @@ -45,6 +45,10 @@ class Policy: strip_emoji: bool = True strip_bidi: bool = True # =False replaces v1's opt-out CONSTANTS.regexes.bidi = False + # in the class body so @dataclass(slots=True) keeps them + __getstate__ = _guarded_getstate + __setstate__ = _guarded_setstate + def __post_init__(self) -> None: order = tuple(self.name_order) if len(order) != 3 or set(order) != _NAME_ROLES: @@ -181,6 +185,10 @@ class PolicyPatch: strip_emoji: bool | _Unset = UNSET strip_bidi: bool | _Unset = UNSET + # in the class body so @dataclass(slots=True) keeps them + __getstate__ = _guarded_getstate + __setstate__ = _guarded_setstate + def __post_init__(self) -> None: # Canonicalize (but do NOT validate) collection fields so a patch # built from a set/list literal is hashable and unions cleanly in diff --git a/nameparser/_types.py b/nameparser/_types.py index 195c7e6..5882919 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -13,6 +13,7 @@ """ from __future__ import annotations +import dataclasses from collections.abc import Mapping from dataclasses import dataclass from enum import Enum, StrEnum @@ -73,6 +74,39 @@ def _coerce_enum(value: object, enum_cls: type[_E], noun: str, plural: str) -> _ ) from None +# Pickle support shared by the frozen slots dataclasses: fail at the +# LOAD site when a pickle's field layout does not match this version of +# the class (version skew) -- silently loading would defer the failure +# to a distant attribute read. Values are deliberately NOT re-validated: +# pickle is not a security boundary (arbitrary pickles can execute code +# anyway), and canonical state only comes from a validated instance. +# These are ASSIGNED IN EACH CLASS BODY (not inherited from a mixin): +# @dataclass(slots=True) regenerates the class and installs its own +# pickle methods unless __getstate__/__setstate__ are in the class's +# own __dict__. Lexicon duplicates this logic by design (its slots also +# carry a rebuilt mappingproxy) -- layering keeps _lexicon import-free +# of _types. + + +def _guarded_getstate(self: object) -> dict[str, object]: + fields = dataclasses.fields(self) # type: ignore[arg-type] + return {f.name: getattr(self, f.name) for f in fields} + + +def _guarded_setstate(self: object, state: dict[str, object]) -> None: + fields = dataclasses.fields(self) # type: ignore[arg-type] + expected = {f.name for f in fields} + if set(state) != expected: + missing = ", ".join(sorted(expected - set(state))) or "none" + unexpected = ", ".join(sorted(set(state) - expected)) or "none" + raise ValueError( + f"incompatible {type(self).__name__} pickle: missing " + f"fields: {missing}; unexpected fields: {unexpected}" + ) + for name, value in state.items(): + object.__setattr__(self, name, value) + + @dataclass(frozen=True, slots=True) class Token: text: str @@ -80,6 +114,10 @@ class Token: role: Role tags: frozenset[str] = frozenset() + # in the class body so @dataclass(slots=True) keeps them + __getstate__ = _guarded_getstate + __setstate__ = _guarded_setstate + def __post_init__(self) -> None: if not isinstance(self.text, str): raise TypeError( @@ -153,6 +191,10 @@ class Ambiguity: detail: str tokens: tuple[Token, ...] + # in the class body so @dataclass(slots=True) keeps them + __getstate__ = _guarded_getstate + __setstate__ = _guarded_setstate + def __post_init__(self) -> None: object.__setattr__( self, "kind", @@ -190,6 +232,10 @@ class ParsedName: tokens: tuple[Token, ...] ambiguities: tuple[Ambiguity, ...] = () + # in the class body so @dataclass(slots=True) keeps them + __getstate__ = _guarded_getstate + __setstate__ = _guarded_setstate + def __post_init__(self) -> None: if not isinstance(self.original, str): raise TypeError( diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index cb7c7c3..63ec122 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -16,7 +16,10 @@ # any config submodule; "data only" holds by convention/review "_lexicon.py": ("nameparser.config.",), "_policy.py": ("nameparser._types",), - "_locale.py": ("nameparser._lexicon", "nameparser._policy"), + # _types is a downward import (bottom of the graph): Locale shares + # the _guarded_getstate/_guarded_setstate pickle functions + "_locale.py": ("nameparser._types", "nameparser._lexicon", + "nameparser._policy"), # _lexicon is needed at runtime: capitalized(lexicon=None) resolves # to Lexicon.default() "_render.py": ("nameparser._types", "nameparser._lexicon"), diff --git a/tests/v2/test_locale.py b/tests/v2/test_locale.py index c900ff5..924a7ce 100644 --- a/tests/v2/test_locale.py +++ b/tests/v2/test_locale.py @@ -66,3 +66,11 @@ def test_locale_code_is_pinned_to_registry_charset() -> None: with pytest.raises(ValueError, match="a-z0-9_"): Locale(code=bad, lexicon=Lexicon.empty()) assert Locale(code="tr_az", lexicon=Lexicon.empty()).code == "tr_az" + + +def test_setstate_rejects_layout_skew() -> None: + loc = Locale(code="ru", lexicon=Lexicon.empty()) + state = dict(loc.__getstate__()) + del state["code"] + with pytest.raises(ValueError, match="code"): + Locale.__new__(Locale).__setstate__(state) diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index 045ac6e..497cf9a 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -206,3 +206,10 @@ def test_policy_and_patch_pickle_round_trip_preserves_unset_identity() -> None: # object() sentinel would break this silently. assert loaded.name_order is UNSET assert loaded.strip_emoji is False + + +def test_setstate_rejects_layout_skew() -> None: + state = dict(Policy().__getstate__()) + del state["name_order"] + with pytest.raises(ValueError, match="name_order"): + Policy.__new__(Policy).__setstate__(state) diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 5211730..84952eb 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -350,3 +350,21 @@ def test_token_rejects_bool_span_coordinates() -> None: # leaking into a coordinate slot, not a span. with pytest.raises(TypeError, match="pair of ints"): Token("x", (False, True), Role.GIVEN) # type: ignore[arg-type] + + +def test_setstate_rejects_layout_skew_on_frozen_types() -> None: + # version-skewed pickles must fail at the LOAD site, naming the + # mismatch -- not at a distant attribute read (same policy as + # Lexicon; values are deliberately NOT re-validated: pickle is not + # a security boundary) + tok = Token("Juan", Span(0, 4), Role.GIVEN) + state = tok.__getstate__() + bad = dict(state) + del bad["tags"] + with pytest.raises(ValueError, match="tags"): + Token.__new__(Token).__setstate__(bad) + pn = _pn("Juan", [tok]) + bad_pn = dict(pn.__getstate__()) + bad_pn["zq_future"] = () + with pytest.raises(ValueError, match="zq_future"): + ParsedName.__new__(ParsedName).__setstate__(bad_pn) From 35e1836bf4349d3028b1f208aa454d184ca1a043 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 17:02:20 -0700 Subject: [PATCH 62/88] Validate rendering arguments eagerly capitalized() with a non-Lexicon argument was a silent no-op on mixed-case input (the gate returned before touching the argument) and a deep AttributeError on single-case input; render()/initials() with non-str spec/delimiter/separator failed inside str.format. All three now raise the taxonomy's eager TypeError naming the argument, matching every constructor on the branch. Co-Authored-By: Claude Fable 5 --- nameparser/_render.py | 10 ++++++++++ tests/v2/test_render.py | 22 ++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/nameparser/_render.py b/nameparser/_render.py index d3c7ea7..d10a56f 100644 --- a/nameparser/_render.py +++ b/nameparser/_render.py @@ -56,6 +56,8 @@ def render(name: ParsedName, spec: str) -> str: """Fill the str.format spec from the seven role fields and the derived views (empty fields substitute ''), then apply the #254 collapse. Unknown keys raise KeyError naming the valid fields.""" + if not isinstance(spec, str): + raise TypeError(f"spec must be a str, got {spec!r}") values = {key: getattr(name, key) for key in _RENDER_KEYS} try: rendered = spec.format(**values) @@ -74,6 +76,10 @@ def initials(name: ParsedName, spec: str, delimiter: str, separator: str) -> str initial in middle/family (given-name tokens always contribute); tags come from the pipeline -- hand-built untagged tokens all contribute. Valid spec keys: given, middle, family.""" + for arg_name, arg in (("spec", spec), ("delimiter", delimiter), + ("separator", separator)): + if not isinstance(arg, str): + raise TypeError(f"{arg_name} must be a str, got {arg!r}") values: dict[str, str] = {} for key in _INITIALS_KEYS: role = Role(key) @@ -126,6 +132,10 @@ def capitalized(name: ParsedName, lexicon: Lexicon | None, *, Idempotent: without force, a capitalized result is mixed-case and the gate returns it unchanged; with force, every _cap_word rule is a fixpoint on its own output.""" + if lexicon is not None and not isinstance(lexicon, Lexicon): + # eager, before the gate: a garbage argument must not become a + # silent no-op on mixed-case input or a deep AttributeError + raise TypeError(f"lexicon must be a Lexicon or None, got {lexicon!r}") lex = Lexicon.default() if lexicon is None else lexicon joined = " ".join(t.text for t in name.tokens) if not force and joined not in (joined.upper(), joined.lower()): diff --git a/tests/v2/test_render.py b/tests/v2/test_render.py index b79db4f..0d14af7 100644 --- a/tests/v2/test_render.py +++ b/tests/v2/test_render.py @@ -221,3 +221,25 @@ def test_capitalized_rebuilds_ambiguity_tokens() -> None: # the ambiguity references the NEW capitalized token (subset invariant) assert out.ambiguities[0].tokens[0] is out.tokens[0] assert out.ambiguities[0].tokens[0].text == "Van" + + +def test_render_and_initials_reject_non_str_arguments() -> None: + # eager, like every constructor: not an AttributeError frames deep + pn = _delavega() + with pytest.raises(TypeError, match="spec must be a str"): + pn.render(7) # type: ignore[arg-type] + with pytest.raises(TypeError, match="spec must be a str"): + pn.initials(7) # type: ignore[arg-type] + with pytest.raises(TypeError, match="delimiter must be a str"): + pn.initials(delimiter=None) # type: ignore[arg-type] + with pytest.raises(TypeError, match="separator must be a str"): + pn.initials(separator=0) # type: ignore[arg-type] + + +def test_capitalized_rejects_non_lexicon_argument() -> None: + # previously a silent no-op on mixed-case input and a deep + # AttributeError on single-case input + with pytest.raises(TypeError, match="must be a Lexicon"): + _delavega().capitalized("garbage") # type: ignore[arg-type] + with pytest.raises(TypeError, match="must be a Lexicon"): + _lowercase_mac().capitalized({"titles": set()}) # type: ignore[arg-type] From 7e8b6fb703d83449387e6f39c37484c0504c52e8 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 17:02:51 -0700 Subject: [PATCH 63/88] Reject bare-string name_order with the taxonomy TypeError tuple('gmf') is ('g', 'm', 'f'), so a bare string fell through to the permutation ValueError instead of the bare-string TypeError every other iterable-valued field raises. Guarded in both Policy and PolicyPatch (whose canonicalization would otherwise store the shredded tuple until apply time). Co-Authored-By: Claude Fable 5 --- nameparser/_policy.py | 12 ++++++++++++ tests/v2/test_policy.py | 10 ++++++++++ 2 files changed, 22 insertions(+) diff --git a/nameparser/_policy.py b/nameparser/_policy.py index f0c375a..624202b 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -50,6 +50,13 @@ class Policy: __setstate__ = _guarded_setstate def __post_init__(self) -> None: + # tuple("gmf") would be ("g", "m", "f") -- catch the bare string + # with the same TypeError every other iterable field raises + if isinstance(self.name_order, str): + raise TypeError( + f"name_order must be an iterable of three Roles, " + f"not a bare string: {self.name_order!r}" + ) order = tuple(self.name_order) if len(order) != 3 or set(order) != _NAME_ROLES: raise ValueError( @@ -196,6 +203,11 @@ def __post_init__(self) -> None: # coerce a list at apply time, but the patch itself (and any # Locale holding it) must already be hashable. if self.name_order is not UNSET: + if isinstance(self.name_order, str): + raise TypeError( + f"name_order must be an iterable of three Roles, " + f"not a bare string: {self.name_order!r}" + ) object.__setattr__(self, "name_order", tuple(self.name_order)) for f in dataclasses.fields(self): if f.metadata.get("compose") != "union": diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index 497cf9a..b84ecac 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -213,3 +213,13 @@ def test_setstate_rejects_layout_skew() -> None: del state["name_order"] with pytest.raises(ValueError, match="name_order"): Policy.__new__(Policy).__setstate__(state) + + +def test_name_order_rejects_bare_string() -> None: + # tuple("gmf") is ("g","m","f"): without the guard a bare string + # fell through to the permutation ValueError instead of the + # taxonomy's bare-string TypeError every other iterable field raises + with pytest.raises(TypeError, match="bare string"): + Policy(name_order="gmf") # type: ignore[arg-type] + with pytest.raises(TypeError, match="bare string"): + PolicyPatch(name_order="gmf") # type: ignore[arg-type] From 4db87b7f7bad4c9e00a0f8cf2d75b9e687970eda Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 17:03:11 -0700 Subject: [PATCH 64/88] Pin two documented rendering contracts The given-group tag immunity in initials() (a tagged given token still contributes -- the PARTICLE_OR_GIVEN reading) and the raw-error contract for malformed render specs (positional fields, bad conversions) were documented but untested. Co-Authored-By: Claude Fable 5 --- tests/v2/test_render.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/v2/test_render.py b/tests/v2/test_render.py index 0d14af7..35ec6f5 100644 --- a/tests/v2/test_render.py +++ b/tests/v2/test_render.py @@ -243,3 +243,23 @@ def test_capitalized_rejects_non_lexicon_argument() -> None: _delavega().capitalized("garbage") # type: ignore[arg-type] with pytest.raises(TypeError, match="must be a Lexicon"): _lowercase_mac().capitalized({"titles": set()}) # type: ignore[arg-type] + + +def test_initials_given_tokens_ignore_skip_tags() -> None: + # documented: a given-name token contributes even when tagged (the + # PARTICLE_OR_GIVEN case -- 'van' read as a given name) + pn = _pn("van Johnson", [ + Token("van", Span(0, 3), Role.GIVEN, frozenset({"particle"})), + Token("Johnson", Span(4, 11), Role.FAMILY), + ]) + assert pn.initials() == "v. J." + + +def test_render_malformed_specs_surface_raw_format_errors() -> None: + # documented contract: only unknown KEYS get the enriched KeyError; + # positional fields and bad conversions raise str.format's own error + pn = _delavega() + with pytest.raises(IndexError): + pn.render("{}") + with pytest.raises(ValueError): + pn.render("{given!q}") From 96b9166e3c11455fe7098ffe4b430ad5fd5bb711 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 17:03:42 -0700 Subject: [PATCH 65/88] Fix three stale doc claims from the PR review AGENTS.md listed _render under 'later' modules one line above the bullet describing its shipped layering; the one-sanctioned-global rule did not acknowledge Lexicon.default()'s functools.cache (a lazily cached FROZEN singleton is a constant, not state); and the Lexicon repr comment cited '~700 entries' when the default vocabulary is ~1460 -- de-numbered so it cannot rot again. Co-Authored-By: Claude Fable 5 --- AGENTS.md | 4 ++-- nameparser/_lexicon.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8b7ed4e..acedeaf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -111,7 +111,7 @@ Each named attribute (`title`, `first`, etc.) is a `@property` that joins its co The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These conventions apply to all new-API code and are stricter than the v1 sections above. The full design record (rationale, settled-decision logs, dated amendments) lives in untracked `docs/superpowers/specs/`; this section is the enforceable subset. **A commit that establishes or amends one of these conventions must update this section in the same commit** — grep-driven staleness sweeps miss paraphrased prose (see Workflow above), so write-time maintenance is the mechanism, audits are the backstop. -- **Module layout**: every new module is underscore-private (`_types.py`, `_lexicon.py`, `_policy.py`, `_locale.py`; later `_pipeline/`, `_parser.py`, `_render.py`). The public import surface is exactly `nameparser` (and later `nameparser.locales`); `nameparser/__init__.py` holds re-exports and `__all__` only — no logic. Old paths (`nameparser.parser`, `nameparser.config`) stay v1-shaped. +- **Module layout**: every new module is underscore-private (`_types.py`, `_lexicon.py`, `_policy.py`, `_locale.py`, `_render.py`; later `_pipeline/`, `_parser.py`). The public import surface is exactly `nameparser` (and later `nameparser.locales`); `nameparser/__init__.py` holds re-exports and `__all__` only — no logic. Old paths (`nameparser.parser`, `nameparser.config`) stay v1-shaped. - **Layering is enforced by `tests/v2/test_layering.py`** (exact-module matching; `if TYPE_CHECKING:` imports don't count): `_types` imports nothing internal at module level — its rendering delegates import `_render` at call time; `_lexicon` and `_policy` sit above `_types` independently (`_lexicon` may import `nameparser.config.*` DATA modules only — vocabulary is single-sourced from the v1 data modules through 2.x, e.g. `config/maiden_markers.py`); `_locale` sits on `_lexicon`+`_policy` (plus `_types` for the shared pickle mixin); `_render` imports `_types` and `_lexicon` (for `Lexicon.default()` and `_normalize`). Extend the test's `ALLOWED` table when adding a module. - **Canonical field order** — the seven roles in `Role` enum declaration order, defined once and derived everywhere (properties, `as_dict`, reprs, `comparison_key`). Never restate the order literally. - **Method organization**, fixed section order in every class: fields + `__post_init__` validation → alternative constructors → dunders (construction/equality → protocol → operators) → properties → public methods by concern (access → editing → comparison → rendering delegates) → private helpers last, except a helper serving exactly one section may sit at that section's head. @@ -119,7 +119,7 @@ The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These - **Reprs are bounded**: render which fields deviate from a named baseline and by how much, never contents (`Lexicon(default + titles: +2)`). - **Typing/docs**: `from __future__ import annotations`; `frozen=True, slots=True` on every public dataclass; strict-profile mypy flags via per-module overrides in pyproject (`strict = true` itself is not valid per-module). Docstrings state contracts in prose with **no doctest blocks** — `--doctest-modules` makes every example a test; behavior examples go to unit tests per the lean-docs rule. - **Pickling**: v2 types must round-trip (`Parser` will be picklable by construction, and it holds a `Lexicon`). Every frozen type assigns `_guarded_getstate`/`_guarded_setstate` (`_types.py`) in its class body (`@dataclass(slots=True)` would override inherited pickle methods) — unpickling fails at the LOAD site on field-layout skew, and values are deliberately NOT re-validated (pickle is not a security boundary; canonical state comes from a validated instance). `Lexicon` keeps its own copy of the guard (layering) plus the `mappingproxy` slot rebuild; a new unpicklable slot type needs the same treatment plus a round-trip test. -- **One sanctioned global**: the (future) cached default `Parser`. Any second piece of module-level mutable state requires amending the conventions doc, on purpose, in review. +- **One sanctioned global**: the (future) cached default `Parser`. Lazily cached FROZEN singletons (`Lexicon.default()`'s `functools.cache`, the future default parser) are constants, not state; any second piece of module-level MUTABLE state requires amending the conventions doc, on purpose, in review. - **Tests**: all v2 tests live in the `tests/v2/` package (its `conftest.py` overrides the v1 dual-run fixture — v2 code never reads shared `CONSTANTS`), one test module per source module plus cross-cutting `test_reprs.py` and `test_layering.py`, names stating behavior. Never assert `Lexicon.default()` contents; the narrow sourcing spot-checks in `test_default_sources_v1_vocabulary` that pin the v1→v2 migration contract (e.g. the flipped `particles_ambiguous` model) are the sanctioned exception. ## Extension Patterns diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index e4f08e9..87ed542 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -177,7 +177,8 @@ def __repr__(self) -> str: # the two named constructors and by how many entries -- never the # entries themselves (design rule, see nameparser._types module # docstring). Diffing empty()-built lexicons against default() - # would tell the wrong story ("default minus ~700 entries"). + # would tell the wrong story ("default minus the entire + # default vocabulary"). if self == Lexicon.default(): return "Lexicon(default)" if self == Lexicon.empty(): From 79277de25e5d1cad8e7362e9454d0c11abd3b1f5 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 17:28:51 -0700 Subject: [PATCH 66/88] Simplify the rendering module Extract _format_spec for the identical fill/enrich/collapse tail of render() and initials() (the two error templates could drift independently); replace initials()'s guarded join-plus-append with separator.join(letter + delimiter ...), which makes the empty-group conditional unnecessary; drop a backslash continuation for the house paren style; record WHY capitalized()'s gate reads joined token texts rather than render() output. Co-Authored-By: Claude Fable 5 --- nameparser/_render.py | 52 +++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/nameparser/_render.py b/nameparser/_render.py index d10a56f..3ce93c6 100644 --- a/nameparser/_render.py +++ b/nameparser/_render.py @@ -52,23 +52,30 @@ def _collapse(rendered: str) -> str: return rendered.strip(", ") -def render(name: ParsedName, spec: str) -> str: - """Fill the str.format spec from the seven role fields and the - derived views (empty fields substitute ''), then apply the #254 - collapse. Unknown keys raise KeyError naming the valid fields.""" +def _format_spec(spec: str, values: dict[str, str], noun: str, + keys: tuple[str, ...]) -> str: + """Shared tail of render()/initials(): fill the spec, enrich + unknown-KEY errors with the valid key list, collapse.""" if not isinstance(spec, str): raise TypeError(f"spec must be a str, got {spec!r}") - values = {key: getattr(name, key) for key in _RENDER_KEYS} try: rendered = spec.format(**values) except KeyError as exc: raise KeyError( - f"unknown render field {exc.args[0]!r}; valid fields: " - f"{', '.join(_RENDER_KEYS)}" + f"unknown {noun} field {exc.args[0]!r}; valid fields: " + f"{', '.join(keys)}" ) from None return _collapse(rendered) +def render(name: ParsedName, spec: str) -> str: + """Fill the str.format spec from the seven role fields and the + derived views (empty fields substitute ''), then apply the #254 + collapse. Unknown keys raise KeyError naming the valid fields.""" + values = {key: getattr(name, key) for key in _RENDER_KEYS} + return _format_spec(spec, values, "render", _RENDER_KEYS) + + def initials(name: ParsedName, spec: str, delimiter: str, separator: str) -> str: """First letter of each contributing token per group, v1 semantics: delimiter follows each initial, separator sits between initials @@ -76,10 +83,10 @@ def initials(name: ParsedName, spec: str, delimiter: str, separator: str) -> str initial in middle/family (given-name tokens always contribute); tags come from the pipeline -- hand-built untagged tokens all contribute. Valid spec keys: given, middle, family.""" - for arg_name, arg in (("spec", spec), ("delimiter", delimiter), - ("separator", separator)): - if not isinstance(arg, str): - raise TypeError(f"{arg_name} must be a str, got {arg!r}") + if not isinstance(delimiter, str): + raise TypeError(f"delimiter must be a str, got {delimiter!r}") + if not isinstance(separator, str): + raise TypeError(f"separator must be a str, got {separator!r}") values: dict[str, str] = {} for key in _INITIALS_KEYS: role = Role(key) @@ -87,26 +94,17 @@ def initials(name: ParsedName, spec: str, delimiter: str, separator: str) -> str if role is not Role.GIVEN: tokens = tuple(t for t in tokens if not (_SKIP_TAGS & t.tags)) - letters = [t.text[0] for t in tokens] - values[key] = ((delimiter + separator).join(letters) + delimiter - if letters else "") - try: - rendered = spec.format(**values) - except KeyError as exc: - raise KeyError( - f"unknown initials field {exc.args[0]!r}; valid fields: " - f"{', '.join(_INITIALS_KEYS)}" - ) from None - return _collapse(rendered) + values[key] = separator.join( + t.text[0] + delimiter for t in tokens) + return _format_spec(spec, values, "initials", _INITIALS_KEYS) def _cap_word(word: str, role: Role, lex: Lexicon) -> str: # v1 cap_word order: particle/conjunction rule first, then the # exceptions map, then Mac/Mc, then str.capitalize normalized = _normalize(word) - if (normalized in lex.particles - and role in (Role.MIDDLE, Role.FAMILY)) \ - or normalized in lex.conjunctions: + if ((normalized in lex.particles and role in (Role.MIDDLE, Role.FAMILY)) + or normalized in lex.conjunctions): return word.lower() exception = lex.capitalization_exceptions_map.get(normalized) if exception is not None: @@ -128,7 +126,9 @@ def capitalized(name: ParsedName, lexicon: Lexicon | None, *, force: bool) -> ParsedName: """Case-fixing transform -> new ParsedName, same spans, new token texts (core spec §5b). Gate (v1 parity): only single-case input is - touched unless force=True; the gate reads the joined token texts. + touched unless force=True; the gate reads the joined token texts + (not render() output -- the case gate stays decoupled from spec + formatting and the #254 collapse). Idempotent: without force, a capitalized result is mixed-case and the gate returns it unchanged; with force, every _cap_word rule is a fixpoint on its own output.""" From 60802f12e93ac4231116b01a94fc2ba690cae507 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 17:28:51 -0700 Subject: [PATCH 67/88] Single-source the bare-string name_order guard The identical TypeError lived in both Policy and PolicyPatch __post_init__ and could drift; extracted to a module helper. Also add the missing back-reference on Lexicon.__setstate__ pointing at its deliberately-duplicated twin in _types (the sync note was previously one-directional). Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 2 ++ nameparser/_policy.py | 25 +++++++++++++------------ 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 87ed542..a1a08fd 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -204,6 +204,8 @@ def __setstate__(self, state: dict[str, object]) -> None: # Fail at the unpickle site if the state comes from a different # Lexicon field layout (version skew) -- silently loading it # would defer the failure to some distant attribute read. + # Message kept in sync with _types._guarded_setstate by design + # (layering keeps this module import-free of _types). expected = {f.name for f in dataclasses.fields(Lexicon)} - {"_cap_map"} if set(state) != expected: missing = ", ".join(sorted(expected - set(state))) or "none" diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 624202b..3ef4233 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -28,6 +28,17 @@ class PatronymicRule(StrEnum): _NAME_ROLES = frozenset({Role.GIVEN, Role.MIDDLE, Role.FAMILY}) +def _reject_bare_string_order(value: object) -> None: + # tuple("gmf") would be ("g", "m", "f") -- catch the bare string + # with the same TypeError every other iterable field raises. + # Single-sourced: called from Policy AND PolicyPatch __post_init__. + if isinstance(value, str): + raise TypeError( + f"name_order must be an iterable of three Roles, " + f"not a bare string: {value!r}" + ) + + @dataclass(frozen=True, slots=True) class Policy: name_order: tuple[Role, Role, Role] = GIVEN_FIRST @@ -50,13 +61,7 @@ class Policy: __setstate__ = _guarded_setstate def __post_init__(self) -> None: - # tuple("gmf") would be ("g", "m", "f") -- catch the bare string - # with the same TypeError every other iterable field raises - if isinstance(self.name_order, str): - raise TypeError( - f"name_order must be an iterable of three Roles, " - f"not a bare string: {self.name_order!r}" - ) + _reject_bare_string_order(self.name_order) order = tuple(self.name_order) if len(order) != 3 or set(order) != _NAME_ROLES: raise ValueError( @@ -203,11 +208,7 @@ def __post_init__(self) -> None: # coerce a list at apply time, but the patch itself (and any # Locale holding it) must already be hashable. if self.name_order is not UNSET: - if isinstance(self.name_order, str): - raise TypeError( - f"name_order must be an iterable of three Roles, " - f"not a bare string: {self.name_order!r}" - ) + _reject_bare_string_order(self.name_order) object.__setattr__(self, "name_order", tuple(self.name_order)) for f in dataclasses.fields(self): if f.metadata.get("compose") != "union": From bc56bd2a5d8cf24c00aa40fbc39c0e5fd427c742 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 17:28:51 -0700 Subject: [PATCH 68/88] Enforce the pickle guards mechanically; flatten test imports Forgetting the two class-body guard assignments on a future frozen type is silent -- @dataclass(slots=True) installs its own working pickle methods without skew detection. A conventions test now enumerates every frozen dataclass in the package and asserts the guards are present (Lexicon allow-listed for its private copy), turning a per-class convention into a CI gate. Also folds five vestigial function-local render imports into the top import block. Co-Authored-By: Claude Fable 5 --- tests/v2/test_layering.py | 33 ++++++++++++++++++++++++++++++++- tests/v2/test_render.py | 7 +------ 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index 63ec122..3968665 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -1,4 +1,5 @@ -"""Enforce the conventions doc's import layering mechanically.""" +"""Enforce the conventions doc's contracts mechanically: import +layering, public exports, and the pickle layout guards.""" import ast import pathlib @@ -108,3 +109,33 @@ def test_type_checking_imports_do_not_count(tmp_path: pathlib.Path) -> None: " from nameparser import _render\n" ) assert _nameparser_imports(mod) == ["nameparser.util", "nameparser"] + + +def test_every_frozen_dataclass_carries_the_pickle_guards() -> None: + # Forgetting the two class-body assignments is SILENT -- + # @dataclass(slots=True) installs its own working pickle methods + # without skew detection. Lexicon keeps a private copy of the guard + # (layering keeps _lexicon import-free of _types). + import dataclasses + import inspect + + import nameparser._lexicon + import nameparser._locale + import nameparser._policy + import nameparser._types + from nameparser._types import _guarded_getstate, _guarded_setstate + + modules = (nameparser._types, nameparser._lexicon, + nameparser._policy, nameparser._locale) + for module in modules: + for _, cls in inspect.getmembers(module, inspect.isclass): + if cls.__module__ != module.__name__: + continue + if not dataclasses.is_dataclass(cls): + continue + if cls.__name__ == "Lexicon": + assert "__getstate__" in cls.__dict__ + assert "__setstate__" in cls.__dict__ + continue + assert cls.__dict__.get("__getstate__") is _guarded_getstate, cls + assert cls.__dict__.get("__setstate__") is _guarded_setstate, cls diff --git a/tests/v2/test_render.py b/tests/v2/test_render.py index 35ec6f5..249e3d4 100644 --- a/tests/v2/test_render.py +++ b/tests/v2/test_render.py @@ -1,7 +1,7 @@ import pytest from nameparser._lexicon import Lexicon -from nameparser._render import _collapse +from nameparser._render import _collapse, render from nameparser._types import Ambiguity, AmbiguityKind, ParsedName, Role, Span, Token @@ -40,7 +40,6 @@ def _delavega() -> ParsedName: def test_render_fills_fields_and_collapses() -> None: - from nameparser._render import render pn = _delavega() assert render(pn, "{title} {given} {middle} {family} {suffix}") \ == "Dr. Juan de la Vega III" @@ -49,7 +48,6 @@ def test_render_fills_fields_and_collapses() -> None: def test_render_accepts_derived_view_keys() -> None: - from nameparser._render import render assert render(_delavega(), "{family_base}, {given} {family_particles}") \ == "Vega, Juan de la" assert render(_delavega(), "{surnames}") == "de la Vega" @@ -57,20 +55,17 @@ def test_render_accepts_derived_view_keys() -> None: def test_render_every_role_key_is_valid() -> None: - from nameparser._render import render pn = _delavega() for role in Role: render(pn, f"{{{role.value}}}") # must not raise def test_render_unknown_key_raises_enriched_keyerror() -> None: - from nameparser._render import render with pytest.raises(KeyError, match="valid fields"): render(_delavega(), "{first}") # v1 spelling: redirected loudly def test_render_empty_parse_is_empty_string() -> None: - from nameparser._render import render assert render(_pn("", []), "{title} {given} {middle} {family} {suffix}") == "" From bf6814796a2dbfdd1158e4eae9d70c9f3907b199 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 20:02:15 -0700 Subject: [PATCH 69/88] Add the pipeline package skeleton: ParseState and layering wiring Co-Authored-By: Claude Fable 5 --- AGENTS.md | 4 +- nameparser/_pipeline/__init__.py | 25 +++++++++++ nameparser/_pipeline/_state.py | 74 ++++++++++++++++++++++++++++++++ pyproject.toml | 2 + tests/v2/pipeline/__init__.py | 0 tests/v2/pipeline/test_state.py | 31 +++++++++++++ tests/v2/test_layering.py | 39 ++++++++++++++++- 7 files changed, 171 insertions(+), 4 deletions(-) create mode 100644 nameparser/_pipeline/__init__.py create mode 100644 nameparser/_pipeline/_state.py create mode 100644 tests/v2/pipeline/__init__.py create mode 100644 tests/v2/pipeline/test_state.py diff --git a/AGENTS.md b/AGENTS.md index acedeaf..5d62952 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -111,8 +111,8 @@ Each named attribute (`title`, `first`, etc.) is a `@property` that joins its co The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These conventions apply to all new-API code and are stricter than the v1 sections above. The full design record (rationale, settled-decision logs, dated amendments) lives in untracked `docs/superpowers/specs/`; this section is the enforceable subset. **A commit that establishes or amends one of these conventions must update this section in the same commit** — grep-driven staleness sweeps miss paraphrased prose (see Workflow above), so write-time maintenance is the mechanism, audits are the backstop. -- **Module layout**: every new module is underscore-private (`_types.py`, `_lexicon.py`, `_policy.py`, `_locale.py`, `_render.py`; later `_pipeline/`, `_parser.py`). The public import surface is exactly `nameparser` (and later `nameparser.locales`); `nameparser/__init__.py` holds re-exports and `__all__` only — no logic. Old paths (`nameparser.parser`, `nameparser.config`) stay v1-shaped. -- **Layering is enforced by `tests/v2/test_layering.py`** (exact-module matching; `if TYPE_CHECKING:` imports don't count): `_types` imports nothing internal at module level — its rendering delegates import `_render` at call time; `_lexicon` and `_policy` sit above `_types` independently (`_lexicon` may import `nameparser.config.*` DATA modules only — vocabulary is single-sourced from the v1 data modules through 2.x, e.g. `config/maiden_markers.py`); `_locale` sits on `_lexicon`+`_policy` (plus `_types` for the shared pickle mixin); `_render` imports `_types` and `_lexicon` (for `Lexicon.default()` and `_normalize`). Extend the test's `ALLOWED` table when adding a module. +- **Module layout**: every new module is underscore-private (`_types.py`, `_lexicon.py`, `_policy.py`, `_locale.py`, `_render.py`, `_pipeline/`, `_parser.py`). The public import surface is exactly `nameparser` (and later `nameparser.locales`); `nameparser/__init__.py` holds re-exports and `__all__` only — no logic. Old paths (`nameparser.parser`, `nameparser.config`) stay v1-shaped. +- **Layering is enforced by `tests/v2/test_layering.py`** (exact-module matching; `if TYPE_CHECKING:` imports don't count): `_types` imports nothing internal at module level — its rendering delegates import `_render` at call time; `_lexicon` and `_policy` sit above `_types` independently (`_lexicon` may import `nameparser.config.*` DATA modules only — vocabulary is single-sourced from the v1 data modules through 2.x, e.g. `config/maiden_markers.py`); `_locale` sits on `_lexicon`+`_policy` (plus `_types` for the shared pickle mixin); `_render` imports `_types` and `_lexicon` (for `Lexicon.default()` and `_normalize`); `_pipeline/*` imports `_types`+`_lexicon`+`_policy` plus in-package `_pipeline` helpers; `_parser` sits on everything except `_render`. Extend the test's `ALLOWED` table when adding a module. - **Canonical field order** — the seven roles in `Role` enum declaration order, defined once and derived everywhere (properties, `as_dict`, reprs, `comparison_key`). Never restate the order literally. - **Method organization**, fixed section order in every class: fields + `__post_init__` validation → alternative constructors → dunders (construction/equality → protocol → operators) → properties → public methods by concern (access → editing → comparison → rendering delegates) → private helpers last, except a helper serving exactly one section may sit at that section's head. - **Validation is eager and fail-loud**: every `raise` states the offending value, the expected form, and the fix. Exception taxonomy: wrong type — including wrong element type inside a collection, bare `str` where an iterable of strings is expected, or a `Mapping` where a plain iterable is expected — raises `TypeError`; well-typed but unacceptable values raise `ValueError`; failed enum lookups stay `ValueError` for any input (stdlib `EnumType` precedent). diff --git a/nameparser/_pipeline/__init__.py b/nameparser/_pipeline/__init__.py new file mode 100644 index 0000000..67fae77 --- /dev/null +++ b/nameparser/_pipeline/__init__.py @@ -0,0 +1,25 @@ +"""The 2.0 parse pipeline: pure stages folded over ParseState. + +Stage names and ParseState are NOT public API (core spec §6). The stage +list is data; runners other than the plain fold (explain(), parse_all()) +arrive in 2.x minors without changing any stage signature. + +Layering: imports only nameparser._pipeline.* (enforced by +tests/v2/test_layering.py). +""" +from __future__ import annotations + +from collections.abc import Callable + +from nameparser._pipeline._state import ParseState + +#: Filled in by later tasks as stages land; the fold is total either way. +STAGES: tuple[Callable[[ParseState], ParseState], ...] = () + + +def run(state: ParseState) -> ParseState: + """Fold the stages over the initial state. Pure: each stage returns + a new ParseState; content never raises (totality, spec §5a).""" + for stage in STAGES: + state = stage(state) + return state diff --git a/nameparser/_pipeline/_state.py b/nameparser/_pipeline/_state.py new file mode 100644 index 0000000..d2a8e2e --- /dev/null +++ b/nameparser/_pipeline/_state.py @@ -0,0 +1,74 @@ +"""Internal pipeline state: WorkToken and ParseState. + +WorkTokens are pipeline-internal (no validation -- the tokenizer is the +only producer) and are addressed BY INDEX in every stage: pieces and +segments are runs of token indices, never joined strings, so value-based +lookup (v1's #100 family) is structurally impossible. + +Layering: imports _types, _lexicon, _policy only (enforced by +tests/v2/test_layering.py). +""" +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum, auto + +from nameparser._lexicon import Lexicon +from nameparser._policy import Policy +from nameparser._types import AmbiguityKind, Role, Span + + +@dataclass(frozen=True, slots=True) +class WorkToken: + """One tokenized word. role stays None until assign; extracted + nickname/maiden tokens arrive with their role pre-set.""" + + text: str + span: Span + tags: frozenset[str] = frozenset() + role: Role | None = None + + +class Structure(Enum): + """segment's comma-structure decision.""" + + NO_COMMA = auto() + FAMILY_COMMA = auto() # "Family, Given ..." (v1 lastname-comma) + SUFFIX_COMMA = auto() # "Given Family, Suffix ..." + + +@dataclass(frozen=True, slots=True) +class PendingAmbiguity: + """An ambiguity recorded mid-pipeline by token INDEX; assemble + materializes real Ambiguity objects over the final tokens.""" + + kind: AmbiguityKind + detail: str + indices: tuple[int, ...] = () + + +@dataclass(frozen=True, slots=True) +class ParseState: + """Carried through the stage fold. Frozen; stages return copies via + dataclasses.replace. Fields are filled progressively: + extract_delimited -> extracted/masked; tokenize -> tokens (span- + sorted)/comma_offsets; segment -> segments/structure; classify -> + token tags; group -> pieces/dropped; assign/post_rules -> token + roles; ambiguities accumulate anywhere.""" + + original: str + lexicon: Lexicon + policy: Policy + extracted: tuple[tuple[Role, Span], ...] = () + masked: tuple[Span, ...] = () + tokens: tuple[WorkToken, ...] = () + comma_offsets: tuple[int, ...] = () + segments: tuple[tuple[int, ...], ...] = () + structure: Structure = Structure.NO_COMMA + # pieces[s][p] = run of token indices: piece p of segment s. + # piece_tags[s][p] = derived flags for that piece ("title", "prefix", + # "suffix", "conjunction") set by group's joins. + pieces: tuple[tuple[tuple[int, ...], ...], ...] = () + piece_tags: tuple[tuple[frozenset[str], ...], ...] = () + dropped: tuple[int, ...] = () # structural tokens (maiden markers) + ambiguities: tuple[PendingAmbiguity, ...] = () diff --git a/pyproject.toml b/pyproject.toml index bba07a3..c56d715 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,6 +62,8 @@ module = [ "nameparser._policy", "nameparser._render", "nameparser._locale", + "nameparser._pipeline.*", + "nameparser._parser", ] disallow_untyped_defs = true disallow_incomplete_defs = true diff --git a/tests/v2/pipeline/__init__.py b/tests/v2/pipeline/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/v2/pipeline/test_state.py b/tests/v2/pipeline/test_state.py new file mode 100644 index 0000000..bbda440 --- /dev/null +++ b/tests/v2/pipeline/test_state.py @@ -0,0 +1,31 @@ +import dataclasses + +from nameparser._lexicon import Lexicon +from nameparser._pipeline._state import ParseState, Structure, WorkToken +from nameparser._policy import Policy +from nameparser._types import Role, Span + + +def _state(text: str) -> ParseState: + return ParseState(original=text, lexicon=Lexicon.empty(), policy=Policy()) + + +def test_state_defaults_are_empty() -> None: + s = _state("John Smith") + assert s.tokens == () and s.segments == () and s.pieces == () + assert s.structure is Structure.NO_COMMA + assert s.ambiguities == () and s.extracted == () and s.masked == () + assert s.comma_offsets == () and s.dropped == () and s.piece_tags == () + + +def test_state_is_frozen_and_replace_works() -> None: + s = _state("x") + tok = WorkToken("x", Span(0, 1)) + s2 = dataclasses.replace(s, tokens=(tok,)) + assert s.tokens == () and s2.tokens == (tok,) + assert s2.tokens[0].role is None and s2.tokens[0].tags == frozenset() + + +def test_worktoken_carries_optional_role() -> None: + t = WorkToken("Jack", Span(6, 10), role=Role.NICKNAME) + assert t.role is Role.NICKNAME diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index 3968665..709a441 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -7,12 +7,26 @@ PKG = pathlib.Path(nameparser.__file__).parent +# ALLOWED keys whose module must exist on disk -- the exists() skip in +# test_layering_contract is for entries that precede their module within +# a plan, and without this list a renamed existing module would silently +# drop out of enforcement. Later tasks add each stage file as it lands. +_MUST_EXIST = {"_types.py", "_lexicon.py", "_policy.py", "_locale.py", + "_render.py", "_pipeline/_state.py", "_pipeline/__init__.py"} + +_PIPELINE_STAGE_ALLOWED = ( + "nameparser._types", "nameparser._lexicon", "nameparser._policy", + # stages share _state plus in-package helpers (_vocab, _group's + # piece predicates); the prefix still forbids _render/_locale/_parser + "nameparser._pipeline.", +) + # module -> prefixes it may import from within nameparser ALLOWED = { # call-time imports only (inside the rendering-delegate method # bodies); module level stays import-free. TYPE_CHECKING imports # are skipped by _nameparser_imports and need no entry. - "_types.py": ("nameparser._render",), + "_types.py": ("nameparser._render", "nameparser._parser"), # intent: DATA modules only, during 2.x -- mechanically this admits # any config submodule; "data only" holds by convention/review "_lexicon.py": ("nameparser.config.",), @@ -24,6 +38,22 @@ # _lexicon is needed at runtime: capitalized(lexicon=None) resolves # to Lexicon.default() "_render.py": ("nameparser._types", "nameparser._lexicon"), + # every stage module: state + the three config/type modules + "_pipeline/_state.py": ("nameparser._types", "nameparser._lexicon", + "nameparser._policy"), + "_pipeline/__init__.py": ("nameparser._pipeline.",), + "_pipeline/_extract.py": _PIPELINE_STAGE_ALLOWED, + "_pipeline/_tokenize.py": _PIPELINE_STAGE_ALLOWED, + "_pipeline/_segment.py": _PIPELINE_STAGE_ALLOWED, + "_pipeline/_classify.py": _PIPELINE_STAGE_ALLOWED, + "_pipeline/_group.py": _PIPELINE_STAGE_ALLOWED, + "_pipeline/_assign.py": _PIPELINE_STAGE_ALLOWED, + "_pipeline/_post_rules.py": _PIPELINE_STAGE_ALLOWED, + "_pipeline/_assemble.py": _PIPELINE_STAGE_ALLOWED, + # Parser sits on everything except _render and the facade + "_parser.py": ("nameparser._types", "nameparser._lexicon", + "nameparser._policy", "nameparser._locale", + "nameparser._pipeline"), } @@ -71,7 +101,12 @@ def _permitted(imported: str, allowed: tuple[str, ...]) -> bool: def test_layering_contract() -> None: for mod, allowed in ALLOWED.items(): - for imported in _nameparser_imports(PKG / mod): + path = PKG / mod + if not path.exists(): + assert mod not in _MUST_EXIST, ( + f"{mod} keyed in ALLOWED but file is missing") + continue # entries may precede their module within a plan + for imported in _nameparser_imports(path): assert _permitted(imported, allowed), ( f"{mod} imports {imported}, which the layering contract " f"forbids (allowed prefixes: {allowed or 'none'})" From 24c5191b8bf70fa7b62b76c839ff52a7fd239579 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 20:11:07 -0700 Subject: [PATCH 70/88] Add extract_delimited: delimiter pairs to nickname/maiden spans Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/__init__.py | 3 +- nameparser/_pipeline/_extract.py | 99 +++++++++++++++++++++++++++++++ tests/v2/pipeline/test_extract.py | 90 ++++++++++++++++++++++++++++ tests/v2/test_layering.py | 3 +- 4 files changed, 193 insertions(+), 2 deletions(-) create mode 100644 nameparser/_pipeline/_extract.py create mode 100644 tests/v2/pipeline/test_extract.py diff --git a/nameparser/_pipeline/__init__.py b/nameparser/_pipeline/__init__.py index 67fae77..7540f5b 100644 --- a/nameparser/_pipeline/__init__.py +++ b/nameparser/_pipeline/__init__.py @@ -11,10 +11,11 @@ from collections.abc import Callable +from nameparser._pipeline._extract import extract_delimited from nameparser._pipeline._state import ParseState #: Filled in by later tasks as stages land; the fold is total either way. -STAGES: tuple[Callable[[ParseState], ParseState], ...] = () +STAGES: tuple[Callable[[ParseState], ParseState], ...] = (extract_delimited,) def run(state: ParseState) -> ParseState: diff --git a/nameparser/_pipeline/_extract.py b/nameparser/_pipeline/_extract.py new file mode 100644 index 0000000..1d0560f --- /dev/null +++ b/nameparser/_pipeline/_extract.py @@ -0,0 +1,99 @@ +"""Stage: extract_delimited. + +Consumes: ParseState.original. +Produces: extracted (role + inner span per delimited region), masked +(full regions incl. delimiter chars, skipped by tokenize), +UNBALANCED_DELIMITER ambiguities for opens with no close. +Reads: Policy.nickname_delimiters, Policy.maiden_delimiters. + +Matching rules (the #273 mechanism): pairs scan the original text left +to right, no nesting. For pairs whose open == close (quotes), the open +must sit at a word boundary (start of text or after whitespace) and the +close before one (end, whitespace, or a comma char) -- this is what +keeps the apostrophe in O'Connor literal. maiden_delimiters are scanned +before nickname_delimiters, so routing a pair to maiden (#274) wins if +a pair appears in both sets. Empty enclosures are masked (removed from +the token stream) but extract nothing. Overlapping candidate regions +resolve by scan order (maiden pairs, then nickname pairs, sorted within +each): the first match masks the region, and any delimiter chars inside +an extracted span stay literal. +""" +from __future__ import annotations + +import dataclasses + +from nameparser._pipeline._state import ParseState, PendingAmbiguity +from nameparser._types import AmbiguityKind, Role, Span + +_CLOSE_BOUNDARY_EXTRAS = {",", "،", "，"} # comma chars end a close + + +def _open_ok(text: str, i: int) -> bool: + return i == 0 or text[i - 1].isspace() + + +def _close_ok(text: str, j: int, width: int) -> bool: + k = j + width + return k >= len(text) or text[k].isspace() or text[k] in _CLOSE_BOUNDARY_EXTRAS + + +def _overlaps(span: Span, taken: list[Span]) -> bool: + return any(span.start < t.end and t.start < span.end for t in taken) + + +def extract_delimited(state: ParseState) -> ParseState: + text = state.original + extracted: list[tuple[Role, Span]] = [] + masked: list[Span] = [] + ambiguities: list[PendingAmbiguity] = [] + for role, pairs in ( + (Role.MAIDEN, state.policy.maiden_delimiters), + (Role.NICKNAME, state.policy.nickname_delimiters), + ): + for open_, close in sorted(pairs): + pos = 0 + while (i := text.find(open_, pos)) != -1: + if open_ == close and not _open_ok(text, i): + pos = i + 1 + continue + j = text.find(close, i + len(open_)) + while (open_ == close and j != -1 + and not _close_ok(text, j, len(close))): + j = text.find(close, j + 1) + if j == -1: + ambiguities.append(PendingAmbiguity( + AmbiguityKind.UNBALANCED_DELIMITER, + f"unmatched {open_!r} at offset {i}; treated as " + f"literal text", + )) + if open_ == close: + # _close_ok is open-independent, so a failed + # close-walk means NO boundary-valid close + # exists anywhere to the right: every remaining + # boundary-valid open is unmatched too. Record + # each in one forward pass without re-walking + # closes (keeps adversarial input linear). + scan = i + len(open_) + while (k := text.find(open_, scan)) != -1: + if _open_ok(text, k): + ambiguities.append(PendingAmbiguity( + AmbiguityKind.UNBALANCED_DELIMITER, + f"unmatched {open_!r} at offset {k}; " + f"treated as literal text", + )) + scan = k + 1 + break + pos = i + len(open_) + continue + full = Span(i, j + len(close)) + if not _overlaps(full, masked): + inner = Span(i + len(open_), j) + if inner.start < inner.end: + extracted.append((role, inner)) + masked.append(full) + pos = j + len(close) + extracted.sort(key=lambda pair: tuple(pair[1])) + masked.sort(key=tuple) + return dataclasses.replace( + state, extracted=tuple(extracted), masked=tuple(masked), + ambiguities=state.ambiguities + tuple(ambiguities)) diff --git a/tests/v2/pipeline/test_extract.py b/tests/v2/pipeline/test_extract.py new file mode 100644 index 0000000..0ee365d --- /dev/null +++ b/tests/v2/pipeline/test_extract.py @@ -0,0 +1,90 @@ +import dataclasses + +from nameparser._lexicon import Lexicon +from nameparser._pipeline._extract import extract_delimited +from nameparser._pipeline._state import ParseState +from nameparser._policy import Policy +from nameparser._types import AmbiguityKind, Role, Span + + +def _state(text: str, policy: Policy | None = None) -> ParseState: + return ParseState(original=text, lexicon=Lexicon.empty(), + policy=policy or Policy()) + + +def test_double_quoted_nickname_extracted() -> None: + # 0123456789012345678 + out = extract_delimited(_state('John "Jack" Kennedy')) + assert out.extracted == ((Role.NICKNAME, Span(6, 10)),) + assert out.masked == (Span(5, 11),) + + +def test_parenthesized_nickname_extracted() -> None: + out = extract_delimited(_state("John (Jack) Kennedy")) + assert out.extracted == ((Role.NICKNAME, Span(6, 10)),) + + +def test_same_char_delimiter_needs_boundaries() -> None: + # the apostrophe in O'Connor is not an opening quote + out = extract_delimited(_state("Sean O'Connor")) + assert out.extracted == () and out.masked == () + + +def test_single_quoted_nickname_at_boundaries() -> None: + # 01234567890123456789 + out = extract_delimited(_state("John 'Jack' Kennedy")) + assert out.extracted == ((Role.NICKNAME, Span(6, 10)),) + + +def test_unbalanced_delimiter_left_literal_with_ambiguity() -> None: + out = extract_delimited(_state('Jon "Nick Smith')) + assert out.extracted == () and out.masked == () + assert len(out.ambiguities) == 1 + assert out.ambiguities[0].kind is AmbiguityKind.UNBALANCED_DELIMITER + + +def test_maiden_delimiters_route_to_maiden() -> None: + policy = dataclasses.replace( + Policy(), + nickname_delimiters=frozenset({("'", "'"), ('"', '"')}), + maiden_delimiters=frozenset({("(", ")")}), + ) + out = extract_delimited(_state("Jane Smith (Jones)", policy)) + assert out.extracted == ((Role.MAIDEN, Span(12, 17)),) + + +def test_empty_enclosure_is_masked_but_not_extracted() -> None: + out = extract_delimited(_state("John () Kennedy")) + assert out.extracted == () + assert out.masked == (Span(5, 7),) + + +def test_multiple_extracts_and_no_overlap() -> None: + # 0123456789012345678901234 + out = extract_delimited(_state('"Jack" John (Jonny) Kim')) + roles = {span: role for role, span in out.extracted} + assert roles == {Span(1, 5): Role.NICKNAME, Span(13, 18): Role.NICKNAME} + + +def test_adjacent_pairs_extract_separately() -> None: + out = extract_delimited(_state('John "A" "B" Kim')) + assert len(out.extracted) == 2 + + +def test_close_at_end_of_string() -> None: + out = extract_delimited(_state('John "Jack"')) + assert out.extracted == ((Role.NICKNAME, Span(6, 10)),) + + +def test_unmatched_open_at_last_char() -> None: + out = extract_delimited(_state('John "')) + assert out.extracted == () + assert len(out.ambiguities) == 1 + + +def test_nested_delimiters_inner_scan_order_pins() -> None: + # parens win over quotes when the quote chars sit flush against the + # parens (their word-boundary test fails); the inner quote chars + # flow into the extracted span verbatim -- v1 parity, deliberate + out = extract_delimited(_state('John ("Jack") Kim')) + assert out.extracted == ((Role.NICKNAME, Span(6, 12)),) diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index 709a441..5c633c6 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -12,7 +12,8 @@ # a plan, and without this list a renamed existing module would silently # drop out of enforcement. Later tasks add each stage file as it lands. _MUST_EXIST = {"_types.py", "_lexicon.py", "_policy.py", "_locale.py", - "_render.py", "_pipeline/_state.py", "_pipeline/__init__.py"} + "_render.py", "_pipeline/_state.py", "_pipeline/__init__.py", + "_pipeline/_extract.py"} _PIPELINE_STAGE_ALLOWED = ( "nameparser._types", "nameparser._lexicon", "nameparser._policy", From 7e341a955e04ab2366be67ab343e5d75aac2feea Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 21:39:04 -0700 Subject: [PATCH 71/88] Add tokenize: char-classification tokenizer with exact spans Ports the emoji/bidi character classes verbatim from config/regexes.py per the layering contract's duplication-by-design note; whitespace, emoji, bidi marks, and comma chars all act as separators without ever entering a token, so token spans always index original exactly. Wires tokenize into STAGES and registers the new module in test_layering's _MUST_EXIST. Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/__init__.py | 4 +- nameparser/_pipeline/_tokenize.py | 86 ++++++++++++++++++++++++++++++ tests/v2/pipeline/test_tokenize.py | 72 +++++++++++++++++++++++++ tests/v2/test_layering.py | 2 +- 4 files changed, 162 insertions(+), 2 deletions(-) create mode 100644 nameparser/_pipeline/_tokenize.py create mode 100644 tests/v2/pipeline/test_tokenize.py diff --git a/nameparser/_pipeline/__init__.py b/nameparser/_pipeline/__init__.py index 7540f5b..961efcd 100644 --- a/nameparser/_pipeline/__init__.py +++ b/nameparser/_pipeline/__init__.py @@ -13,9 +13,11 @@ from nameparser._pipeline._extract import extract_delimited from nameparser._pipeline._state import ParseState +from nameparser._pipeline._tokenize import tokenize #: Filled in by later tasks as stages land; the fold is total either way. -STAGES: tuple[Callable[[ParseState], ParseState], ...] = (extract_delimited,) +STAGES: tuple[Callable[[ParseState], ParseState], ...] = ( + extract_delimited, tokenize) def run(state: ParseState) -> ParseState: diff --git a/nameparser/_pipeline/_tokenize.py b/nameparser/_pipeline/_tokenize.py new file mode 100644 index 0000000..712e094 --- /dev/null +++ b/nameparser/_pipeline/_tokenize.py @@ -0,0 +1,86 @@ +"""Stage: tokenize. + +Consumes: original, masked (regions to skip), extracted (regions that +tokenize with a pre-set role). +Produces: tokens (span-sorted WorkTokens; text always == original +slice), comma_offsets (segmentation points; never tokens). +Reads: Policy.strip_emoji, Policy.strip_bidi. + +There is NO text-rewriting normalize stage (core spec §6): whitespace +collapsing and emoji/bidi stripping are character-classification rules +here -- ignorable characters act as separators and never enter a token, +so spans always index the original exactly as given. + +v1's squash_emoji/squash_bidi REMOVED the char and joined neighbors +('A\U0001f600B' -> 'AB'); here an ignorable char is a SEPARATOR +('A\U0001f600B' -> 'A', 'B') -- the unavoidable consequence of spans +indexing the original exactly. +""" +from __future__ import annotations + +import dataclasses +import re + +from nameparser._pipeline._state import ParseState, WorkToken +from nameparser._types import Role, Span + +_COMMA_CHARS = {",", "،", "，"} # ASCII, Arabic, fullwidth + +# Ported verbatim from v1 (nameparser/config/regexes.py, "emoji" and +# "bidi") -- layering forbids importing the config package here, so the +# patterns are duplicated by design with this provenance note. When +# editing, keep both copies in sync. +_EMOJI = re.compile('[' # lgtm[py/overly-large-range] + '\U0001F300-\U0001F64F' + '\U0001F680-\U0001F6FF' + '\u2600-\u26FF\u2700-\u27BF]+') +_BIDI = re.compile('[\u061C\u200E\u200F\u202A-\u202E\u2066-\u2069]+') + + +def _ignorable(ch: str, state: ParseState) -> bool: + if ch.isspace(): + return True + if state.policy.strip_bidi and _BIDI.match(ch): + return True + return bool(state.policy.strip_emoji and _EMOJI.match(ch)) + + +def _tokenize_region(state: ParseState, start: int, end: int, + role: Role | None, record_commas: bool, + tokens: list[WorkToken], commas: list[int]) -> None: + text = state.original + tok_start: int | None = None + for i in range(start, end): + ch = text[i] + if ch in _COMMA_CHARS or _ignorable(ch, state): + if tok_start is not None: + tokens.append(WorkToken(text[tok_start:i], + Span(tok_start, i), role=role)) + tok_start = None + if ch in _COMMA_CHARS and record_commas: + commas.append(i) + continue + if tok_start is None: + tok_start = i + if tok_start is not None: + tokens.append(WorkToken(text[tok_start:end], + Span(tok_start, end), role=role)) + + +def tokenize(state: ParseState) -> ParseState: + tokens: list[WorkToken] = [] + commas: list[int] = [] + # main stream: everything outside masked regions + boundaries = [0] + for m in state.masked: + boundaries.extend((m.start, m.end)) + boundaries.append(len(state.original)) + for start, end in zip(boundaries[::2], boundaries[1::2]): + _tokenize_region(state, start, end, None, True, tokens, commas) + # extracted regions: pre-set role, commas are mere separators + for role, inner in state.extracted: + _tokenize_region(state, inner.start, inner.end, role, False, + tokens, commas) + tokens.sort(key=lambda t: tuple(t.span)) + return dataclasses.replace(state, tokens=tuple(tokens), + comma_offsets=tuple(sorted(commas))) diff --git a/tests/v2/pipeline/test_tokenize.py b/tests/v2/pipeline/test_tokenize.py new file mode 100644 index 0000000..3e94e18 --- /dev/null +++ b/tests/v2/pipeline/test_tokenize.py @@ -0,0 +1,72 @@ +import dataclasses + +from nameparser._lexicon import Lexicon +from nameparser._pipeline._extract import extract_delimited +from nameparser._pipeline._state import ParseState +from nameparser._pipeline._tokenize import tokenize +from nameparser._policy import Policy +from nameparser._types import Role + + +def _tokenized(text: str, policy: Policy | None = None) -> ParseState: + state = ParseState(original=text, lexicon=Lexicon.empty(), + policy=policy or Policy()) + return tokenize(extract_delimited(state)) + + +def test_whitespace_split_with_spans() -> None: + out = _tokenized("John Smith") + assert [(t.text, tuple(t.span)) for t in out.tokens] == [ + ("John", (0, 4)), ("Smith", (6, 11)), + ] + assert all(t.role is None for t in out.tokens) + + +def test_provenance_text_equals_original_slice() -> None: + out = _tokenized(" Dr. Juan de la Vega ") + for t in out.tokens: + assert t.text == out.original[t.span.start:t.span.end] + + +def test_commas_are_separators_and_recorded() -> None: + out = _tokenized("Smith, John") + assert [t.text for t in out.tokens] == ["Smith", "John"] + assert out.comma_offsets == (5,) + + +def test_fullwidth_and_arabic_commas_segment() -> None: + out = _tokenized("سميث، جون") + assert out.comma_offsets == (4,) + out2 = _tokenized("山田，太郎") + assert out2.comma_offsets == (2,) + + +def test_extracted_regions_are_skipped_and_tokenized_with_role() -> None: + out = _tokenized('John "Jack Jr" Kennedy') + main = [(t.text, t.role) for t in out.tokens if t.role is None] + nick = [(t.text, t.role) for t in out.tokens if t.role is Role.NICKNAME] + assert main == [("John", None), ("Kennedy", None)] + assert nick == [("Jack", Role.NICKNAME), ("Jr", Role.NICKNAME)] + # tokens are span-sorted overall + starts = [t.span.start for t in out.tokens] + assert starts == sorted(starts) + + +def test_comma_inside_extracted_region_is_not_an_offset() -> None: + out = _tokenized('John "Jack, Jr" Kim') + assert out.comma_offsets == () + nick = [t.text for t in out.tokens if t.role is Role.NICKNAME] + assert nick == ["Jack", "Jr"] + + +def test_emoji_and_bidi_are_ignorable_by_policy() -> None: + out = _tokenized("John‏ \U0001f600Smith") + assert [t.text for t in out.tokens] == ["John", "Smith"] + keep = dataclasses.replace(Policy(), strip_emoji=False, strip_bidi=False) + out2 = _tokenized("John \U0001f600Smith", keep) + assert [t.text for t in out2.tokens] == ["John", "\U0001f600Smith"] + + +def test_empty_and_whitespace_yield_no_tokens() -> None: + assert _tokenized("").tokens == () + assert _tokenized(" ").tokens == () diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index 5c633c6..fe4758a 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -13,7 +13,7 @@ # drop out of enforcement. Later tasks add each stage file as it lands. _MUST_EXIST = {"_types.py", "_lexicon.py", "_policy.py", "_locale.py", "_render.py", "_pipeline/_state.py", "_pipeline/__init__.py", - "_pipeline/_extract.py"} + "_pipeline/_extract.py", "_pipeline/_tokenize.py"} _PIPELINE_STAGE_ALLOWED = ( "nameparser._types", "nameparser._lexicon", "nameparser._policy", From 29ceb1880f34f6c78dae5d52e575f0f1774a959d Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 21:48:31 -0700 Subject: [PATCH 72/88] Add segment: comma-structure decision over token index runs Adds _vocab.py (is_initial/is_suffix_strict/is_suffix_lenient, shared across stages) and _segment.py (the family-comma/suffix-comma/no-comma decision). segment reads Lexicon suffix vocabulary directly rather than waiting for classify (recorded plan deviation #3: the suffix-comma vs family-comma call is definitionally vocabulary-dependent, mirroring v1's are_suffixes_after_comma). Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/__init__.py | 3 +- nameparser/_pipeline/_segment.py | 66 +++++++++++++++++++++++++++ nameparser/_pipeline/_vocab.py | 44 ++++++++++++++++++ tests/v2/pipeline/test_segment.py | 76 +++++++++++++++++++++++++++++++ tests/v2/pipeline/test_vocab.py | 34 ++++++++++++++ tests/v2/test_layering.py | 4 +- 6 files changed, 225 insertions(+), 2 deletions(-) create mode 100644 nameparser/_pipeline/_segment.py create mode 100644 nameparser/_pipeline/_vocab.py create mode 100644 tests/v2/pipeline/test_segment.py create mode 100644 tests/v2/pipeline/test_vocab.py diff --git a/nameparser/_pipeline/__init__.py b/nameparser/_pipeline/__init__.py index 961efcd..863bd8f 100644 --- a/nameparser/_pipeline/__init__.py +++ b/nameparser/_pipeline/__init__.py @@ -12,12 +12,13 @@ from collections.abc import Callable from nameparser._pipeline._extract import extract_delimited +from nameparser._pipeline._segment import segment from nameparser._pipeline._state import ParseState from nameparser._pipeline._tokenize import tokenize #: Filled in by later tasks as stages land; the fold is total either way. STAGES: tuple[Callable[[ParseState], ParseState], ...] = ( - extract_delimited, tokenize) + extract_delimited, tokenize, segment) def run(state: ParseState) -> ParseState: diff --git a/nameparser/_pipeline/_segment.py b/nameparser/_pipeline/_segment.py new file mode 100644 index 0000000..961f720 --- /dev/null +++ b/nameparser/_pipeline/_segment.py @@ -0,0 +1,66 @@ +"""Stage: segment. + +Consumes: tokens (role-None main stream), comma_offsets. +Produces: segments (runs of main-token indices), structure, +COMMA_STRUCTURE ambiguities for unrecognized extra segments. +Reads: Lexicon suffix vocabulary (via _vocab.is_suffix_lenient) -- +the suffix-comma decision is definitionally vocabulary-dependent +(recorded plan deviation #3); Policy is not consulted here. + +Decision (v1 parity): >=1 comma and every post-first segment entirely +lenient-suffix AND >1 word before the first comma -> SUFFIX_COMMA; +otherwise FAMILY_COMMA ("Family, Given ..."), with segments beyond the +second that are not lenient-suffix flagged COMMA_STRUCTURE (they are +still best-effort consumed as suffixes by assign, spec §5a). +""" +from __future__ import annotations + +import bisect +import dataclasses + +from nameparser._pipeline._state import ParseState, PendingAmbiguity, Structure +from nameparser._pipeline._vocab import is_suffix_lenient +from nameparser._types import AmbiguityKind + + +def segment(state: ParseState) -> ParseState: + main = [i for i, t in enumerate(state.tokens) if t.role is None] + if not main: + return dataclasses.replace(state, segments=(), + structure=Structure.NO_COMMA) + if not state.comma_offsets: + return dataclasses.replace(state, segments=(tuple(main),), + structure=Structure.NO_COMMA) + buckets: list[list[int]] = [[] for _ in range(len(state.comma_offsets) + 1)] + for i in main: + # comma_offsets is sorted and no offset ever equals a token + # start, so bisect_left counts the commas before this token + start = state.tokens[i].span.start + bucket = bisect.bisect_left(state.comma_offsets, start) + buckets[bucket].append(i) + groups = [tuple(b) for b in buckets if b] + if len(groups) <= 1: + segs = tuple(groups) if groups else (tuple(main),) + return dataclasses.replace(state, segments=segs, + structure=Structure.NO_COMMA) + + def suffixy(seg: tuple[int, ...]) -> bool: + return all(is_suffix_lenient(state.tokens[i].text, state.lexicon) + for i in seg) + + rest = groups[1:] + if all(suffixy(s) for s in rest) and len(groups[0]) > 1: + return dataclasses.replace(state, segments=tuple(groups), + structure=Structure.SUFFIX_COMMA) + ambiguities = list(state.ambiguities) + for seg in groups[2:]: + if not suffixy(seg): + texts = " ".join(state.tokens[i].text for i in seg) + ambiguities.append(PendingAmbiguity( + AmbiguityKind.COMMA_STRUCTURE, + f"segment {texts!r} beyond the recognized comma " + f"structures; consumed as suffix best-effort", + tuple(seg))) + return dataclasses.replace(state, segments=tuple(groups), + structure=Structure.FAMILY_COMMA, + ambiguities=tuple(ambiguities)) diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py new file mode 100644 index 0000000..48475fb --- /dev/null +++ b/nameparser/_pipeline/_vocab.py @@ -0,0 +1,44 @@ +"""Shared vocabulary predicates for pipeline stages. + +Text-level tests used by more than one stage; token/piece-level +predicates live with their stage. All take normalized-or-raw text +explicitly -- no state. + +Layering: imports _lexicon and _types only. +""" +from __future__ import annotations + +import re + +from nameparser._lexicon import Lexicon, _normalize + +# Ported verbatim from v1 (nameparser/config/regexes.py "initial") minus +# its empty-string alternative -- WorkToken text is never empty. Kept in +# sync by hand; layering forbids importing the config package here. +_INITIAL = re.compile(r"^(\w\.|[A-Z])$") + + +def is_initial(text: str) -> bool: + """'A.' / 'j.' / bare capital -- v1's is_an_initial.""" + return bool(_INITIAL.fullmatch(text)) + + +def is_suffix_strict(text: str, lexicon: Lexicon) -> bool: + """v1's is_suffix: suffix vocabulary with the initial veto ('V.' in + 'John V. Smith' is a middle initial, not roman five); ambiguous + acronyms count only when written with periods ('M.A.' yes, 'Ma' no). + """ + n = _normalize(text) + if "." in text and n in lexicon.suffix_acronyms_ambiguous: + return True + if is_initial(text): + return False + return n in lexicon.suffix_acronyms or n in lexicon.suffix_words + + +def is_suffix_lenient(text: str, lexicon: Lexicon) -> bool: + """v1's is_suffix_lenient: suffix_words accepted unconditionally, + bypassing the initial veto -- only safe in unambiguous positions + (after a comma).""" + return _normalize(text) in lexicon.suffix_words or \ + is_suffix_strict(text, lexicon) diff --git a/tests/v2/pipeline/test_segment.py b/tests/v2/pipeline/test_segment.py new file mode 100644 index 0000000..aad0567 --- /dev/null +++ b/tests/v2/pipeline/test_segment.py @@ -0,0 +1,76 @@ +from nameparser._lexicon import Lexicon +from nameparser._pipeline._extract import extract_delimited +from nameparser._pipeline._segment import segment +from nameparser._pipeline._state import ParseState, Structure +from nameparser._pipeline._tokenize import tokenize +from nameparser._policy import Policy +from nameparser._types import AmbiguityKind + +# synthetic vocabulary: behavior given a lexicon, never default() contents +_LEX = Lexicon( + suffix_acronyms=frozenset({"phd"}), + suffix_words=frozenset({"jr", "v"}), +) + + +def _segmented(text: str) -> ParseState: + state = ParseState(original=text, lexicon=_LEX, policy=Policy()) + return segment(tokenize(extract_delimited(state))) + + +def _texts(state: ParseState, seg: tuple[int, ...]) -> list[str]: + return [state.tokens[i].text for i in seg] + + +def test_no_comma() -> None: + out = _segmented("John Smith") + assert out.structure is Structure.NO_COMMA + assert [_texts(out, s) for s in out.segments] == [["John", "Smith"]] + + +def test_family_comma() -> None: + out = _segmented("Smith, John") + assert out.structure is Structure.FAMILY_COMMA + assert [_texts(out, s) for s in out.segments] == [["Smith"], ["John"]] + + +def test_suffix_comma_when_all_rest_groups_are_suffixes() -> None: + out = _segmented("John Smith, PhD") + assert out.structure is Structure.SUFFIX_COMMA + assert [_texts(out, s) for s in out.segments] == [["John", "Smith"], ["PhD"]] + + +def test_suffix_comma_lenient_accepts_initial_shaped_suffix_word() -> None: + # "V" is initial-shaped; the strict test vetoes it, the post-comma + # lenient test accepts suffix_words unconditionally (v1 parity) + out = _segmented("John Ingram, V") + assert out.structure is Structure.SUFFIX_COMMA + + +def test_family_comma_with_trailing_suffix_segment() -> None: + out = _segmented("Smith, John, Jr.") + assert out.structure is Structure.FAMILY_COMMA + assert [_texts(out, s) for s in out.segments] == [["Smith"], ["John"], ["Jr."]] + + +def test_single_pre_comma_word_never_suffix_comma() -> None: + # v1: suffix-comma requires >1 word before the comma + out = _segmented("Johnson, Jr.") + assert out.structure is Structure.FAMILY_COMMA + + +def test_excess_non_suffix_segment_flags_comma_structure() -> None: + out = _segmented("Smith, John, Extra, Jr.") + assert out.structure is Structure.FAMILY_COMMA + kinds = [a.kind for a in out.ambiguities] + assert AmbiguityKind.COMMA_STRUCTURE in kinds + + +def test_empty_input_yields_no_segments() -> None: + assert _segmented("").segments == () + + +def test_comma_only_input_is_no_comma_structure() -> None: + out = _segmented(",,,") + assert out.structure is Structure.NO_COMMA + assert out.segments == () diff --git a/tests/v2/pipeline/test_vocab.py b/tests/v2/pipeline/test_vocab.py new file mode 100644 index 0000000..54d4cfc --- /dev/null +++ b/tests/v2/pipeline/test_vocab.py @@ -0,0 +1,34 @@ +from nameparser._lexicon import Lexicon +from nameparser._pipeline._vocab import is_initial, is_suffix_lenient, is_suffix_strict + +_LEX = Lexicon( + suffix_acronyms=frozenset({"phd"}), + suffix_words=frozenset({"jr", "v"}), + suffix_acronyms_ambiguous=frozenset({"ma"}), +) + + +def test_is_initial() -> None: + assert is_initial("A.") + assert is_initial("j.") + assert is_initial("B") + assert not is_initial("Jo") + assert not is_initial("b") # bare lowercase letter is not an initial + + +def test_strict_suffix_initial_veto() -> None: + assert is_suffix_strict("PhD", _LEX) + assert not is_suffix_strict("V.", _LEX) # initial veto + assert not is_suffix_strict("V", _LEX) # initial veto + assert is_suffix_strict("Jr", _LEX) + + +def test_ambiguous_acronym_needs_periods_and_beats_the_veto() -> None: + assert is_suffix_strict("M.A.", _LEX) + assert not is_suffix_strict("Ma", _LEX) + + +def test_lenient_accepts_suffix_words_unconditionally() -> None: + assert is_suffix_lenient("V", _LEX) + assert is_suffix_lenient("V.", _LEX) + assert not is_suffix_lenient("Ma", _LEX) diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index fe4758a..1fd7ee8 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -13,7 +13,8 @@ # drop out of enforcement. Later tasks add each stage file as it lands. _MUST_EXIST = {"_types.py", "_lexicon.py", "_policy.py", "_locale.py", "_render.py", "_pipeline/_state.py", "_pipeline/__init__.py", - "_pipeline/_extract.py", "_pipeline/_tokenize.py"} + "_pipeline/_extract.py", "_pipeline/_tokenize.py", + "_pipeline/_vocab.py", "_pipeline/_segment.py"} _PIPELINE_STAGE_ALLOWED = ( "nameparser._types", "nameparser._lexicon", "nameparser._policy", @@ -43,6 +44,7 @@ "_pipeline/_state.py": ("nameparser._types", "nameparser._lexicon", "nameparser._policy"), "_pipeline/__init__.py": ("nameparser._pipeline.",), + "_pipeline/_vocab.py": _PIPELINE_STAGE_ALLOWED, "_pipeline/_extract.py": _PIPELINE_STAGE_ALLOWED, "_pipeline/_tokenize.py": _PIPELINE_STAGE_ALLOWED, "_pipeline/_segment.py": _PIPELINE_STAGE_ALLOWED, From a91e46562794fb3ca7d51bb3087afb33f44ff65d Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 21:58:49 -0700 Subject: [PATCH 73/88] Add classify: vocabulary tags from the lexicon Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/__init__.py | 3 +- nameparser/_pipeline/_classify.py | 61 +++++++++++++++++++++++++++ tests/v2/pipeline/test_classify.py | 68 ++++++++++++++++++++++++++++++ tests/v2/test_layering.py | 3 +- 4 files changed, 133 insertions(+), 2 deletions(-) create mode 100644 nameparser/_pipeline/_classify.py create mode 100644 tests/v2/pipeline/test_classify.py diff --git a/nameparser/_pipeline/__init__.py b/nameparser/_pipeline/__init__.py index 863bd8f..b964120 100644 --- a/nameparser/_pipeline/__init__.py +++ b/nameparser/_pipeline/__init__.py @@ -11,6 +11,7 @@ from collections.abc import Callable +from nameparser._pipeline._classify import classify from nameparser._pipeline._extract import extract_delimited from nameparser._pipeline._segment import segment from nameparser._pipeline._state import ParseState @@ -18,7 +19,7 @@ #: Filled in by later tasks as stages land; the fold is total either way. STAGES: tuple[Callable[[ParseState], ParseState], ...] = ( - extract_delimited, tokenize, segment) + extract_delimited, tokenize, segment, classify) def run(state: ParseState) -> ParseState: diff --git a/nameparser/_pipeline/_classify.py b/nameparser/_pipeline/_classify.py new file mode 100644 index 0000000..60c3506 --- /dev/null +++ b/nameparser/_pipeline/_classify.py @@ -0,0 +1,61 @@ +"""Stage: classify. + +Consumes: tokens. +Produces: tokens with vocabulary tags added (text/span/role unchanged). +Reads: every Lexicon vocabulary field; Policy is not consulted. + +Tags emitted -- stable (API): "particle", "conjunction", "initial"; +namespaced (unstable): "vocab:title", "vocab:given-title", +"vocab:suffix", "vocab:suffix-word", "vocab:suffix-ambiguous", +"vocab:particle-ambiguous", "vocab:bound-given", "vocab:maiden-marker". +"vocab:suffix" means "counts as a suffix as written": unambiguous +suffix vocabulary, or an ambiguous acronym written with periods +('M.A.' yes, 'Ma' no -- 'Ma' gets only "vocab:suffix-ambiguous"). +The initial veto is assign's job, not classify's: 'V' carries both +"vocab:suffix" and "initial". +""" +from __future__ import annotations + +import dataclasses + +from nameparser._lexicon import _normalize +from nameparser._pipeline._state import ParseState, WorkToken +from nameparser._pipeline._vocab import is_initial + + +def _tags_for(token: WorkToken, state: ParseState) -> frozenset[str]: + lex = state.lexicon + n = _normalize(token.text) + tags = set(token.tags) + if n in lex.titles: + tags.add("vocab:title") + if n in lex.given_name_titles: + tags.add("vocab:given-title") + if n in lex.suffix_acronyms or n in lex.suffix_words: + tags.add("vocab:suffix") + if n in lex.suffix_words: + tags.add("vocab:suffix-word") + if n in lex.suffix_acronyms_ambiguous: + tags.add("vocab:suffix-ambiguous") + if "." in token.text: + tags.add("vocab:suffix") + if n in lex.particles: + tags.add("particle") + if n in lex.particles_ambiguous: + tags.add("vocab:particle-ambiguous") + if n in lex.conjunctions: + tags.add("conjunction") + if n in lex.bound_given_names: + tags.add("vocab:bound-given") + if n in lex.maiden_markers: + tags.add("vocab:maiden-marker") + if is_initial(token.text): + tags.add("initial") + return frozenset(tags) + + +def classify(state: ParseState) -> ParseState: + tokens = tuple( + dataclasses.replace(t, tags=_tags_for(t, state)) + for t in state.tokens) + return dataclasses.replace(state, tokens=tokens) diff --git a/tests/v2/pipeline/test_classify.py b/tests/v2/pipeline/test_classify.py new file mode 100644 index 0000000..1c4f0a4 --- /dev/null +++ b/tests/v2/pipeline/test_classify.py @@ -0,0 +1,68 @@ +from nameparser._lexicon import Lexicon +from nameparser._pipeline._classify import classify +from nameparser._pipeline._extract import extract_delimited +from nameparser._pipeline._segment import segment +from nameparser._pipeline._state import ParseState +from nameparser._pipeline._tokenize import tokenize +from nameparser._policy import Policy + +_LEX = Lexicon( + titles=frozenset({"dr", "sir"}), + given_name_titles=frozenset({"sir"}), + suffix_acronyms=frozenset({"phd"}), + suffix_words=frozenset({"jr", "v"}), + suffix_acronyms_ambiguous=frozenset({"ma"}), + particles=frozenset({"de", "la", "van"}), + particles_ambiguous=frozenset({"van"}), + conjunctions=frozenset({"and", "y"}), + bound_given_names=frozenset({"abdul"}), + maiden_markers=frozenset({"née"}), +) + + +def _classified(text: str) -> ParseState: + state = ParseState(original=text, lexicon=_LEX, policy=Policy()) + return classify(segment(tokenize(extract_delimited(state)))) + + +def _tags(state: ParseState, text: str) -> frozenset[str]: + return next(t.tags for t in state.tokens if t.text == text) + + +def test_vocabulary_tags() -> None: + out = _classified("Dr. van de la Smith and abdul née PhD Jr") + assert "vocab:title" in _tags(out, "Dr.") + assert {"particle", "vocab:particle-ambiguous"} <= _tags(out, "van") + assert "particle" in _tags(out, "de") + assert "vocab:particle-ambiguous" not in _tags(out, "de") + assert "conjunction" in _tags(out, "and") + assert "vocab:bound-given" in _tags(out, "abdul") + assert "vocab:maiden-marker" in _tags(out, "née") + assert "vocab:suffix" in _tags(out, "PhD") + assert {"vocab:suffix", "vocab:suffix-word"} <= _tags(out, "Jr") + assert _tags(out, "Smith") == frozenset() + + +def test_given_title_tag() -> None: + out = _classified("Sir John") + assert {"vocab:title", "vocab:given-title"} <= _tags(out, "Sir") + + +def test_initial_tag() -> None: + out = _classified("John A. B Smith") + assert "initial" in _tags(out, "A.") + assert "initial" in _tags(out, "B") + assert "initial" not in _tags(out, "John") + + +def test_ambiguous_suffix_acronym_needs_periods() -> None: + out = _classified("M.A. Ma") + assert "vocab:suffix" in _tags(out, "M.A.") + assert "vocab:suffix" not in _tags(out, "Ma") + assert "vocab:suffix-ambiguous" in _tags(out, "Ma") + + +def test_v_is_suffix_word_and_initial() -> None: + # both tags present; assign applies the veto, not classify + out = _classified("John V Smith") + assert {"vocab:suffix", "vocab:suffix-word", "initial"} <= _tags(out, "V") diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index 1fd7ee8..98112ed 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -14,7 +14,8 @@ _MUST_EXIST = {"_types.py", "_lexicon.py", "_policy.py", "_locale.py", "_render.py", "_pipeline/_state.py", "_pipeline/__init__.py", "_pipeline/_extract.py", "_pipeline/_tokenize.py", - "_pipeline/_vocab.py", "_pipeline/_segment.py"} + "_pipeline/_vocab.py", "_pipeline/_segment.py", + "_pipeline/_classify.py"} _PIPELINE_STAGE_ALLOWED = ( "nameparser._types", "nameparser._lexicon", "nameparser._policy", From 20e6bf644813b36d25647c5b36d34651612dd2e1 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 22:06:18 -0700 Subject: [PATCH 74/88] Add group: piece boundaries by index; joins ported from v1 Ports v1's join_on_conjunctions, prefix chains, and _join_bound_first_name as pure index operations over WorkToken runs (the anti-#100 invariant: no value-based lookup, no string joins). Adds two v2 additions from the plan: the "Ph. D."-split merge (plan deviation #1 -- lands in group since assign needs the merged piece to exist first) and the maiden-marker consuming rule (#274). Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/__init__.py | 3 +- nameparser/_pipeline/_group.py | 211 +++++++++++++++++++++++++++++++ tests/v2/pipeline/test_group.py | 129 +++++++++++++++++++ tests/v2/test_layering.py | 2 +- 4 files changed, 343 insertions(+), 2 deletions(-) create mode 100644 nameparser/_pipeline/_group.py create mode 100644 tests/v2/pipeline/test_group.py diff --git a/nameparser/_pipeline/__init__.py b/nameparser/_pipeline/__init__.py index b964120..d4f3fb9 100644 --- a/nameparser/_pipeline/__init__.py +++ b/nameparser/_pipeline/__init__.py @@ -13,13 +13,14 @@ from nameparser._pipeline._classify import classify from nameparser._pipeline._extract import extract_delimited +from nameparser._pipeline._group import group from nameparser._pipeline._segment import segment from nameparser._pipeline._state import ParseState from nameparser._pipeline._tokenize import tokenize #: Filled in by later tasks as stages land; the fold is total either way. STAGES: tuple[Callable[[ParseState], ParseState], ...] = ( - extract_delimited, tokenize, segment, classify) + extract_delimited, tokenize, segment, classify, group) def run(state: ParseState) -> ParseState: diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py new file mode 100644 index 0000000..4023c7e --- /dev/null +++ b/nameparser/_pipeline/_group.py @@ -0,0 +1,211 @@ +"""Stage: group. + +Consumes: tokens (classified), segments, structure. +Produces: pieces + piece_tags per segment (runs of token indices -- +tokens are NEVER joined into strings: the anti-#100 invariant); maiden +tail tokens get role=MAIDEN; marker tokens land in dropped. +Reads: token tags (from classify); Policy is not consulted. The v1 +"derived titles/prefixes" registration becomes piece_tags entries -- +per-parse state that dissolves with the state (v1 kept per-parse sets +for the same reason). + +Ports v1's join_on_conjunctions + prefix chains + _join_bound_first_name +plus two additions: the "Ph. D."-split merge (v1 fix_phd, recorded plan +deviation #1) and the maiden-marker consuming rule (#274: marker plus +following pieces until a suffix become maiden; the marker itself is +structural, like a delimiter char, and is dropped from assembly). +""" +from __future__ import annotations + +import dataclasses +import re + +from nameparser._pipeline._state import ParseState, Structure, WorkToken +from nameparser._types import Role + +_PH = re.compile(r"^ph\.?$", re.IGNORECASE) +_D = re.compile(r"^d\.?$", re.IGNORECASE) + +Piece = list[int] + + +def _is_title_piece(piece: Piece, ptags: set[str], + tokens: tuple[WorkToken, ...]) -> bool: + if "title" in ptags: + return True + return len(piece) == 1 and "vocab:title" in tokens[piece[0]].tags + + +def _is_prefix_piece(piece: Piece, ptags: set[str], + tokens: tuple[WorkToken, ...]) -> bool: + if "prefix" in ptags: + return True + return len(piece) == 1 and "particle" in tokens[piece[0]].tags + + +def _is_suffix_piece(piece: Piece, ptags: set[str], + tokens: tuple[WorkToken, ...]) -> bool: + if "suffix" in ptags: + return True + if len(piece) != 1: + return False + tags = tokens[piece[0]].tags + return "vocab:suffix" in tags and "initial" not in tags + + +def _is_conj_piece(piece: Piece, ptags: set[str], + tokens: tuple[WorkToken, ...]) -> bool: + if "conjunction" in ptags: + return True + return len(piece) == 1 and "conjunction" in tokens[piece[0]].tags + + +def _is_rootname(piece: Piece, ptags: set[str], + tokens: tuple[WorkToken, ...]) -> bool: + if len(piece) == 1 and "initial" in tokens[piece[0]].tags: + return False + return not (_is_title_piece(piece, ptags, tokens) + or _is_prefix_piece(piece, ptags, tokens) + or _is_suffix_piece(piece, ptags, tokens)) + + +def _group_segment(seg: tuple[int, ...], additional: int, + tokens: tuple[WorkToken, ...], + ) -> tuple[list[Piece], list[set[str]]]: + pieces: list[Piece] = [[i] for i in seg] + ptags: list[set[str]] = [set() for _ in seg] + + def title(k: int) -> bool: + return _is_title_piece(pieces[k], ptags[k], tokens) + + def prefix(k: int) -> bool: + return _is_prefix_piece(pieces[k], ptags[k], tokens) + + def suffix(k: int) -> bool: + return _is_suffix_piece(pieces[k], ptags[k], tokens) + + def conj(k: int) -> bool: + return _is_conj_piece(pieces[k], ptags[k], tokens) + + # ph-d merge first: "Ph." "D." adjacent -> one suffix piece (plan + # deviation #1; v1 fix_phd did this by regex on the raw string) + k = 0 + while k < len(pieces) - 1: + a, b = pieces[k], pieces[k + 1] + if (len(a) == 1 and len(b) == 1 + and _PH.fullmatch(tokens[a[0]].text) + and _D.fullmatch(tokens[b[0]].text)): + pieces[k:k + 2] = [a + b] + merged = ptags[k] | ptags[k + 1] | {"suffix"} + ptags[k:k + 2] = [merged] + else: + k += 1 + + if len(pieces) + additional >= 3: + total = sum(_is_rootname(p, t, tokens) + for p, t in zip(pieces, ptags)) + additional + # contiguous conjunction runs merge first (v1: "of the") + k = 0 + while k < len(pieces) - 1: + if conj(k) and conj(k + 1): + pieces[k:k + 2] = [pieces[k] + pieces[k + 1]] + ptags[k:k + 2] = [ptags[k] | ptags[k + 1] | {"conjunction"}] + else: + k += 1 + # each conjunction joins its neighbors (v1 issue #11 carve-out: + # a single-letter alphabetic conjunction in a short name is more + # likely an initial) + k = 0 + while k < len(pieces): + if not conj(k): + k += 1 + continue + text = " ".join(tokens[i].text for i in pieces[k]) + if len(text) == 1 and total < 4 and text.isalpha(): + k += 1 + continue + start = max(0, k - 1) + end = min(len(pieces), k + 2) + neighbor = start if start < k else end - 1 + new_tags = set().union(*ptags[start:end]) + if title(neighbor): + new_tags.add("title") + if prefix(neighbor): + new_tags.add("prefix") + merged_piece = [i for p in pieces[start:end] for i in p] + pieces[start:end] = [merged_piece] + ptags[start:end] = [new_tags] + k = start + 1 + # prefix chains: a non-leading prefix run absorbs everything to + # the next prefix or suffix (v1's leading_first_name rule keeps + # the first piece a name: "Van Johnson") + k = 0 + while k < len(pieces): + if k == 0 or not prefix(k): + k += 1 + continue + j = k + 1 + while j < len(pieces) and prefix(j): + j += 1 + while j < len(pieces) and not prefix(j) and not suffix(j): + j += 1 + merged_piece = [i for p in pieces[k:j] for i in p] + merged_tags = set().union(*ptags[k:j]) - {"prefix"} + pieces[k:j] = [merged_piece] + ptags[k:j] = [merged_tags] + k += 1 + # bound given names: first non-title piece joins the next when + # enough rootname pieces remain (v1 reserve_last) + first_name_k = next( + (k for k in range(len(pieces)) if not title(k)), None) + if (first_name_k is not None + and first_name_k + 1 < len(pieces) + and len(pieces[first_name_k]) == 1 + and "vocab:bound-given" + in tokens[pieces[first_name_k][0]].tags): + non_suffix = sum(1 for k in range(len(pieces)) + if not title(k) and not suffix(k)) + if non_suffix >= 3: + bg = first_name_k + pieces[bg:bg + 2] = [pieces[bg] + pieces[bg + 1]] + ptags[bg:bg + 2] = [ptags[bg] | ptags[bg + 1]] + return pieces, ptags + + +def group(state: ParseState) -> ParseState: + tokens = list(state.tokens) + dropped = list(state.dropped) + all_pieces: list[tuple[tuple[int, ...], ...]] = [] + all_ptags: list[tuple[frozenset[str], ...]] = [] + # v1 parity: additional_parts_count=1 applies only to FAMILY_COMMA + # parts (parser.py:1333); the SUFFIX_COMMA pre-comma segment gets 0. + additional = 1 if state.structure is Structure.FAMILY_COMMA else 0 + for seg in state.segments: + pieces, ptags = _group_segment(seg, additional, tuple(tokens)) + # maiden markers: a non-leading marker piece consumes following + # pieces until a suffix; consumed tokens become MAIDEN, the + # marker is dropped (#274) + m = next( + (k for k in range(1, len(pieces)) + if len(pieces[k]) == 1 + and "vocab:maiden-marker" in tokens[pieces[k][0]].tags), + None) + if m is not None: + j = m + 1 + consumed: list[int] = [] + while j < len(pieces) and not _is_suffix_piece( + pieces[j], ptags[j], tuple(tokens)): + consumed.extend(pieces[j]) + j += 1 + if consumed: + dropped.extend(pieces[m]) + for i in consumed: + tokens[i] = dataclasses.replace( + tokens[i], role=Role.MAIDEN) + pieces[m:j] = [] + ptags[m:j] = [] + all_pieces.append(tuple(tuple(p) for p in pieces)) + all_ptags.append(tuple(frozenset(t) for t in ptags)) + return dataclasses.replace( + state, tokens=tuple(tokens), pieces=tuple(all_pieces), + piece_tags=tuple(all_ptags), dropped=tuple(dropped)) diff --git a/tests/v2/pipeline/test_group.py b/tests/v2/pipeline/test_group.py new file mode 100644 index 0000000..f5fecc4 --- /dev/null +++ b/tests/v2/pipeline/test_group.py @@ -0,0 +1,129 @@ +from nameparser._lexicon import Lexicon +from nameparser._pipeline._classify import classify +from nameparser._pipeline._extract import extract_delimited +from nameparser._pipeline._group import group +from nameparser._pipeline._segment import segment +from nameparser._pipeline._state import ParseState +from nameparser._pipeline._tokenize import tokenize +from nameparser._policy import Policy +from nameparser._types import Role + +_LEX = Lexicon( + titles=frozenset({"mr", "mrs", "secretary", "the", "state"}), + suffix_acronyms=frozenset({"phd"}), + suffix_words=frozenset({"jr"}), + particles=frozenset({"de", "la", "van", "von", "und", "zu"}), + particles_ambiguous=frozenset({"van"}), + conjunctions=frozenset({"and", "of", "the", "y"}), + bound_given_names=frozenset({"abdul"}), + maiden_markers=frozenset({"née", "geb"}), +) +# NOTE: "the" sits in both titles and conjunctions here, mirroring v1's +# real vocabulary overlap; the subset rule only constrains particles. + + +def _grouped(text: str) -> ParseState: + state = ParseState(original=text, lexicon=_LEX, policy=Policy()) + return group(classify(segment(tokenize(extract_delimited(state))))) + + +def _piece_texts(state: ParseState) -> list[list[str]]: + return [[" ".join(state.tokens[i].text for i in piece) + for piece in seg] for seg in state.pieces] + + +def test_no_joins_pass_through() -> None: + out = _grouped("John Smith") + assert _piece_texts(out) == [["John", "Smith"]] + + +def test_conjunction_joins_neighbors() -> None: + out = _grouped("Mr. and Mrs. John Smith") + assert _piece_texts(out) == [["Mr. and Mrs.", "John", "Smith"]] + # joined-to-a-title piece is flagged a derived title + assert "title" in out.piece_tags[0][0] + + +def test_contiguous_conjunctions_join_first() -> None: + out = _grouped("The Secretary of State Hillary Clinton") + assert _piece_texts(out) == [["The Secretary of State", "Hillary", "Clinton"]] + assert "title" in out.piece_tags[0][0] + + +def test_single_letter_conjunction_prefers_initial_when_short() -> None: + # v1 issue #11: 3 rootname parts, single-letter conjunction "y" + out = _grouped("John y Smith") + assert _piece_texts(out) == [["John", "y", "Smith"]] + + +def test_prefix_chain_joins_to_following() -> None: + out = _grouped("Juan de la Vega") + assert _piece_texts(out) == [["Juan", "de la Vega"]] + + +def test_prefix_chain_absorbs_through_to_next_suffix() -> None: + out = _grouped("Juan de la Vega Martinez PhD") + assert _piece_texts(out) == [["Juan", "de la Vega Martinez", "PhD"]] + + +def test_leading_prefix_is_never_chained() -> None: + # "Van Johnson": the leading piece is a first name, not a particle + out = _grouped("Van Johnson") + assert _piece_texts(out) == [["Van", "Johnson"]] + + +def test_von_und_zu_bridges() -> None: + # conjunction "und" joins two prefixes; the joined piece is a derived + # prefix and still chains onto the following name (v1 PR #191) + out = _grouped("Otto von und zu Habsburg") + assert _piece_texts(out) == [["Otto", "von und zu Habsburg"]] + + +def test_bound_given_joins_with_three_rootnames() -> None: + assert _piece_texts(_grouped("abdul rahman al-said")) == \ + [["abdul rahman", "al-said"]] + # only two rootname pieces: no join (v1 reserve_last) + assert _piece_texts(_grouped("abdul rahman")) == [["abdul", "rahman"]] + + +def test_phd_split_across_tokens_merges_as_suffix() -> None: + out = _grouped("John Smith Ph. D.") + assert _piece_texts(out) == [["John", "Smith", "Ph. D."]] + assert "suffix" in out.piece_tags[0][2] + + +def test_maiden_marker_consumes_tail() -> None: + out = _grouped("Jane Smith née Jones") + assert _piece_texts(out) == [["Jane", "Smith"]] + maiden = [t.text for t in out.tokens if t.role is Role.MAIDEN] + assert maiden == ["Jones"] + # the marker token itself is structural: dropped from assembly + née_idx = next(i for i, t in enumerate(out.tokens) if t.text == "née") + assert née_idx in out.dropped + + +def test_maiden_marker_stops_at_suffix() -> None: + out = _grouped("Jane Smith née Jones PhD") + maiden = [t.text for t in out.tokens if t.role is Role.MAIDEN] + assert maiden == ["Jones"] + assert _piece_texts(out)[0][-1] == "PhD" + + +def test_leading_marker_is_not_consumed() -> None: + # "née Jones" alone: marker at piece 0 has no name before it + out = _grouped("née Jones") + assert _piece_texts(out) == [["née", "Jones"]] + + +def test_initials_do_not_count_as_rootnames_for_conjunction_carveout() -> None: + # v1 parity: 'J.' is an initial, so total rootnames stay under 4 and + # the single-letter conjunction 'y' is treated as an initial, not joined + out = _grouped("J. Ruiz y Gomez") + assert _piece_texts(out) == [["J.", "Ruiz", "y", "Gomez"]] + + +def test_suffix_comma_name_segment_gets_no_additional_count() -> None: + # v1 parity: additional_parts_count applies to FAMILY_COMMA parts only; + # ', PhD' must not tip the single-letter-conjunction carve-out + out = _grouped("John y Smith, PhD") + assert _piece_texts(out) == [["John", "y", "Smith"], ["PhD"]] diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index 98112ed..ad6f777 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -15,7 +15,7 @@ "_render.py", "_pipeline/_state.py", "_pipeline/__init__.py", "_pipeline/_extract.py", "_pipeline/_tokenize.py", "_pipeline/_vocab.py", "_pipeline/_segment.py", - "_pipeline/_classify.py"} + "_pipeline/_classify.py", "_pipeline/_group.py"} _PIPELINE_STAGE_ALLOWED = ( "nameparser._types", "nameparser._lexicon", "nameparser._policy", From 5e22e78262013ace77f4da43d76b24f648fc151e Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 22:25:59 -0700 Subject: [PATCH 75/88] Add assign: positional roles per name_order over pieces Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/__init__.py | 3 +- nameparser/_pipeline/_assign.py | 183 +++++++++++++++++++++++++++++++ tests/v2/pipeline/test_assign.py | 158 ++++++++++++++++++++++++++ tests/v2/test_layering.py | 3 +- 4 files changed, 345 insertions(+), 2 deletions(-) create mode 100644 nameparser/_pipeline/_assign.py create mode 100644 tests/v2/pipeline/test_assign.py diff --git a/nameparser/_pipeline/__init__.py b/nameparser/_pipeline/__init__.py index d4f3fb9..25f8149 100644 --- a/nameparser/_pipeline/__init__.py +++ b/nameparser/_pipeline/__init__.py @@ -11,6 +11,7 @@ from collections.abc import Callable +from nameparser._pipeline._assign import assign from nameparser._pipeline._classify import classify from nameparser._pipeline._extract import extract_delimited from nameparser._pipeline._group import group @@ -20,7 +21,7 @@ #: Filled in by later tasks as stages land; the fold is total either way. STAGES: tuple[Callable[[ParseState], ParseState], ...] = ( - extract_delimited, tokenize, segment, classify, group) + extract_delimited, tokenize, segment, classify, group, assign) def run(state: ParseState) -> ParseState: diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py new file mode 100644 index 0000000..706294d --- /dev/null +++ b/nameparser/_pipeline/_assign.py @@ -0,0 +1,183 @@ +"""Stage: assign. + +Consumes: pieces + piece_tags (grouped), segments, structure, tokens. +Produces: tokens with roles set on every main-stream token. +Reads: Policy.name_order (#270); token/piece tags; Lexicon only through +tags already applied by classify (plus the leading-title period rule). + +Ports v1's assignment loops. NO_COMMA (per name_order): +leading title pieces chain while no given-position name has been seen +(a title needs a following piece, unless the whole name is one title); +then positional assignment per name_order with the trailing-suffix +rule: the piece from which everything after is a strict suffix is the +last name-position piece, the rest are suffixes. The v1 single-name+ +nickname rule lives here (plan deviation #2): one non-title piece plus +a nonempty nickname puts that piece in FAMILY. +FAMILY_COMMA: segment 0 wholly FAMILY (v1 parity); segment 1 gets +leading titles, then given, then middles with strict-suffix pieces to +suffix; segments 2+ are suffixes (lenient -- segment already flagged +non-suffixy ones COMMA_STRUCTURE). +SUFFIX_COMMA: segment 0 as NO_COMMA; segments 1+ wholly SUFFIX. +Emits PARTICLE_OR_GIVEN when the given position consumed a leading +particles_ambiguous token with more pieces following ("Van Johnson"). +""" +from __future__ import annotations + +import dataclasses +import re + +from nameparser._pipeline._group import ( + _is_suffix_piece, _is_title_piece, +) +from nameparser._pipeline._state import ( + ParseState, PendingAmbiguity, Structure, WorkToken, +) +from nameparser._types import AmbiguityKind, Role + +# Ported verbatim from v1 (nameparser/config/regexes.py +# "period_abbreviation" and "roman_numeral") -- layering forbids the +# config import; keep in sync by hand. +_PERIOD_ABBREV = re.compile(r'^[^\W\d_]{2,}\.$') +_ROMAN = re.compile(r'^(X|IX|IV|V?I{0,3})$', re.I) + + +def _set_roles(tokens: list[WorkToken], piece: tuple[int, ...], + role: Role) -> None: + for i in piece: + tokens[i] = dataclasses.replace(tokens[i], role=role) + + +def _is_leading_title(piece: tuple[int, ...], ptags: frozenset[str], + tokens: list[WorkToken]) -> bool: + if _is_title_piece(list(piece), set(ptags), tuple(tokens)): + return True + return (len(piece) == 1 + and bool(_PERIOD_ABBREV.match(tokens[piece[0]].text))) + + +def _name_positions(order: tuple[Role, Role, Role], + count: int) -> list[Role]: + """Roles for `count` name pieces (titles/suffixes already peeled), + per name_order. GIVEN_FIRST: given, middles..., family. + FAMILY_FIRST: family, given, middles... FAMILY_FIRST_GIVEN_LAST: + family, middles..., given. One piece takes order[0]'s role + (spec §5a); two pieces take order[0] and the other primary.""" + first, second = order[0], order[1] + if count == 1: + return [first] + if first is Role.GIVEN: # GIVEN_FIRST + return ([Role.GIVEN] + [Role.MIDDLE] * (count - 2) + + [Role.FAMILY]) + if second is Role.GIVEN: # FAMILY_FIRST + return ([Role.FAMILY, Role.GIVEN] + + [Role.MIDDLE] * (count - 2)) + return ([Role.FAMILY] + [Role.MIDDLE] * (count - 2) # F_F_GIVEN_LAST + + [Role.GIVEN]) + + +def _assign_main(seg_idx: int, state: ParseState, + tokens: list[WorkToken], + ambiguities: list[PendingAmbiguity]) -> None: + pieces = state.pieces[seg_idx] + ptags = state.piece_tags[seg_idx] + has_nickname = any(t.role is Role.NICKNAME for t in tokens) + # peel leading titles + n = 0 + while n < len(pieces): + has_next = n + 1 < len(pieces) + if ((has_next or len(pieces) == 1) + and _is_leading_title(pieces[n], ptags[n], tokens)): + _set_roles(tokens, pieces[n], Role.TITLE) + n += 1 + continue + break + rest = list(range(n, len(pieces))) + if not rest: + return + # v1 nickname rule (plan deviation #2) + if len(rest) == 1 and has_nickname: + _set_roles(tokens, pieces[rest[0]], Role.FAMILY) + return + # peel the trailing suffix run: k = first index in rest from which + # every piece is a strict suffix (v1's are_suffixes tail rule, with + # the roman-numeral special: a final roman numeral after a + # non-initial piece is a suffix) + k = len(rest) + while k > 0: + piece = pieces[rest[k - 1]] + tags = ptags[rest[k - 1]] + if _is_suffix_piece(list(piece), set(tags), tuple(tokens)): + k -= 1 + continue + if (k == len(rest) and k >= 2 and len(piece) == 1 + and _ROMAN.match(tokens[piece[0]].text) + and "initial" not in tokens[pieces[rest[k - 2]][0]].tags): + k -= 1 + continue + break + name_pieces, suffix_pieces = rest[:k], rest[k:] + if not name_pieces and suffix_pieces: + # everything suffix-shaped after titles: first one is the name + name_pieces, suffix_pieces = suffix_pieces[:1], suffix_pieces[1:] + roles = _name_positions(state.policy.name_order, len(name_pieces)) + for pos, piece_idx in enumerate(name_pieces): + _set_roles(tokens, pieces[piece_idx], roles[pos]) + for piece_idx in suffix_pieces: + _set_roles(tokens, pieces[piece_idx], Role.SUFFIX) + # leading ambiguous particle read as a name (#121 surfaced) + if name_pieces: + head = pieces[name_pieces[0]] + if (len(head) == 1 and len(name_pieces) > 1 + and "vocab:particle-ambiguous" in tokens[head[0]].tags): + ambiguities.append(PendingAmbiguity( + AmbiguityKind.PARTICLE_OR_GIVEN, + f"leading {tokens[head[0]].text!r} may be a family-name " + f"particle; read as a given name", + tuple(head))) + + +def assign(state: ParseState) -> ParseState: + tokens = list(state.tokens) + ambiguities = list(state.ambiguities) + if not state.segments: + return state + if state.structure is Structure.NO_COMMA: + _assign_main(0, state, tokens, ambiguities) + elif state.structure is Structure.SUFFIX_COMMA: + _assign_main(0, state, tokens, ambiguities) + for seg_idx in range(1, len(state.segments)): + for piece in state.pieces[seg_idx]: + _set_roles(tokens, piece, Role.SUFFIX) + else: # FAMILY_COMMA + # PARTICLE_OR_GIVEN is deliberately not emitted here: after a + # comma the family is already fixed, so a leading given-position + # particle is not meaningfully ambiguous. + for piece in state.pieces[0]: + _set_roles(tokens, piece, Role.FAMILY) + if len(state.segments) > 1: + pieces = state.pieces[1] + ptags = state.piece_tags[1] + given_done = False + n = 0 + while n < len(pieces): + if (not given_done + and _is_leading_title(pieces[n], ptags[n], tokens) + and (n + 1 < len(pieces) or len(pieces) == 1)): + _set_roles(tokens, pieces[n], Role.TITLE) + n += 1 + continue + break + for m in range(n, len(pieces)): + if _is_suffix_piece(list(pieces[m]), set(ptags[m]), + tuple(tokens)): + _set_roles(tokens, pieces[m], Role.SUFFIX) + elif not given_done: + _set_roles(tokens, pieces[m], Role.GIVEN) + given_done = True + else: + _set_roles(tokens, pieces[m], Role.MIDDLE) + for seg_idx in range(2, len(state.segments)): + for piece in state.pieces[seg_idx]: + _set_roles(tokens, piece, Role.SUFFIX) + return dataclasses.replace(state, tokens=tuple(tokens), + ambiguities=tuple(ambiguities)) diff --git a/tests/v2/pipeline/test_assign.py b/tests/v2/pipeline/test_assign.py new file mode 100644 index 0000000..88bb435 --- /dev/null +++ b/tests/v2/pipeline/test_assign.py @@ -0,0 +1,158 @@ +# tests/v2/pipeline/test_assign.py +from nameparser._lexicon import Lexicon +from nameparser._pipeline._assign import assign +from nameparser._pipeline._classify import classify +from nameparser._pipeline._extract import extract_delimited +from nameparser._pipeline._group import group +from nameparser._pipeline._segment import segment +from nameparser._pipeline._state import ParseState +from nameparser._pipeline._tokenize import tokenize +from nameparser._policy import FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, Policy +from nameparser._types import AmbiguityKind, Role + +_LEX = Lexicon( + titles=frozenset({"dr", "mr", "mrs", "sir"}), + given_name_titles=frozenset({"sir"}), + suffix_acronyms=frozenset({"phd", "md"}), + suffix_words=frozenset({"jr", "iii", "v"}), + particles=frozenset({"de", "la", "van"}), + particles_ambiguous=frozenset({"van"}), + conjunctions=frozenset({"and"}), + maiden_markers=frozenset({"née"}), +) + + +def _assigned(text: str, policy: Policy | None = None) -> ParseState: + state = ParseState(original=text, lexicon=_LEX, + policy=policy or Policy()) + return assign(group(classify(segment(tokenize( + extract_delimited(state)))))) + + +def _by_role(state: ParseState, role: Role) -> str: + return " ".join(t.text for t in state.tokens if t.role is role) + + +def test_given_first_positional() -> None: + out = _assigned("Dr. Juan de la Vega III") + assert _by_role(out, Role.TITLE) == "Dr." + assert _by_role(out, Role.GIVEN) == "Juan" + assert _by_role(out, Role.FAMILY) == "de la Vega" + assert _by_role(out, Role.SUFFIX) == "III" + + +def test_middles() -> None: + out = _assigned("John Quincy Adams Smith") + assert _by_role(out, Role.GIVEN) == "John" + assert _by_role(out, Role.MIDDLE) == "Quincy Adams" + assert _by_role(out, Role.FAMILY) == "Smith" + + +def test_single_token_takes_name_order_head() -> None: + assert _by_role(_assigned("John"), Role.GIVEN) == "John" + out = _assigned("John", Policy(name_order=FAMILY_FIRST)) + assert _by_role(out, Role.FAMILY) == "John" + + +def test_title_only() -> None: + out = _assigned("Dr.") + assert _by_role(out, Role.TITLE) == "Dr." + assert not _by_role(out, Role.GIVEN) + + +def test_leading_ambiguous_particle_reads_as_given_with_ambiguity() -> None: + out = _assigned("Van Johnson") + assert _by_role(out, Role.GIVEN) == "Van" + assert _by_role(out, Role.FAMILY) == "Johnson" + assert any(a.kind is AmbiguityKind.PARTICLE_OR_GIVEN + for a in out.ambiguities) + # unambiguous input: no ambiguity recorded + assert not _assigned("John Smith").ambiguities + + +def test_family_comma() -> None: + out = _assigned("de la Vega, Juan") + assert _by_role(out, Role.FAMILY) == "de la Vega" + assert _by_role(out, Role.GIVEN) == "Juan" + + +def test_family_comma_with_title_and_middle() -> None: + out = _assigned("Smith, Dr. John A.") + assert _by_role(out, Role.TITLE) == "Dr." + assert _by_role(out, Role.GIVEN) == "John" + assert _by_role(out, Role.MIDDLE) == "A." + assert _by_role(out, Role.FAMILY) == "Smith" + + +def test_family_comma_lone_post_comma_title() -> None: + # v1 parity: a lone post-comma title is a TITLE ('Smith, Dr.'), + # not a given name or suffix; post_rules later applies the + # title+lone-family handling + out = _assigned("Smith, Dr.") + assert _by_role(out, Role.TITLE) == "Dr." + assert _by_role(out, Role.FAMILY) == "Smith" + assert not _by_role(out, Role.GIVEN) + + +def test_suffix_comma_and_extra_segments() -> None: + out = _assigned("Smith, John, Jr.") + assert _by_role(out, Role.GIVEN) == "John" + assert _by_role(out, Role.FAMILY) == "Smith" + assert _by_role(out, Role.SUFFIX) == "Jr." + + +def test_trailing_suffix_run_no_comma() -> None: + out = _assigned("John Jack Kennedy PhD MD") + assert _by_role(out, Role.MIDDLE) == "Jack" + assert _by_role(out, Role.FAMILY) == "Kennedy" + assert _by_role(out, Role.SUFFIX) == "PhD MD" + + +def test_initial_veto_keeps_v_in_middle() -> None: + out = _assigned("John V. Smith") + assert _by_role(out, Role.MIDDLE) == "V." + assert _by_role(out, Role.FAMILY) == "Smith" + + +def test_nickname_only_leaves_name_fields_empty() -> None: + out = _assigned("(Jack)") + assert _by_role(out, Role.NICKNAME) == "Jack" + assert not _by_role(out, Role.GIVEN) and not _by_role(out, Role.FAMILY) + + +def test_single_name_with_nickname_goes_to_family() -> None: + out = _assigned("John (Jack)") + assert _by_role(out, Role.FAMILY) == "John" + assert not _by_role(out, Role.GIVEN) + + +def test_family_first_order() -> None: + out = _assigned("Yamada Taro", Policy(name_order=FAMILY_FIRST)) + assert _by_role(out, Role.FAMILY) == "Yamada" + assert _by_role(out, Role.GIVEN) == "Taro" + out2 = _assigned("Yamada Hanako Taro", + Policy(name_order=FAMILY_FIRST)) + assert _by_role(out2, Role.FAMILY) == "Yamada" + assert _by_role(out2, Role.GIVEN) == "Hanako" + assert _by_role(out2, Role.MIDDLE) == "Taro" + + +def test_family_first_given_last_order() -> None: + # Pinned against the actual built pipeline (2026-07-12), not guessed: + # 'Van' is particles_ambiguous and non-leading, so group's prefix + # chain (which is name_order-agnostic -- it lands upstream of assign + # and doesn't consult Policy) absorbs 'Van Anh Thu' into ONE piece + # before assign ever sees it. With only two name pieces left, + # FAMILY_FIRST_GIVEN_LAST's positional rule (family, then given for + # count==2) correctly assigns Nguyen -> FAMILY and the whole merged + # piece -> GIVEN; there is no MIDDLE piece to assign. This is a + # sensible consequence of the current, order-agnostic group stage -- + # not a contract violation of assign -- and is the exact kind of gap + # the Vietnamese-aware locale pack (#270/#272 follow-ups) is meant to + # close by teaching group to suppress the particle chain under this + # name_order. Do not "fix" this by changing assign. + out = _assigned("Nguyen Van Anh Thu", + Policy(name_order=FAMILY_FIRST_GIVEN_LAST)) + assert _by_role(out, Role.FAMILY) == "Nguyen" + assert _by_role(out, Role.GIVEN) == "Van Anh Thu" + assert _by_role(out, Role.MIDDLE) == "" diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index ad6f777..cb9749f 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -15,7 +15,8 @@ "_render.py", "_pipeline/_state.py", "_pipeline/__init__.py", "_pipeline/_extract.py", "_pipeline/_tokenize.py", "_pipeline/_vocab.py", "_pipeline/_segment.py", - "_pipeline/_classify.py", "_pipeline/_group.py"} + "_pipeline/_classify.py", "_pipeline/_group.py", + "_pipeline/_assign.py"} _PIPELINE_STAGE_ALLOWED = ( "nameparser._types", "nameparser._lexicon", "nameparser._policy", From 4f3a70766fd05db1b059c7d675e0e5749bff9a13 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 22:43:00 -0700 Subject: [PATCH 76/88] Add post_rules: firstname swap and opt-in patronymic rotations Completes the seven-stage STAGES tuple. Patronymic regexes verified byte-for-byte against nameparser/config/regexes.py's east_slavic/turkic entries -- no mismatch found. Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/__init__.py | 7 +- nameparser/_pipeline/_post_rules.py | 98 ++++++++++++++++++++++++++++ tests/v2/pipeline/test_post_rules.py | 83 +++++++++++++++++++++++ tests/v2/test_layering.py | 2 +- 4 files changed, 187 insertions(+), 3 deletions(-) create mode 100644 nameparser/_pipeline/_post_rules.py create mode 100644 tests/v2/pipeline/test_post_rules.py diff --git a/nameparser/_pipeline/__init__.py b/nameparser/_pipeline/__init__.py index 25f8149..d9a3bfe 100644 --- a/nameparser/_pipeline/__init__.py +++ b/nameparser/_pipeline/__init__.py @@ -15,13 +15,16 @@ from nameparser._pipeline._classify import classify from nameparser._pipeline._extract import extract_delimited from nameparser._pipeline._group import group +from nameparser._pipeline._post_rules import post_rules from nameparser._pipeline._segment import segment from nameparser._pipeline._state import ParseState from nameparser._pipeline._tokenize import tokenize -#: Filled in by later tasks as stages land; the fold is total either way. +#: The full seven-stage fold (spec §6). STAGES: tuple[Callable[[ParseState], ParseState], ...] = ( - extract_delimited, tokenize, segment, classify, group, assign) + extract_delimited, tokenize, segment, classify, group, assign, + post_rules, +) def run(state: ParseState) -> ParseState: diff --git a/nameparser/_pipeline/_post_rules.py b/nameparser/_pipeline/_post_rules.py new file mode 100644 index 0000000..4edfb39 --- /dev/null +++ b/nameparser/_pipeline/_post_rules.py @@ -0,0 +1,98 @@ +"""Stage: post_rules. + +Consumes: tokens (roles assigned). +Produces: tokens with roles adjusted by the post rules. +Reads: Policy.patronymic_rules; Lexicon.given_name_titles. + +Rules (each a small pure function over the role-bearing tokens): +1. v1 handle_firstnames: when the parse is exactly a title plus ONE + given token (no other roles), and the title is not a given-name + title ('Sir'), that token is a family name -- "Mr. Johnson". +2. EAST_SLAVIC (opt-in): positional GIVEN/MIDDLE/FAMILY each exactly + one token, the FAMILY-position token carries an East Slavic + patronymic ending, and the MIDDLE-position token does NOT (given + + patronymic + patronymic-derived surname like Abramovich must not + rotate) -> rotate: given<-old MIDDLE, middle<-old FAMILY (the + patronymic), family<-old GIVEN (v1 parity, pinned live). +3. TURKIC (opt-in): exactly 1 GIVEN + 2 MIDDLE + 1 FAMILY tokens and + the FAMILY-position token is a standalone Turkic marker -> + given<-first MIDDLE, middle<-(second MIDDLE, marker), family<-old + GIVEN. + +Both rotations fire only on Structure.NO_COMMA (v1 gates them on +`not self._had_comma`): a comma already established the family. +""" +from __future__ import annotations + +import dataclasses +import re + +from nameparser._lexicon import _normalize +from nameparser._pipeline._state import ParseState, Structure, WorkToken +from nameparser._policy import PatronymicRule +from nameparser._types import Role + +# Ported verbatim from v1 (nameparser/config/regexes.py) -- layering +# forbids the config import; keep in sync by hand. +_EAST_SLAVIC = re.compile( + r"(ovich|ovna|evich|evna|ichna|ilyich|kuzmich|lukich|fomich|fokich)$", + re.I) +_EAST_SLAVIC_CYR = re.compile( + r"(ович|овна|евич|евна|ична|ильич|кузьмич|лукич|фомич|фокич)$", + re.I) +_TURKIC = re.compile( + r"^(oglu|oğlu|ogly|ogli|o['’ʻ]g['’ʻ]li" + r"|qizi|qızı|kizi|kyzy|gyzy|uly|uulu)$", re.I) +_TURKIC_CYR = re.compile( + r"^(оглу|оглы|оғлу|ўғли|угли|кызы|гызы|қызы|қизи|улы|ұлы|уулу)$", re.I) + + +def _idx(tokens: list[WorkToken], role: Role) -> list[int]: + return [i for i, t in enumerate(tokens) if t.role is role] + + +def _retag(tokens: list[WorkToken], i: int, role: Role) -> None: + tokens[i] = dataclasses.replace(tokens[i], role=role) + + +def post_rules(state: ParseState) -> ParseState: + tokens = list(state.tokens) + titles = _idx(tokens, Role.TITLE) + givens = _idx(tokens, Role.GIVEN) + middles = _idx(tokens, Role.MIDDLE) + families = _idx(tokens, Role.FAMILY) + others = [t for t in tokens + if t.role in (Role.SUFFIX, Role.NICKNAME, Role.MAIDEN)] + + # rule 1: title + lone given -> family (v1 handle_firstnames) + if titles and givens and not middles and not families and not others: + joined = " ".join(_normalize(tokens[i].text) for i in titles) + if joined not in state.lexicon.given_name_titles: + for i in givens: + _retag(tokens, i, Role.FAMILY) + + # v1 gates both rotations on `not self._had_comma` + if state.structure is not Structure.NO_COMMA: + return dataclasses.replace(state, tokens=tuple(tokens)) + rules = state.policy.patronymic_rules + if PatronymicRule.EAST_SLAVIC in rules and \ + len(givens) == 1 and len(middles) == 1 and len(families) == 1: + tail = tokens[families[0]].text + mid = tokens[middles[0]].text + if (_EAST_SLAVIC.search(tail) or _EAST_SLAVIC_CYR.search(tail)) \ + and not (_EAST_SLAVIC.search(mid) + or _EAST_SLAVIC_CYR.search(mid)): + g, m, f = givens[0], middles[0], families[0] + _retag(tokens, m, Role.GIVEN) + _retag(tokens, f, Role.MIDDLE) + _retag(tokens, g, Role.FAMILY) + if PatronymicRule.TURKIC in rules and \ + len(givens) == 1 and len(middles) == 2 and len(families) == 1: + tail = tokens[families[0]].text + if _TURKIC.match(tail) or _TURKIC_CYR.match(tail): + g, m1, m2, f = givens[0], middles[0], middles[1], families[0] + _retag(tokens, m1, Role.GIVEN) + _retag(tokens, m2, Role.MIDDLE) + _retag(tokens, f, Role.MIDDLE) + _retag(tokens, g, Role.FAMILY) + return dataclasses.replace(state, tokens=tuple(tokens)) diff --git a/tests/v2/pipeline/test_post_rules.py b/tests/v2/pipeline/test_post_rules.py new file mode 100644 index 0000000..12634ca --- /dev/null +++ b/tests/v2/pipeline/test_post_rules.py @@ -0,0 +1,83 @@ +from nameparser._lexicon import Lexicon +from nameparser._pipeline import run +from nameparser._pipeline._state import ParseState +from nameparser._policy import PatronymicRule, Policy +from nameparser._types import Role + +_LEX = Lexicon( + titles=frozenset({"mr", "sir"}), + given_name_titles=frozenset({"sir"}), +) + + +def _parsed(text: str, policy: Policy | None = None) -> ParseState: + return run(ParseState(original=text, lexicon=_LEX, + policy=policy or Policy())) + + +def _by_role(state: ParseState, role: Role) -> str: + return " ".join(t.text for t in state.tokens if t.role is role) + + +def test_plain_title_with_single_name_swaps_to_family() -> None: + out = _parsed("Mr. Johnson") + assert _by_role(out, Role.FAMILY) == "Johnson" + assert not _by_role(out, Role.GIVEN) + + +def test_given_name_title_keeps_given() -> None: + out = _parsed("Sir Bob") + assert _by_role(out, Role.GIVEN) == "Bob" + assert not _by_role(out, Role.FAMILY) + + +def test_no_swap_when_more_fields_present() -> None: + out = _parsed("Mr. John Johnson") + assert _by_role(out, Role.GIVEN) == "John" + assert _by_role(out, Role.FAMILY) == "Johnson" + + +_ES = Policy(patronymic_rules=frozenset({PatronymicRule.EAST_SLAVIC})) +_TK = Policy(patronymic_rules=frozenset({PatronymicRule.TURKIC})) + + +def test_east_slavic_rotation() -> None: + out = _parsed("Сидоров Иван Петрович", _ES) + assert _by_role(out, Role.GIVEN) == "Иван" + assert _by_role(out, Role.MIDDLE) == "Петрович" + assert _by_role(out, Role.FAMILY) == "Сидоров" + + +def test_east_slavic_needs_one_one_one() -> None: + # four tokens: left unchanged (v1 parity) + out = _parsed("Anna Maria Petrova Ivanovna", _ES) + assert _by_role(out, Role.GIVEN) == "Anna" + + +def test_east_slavic_skips_comma_forms() -> None: + # v1 parity: patronymic reorder never fires on comma input -- + # the comma already established the family + out = _parsed("Abramovich, Roman Petrovich", _ES) + assert _by_role(out, Role.FAMILY) == "Abramovich" + assert _by_role(out, Role.GIVEN) == "Roman" + + +def test_east_slavic_skips_when_middle_is_also_patronymic() -> None: + # v1 parity: given + patronymic + patronymic-derived surname + # (Abramovich) must not rotate + out = _parsed("Roman Petrovich Abramovich", _ES) + assert _by_role(out, Role.GIVEN) == "Roman" + assert _by_role(out, Role.MIDDLE) == "Petrovich" + assert _by_role(out, Role.FAMILY) == "Abramovich" + + +def test_east_slavic_off_by_default() -> None: + out = _parsed("Сидоров Иван Петрович") + assert _by_role(out, Role.GIVEN) == "Сидоров" + + +def test_turkic_rotation() -> None: + out = _parsed("Mammadova Aygun Ali kizi", _TK) + assert _by_role(out, Role.GIVEN) == "Aygun" + assert _by_role(out, Role.MIDDLE) == "Ali kizi" + assert _by_role(out, Role.FAMILY) == "Mammadova" diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index cb9749f..f16838b 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -16,7 +16,7 @@ "_pipeline/_extract.py", "_pipeline/_tokenize.py", "_pipeline/_vocab.py", "_pipeline/_segment.py", "_pipeline/_classify.py", "_pipeline/_group.py", - "_pipeline/_assign.py"} + "_pipeline/_assign.py", "_pipeline/_post_rules.py"} _PIPELINE_STAGE_ALLOWED = ( "nameparser._types", "nameparser._lexicon", "nameparser._policy", From bb80c8d0ba7a5678b2c22a73c320fffe5254628d Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 22:57:56 -0700 Subject: [PATCH 77/88] Add assemble: final ParseState to validated ParsedName Not a pipeline stage but the runner's tail: converts the seven-stage fold's ParseState into a public ParsedName, dropping structural marker tokens (e.g. maiden markers) and materializing PendingAmbiguity indices into real Ambiguity objects over the final token tuple. Also adds _pipeline/_assemble.py to test_layering's _MUST_EXIST now that the file exists. Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/_assemble.py | 33 +++++++++++++++++++ tests/v2/pipeline/test_assemble.py | 52 ++++++++++++++++++++++++++++++ tests/v2/test_layering.py | 3 +- 3 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 nameparser/_pipeline/_assemble.py create mode 100644 tests/v2/pipeline/test_assemble.py diff --git a/nameparser/_pipeline/_assemble.py b/nameparser/_pipeline/_assemble.py new file mode 100644 index 0000000..0eb4897 --- /dev/null +++ b/nameparser/_pipeline/_assemble.py @@ -0,0 +1,33 @@ +"""Not a stage: converts the final ParseState into a public ParsedName. + +Consumes: tokens (all roles set), dropped, ambiguities (by index). +Produces: a validated ParsedName -- the constructor re-checks every +invariant (span order/bounds, ambiguity subset), so a pipeline bug +that would produce an invalid result dies HERE, not in a renderer +three layers away. + +Structural tokens (dropped maiden markers) are omitted, like delimiter +characters. A main-stream token that somehow reaches here with no role +takes Role.GIVEN -- parse is total over str (spec §5a) and must not +raise on content; the fallback is deliberately boring. +""" +from __future__ import annotations + +from nameparser._pipeline._state import ParseState +from nameparser._types import Ambiguity, ParsedName, Role, Token + + +def assemble(state: ParseState) -> ParsedName: + dropped = set(state.dropped) + final: dict[int, Token] = {} + for i, t in enumerate(state.tokens): + if i in dropped: + continue + final[i] = Token(t.text, t.span, t.role or Role.GIVEN, t.tags) + ambiguities = tuple( + Ambiguity(p.kind, p.detail, + tuple(final[i] for i in p.indices if i in final)) + for p in state.ambiguities) + return ParsedName(original=state.original, + tokens=tuple(final.values()), + ambiguities=ambiguities) diff --git a/tests/v2/pipeline/test_assemble.py b/tests/v2/pipeline/test_assemble.py new file mode 100644 index 0000000..d1aaae9 --- /dev/null +++ b/tests/v2/pipeline/test_assemble.py @@ -0,0 +1,52 @@ +from nameparser._lexicon import Lexicon +from nameparser._pipeline import run +from nameparser._pipeline._assemble import assemble +from nameparser._pipeline._state import ParseState +from nameparser._policy import Policy +from nameparser._types import AmbiguityKind, ParsedName + +_LEX = Lexicon( + titles=frozenset({"dr"}), + particles=frozenset({"de", "la", "van"}), + particles_ambiguous=frozenset({"van"}), + suffix_words=frozenset({"iii"}), + maiden_markers=frozenset({"née"}), +) + + +def _parse(text: str) -> ParsedName: + return assemble(run(ParseState(original=text, lexicon=_LEX, + policy=Policy()))) + + +def test_assemble_produces_validated_parsedname() -> None: + pn = _parse("Dr. Juan de la Vega III") + assert pn.title == "Dr." + assert pn.given == "Juan" + assert pn.family == "de la Vega" + assert pn.family_base == "Vega" # particle tags carried over + assert pn.family_particles == "de la" + assert pn.suffix == "III" + assert pn.original == "Dr. Juan de la Vega III" + for t in pn.tokens: + assert t.span is not None + assert t.text == pn.original[t.span.start:t.span.end] + + +def test_assemble_materializes_ambiguities_on_final_tokens() -> None: + pn = _parse("Van Johnson") + assert len(pn.ambiguities) == 1 + amb = pn.ambiguities[0] + assert amb.kind is AmbiguityKind.PARTICLE_OR_GIVEN + assert amb.tokens[0] is pn.tokens[0] + + +def test_assemble_drops_structural_marker_tokens() -> None: + pn = _parse("Jane Smith née Jones") + assert [t.text for t in pn.tokens] == ["Jane", "Smith", "Jones"] + assert pn.maiden == "Jones" + + +def test_empty_parse_is_falsy() -> None: + assert not _parse("") + assert not _parse(" ") diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index f16838b..5c8ed72 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -16,7 +16,8 @@ "_pipeline/_extract.py", "_pipeline/_tokenize.py", "_pipeline/_vocab.py", "_pipeline/_segment.py", "_pipeline/_classify.py", "_pipeline/_group.py", - "_pipeline/_assign.py", "_pipeline/_post_rules.py"} + "_pipeline/_assign.py", "_pipeline/_post_rules.py", + "_pipeline/_assemble.py"} _PIPELINE_STAGE_ALLOWED = ( "nameparser._types", "nameparser._lexicon", "nameparser._policy", From 9bcd0073fda0554d78b817a011c4a036f21a2549 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 23:07:24 -0700 Subject: [PATCH 78/88] Add Parser and parse(): the 2.0 pipeline goes end to end Co-Authored-By: Claude Fable 5 --- nameparser/__init__.py | 2 + nameparser/_parser.py | 73 +++++++++++++++++++++++++++++++++++ tests/v2/test_layering.py | 8 +++- tests/v2/test_parser.py | 80 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 161 insertions(+), 2 deletions(-) create mode 100644 nameparser/_parser.py create mode 100644 tests/v2/test_parser.py diff --git a/nameparser/__init__.py b/nameparser/__init__.py index 5e3adcb..1b49624 100644 --- a/nameparser/__init__.py +++ b/nameparser/__init__.py @@ -8,6 +8,7 @@ from nameparser._lexicon import Lexicon from nameparser._locale import Locale +from nameparser._parser import Parser, parse from nameparser._policy import ( FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, @@ -33,4 +34,5 @@ "Span", "Role", "Token", "Ambiguity", "AmbiguityKind", "ParsedName", "Lexicon", "Policy", "PolicyPatch", "PatronymicRule", "UNSET", "GIVEN_FIRST", "FAMILY_FIRST", "FAMILY_FIRST_GIVEN_LAST", "Locale", + "Parser", "parse", ] diff --git a/nameparser/_parser.py b/nameparser/_parser.py new file mode 100644 index 0000000..f6562b0 --- /dev/null +++ b/nameparser/_parser.py @@ -0,0 +1,73 @@ +"""Parser and the module-level parse() for the 2.0 API. + +Layering: sits on _types/_lexicon/_policy/_locale/_pipeline; never +imports _render or the v1 facade (enforced by tests/v2/test_layering.py). + +_default_parser is THE one sanctioned module-level global (conventions +§8): a functools.cache'd frozen Parser over default config. +""" +from __future__ import annotations + +import functools +from dataclasses import dataclass + +from nameparser._lexicon import Lexicon +from nameparser._pipeline import run +from nameparser._pipeline._assemble import assemble +from nameparser._pipeline._state import ParseState +from nameparser._policy import Policy +from nameparser._types import ParsedName, _guarded_getstate, _guarded_setstate + + +@dataclass(frozen=True, slots=True) +class Parser: + """Immutable, thread-safe, picklable by construction (spec §4): all + validity checking happens at construction; a Parser that constructs + successfully cannot fail at parse time on any str content.""" + + lexicon: Lexicon = None # type: ignore[assignment] # None -> default() + policy: Policy = None # type: ignore[assignment] # None -> Policy() + + # in the class body so @dataclass(slots=True) keeps them + __getstate__ = _guarded_getstate + __setstate__ = _guarded_setstate + + def __post_init__(self) -> None: + if self.lexicon is None: + object.__setattr__(self, "lexicon", Lexicon.default()) + elif not isinstance(self.lexicon, Lexicon): + raise TypeError( + f"lexicon must be a Lexicon or None, got {self.lexicon!r}") + if self.policy is None: + object.__setattr__(self, "policy", Policy()) + elif not isinstance(self.policy, Policy): + raise TypeError( + f"policy must be a Policy or None, got {self.policy!r}") + + def __repr__(self) -> str: + # composes the two bounded component reprs (spec §2 reprs) + return f"Parser({self.lexicon!r}, {self.policy!r})" + + def parse(self, text: str) -> ParsedName: + """Total over str (spec §5a): content never raises; non-str + raises TypeError eagerly, with a decode hint for bytes (#245: + bytes support ended with 1.x).""" + if isinstance(text, bytes): + raise TypeError( + "parse() takes str, not bytes -- decode first, e.g. " + "raw.decode('utf-8')") + if not isinstance(text, str): + raise TypeError(f"parse() takes str, got {text!r}") + state = ParseState(original=text, lexicon=self.lexicon, + policy=self.policy) + return assemble(run(state)) + + +@functools.cache +def _default_parser() -> Parser: + return Parser() + + +def parse(text: str) -> ParsedName: + """Module-level convenience over a lazily created default Parser.""" + return _default_parser().parse(text) diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index 5c8ed72..9d84758 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -17,7 +17,7 @@ "_pipeline/_vocab.py", "_pipeline/_segment.py", "_pipeline/_classify.py", "_pipeline/_group.py", "_pipeline/_assign.py", "_pipeline/_post_rules.py", - "_pipeline/_assemble.py"} + "_pipeline/_assemble.py", "_parser.py"} _PIPELINE_STAGE_ALLOWED = ( "nameparser._types", "nameparser._lexicon", "nameparser._policy", @@ -130,6 +130,7 @@ def test_public_exports() -> None: "Span", "Role", "Token", "Ambiguity", "AmbiguityKind", "ParsedName", "Lexicon", "Policy", "PolicyPatch", "PatronymicRule", "UNSET", "GIVEN_FIRST", "FAMILY_FIRST", "FAMILY_FIRST_GIVEN_LAST", "Locale", + "Parser", "parse", } assert expected <= set(nameparser.__all__) for name in expected: @@ -162,12 +163,15 @@ def test_every_frozen_dataclass_carries_the_pickle_guards() -> None: import nameparser._lexicon import nameparser._locale + import nameparser._parser import nameparser._policy import nameparser._types from nameparser._types import _guarded_getstate, _guarded_setstate + # _pipeline internals (ParseState, WorkToken, ...) are exempt: they + # are never pickled and are not public API. modules = (nameparser._types, nameparser._lexicon, - nameparser._policy, nameparser._locale) + nameparser._policy, nameparser._locale, nameparser._parser) for module in modules: for _, cls in inspect.getmembers(module, inspect.isclass): if cls.__module__ != module.__name__: diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py new file mode 100644 index 0000000..dc31655 --- /dev/null +++ b/tests/v2/test_parser.py @@ -0,0 +1,80 @@ +import pickle + +import pytest + +from nameparser import Lexicon, Parser, Policy, parse +from nameparser._policy import FAMILY_FIRST +from nameparser._types import AmbiguityKind + + +def test_parser_defaults_and_properties() -> None: + p = Parser() + assert p.lexicon == Lexicon.default() + assert p.policy == Policy() + + +def test_parser_rejects_wrong_types_eagerly() -> None: + with pytest.raises(TypeError, match="lexicon"): + Parser(lexicon={"titles": set()}) # type: ignore[arg-type] + with pytest.raises(TypeError, match="policy"): + Parser(policy="strict") # type: ignore[arg-type] + + +def test_parse_end_to_end_with_default_vocabulary() -> None: + pn = parse("Dr. Juan de la Vega III") + assert pn.title == "Dr." + assert pn.given == "Juan" + assert pn.family == "de la Vega" + assert pn.suffix == "III" + assert str(pn) == "Dr. Juan de la Vega III" + + +def test_parse_rejects_non_str_with_decode_hint() -> None: + with pytest.raises(TypeError, match="decode"): + parse(b"John Smith") # type: ignore[arg-type] + with pytest.raises(TypeError, match="str"): + parse(None) # type: ignore[arg-type] + + +def test_degenerate_inputs_are_total() -> None: + # spec §5a table + assert not parse("") + assert not parse(" ") + assert parse("").original == "" + single = parse("John") + assert single.given == "John" + family_first = Parser(policy=Policy(name_order=FAMILY_FIRST)) + assert family_first.parse("Yamada").family == "Yamada" + title_only = parse("Dr.") + assert title_only.title == "Dr." and not title_only.given + unbalanced = parse('Jon "Nick Smith') + kinds = {a.kind for a in unbalanced.ambiguities} + assert AmbiguityKind.UNBALANCED_DELIMITER in kinds + assert '"Nick' in [t.text for t in unbalanced.tokens] # literal + + +def test_parser_is_picklable_and_frozen() -> None: + p = Parser(policy=Policy(name_order=FAMILY_FIRST)) + loaded = pickle.loads(pickle.dumps(p)) + assert loaded == p + assert loaded.parse("Yamada Taro").family == "Yamada" + with pytest.raises(AttributeError): + p.policy = Policy() # type: ignore[misc] + + +def test_parser_repr_composes_component_reprs() -> None: + assert repr(Parser()) == "Parser(Lexicon(default), Policy())" + p = Parser(policy=Policy(name_order=FAMILY_FIRST)) + assert repr(p) == "Parser(Lexicon(default), Policy(name_order=FAMILY_FIRST))" + + +def test_parsedname_repr_includes_ambiguities_line() -> None: + pn = parse("Van Johnson") + r = repr(pn) + assert "given: 'Van'" in r + assert "ambiguities:" in r and "particle-or-given" in r + + +def test_module_parse_reuses_the_default_parser() -> None: + import nameparser._parser as parser_mod + assert parser_mod._default_parser() is parser_mod._default_parser() From 8ad0247fc3a02d0a03c55f030015163f73d0e438 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 23:16:34 -0700 Subject: [PATCH 79/88] Add parser_for() with locale-identified errors and matches() Co-Authored-By: Claude Fable 5 --- nameparser/__init__.py | 4 +-- nameparser/_parser.py | 46 ++++++++++++++++++++++++++++++- nameparser/_types.py | 15 +++++++++++ tests/v2/test_layering.py | 2 +- tests/v2/test_parser.py | 57 +++++++++++++++++++++++++++++++++++++-- 5 files changed, 118 insertions(+), 6 deletions(-) diff --git a/nameparser/__init__.py b/nameparser/__init__.py index 1b49624..fb9575f 100644 --- a/nameparser/__init__.py +++ b/nameparser/__init__.py @@ -8,7 +8,7 @@ from nameparser._lexicon import Lexicon from nameparser._locale import Locale -from nameparser._parser import Parser, parse +from nameparser._parser import Parser, parse, parser_for from nameparser._policy import ( FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, @@ -34,5 +34,5 @@ "Span", "Role", "Token", "Ambiguity", "AmbiguityKind", "ParsedName", "Lexicon", "Policy", "PolicyPatch", "PatronymicRule", "UNSET", "GIVEN_FIRST", "FAMILY_FIRST", "FAMILY_FIRST_GIVEN_LAST", "Locale", - "Parser", "parse", + "Parser", "parse", "parser_for", ] diff --git a/nameparser/_parser.py b/nameparser/_parser.py index f6562b0..b94039c 100644 --- a/nameparser/_parser.py +++ b/nameparser/_parser.py @@ -8,14 +8,17 @@ """ from __future__ import annotations +import dataclasses import functools +import warnings from dataclasses import dataclass from nameparser._lexicon import Lexicon +from nameparser._locale import Locale from nameparser._pipeline import run from nameparser._pipeline._assemble import assemble from nameparser._pipeline._state import ParseState -from nameparser._policy import Policy +from nameparser._policy import UNSET, Policy, PolicyPatch, apply_patch from nameparser._types import ParsedName, _guarded_getstate, _guarded_setstate @@ -71,3 +74,44 @@ def _default_parser() -> Parser: def parse(text: str) -> ParsedName: """Module-level convenience over a lazily created default Parser.""" return _default_parser().parse(text) + + +def parser_for(*locales: Locale, base: Parser | None = None) -> Parser: + """Lexicon fragments unioned left-to-right onto base's; policy + patches applied left-to-right (later wins; set-valued fields union + per the patch metadata). Validation errors raised while applying a + pack are wrapped with that pack's identity (spec §4 amendment) -- + PolicyPatch validates lazily, so with stacked packs the raw error + would otherwise point at nothing. Two packs setting the same SCALAR + field is a declared conflict: UserWarning, later wins.""" + if base is not None and not isinstance(base, Parser): + raise TypeError(f"base must be a Parser or None, got {base!r}") + for loc in locales: + if not isinstance(loc, Locale): + raise TypeError(f"parser_for() takes Locale packs, got {loc!r}") + lexicon = base.lexicon if base is not None else Lexicon.default() + policy = base.policy if base is not None else Policy() + scalar_setters: dict[str, str] = {} + for loc in locales: + for f in dataclasses.fields(PolicyPatch): + if f.metadata.get("compose") == "union": + continue + if getattr(loc.policy, f.name) is UNSET: + continue + if f.name in scalar_setters and scalar_setters[f.name] != loc.code: + warnings.warn( + f"locale {loc.code!r} overrides scalar policy field " + f"{f.name!r} already set by locale " + f"{scalar_setters[f.name]!r}; later wins", + UserWarning, stacklevel=2) + scalar_setters[f.name] = loc.code + try: + lexicon = lexicon | loc.lexicon + policy = apply_patch(policy, loc.policy) + except (TypeError, ValueError) as exc: + # safe: every raise in the apply path (Policy.__post_init__, + # Lexicon.__or__, apply_patch) is a PLAIN TypeError/ValueError -- + # a subclass with extra mandatory args would break this rewrap + raise type(exc)( + f"while applying locale {loc.code!r}: {exc}") from exc + return Parser(lexicon=lexicon, policy=policy) diff --git a/nameparser/_types.py b/nameparser/_types.py index 5882919..af6789a 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -21,6 +21,7 @@ if TYPE_CHECKING: from nameparser._lexicon import Lexicon + from nameparser._parser import Parser class Role(Enum): @@ -422,6 +423,20 @@ def comparison_key(self) -> tuple[str, ...]: """ return tuple(self._text_for(role).casefold() for role in Role) + def matches(self, other: str | ParsedName, *, + parser: Parser | None = None) -> bool: + """Component-wise case-insensitive comparison (the semantic + layer; __eq__ stays strict). A str argument is parsed with + `parser`, or the default parser when None.""" + if isinstance(other, str): + import nameparser._parser as _parser + active = parser if parser is not None else _parser._default_parser() + other = active.parse(other) + if not isinstance(other, ParsedName): + raise TypeError( + f"matches() takes a str or ParsedName, got {other!r}") + return self.comparison_key() == other.comparison_key() + # -- rendering delegates ---------------------------------------------- # One-line delegation to nameparser._render (core spec §5b): parsing # code physically cannot import formatting logic, so these import at diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index 9d84758..5b85218 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -130,7 +130,7 @@ def test_public_exports() -> None: "Span", "Role", "Token", "Ambiguity", "AmbiguityKind", "ParsedName", "Lexicon", "Policy", "PolicyPatch", "PatronymicRule", "UNSET", "GIVEN_FIRST", "FAMILY_FIRST", "FAMILY_FIRST_GIVEN_LAST", "Locale", - "Parser", "parse", + "Parser", "parse", "parser_for", } assert expected <= set(nameparser.__all__) for name in expected: diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index dc31655..d82aa67 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -2,8 +2,8 @@ import pytest -from nameparser import Lexicon, Parser, Policy, parse -from nameparser._policy import FAMILY_FIRST +from nameparser import Lexicon, Locale, Parser, Policy, PolicyPatch, parse, parser_for +from nameparser._policy import FAMILY_FIRST, PatronymicRule from nameparser._types import AmbiguityKind @@ -78,3 +78,56 @@ def test_parsedname_repr_includes_ambiguities_line() -> None: def test_module_parse_reuses_the_default_parser() -> None: import nameparser._parser as parser_mod assert parser_mod._default_parser() is parser_mod._default_parser() + + +def test_parser_for_stacks_locales() -> None: + ru = Locale(code="ru", + lexicon=Lexicon.empty().add(titles={"г-н"}), + policy=PolicyPatch(patronymic_rules=frozenset( + {PatronymicRule.EAST_SLAVIC}))) + p = parser_for(ru) + assert PatronymicRule.EAST_SLAVIC in p.policy.patronymic_rules + pn = p.parse("г-н Сидоров Иван Петрович") + assert pn.title == "г-н" + assert pn.given == "Иван" + assert pn.family == "Сидоров" + + +def test_parser_for_rejects_non_locales() -> None: + with pytest.raises(TypeError, match="Locale"): + parser_for("ru") # type: ignore[arg-type] + + +def test_parser_for_wraps_pack_errors_with_identity() -> None: + # PolicyPatch validates lazily (by design), so an invalid value sits + # latent in a perfectly constructible Locale until apply time + bad = Locale(code="xx", lexicon=Lexicon.empty(), + policy=PolicyPatch(name_order=(1, 2, 3))) # type: ignore[arg-type] + with pytest.raises(ValueError, match="while applying locale 'xx'"): + parser_for(bad) + + +def test_parser_for_warns_on_scalar_conflict() -> None: + a = Locale(code="aa", lexicon=Lexicon.empty(), + policy=PolicyPatch(strip_emoji=False)) + b = Locale(code="bb", lexicon=Lexicon.empty(), + policy=PolicyPatch(strip_emoji=True)) + with pytest.warns(UserWarning, match="strip_emoji"): + p = parser_for(a, b) + assert p.policy.strip_emoji is True # later wins + + +def test_matches_component_wise_case_insensitive() -> None: + pn = parse("John Smith") + assert pn.matches("JOHN SMITH") + assert pn.matches(parse("john smith")) + assert not pn.matches("John Smythe") + with pytest.raises(TypeError, match="str or ParsedName"): + pn.matches(42) # type: ignore[arg-type] + + +def test_matches_accepts_explicit_parser() -> None: + family_first = Parser(policy=Policy(name_order=FAMILY_FIRST)) + pn = family_first.parse("Yamada Taro") + assert pn.matches("Yamada Taro", parser=family_first) + assert not pn.matches("Yamada Taro") # default parser reads given-first From c7e3fb011b151a3515da393ac73448daaa2395a8 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 23:25:39 -0700 Subject: [PATCH 80/88] Fix the shared case-table format and seed it with the pinned corpus Adds tests/v2/cases.py (Case dataclass + CASES tuple) and tests/v2/test_cases.py (the core runner), seeded with the v1 parity battery pinned in the plan header plus three carried-forward rows verified live against v1.4.0: - suffix_stays_suffix / suffix_stays_suffix_title: v1 routes a lone trailing recognized suffix (PhD) to the family/last field; v2 keeps it in suffix. Classified fix(suffix-routing). - family_comma_lone_title: v1 puts the pre-comma piece in first when it's followed only by a title; v2 treats pre-comma as definitionally family. Classified fix(comma-family). All 35 case rows pass against the pipeline as built through Task 11 with no corrections needed to the plan's predicted values; the East Slavic/Turkic policy rows were cross-checked against v1's equivalent patronymic_name_order opt-in and confirmed as true parity. Co-Authored-By: Claude Fable 5 --- tests/v2/cases.py | 125 +++++++++++++++++++++++++++++++++++++++++ tests/v2/test_cases.py | 21 +++++++ 2 files changed, 146 insertions(+) create mode 100644 tests/v2/cases.py create mode 100644 tests/v2/test_cases.py diff --git a/tests/v2/cases.py b/tests/v2/cases.py new file mode 100644 index 0000000..0157931 --- /dev/null +++ b/tests/v2/cases.py @@ -0,0 +1,125 @@ +"""THE shared behavior case table (core spec §7.2). + +Format is fixed here, in the first pipeline PR, and never per-PR: +one Case per input, expected values for exactly the non-empty fields, +optional Policy/Locale context, and a mandatory classification -- +"parity" (matches v1.4.0, pinned live 2026-07-12) or "fix(#N)" / +"fix()" (an intentional 2.0 behavior change, annotated with its +issue or a design-decision slug). No silent expectation edits: +changing a row means changing its classification. + +The v1 suite's full corpus is extracted into this table by the +migration plan (facade runner consumes the same rows); this file seeds +it with the pinned battery. +""" +from __future__ import annotations + +from dataclasses import dataclass + +from nameparser import Policy +from nameparser._policy import PatronymicRule + + +@dataclass(frozen=True) +class Case: + id: str + text: str + expect: dict[str, str] # field -> value; absent fields == "" + policy: Policy | None = None + classification: str = "parity" + ambiguities: tuple[str, ...] = () # expected AmbiguityKind values + notes: str = "" + + +_ES = Policy(patronymic_rules=frozenset({PatronymicRule.EAST_SLAVIC})) +_TK = Policy(patronymic_rules=frozenset({PatronymicRule.TURKIC})) + +CASES: tuple[Case, ...] = ( + Case("plain", "John Smith", {"given": "John", "family": "Smith"}), + Case("family_comma", "Smith, John", + {"given": "John", "family": "Smith"}), + Case("suffix_comma", "John Smith, PhD", + {"given": "John", "family": "Smith", "suffix": "PhD"}), + Case("delavega", "Dr. Juan de la Vega III", + {"title": "Dr.", "given": "Juan", "family": "de la Vega", + "suffix": "III"}), + Case("prefix_chain_to_end", "Juan de la Vega Martinez", + {"given": "Juan", "family": "de la Vega Martinez"}), + Case("van_johnson", "Van Johnson", + {"given": "Van", "family": "Johnson"}, + ambiguities=("particle-or-given",), + notes="v2 surfaces #121's irreducible ambiguity"), + Case("family_comma_particles", "de la Vega, Juan", + {"given": "Juan", "family": "de la Vega"}), + Case("nickname_quotes", 'John "Jack" Kennedy', + {"given": "John", "family": "Kennedy", "nickname": "Jack"}), + Case("nickname_parens", "John (Jack) Kennedy", + {"given": "John", "family": "Kennedy", "nickname": "Jack"}), + Case("sir_bob", "Sir Bob Andrew Dole", + {"title": "Sir", "given": "Bob", "middle": "Andrew", + "family": "Dole"}), + Case("long_title", "President of the United States Barack Obama", + {"title": "President of the United States", + "given": "Barack", "family": "Obama"}), + Case("secretary", "The Secretary of State Hillary Clinton", + {"title": "The Secretary of State", "given": "Hillary", + "family": "Clinton"}), + Case("comma_middle_initial", "Doe, John A.", + {"given": "John", "middle": "A.", "family": "Doe"}), + Case("single", "John", {"given": "John"}), + Case("title_only", "Dr.", {"title": "Dr."}), + Case("double_comma_suffix", "Smith, John, Jr.", + {"given": "John", "family": "Smith", "suffix": "Jr."}), + Case("bound_given_two", "abdul rahman", + {"given": "abdul", "family": "rahman"}), + Case("bound_given_three", "abdul rahman al-said", + {"given": "abdul rahman", "family": "al-said"}), + Case("mr_and_mrs", "Mr. and Mrs. John Smith", + {"title": "Mr. and Mrs.", "given": "John", "family": "Smith"}), + Case("roman_suffix", "John Smith V", + {"given": "John", "family": "Smith", "suffix": "V"}), + Case("initial_not_suffix", "John V. Smith", + {"given": "John", "middle": "V.", "family": "Smith"}), + Case("lenient_after_comma", "John Ingram, V", + {"given": "John", "family": "Ingram", "suffix": "V"}), + Case("comma_then_title", "Smith, Dr. John", + {"title": "Dr.", "given": "John", "family": "Smith"}), + Case("nickname_single_name", "John (Jack)", + {"family": "John", "nickname": "Jack"}), + Case("nickname_only", "(Jack)", {"nickname": "Jack"}), + Case("suffix_run", "John Jack Kennedy PhD MD", + {"given": "John", "middle": "Jack", "family": "Kennedy", + "suffix": "PhD, MD"}), + Case("maiden_marker", "Jane Smith née Jones", + {"given": "Jane", "family": "Smith", "maiden": "Jones"}, + classification="fix(#274)", + notes="v1 mangles to middle='Smith née'"), + Case("east_slavic", "Сидоров Иван Петрович", + {"given": "Иван", "middle": "Петрович", "family": "Сидоров"}, + policy=_ES), + Case("turkic", "Mammadova Aygun Ali kizi", + {"given": "Aygun", "middle": "Ali kizi", "family": "Mammadova"}, + policy=_TK), + Case("empty", "", {}), + Case("whitespace", " ", {}), + Case("unbalanced_quote", 'Jon "Nick Smith', + {"given": "Jon", "middle": '"Nick', "family": "Smith"}, + ambiguities=("unbalanced-delimiter",), + notes="quote char stays literal (spec §5a)"), + Case("suffix_stays_suffix", "Johnson PhD", + {"given": "Johnson", "suffix": "PhD"}, + classification="fix(suffix-routing)", + notes="v1 routes a lone trailing suffix to family " + "(first=Johnson last=PhD); v2 keeps recognized " + "suffixes in suffix"), + Case("suffix_stays_suffix_title", "Mr. Johnson PhD", + {"title": "Mr.", "given": "Johnson", "suffix": "PhD"}, + classification="fix(suffix-routing)", + notes="v1 routes a lone trailing suffix to family " + "(title=Mr. first=Johnson last=PhD); v2 keeps " + "recognized suffixes in suffix"), + Case("family_comma_lone_title", "Smith, Dr.", + {"title": "Dr.", "family": "Smith"}, + classification="fix(comma-family)", + notes="pre-comma is definitionally family; v1 put it in first"), +) diff --git a/tests/v2/test_cases.py b/tests/v2/test_cases.py new file mode 100644 index 0000000..aea0e19 --- /dev/null +++ b/tests/v2/test_cases.py @@ -0,0 +1,21 @@ +"""Core runner over the shared case table (spec §7.2). The facade +runner (migration plan) consumes the same CASES.""" +import pytest + +from nameparser import Parser + +from .cases import CASES, Case + +_FIELDS = ("title", "given", "middle", "family", "suffix", "nickname", + "maiden") + + +@pytest.mark.parametrize("case", CASES, ids=lambda c: c.id) +def test_case(case: Case) -> None: + parser = Parser(policy=case.policy) if case.policy else Parser() + pn = parser.parse(case.text) + actual = {f: getattr(pn, f) for f in _FIELDS if getattr(pn, f)} + assert actual == case.expect, f"{case.text!r} ({case.classification})" + kinds = sorted(a.kind.value for a in pn.ambiguities) + assert kinds == sorted(case.ambiguities), \ + f"{case.text!r} ({case.classification})" From 93478485a6ffdef8897c46a0fc039749c5460ba8 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 12 Jul 2026 23:33:48 -0700 Subject: [PATCH 81/88] Add property, contract, and benchmark test layers Co-Authored-By: Claude Fable 5 --- pyproject.toml | 1 + tests/v2/test_benchmark.py | 15 ++++++++ tests/v2/test_contracts.py | 67 ++++++++++++++++++++++++++++++++++ tests/v2/test_properties.py | 52 +++++++++++++++++++++++++++ uv.lock | 72 +++++++++++++++++++++++++++++++++++++ 5 files changed, 207 insertions(+) create mode 100644 tests/v2/test_benchmark.py create mode 100644 tests/v2/test_contracts.py create mode 100644 tests/v2/test_properties.py diff --git a/pyproject.toml b/pyproject.toml index c56d715..6a86e4b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,7 @@ dev = [ "ruff (>=0.15)", "pytest-timeout>=2.4.0", "pytest-cov>=6", + "hypothesis", ] [tool.mypy] diff --git a/tests/v2/test_benchmark.py b/tests/v2/test_benchmark.py new file mode 100644 index 0000000..f371699 --- /dev/null +++ b/tests/v2/test_benchmark.py @@ -0,0 +1,15 @@ +"""Perf smoke (core spec §7 tail): parse cost stays v1-comparable +(microseconds per name). Deliberately generous bound -- guards against +order-of-magnitude regressions, does not gate normal variance.""" +import time + +from nameparser import parse + + +def test_parse_thousand_names_under_a_second() -> None: + parse("warm up the default parser cache") + start = time.perf_counter() + for i in range(1000): + parse(f"Dr. Juan{i} de la Vega III") + elapsed = time.perf_counter() - start + assert elapsed < 1.0, f"1000 parses took {elapsed:.2f}s" diff --git a/tests/v2/test_contracts.py b/tests/v2/test_contracts.py new file mode 100644 index 0000000..78f284d --- /dev/null +++ b/tests/v2/test_contracts.py @@ -0,0 +1,67 @@ +"""Stable-string contract tests (core spec §7.4): every enum member and +stable tag has a canonical triggering input, parametrized by iterating +the registries -- a new member without an entry here fails loudly.""" +import pytest + +from nameparser import Parser, Policy, parse +from nameparser._policy import PatronymicRule +from nameparser._types import STABLE_TAGS, AmbiguityKind + +_AMBIGUITY_TRIGGERS: dict[AmbiguityKind, str | None] = { + AmbiguityKind.PARTICLE_OR_GIVEN: "Van Johnson", + AmbiguityKind.UNBALANCED_DELIMITER: 'Jon "Nick Smith', + AmbiguityKind.COMMA_STRUCTURE: "Smith, John, Extra, Jr.", + # no emitter yet -- arrives with locale-pack order detection (2.x) + AmbiguityKind.ORDER: None, + # no emitter yet -- arrives with suffix/nickname refinement (2.x) + AmbiguityKind.SUFFIX_OR_NICKNAME: None, +} + + +@pytest.mark.parametrize("kind", [ + pytest.param(k, marks=pytest.mark.xfail( + strict=True, reason=f"{k.value}: emitter not yet implemented")) + if k in _AMBIGUITY_TRIGGERS and _AMBIGUITY_TRIGGERS[k] is None else k + for k in AmbiguityKind +]) +def test_every_ambiguity_kind_has_a_registered_trigger( + kind: AmbiguityKind) -> None: + assert kind in _AMBIGUITY_TRIGGERS, ( + f"new AmbiguityKind {kind.value!r} needs a canonical trigger " + f"(or an explicit None with its planned emitter)") + trigger = _AMBIGUITY_TRIGGERS[kind] + assert trigger is not None # None triggers are strict-xfail marked + kinds = {a.kind for a in parse(trigger).ambiguities} + assert kind in kinds + + +_PATRONYMIC_TRIGGERS: dict[PatronymicRule, tuple[str, str]] = { + # rule -> (input, expected given) + PatronymicRule.EAST_SLAVIC: ("Сидоров Иван Петрович", "Иван"), + PatronymicRule.TURKIC: ("Mammadova Aygun Ali kizi", "Aygun"), +} + + +@pytest.mark.parametrize("rule", list(PatronymicRule)) +def test_every_patronymic_rule_has_a_trigger(rule: PatronymicRule) -> None: + assert rule in _PATRONYMIC_TRIGGERS + text, expected_given = _PATRONYMIC_TRIGGERS[rule] + p = Parser(policy=Policy(patronymic_rules=frozenset({rule}))) + assert p.parse(text).given == expected_given + + +_TAG_TRIGGERS: dict[str, tuple[str, str]] = { + # tag -> (input, token text carrying the tag) + "particle": ("Juan de la Vega", "de"), + "conjunction": ("Mr. and Mrs. John Smith", "and"), + "initial": ("John A. Smith", "A."), +} + + +@pytest.mark.parametrize("tag", sorted(STABLE_TAGS)) +def test_every_stable_tag_has_a_trigger(tag: str) -> None: + assert tag in _TAG_TRIGGERS + text, token_text = _TAG_TRIGGERS[tag] + pn = parse(text) + tagged = next(t for t in pn.tokens if t.text == token_text) + assert tag in tagged.tags diff --git a/tests/v2/test_properties.py b/tests/v2/test_properties.py new file mode 100644 index 0000000..157116d --- /dev/null +++ b/tests/v2/test_properties.py @@ -0,0 +1,52 @@ +"""Property layer (core spec §7.3). Hypothesis is a dev dependency only. + +The alphabet is punctuation-heavy on purpose: plain st.text() spreads +over all of Unicode, so commas, quotes, and delimiters almost never +appear and the interesting planes go unexercised. derandomize=True +keeps runs reproducible on shared CI runners -- this layer guards +against regressions; exploratory fuzzing happened during review. +""" +from hypothesis import given, settings +from hypothesis import strategies as st + +from nameparser import parse + +_ALPHABET = st.sampled_from( + 'abcdefgh ABC .,،，\'"()«»‏‏\U0001f600éñßЖ-') + + +@given(st.text(alphabet=_ALPHABET, max_size=200)) +@settings(max_examples=300, deadline=None, derandomize=True) +def test_parse_never_raises_on_str(text: str) -> None: + parse(text) + + +@given(st.text(alphabet=_ALPHABET, max_size=200)) +@settings(max_examples=300, deadline=None, derandomize=True) +def test_provenance_for_parser_produced_names(text: str) -> None: + pn = parse(text) + for t in pn.tokens: + assert t.span is not None + assert t.text == pn.original[t.span.start:t.span.end] + + +@given(st.text(alphabet=_ALPHABET, max_size=100)) +@settings(max_examples=200, deadline=None, derandomize=True) +def test_capitalized_idempotent(text: str) -> None: + once = parse(text).capitalized() + assert once.capitalized() == once + + +@given(st.text(alphabet=_ALPHABET, max_size=100)) +@settings(max_examples=200, deadline=None, derandomize=True) +def test_render_reparse_reaches_fixpoint(text: str) -> None: + # render/reparse legitimately takes several rounds to stabilize on + # comma-heavy input (each round can re-segment); the invariant is + # BOUNDED CONVERGENCE, not one-step idempotence + s = str(parse(text)) + for _ in range(10): + nxt = str(parse(s)) + if nxt == s: + break + s = nxt + assert str(parse(s)) == s, f"no fixpoint within 10 rounds: {s!r}" diff --git a/uv.lock b/uv.lock index fe84787..7ac65e1 100644 --- a/uv.lock +++ b/uv.lock @@ -321,6 +321,67 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f4/b2/50e9b292b5cac13e9e81272c7171301abc753a60460d21505b606e15cf21/furo-2025.12.19-py3-none-any.whl", hash = "sha256:bb0ead5309f9500130665a26bee87693c41ce4dbdff864dbfb6b0dae4673d24f", size = 339262, upload-time = "2025-12-19T17:34:38.905Z" }, ] +[[package]] +name = "hypothesis" +version = "6.156.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/20/83/8dbe89bdb8c6f25a7a52e7898af6d82fe35dfef08e5c702f6e33231ce6c6/hypothesis-6.156.6.tar.gz", hash = "sha256:96de02faefa3ce079873541da96f42595583bb001e8e4219294ed7d4501cc4cc", size = 476304, upload-time = "2026-07-10T20:56:49.96Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1a/dc/0c2a851f06c91d5ac9ef0f3b9615efc1ed650411d2eee23b6334f491c85e/hypothesis-6.156.6-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:caf6a93d011c10972da111c38ceb34ced20feaa8581e2b350c0655b022e27875", size = 747998, upload-time = "2026-07-10T20:56:16.311Z" }, + { url = "https://files.pythonhosted.org/packages/8e/f8/59203ca978ab51595d12d6bc7e7a63300d7373431ab42ca3f1742e45db68/hypothesis-6.156.6-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:07f2bc9df1aeba80e12029c1618e2ee54abc440068c305d7075ffd6b85251843", size = 743073, upload-time = "2026-07-10T20:55:36.825Z" }, + { url = "https://files.pythonhosted.org/packages/68/d8/86a0023740434098d1b187a62bd5f99b198f098fb43e7fc58342283a8270/hypothesis-6.156.6-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7baca17f4803ad4aa151732326f3990baf54c3127df44aa872ac5bdf8a98a9a6", size = 1070169, upload-time = "2026-07-10T20:55:49.47Z" }, + { url = "https://files.pythonhosted.org/packages/9b/82/673453915fd0c67673f35a4876ba88f48c621335f293f3537d77b27d4286/hypothesis-6.156.6-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8083806645f84243aade727f4978185caaa0b7190af4318673999ee15fdbf424", size = 1121760, upload-time = "2026-07-10T20:55:53.502Z" }, + { url = "https://files.pythonhosted.org/packages/8a/c3/3a5557f52912f2fecc6ed59642dcf80dd8e89d0d9664502b68e23d66bf3d/hypothesis-6.156.6-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a922eedcd8618f9c2e17b79fa7b3f3f0b2df34e201958611cc3f0f46cca33c10", size = 1111440, upload-time = "2026-07-10T20:55:43.054Z" }, + { url = "https://files.pythonhosted.org/packages/38/a6/ae636d4ca7f996a1ccb4b3d5997d949f1718fba52b01559b3ab53b237b3f/hypothesis-6.156.6-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:5291bd33c4704d274d7c214d5c200e77f372a06644f5cbbe96dcbe53cb2fbf10", size = 1244944, upload-time = "2026-07-10T20:55:56.109Z" }, + { url = "https://files.pythonhosted.org/packages/1e/79/c425d22d734be0268ca60d120c6296299e4220a1783cb1a4cc76232807bb/hypothesis-6.156.6-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:55f3ec50161b4a95bae63bff2b5166e45935b493013d3be30ede279bf6192318", size = 1288808, upload-time = "2026-07-10T20:56:06.249Z" }, + { url = "https://files.pythonhosted.org/packages/c5/3a/cc9f479d22cbdd36ddfc55a978378eddadd183b09339ebdb81be33bb18e7/hypothesis-6.156.6-cp310-abi3-win32.whl", hash = "sha256:e96570ca5cdd9a5f2ff9e80a6fb2fd5420ebf33b833d7de5b09b6ebb26a3eb6c", size = 634868, upload-time = "2026-07-10T20:55:37.959Z" }, + { url = "https://files.pythonhosted.org/packages/d6/89/2008d287289841a936456cb13443ca89d88da6e4527d611d482e9544164d/hypothesis-6.156.6-cp310-abi3-win_amd64.whl", hash = "sha256:32710718c22fe8c5571464e898bb87d282837b02617d6ad68130abf7cb4843cb", size = 640382, upload-time = "2026-07-10T20:55:30.634Z" }, + { url = "https://files.pythonhosted.org/packages/13/64/e4a0796190d8089e85f06731e21fdddd7e8edd3a4e562101527a048e21c4/hypothesis-6.156.6-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:5baec7943a14d106e982121dd4f74cfc5ef45e37c17f94fe49338d3d1377f38c", size = 748988, upload-time = "2026-07-10T20:56:48.22Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a2/4a789b286cd2cced31992e1f683036b51dd6909b934ea007ffb43aa3a32f/hypothesis-6.156.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b5f519905ddeb10e23b8ba2c254541a5b1a8f146fe0551be94d972f4a77226f4", size = 743754, upload-time = "2026-07-10T20:56:09.113Z" }, + { url = "https://files.pythonhosted.org/packages/d2/f7/3dd36c1c03d24ae3ffc3c5b0eca8cc4ae90c07abc320f76509eceb37019a/hypothesis-6.156.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c108655960b58ded3ca71b2dc5c69fb2ba7e9c723aeb6106facec3892d09087", size = 1070732, upload-time = "2026-07-10T20:56:28.738Z" }, + { url = "https://files.pythonhosted.org/packages/57/51/befc4b816b471078034a875eb1ef69e0411ab84bcce582b4be173258785a/hypothesis-6.156.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7c8d37bebb6924729bc0bbf5852689df568842948abe4d93dd0ad3377adf76fa", size = 1121988, upload-time = "2026-07-10T20:56:07.676Z" }, + { url = "https://files.pythonhosted.org/packages/05/b7/a796f5e3e4b7cb911ff346008d49720296d1f4073490b8bc1cce6b3fbb07/hypothesis-6.156.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:74137bef6d502305c3648b2ed1a9bb4bc05fb1025e96b30a2c092204c40fe097", size = 1245596, upload-time = "2026-07-10T20:55:50.662Z" }, + { url = "https://files.pythonhosted.org/packages/37/65/849c4cba44a6f6cc888fd931124429b24180234ccc4883abab8cad5fcfcb/hypothesis-6.156.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5e0afdf79cceed20fcf0a9fb80d4064a9b2b53d4d4eecbac0e21208a13f5a31b", size = 1289172, upload-time = "2026-07-10T20:55:58.915Z" }, + { url = "https://files.pythonhosted.org/packages/13/03/7106a110df29eb631d66776e8aa8128f82f04a9dd2b6b22b612e6025e3a2/hypothesis-6.156.6-cp311-cp311-win_amd64.whl", hash = "sha256:84dc89caaf741a02f904ca7bd02b1af99650c75552868162290208aeecb70858", size = 640222, upload-time = "2026-07-10T20:56:10.396Z" }, + { url = "https://files.pythonhosted.org/packages/8c/45/9f009005b9c796f4a40424484ac7e70847bc088456fd940a937f96bb4b6d/hypothesis-6.156.6-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a2a728b514fceb81e3f0464508911d5220fd74dadc3270f859427a686b60c4cf", size = 748844, upload-time = "2026-07-10T20:56:38.036Z" }, + { url = "https://files.pythonhosted.org/packages/02/2f/4d852bb8a9c73a68b18eca9b5b085285282122166e158f4d2a477639bfee/hypothesis-6.156.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7489b9a8f9df8227edd6c7cd8b9ccfab2483bab24da6a474c175973ca2294f58", size = 741936, upload-time = "2026-07-10T20:55:27.539Z" }, + { url = "https://files.pythonhosted.org/packages/74/89/b9968070ae042f9bf3149bb6ba6399d5f28f452e0fb7f638cafc69ff0b9a/hypothesis-6.156.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42760873d6db1069d6edbaa355a61b9078a9950259efcfc72fc695741d7db7cd", size = 1069749, upload-time = "2026-07-10T20:56:43.017Z" }, + { url = "https://files.pythonhosted.org/packages/00/a9/753806f5292b40aeab1d269e408e3a7e85be3c0d88828fb78ab4a34d6626/hypothesis-6.156.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b4e66aaa7385538a5d617174d47c198ee807f06de99e282a67c6cb724c69340d", size = 1120983, upload-time = "2026-07-10T20:56:25.424Z" }, + { url = "https://files.pythonhosted.org/packages/85/88/8386d064d680be27e936eba94f1448bc93ef6fa05473ee5034139f1c4284/hypothesis-6.156.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:08796b674c0b31a5dd4119b2173823390055921588d13eb77324e861b00fd7f8", size = 1243911, upload-time = "2026-07-10T20:55:54.799Z" }, + { url = "https://files.pythonhosted.org/packages/c1/8c/7524c1e5279e7728eb47c99f2357cbc5f08ae92e9bce49bf50118b53f9c9/hypothesis-6.156.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4ca8cc26ea2d31d22cf7710e92951cfaa921f0f8aa1b6db33a5176335f583a4f", size = 1287806, upload-time = "2026-07-10T20:56:02.176Z" }, + { url = "https://files.pythonhosted.org/packages/5a/b3/c347ad913e1c5f2988956fe17826c0400b4ce470b973e6c248e97b6a0acf/hypothesis-6.156.6-cp312-cp312-win_amd64.whl", hash = "sha256:c3363d3fb8015594636689572510bb6090602d8e8e838a5693c2d52d3b5b09d8", size = 637679, upload-time = "2026-07-10T20:55:39.056Z" }, + { url = "https://files.pythonhosted.org/packages/70/5d/9583fe153573523dac27226c89e041a86ad4aeeae08c868160cbb93d39d2/hypothesis-6.156.6-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:59a8def90d9a5a9b67e1ac529e903a2363ceb6cf873c209da6b4284c5daab671", size = 749264, upload-time = "2026-07-10T20:56:46.118Z" }, + { url = "https://files.pythonhosted.org/packages/86/35/e4113d06769b544f0fb77ffea9195b598b4c56a298905c21fd47c4eed388/hypothesis-6.156.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c574c3224563d730848bc5d1ef1683c4f83993400c0167899fe328f4bfcd4725", size = 742095, upload-time = "2026-07-10T20:56:41.412Z" }, + { url = "https://files.pythonhosted.org/packages/d8/5c/a47666ede10384e8978722cade7ab96a42df71d2ab577317092d0fed7c8a/hypothesis-6.156.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01bb8270c46b3ef53b0c2d23ff613ea506d609d06f936d823ea57c58b66b05f7", size = 1069917, upload-time = "2026-07-10T20:56:04.845Z" }, + { url = "https://files.pythonhosted.org/packages/79/93/75f6057dadd9dc0134f37c08d5d14d04d3cd7374debbcb0cc4569c6712f1/hypothesis-6.156.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d4ea6559c13606e13b645927f2e0906e52b5ac5d99b40d3abaaeb2e8c7ceeb75", size = 1121204, upload-time = "2026-07-10T20:55:52.008Z" }, + { url = "https://files.pythonhosted.org/packages/62/87/308efef08bc60d1e673d035e8ca8e9663f4b6b3ba519c3cdebf6583c2b76/hypothesis-6.156.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:2d47054d0230f0dd9b6868fc030126c7a6c25527144272ff376cc4e9c39f7540", size = 1244168, upload-time = "2026-07-10T20:55:40.288Z" }, + { url = "https://files.pythonhosted.org/packages/3b/66/de8fff5bd9a40a4056dafbe7f904887ef12632282bbbac90f1977c30dd3b/hypothesis-6.156.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:050c8c0815f88d47dd0875a92698d20d61639b7b721ee043a6d687c7f14ff7d8", size = 1288127, upload-time = "2026-07-10T20:56:00.541Z" }, + { url = "https://files.pythonhosted.org/packages/cf/8d/794fb26e1fd3ff004978f8f18b7aa7e1c2270ba72e1f977b987a812064f8/hypothesis-6.156.6-cp313-cp313-win_amd64.whl", hash = "sha256:f0d73edab7b8a0051b3634f2d04d62b7e7282f8f274963b11188ee4957d672ef", size = 637954, upload-time = "2026-07-10T20:56:33.35Z" }, + { url = "https://files.pythonhosted.org/packages/a5/be/5b4b27984cb43c60e95f570b069660335dad34cb67f7d226017c5d35d31e/hypothesis-6.156.6-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:34a70a7b8226e34d658072d8fb81d03f97f0a75ceb536329a321b94ce2232fd6", size = 749312, upload-time = "2026-07-10T20:55:46.902Z" }, + { url = "https://files.pythonhosted.org/packages/31/11/709cceffc28666c9d4cb75ffc6df5ce30db8c7dd5cc2c8b38a2fd837427f/hypothesis-6.156.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:f1969646beead7d8cf6a2537d2765af89d73056e2cb218e7fae92b83802250a3", size = 742332, upload-time = "2026-07-10T20:56:30.254Z" }, + { url = "https://files.pythonhosted.org/packages/84/52/cfc79b13d8dd3cd6de6b9df921c557efe8528a9c90a3a7cd93b37188d57e/hypothesis-6.156.6-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cbc2ec7b7d905e6b6ec1635f6340bfa52aaab718101c59f052bc012a6b486cd8", size = 1070109, upload-time = "2026-07-10T20:55:48.244Z" }, + { url = "https://files.pythonhosted.org/packages/ca/ac/1da4def1f006b5ad01187ff96379e24c37439d659ec10c3e944c03436c0f/hypothesis-6.156.6-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9367ae25dfa6dc1af37904785e43c4f8fe1c4118cafdc2f06514154fbdd90992", size = 1121528, upload-time = "2026-07-10T20:56:39.665Z" }, + { url = "https://files.pythonhosted.org/packages/68/47/744e4f5e3d635dea20dbedf3fa486e2a6fa5210e0a52a0d5c4da56babd84/hypothesis-6.156.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:455f09107ec07c78f2a83cb8fc19e23879c9d51cdc831de6f9cb6ec4059cb9af", size = 1244690, upload-time = "2026-07-10T20:56:31.854Z" }, + { url = "https://files.pythonhosted.org/packages/25/8a/42252fcd5e521d140dac532f29c2a13ca8f22908cb545ffdd64b5e225680/hypothesis-6.156.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c76634c45a3ceee4c4fdfed39aebd08b8b822ec8b0c556877ef82846fd777730", size = 1288519, upload-time = "2026-07-10T20:56:03.429Z" }, + { url = "https://files.pythonhosted.org/packages/44/e7/176df9e47cf583d2b8d234b78c0aac3a47075ad5d147e60b2c21a1338bb1/hypothesis-6.156.6-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:eb7e9f8343bc6b948937e6ec12e6879ed25a17b53ceccbd2b84adadd3d511698", size = 586452, upload-time = "2026-07-10T20:56:22.285Z" }, + { url = "https://files.pythonhosted.org/packages/8c/75/2c8a0411bbe76429f3ae738ef9a00107201bf6146d9534350014ce369d98/hypothesis-6.156.6-cp314-cp314-win_amd64.whl", hash = "sha256:f9631cd604ae6032c3edf99160dc1b9e33873f2e52762246b24f07fb758652ae", size = 637774, upload-time = "2026-07-10T20:56:34.97Z" }, + { url = "https://files.pythonhosted.org/packages/2a/22/8115005e9aa72c8d63d90e9db5e0b8425fd8950fbc5d6e332805d4d32c9e/hypothesis-6.156.6-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:1f81163d36d3763b09ffaef7c3a71e88174ca3e6816201fca9d1d159f448fdb5", size = 747428, upload-time = "2026-07-10T20:56:44.611Z" }, + { url = "https://files.pythonhosted.org/packages/e8/c2/66bfe9337f4a4b1f7754ee6d01d950280152a81d0d797e6c1d9eb0909750/hypothesis-6.156.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:556b905767e36147918634a64356aa05d8c956576f00aee01eb351678f193908", size = 740889, upload-time = "2026-07-10T20:55:57.656Z" }, + { url = "https://files.pythonhosted.org/packages/95/3b/69f45af2d4f0950b7d1af3cdbdd800b88a6c2370331481eda79d6171fbe3/hypothesis-6.156.6-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3f2604b28d16d696aaaf4954d20f907b27e54034df98e64746a20c74c319f03", size = 1069270, upload-time = "2026-07-10T20:56:12.024Z" }, + { url = "https://files.pythonhosted.org/packages/f8/43/6b2549885da08f5e50ba34fb8e0d0a60b2f190ffd516fac220f8db5b5869/hypothesis-6.156.6-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffe012ad66dbe7b8e8ddef6f6992ab1b36719ea64430c2bf1ff7135521052a15", size = 1120409, upload-time = "2026-07-10T20:55:34.551Z" }, + { url = "https://files.pythonhosted.org/packages/70/97/745c778c3eb29befa2367b1ded8437eecfbbe6932359d0f831275bc32170/hypothesis-6.156.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5bfa3c7b758f7278081c6bfec5f89b43c4eb075c0c9eb095323f7a9eb019b513", size = 1243111, upload-time = "2026-07-10T20:56:17.83Z" }, + { url = "https://files.pythonhosted.org/packages/ab/d7/c5ec6a442dc9b822f47064bda4b6d3e739dccdd1c5bf44c9a57fb6136830/hypothesis-6.156.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:d0db1f4573800c618773622f03cb6533bb3377430ef938c9476ba10c39d22591", size = 1287262, upload-time = "2026-07-10T20:56:23.749Z" }, + { url = "https://files.pythonhosted.org/packages/11/0c/c134d61710e14b68b010215dcf6bd57d2ec05cd169dff8bfab8fefc2d410/hypothesis-6.156.6-cp314-cp314t-win_amd64.whl", hash = "sha256:38cd0c4a7b9f809f1e23a4d15adfa9c5d99869b9afc327350a5e563350b78e48", size = 637862, upload-time = "2026-07-10T20:56:13.347Z" }, + { url = "https://files.pythonhosted.org/packages/59/9c/b94f3a31665527b6181616b72990fcf8d6d5fa82b4187aab104ab5f548f0/hypothesis-6.156.6-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:8893b4da90e06828846c1b100c3414a7729d047a020d854c0899ae9339df0e70", size = 749575, upload-time = "2026-07-10T20:55:29.371Z" }, + { url = "https://files.pythonhosted.org/packages/21/74/dcf695f79f526543ae5d0f8c1325508e9fe990a996c0e0853129a9a5d81d/hypothesis-6.156.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:b7fc0b7df9b28d028e4cc295b2ac8fbbbc22e090a23382c92fff5e37696be74f", size = 744351, upload-time = "2026-07-10T20:56:36.46Z" }, + { url = "https://files.pythonhosted.org/packages/11/d3/5bff4c55c6995a6c43f66ec8e5866b56e34f03837fd0be0e4922f3bab168/hypothesis-6.156.6-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38ed3178526382d392d04ad699ad7a2e53845e521a09d40f1cbbc1e1ff63ba48", size = 1070916, upload-time = "2026-07-10T20:56:19.256Z" }, + { url = "https://files.pythonhosted.org/packages/ba/af/5ed42117a69221ea118caaff933d8212039a0ac0bc15afa915635f13984c/hypothesis-6.156.6-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ad94e28aabf4db0d479297d43b8a2a01e7caaa9bdfccfdac7a4a3717e05b993", size = 1122625, upload-time = "2026-07-10T20:56:14.758Z" }, + { url = "https://files.pythonhosted.org/packages/3b/d3/02499badc6e3f3e980941021edf5fd780c895d8d08c9015e78516340ed83/hypothesis-6.156.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:983a5cfd955994bffc7eb02976241f7a1f3c2d94dbc3389430c45858fa5c1ae0", size = 640823, upload-time = "2026-07-10T20:55:45.461Z" }, +] + [[package]] name = "idna" version = "3.18" @@ -575,6 +636,7 @@ source = { editable = "." } dev = [ { name = "dill" }, { name = "furo" }, + { name = "hypothesis" }, { name = "mypy" }, { name = "pytest" }, { name = "pytest-cov" }, @@ -590,6 +652,7 @@ dev = [ dev = [ { name = "dill", specifier = ">=0.2.5" }, { name = "furo" }, + { name = "hypothesis" }, { name = "mypy", specifier = ">=2.1" }, { name = "pytest", specifier = ">=8" }, { name = "pytest-cov", specifier = ">=6" }, @@ -734,6 +797,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4c/07/2ebca9b11fb9be7340a818d8d6f63feaebb146be2c4afbd6061701d6df6e/snowballstemmer-3.1.1-py3-none-any.whl", hash = "sha256:7e207fa178741da09cdee59d3ecec3827ad5f92b1fc5c9ff3755b639f71f5752", size = 104164, upload-time = "2026-06-03T00:56:38.614Z" }, ] +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, +] + [[package]] name = "soupsieve" version = "2.8.4" From bacb42ba2376602c009a349138573bbd239eb526 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 13 Jul 2026 02:28:22 -0700 Subject: [PATCH 82/88] Apply type-design review fixes to the pipeline state contract The ParseState ownership map said roles belong to assign/post_rules exclusively, but group writes maiden roles -- an ownership contract wrong in one place teaches readers to distrust it everywhere. Fixed the map, noted post-group segments staleness and the sole-producer text invariant on WorkToken, documented PendingAmbiguity's dangling- index tolerance and Parser's None-default resolution, made assemble's role fallback say what it means, and pinned the whole ownership map mechanically: a new test folds the case corpus stage by stage and asserts each stage changes only the fields it owns. Co-Authored-By: Claude Fable 5 --- nameparser/_parser.py | 5 ++++- nameparser/_pipeline/_assemble.py | 3 ++- nameparser/_pipeline/_state.py | 11 +++++++--- tests/v2/pipeline/test_state.py | 36 +++++++++++++++++++++++++++++++ 4 files changed, 50 insertions(+), 5 deletions(-) diff --git a/nameparser/_parser.py b/nameparser/_parser.py index b94039c..21e6073 100644 --- a/nameparser/_parser.py +++ b/nameparser/_parser.py @@ -26,7 +26,10 @@ class Parser: """Immutable, thread-safe, picklable by construction (spec §4): all validity checking happens at construction; a Parser that constructs - successfully cannot fail at parse time on any str content.""" + successfully cannot fail at parse time on any str content. The None + field defaults resolve in __post_init__; after construction both + fields are always non-None (the annotations state the steady-state + truth, hence the assignment ignores on the defaults).""" lexicon: Lexicon = None # type: ignore[assignment] # None -> default() policy: Policy = None # type: ignore[assignment] # None -> Policy() diff --git a/nameparser/_pipeline/_assemble.py b/nameparser/_pipeline/_assemble.py index 0eb4897..485f0ad 100644 --- a/nameparser/_pipeline/_assemble.py +++ b/nameparser/_pipeline/_assemble.py @@ -23,7 +23,8 @@ def assemble(state: ParseState) -> ParsedName: for i, t in enumerate(state.tokens): if i in dropped: continue - final[i] = Token(t.text, t.span, t.role or Role.GIVEN, t.tags) + role = t.role if t.role is not None else Role.GIVEN + final[i] = Token(t.text, t.span, role, t.tags) ambiguities = tuple( Ambiguity(p.kind, p.detail, tuple(final[i] for i in p.indices if i in final)) diff --git a/nameparser/_pipeline/_state.py b/nameparser/_pipeline/_state.py index d2a8e2e..afa5092 100644 --- a/nameparser/_pipeline/_state.py +++ b/nameparser/_pipeline/_state.py @@ -21,7 +21,9 @@ @dataclass(frozen=True, slots=True) class WorkToken: """One tokenized word. role stays None until assign; extracted - nickname/maiden tokens arrive with their role pre-set.""" + nickname/maiden tokens arrive with their role pre-set. text is + always the exact original slice (tokenize is the sole producer; + the anti-#100 invariant depends on it).""" text: str span: Span @@ -53,8 +55,11 @@ class ParseState: dataclasses.replace. Fields are filled progressively: extract_delimited -> extracted/masked; tokenize -> tokens (span- sorted)/comma_offsets; segment -> segments/structure; classify -> - token tags; group -> pieces/dropped; assign/post_rules -> token - roles; ambiguities accumulate anywhere.""" + token tags; group -> pieces/piece_tags/dropped AND maiden token + roles; assign/post_rules -> the remaining token roles; ambiguities + accumulate anywhere. Post-group, segments may retain indices of + dropped tokens -- assign iterates pieces, never segments. This + ownership map is pinned by tests/v2/pipeline/test_state.py.""" original: str lexicon: Lexicon diff --git a/tests/v2/pipeline/test_state.py b/tests/v2/pipeline/test_state.py index bbda440..b01433c 100644 --- a/tests/v2/pipeline/test_state.py +++ b/tests/v2/pipeline/test_state.py @@ -29,3 +29,39 @@ def test_state_is_frozen_and_replace_works() -> None: def test_worktoken_carries_optional_role() -> None: t = WorkToken("Jack", Span(6, 10), role=Role.NICKNAME) assert t.role is Role.NICKNAME + + +def test_stage_field_ownership() -> None: + # The ParseState docstring's ownership map, pinned mechanically: run + # the whole case corpus through the fold stage by stage and assert + # each stage only changes the fields it owns. Converts the prose + # contract into a test (a future stage clobbering another stage's + # field fails here, not in a distant assertion). + import dataclasses as _dc + + from nameparser import Lexicon as _Lexicon + from nameparser._pipeline import STAGES + + from ..cases import CASES + + ownership = { + "extract_delimited": {"extracted", "masked", "ambiguities"}, + "tokenize": {"tokens", "comma_offsets"}, + "segment": {"segments", "structure", "ambiguities"}, + "classify": {"tokens"}, + "group": {"tokens", "pieces", "piece_tags", "dropped"}, + "assign": {"tokens", "ambiguities"}, + "post_rules": {"tokens"}, + } + assert {s.__name__ for s in STAGES} == set(ownership) + for case in CASES: + state = ParseState(original=case.text, lexicon=_Lexicon.default(), + policy=case.policy or Policy()) + for stage in STAGES: + before = {f.name: getattr(state, f.name) + for f in _dc.fields(state)} + state = stage(state) + changed = {name for name, value in before.items() + if getattr(state, name) != value} + assert changed <= ownership[stage.__name__], ( + f"{case.id}: {stage.__name__} changed {changed - ownership[stage.__name__]}") From f65bd7ced6e504801227224ff1b65db9e21564de Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 13 Jul 2026 02:54:12 -0700 Subject: [PATCH 83/88] Exclude the ambiguous subset from the plain suffix-acronym test In the real data suffix_acronyms_ambiguous is a SUBSET of suffix_acronyms (v1 shape), so the plain membership test in classify and is_suffix_strict fired before the period gate could withhold vocab:suffix -- the gate was dead code for the overlap members ({'ed', 'jd'}), and 'Smith, Ed' silently dropped the given name into suffix. Both sites now exclude the ambiguous subset, Lexicon enforces the subset invariant like particles_ambiguous, the classify/vocab fixtures mirror the real subset shape instead of hiding the bug with disjoint sets, and three v1-verified case rows pin the parity. Found by cross-session PR review (C1/I1). Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 7 +++++++ nameparser/_pipeline/_classify.py | 8 +++++++- nameparser/_pipeline/_vocab.py | 5 ++++- tests/v2/cases.py | 7 +++++++ tests/v2/pipeline/test_classify.py | 12 +++++++++++- tests/v2/pipeline/test_vocab.py | 8 +++++++- tests/v2/test_lexicon.py | 8 ++++++++ 7 files changed, 51 insertions(+), 4 deletions(-) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index a1a08fd..1647546 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -149,6 +149,13 @@ def __post_init__(self) -> None: f"particles_ambiguous must be a subset of particles; " f"not in particles: {extra}" ) + if not self.suffix_acronyms_ambiguous <= self.suffix_acronyms: + extra = ", ".join(sorted( + self.suffix_acronyms_ambiguous - self.suffix_acronyms)) + raise ValueError( + f"suffix_acronyms_ambiguous must be a subset of " + f"suffix_acronyms; not in suffix_acronyms: {extra}" + ) # -- constructors ---------------------------------------------------- diff --git a/nameparser/_pipeline/_classify.py b/nameparser/_pipeline/_classify.py index 60c3506..3322678 100644 --- a/nameparser/_pipeline/_classify.py +++ b/nameparser/_pipeline/_classify.py @@ -31,7 +31,13 @@ def _tags_for(token: WorkToken, state: ParseState) -> frozenset[str]: tags.add("vocab:title") if n in lex.given_name_titles: tags.add("vocab:given-title") - if n in lex.suffix_acronyms or n in lex.suffix_words: + # the ambiguous subset is EXCLUDED from the plain membership test: + # in the real data suffix_acronyms_ambiguous is a subset of + # suffix_acronyms, and without the exclusion the period gate below + # is dead code (bare 'Ed'/'Jd' would silently become suffixes) + if (n in lex.suffix_acronyms + and n not in lex.suffix_acronyms_ambiguous) \ + or n in lex.suffix_words: tags.add("vocab:suffix") if n in lex.suffix_words: tags.add("vocab:suffix-word") diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index 48475fb..57c77cd 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -33,7 +33,10 @@ def is_suffix_strict(text: str, lexicon: Lexicon) -> bool: return True if is_initial(text): return False - return n in lexicon.suffix_acronyms or n in lexicon.suffix_words + # ambiguous subset excluded from the plain test (see _classify) + return (n in lexicon.suffix_acronyms + and n not in lexicon.suffix_acronyms_ambiguous) \ + or n in lexicon.suffix_words def is_suffix_lenient(text: str, lexicon: Lexicon) -> bool: diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 0157931..2becd4a 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -102,6 +102,13 @@ class Case: policy=_TK), Case("empty", "", {}), Case("whitespace", " ", {}), + Case("bare_ambiguous_acronym", "John Ed", + {"given": "John", "family": "Ed"}, + notes="'ed' is an ambiguous acronym; bare form is a name (C1)"), + Case("comma_ambiguous_acronym", "Smith, Ed", + {"given": "Ed", "family": "Smith"}), + Case("ambiguous_acronym_with_suffix", "John Ed III", + {"given": "John", "family": "Ed", "suffix": "III"}), Case("unbalanced_quote", 'Jon "Nick Smith', {"given": "Jon", "middle": '"Nick', "family": "Smith"}, ambiguities=("unbalanced-delimiter",), diff --git a/tests/v2/pipeline/test_classify.py b/tests/v2/pipeline/test_classify.py index 1c4f0a4..ef0ace1 100644 --- a/tests/v2/pipeline/test_classify.py +++ b/tests/v2/pipeline/test_classify.py @@ -9,7 +9,7 @@ _LEX = Lexicon( titles=frozenset({"dr", "sir"}), given_name_titles=frozenset({"sir"}), - suffix_acronyms=frozenset({"phd"}), + suffix_acronyms=frozenset({"phd", "ma"}), suffix_words=frozenset({"jr", "v"}), suffix_acronyms_ambiguous=frozenset({"ma"}), particles=frozenset({"de", "la", "van"}), @@ -66,3 +66,13 @@ def test_v_is_suffix_word_and_initial() -> None: # both tags present; assign applies the veto, not classify out = _classified("John V Smith") assert {"vocab:suffix", "vocab:suffix-word", "initial"} <= _tags(out, "V") + + +def test_bare_ambiguous_acronym_in_acronyms_is_not_suffix() -> None: + # the default lexicon has suffix_acronyms_ambiguous SUBSET OF + # suffix_acronyms (v1 data shape); the plain membership test must + # exclude the ambiguous members or the period gate is dead code + # and bare 'Ed'/'Jd' silently become suffixes (PR review C1) + out = _classified("Ma M.A.") + assert "vocab:suffix" not in _tags(out, "Ma") + assert "vocab:suffix" in _tags(out, "M.A.") diff --git a/tests/v2/pipeline/test_vocab.py b/tests/v2/pipeline/test_vocab.py index 54d4cfc..a8e2132 100644 --- a/tests/v2/pipeline/test_vocab.py +++ b/tests/v2/pipeline/test_vocab.py @@ -2,7 +2,7 @@ from nameparser._pipeline._vocab import is_initial, is_suffix_lenient, is_suffix_strict _LEX = Lexicon( - suffix_acronyms=frozenset({"phd"}), + suffix_acronyms=frozenset({"phd", "ma"}), suffix_words=frozenset({"jr", "v"}), suffix_acronyms_ambiguous=frozenset({"ma"}), ) @@ -32,3 +32,9 @@ def test_lenient_accepts_suffix_words_unconditionally() -> None: assert is_suffix_lenient("V", _LEX) assert is_suffix_lenient("V.", _LEX) assert not is_suffix_lenient("Ma", _LEX) + + +def test_strict_excludes_bare_ambiguous_even_when_in_acronyms() -> None: + # mirrors the real data shape: ambiguous is a SUBSET of acronyms + assert not is_suffix_strict("Ma", _LEX) + assert is_suffix_strict("M.A.", _LEX) diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index 99bf095..690fc92 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -186,3 +186,11 @@ def test_normalization_casefolds_and_strips_interior_periods() -> None: # A "simplify to .lower()/.strip('.')" regression must fail here. lex = Lexicon(titles=frozenset({"STRAßE", "Ph.D"})) assert lex.titles == frozenset({"strasse", "phd"}) + + +def test_suffix_ambiguous_must_be_subset_of_acronyms() -> None: + # same invariant as particles_ambiguous <= particles: the ambiguous + # set marks a subset of suffix_acronyms, and classify's period gate + # depends on the membership tests agreeing about it + with pytest.raises(ValueError, match="subset"): + Lexicon(suffix_acronyms_ambiguous=frozenset({"ma"})) From c66be143b72db8f924ae00d24fdde47f94525cf9 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 13 Jul 2026 02:57:03 -0700 Subject: [PATCH 84/88] Heal the split Ph. D. credential across the pipeline (I2/I3) The group-stage merge reunited the two tokens as a piece, but the suffix view joins SUFFIX tokens with ', ' -- 'John Ph. D.' rendered as suffix 'Ph., D.' -- and a mid-name credential never reached the trailing suffix peel, landing in MIDDLE. Continuation tokens of a suffix-merged piece now carry a new stable 'joined' tag (spec's provenance-tag sketch anticipated it) which the suffix view uses to attach with a space; assign peels group-flagged suffix pieces at ANY position, matching v1's fix_phd which extracted the credential before parsing. Contract-test trigger added for the new stable tag; two v1-verified case rows pin the behavior. Found by cross-session PR review (I2/I3). Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/_assign.py | 9 +++++++++ nameparser/_pipeline/_group.py | 8 ++++++++ nameparser/_types.py | 20 ++++++++++++++------ tests/v2/cases.py | 6 ++++++ tests/v2/pipeline/test_group.py | 5 +++++ tests/v2/test_contracts.py | 1 + tests/v2/test_parser.py | 15 +++++++++++++++ 7 files changed, 58 insertions(+), 6 deletions(-) diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index 706294d..e4a9aa2 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -92,6 +92,15 @@ def _assign_main(seg_idx: int, state: ParseState, continue break rest = list(range(n, len(pieces))) + if not rest: + return + # group-flagged suffix pieces (the ph-d merge) are suffixes at ANY + # position -- v1's fix_phd extracted the credential from the string + # before parsing, so position never mattered (PR review I3) + flagged = [k for k in rest if "suffix" in ptags[k]] + for k in flagged: + _set_roles(tokens, pieces[k], Role.SUFFIX) + rest = [k for k in rest if "suffix" not in ptags[k]] if not rest: return # v1 nickname rule (plan deviation #2) diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index 4023c7e..c64a779 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -182,6 +182,14 @@ def group(state: ParseState) -> ParseState: additional = 1 if state.structure is Structure.FAMILY_COMMA else 0 for seg in state.segments: pieces, ptags = _group_segment(seg, additional, tuple(tokens)) + # continuation tokens of a suffix-merged piece (the ph-d merge) + # carry the stable "joined" tag: the suffix string view joins + # SUFFIX tokens with ", ", and the tag lets it heal the split + for piece, piece_tags_ in zip(pieces, ptags): + if "suffix" in piece_tags_ and len(piece) > 1: + for i in piece[1:]: + tokens[i] = dataclasses.replace( + tokens[i], tags=tokens[i].tags | {"joined"}) # maiden markers: a non-leading marker piece consumes following # pieces until a suffix; consumed tokens become MAIDEN, the # marker is dropped (#274) diff --git a/nameparser/_types.py b/nameparser/_types.py index af6789a..b8065b6 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -54,8 +54,10 @@ def __add__(self, other: object) -> NoReturn: # type: ignore[override] #: Stable, documented tag vocabulary (API). All other tags are -#: namespaced ("vocab:...", "patronymic:...") and unstable. -STABLE_TAGS = frozenset({"particle", "conjunction", "initial"}) +#: namespaced ("vocab:...", "patronymic:...") and unstable. "joined" +#: marks a continuation token of a merged piece (the ph-d merge) and +#: drives the suffix view's space-vs-comma join. +STABLE_TAGS = frozenset({"particle", "conjunction", "initial", "joined"}) _E = TypeVar("_E", bound=Enum) @@ -302,8 +304,8 @@ def __repr__(self) -> str: def _text_for(self, *roles: Role, tag: str | None = None, without_tag: str | None = None) -> str: - joiner = ", " if roles == (Role.SUFFIX,) else " " - parts = [] + suffix_join = roles == (Role.SUFFIX,) + parts: list[str] = [] for tok in self.tokens: if tok.role not in roles: continue @@ -311,8 +313,14 @@ def _text_for(self, *roles: Role, tag: str | None = None, continue if without_tag is not None and without_tag in tok.tags: continue - parts.append(tok.text) - return joiner.join(parts) + # "joined" (stable tag) marks a continuation of the previous + # token ("Ph." + "D."): attach with a space so the suffix + # view's ", " join does not split one credential in two + if suffix_join and "joined" in tok.tags and parts: + parts[-1] += " " + tok.text + else: + parts.append(tok.text) + return (", " if suffix_join else " ").join(parts) @property def title(self) -> str: diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 2becd4a..b0c174d 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -109,6 +109,12 @@ class Case: {"given": "Ed", "family": "Smith"}), Case("ambiguous_acronym_with_suffix", "John Ed III", {"given": "John", "family": "Ed", "suffix": "III"}), + Case("phd_split", "John Ph. D.", + {"given": "John", "suffix": "Ph. D."}, + notes="v1 fix_phd; healed via the stable 'joined' tag"), + Case("phd_split_mid_name", "Dr. John Ph. D. Smith", + {"title": "Dr.", "given": "John", "family": "Smith", + "suffix": "Ph. D."}), Case("unbalanced_quote", 'Jon "Nick Smith', {"given": "Jon", "middle": '"Nick', "family": "Smith"}, ambiguities=("unbalanced-delimiter",), diff --git a/tests/v2/pipeline/test_group.py b/tests/v2/pipeline/test_group.py index f5fecc4..0d3150d 100644 --- a/tests/v2/pipeline/test_group.py +++ b/tests/v2/pipeline/test_group.py @@ -90,6 +90,11 @@ def test_phd_split_across_tokens_merges_as_suffix() -> None: out = _grouped("John Smith Ph. D.") assert _piece_texts(out) == [["John", "Smith", "Ph. D."]] assert "suffix" in out.piece_tags[0][2] + # continuation tokens of the merged piece carry the stable "joined" + # tag so the suffix string view can heal the split (", " join would + # otherwise render 'Ph., D.') + d_tok = next(t for t in out.tokens if t.text == "D.") + assert "joined" in d_tok.tags def test_maiden_marker_consumes_tail() -> None: diff --git a/tests/v2/test_contracts.py b/tests/v2/test_contracts.py index 78f284d..be4381b 100644 --- a/tests/v2/test_contracts.py +++ b/tests/v2/test_contracts.py @@ -55,6 +55,7 @@ def test_every_patronymic_rule_has_a_trigger(rule: PatronymicRule) -> None: "particle": ("Juan de la Vega", "de"), "conjunction": ("Mr. and Mrs. John Smith", "and"), "initial": ("John A. Smith", "A."), + "joined": ("John Ph. D.", "D."), } diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index d82aa67..b215bd3 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -131,3 +131,18 @@ def test_matches_accepts_explicit_parser() -> None: pn = family_first.parse("Yamada Taro") assert pn.matches("Yamada Taro", parser=family_first) assert not pn.matches("Yamada Taro") # default parser reads given-first + + +def test_phd_split_heals_in_the_suffix_view() -> None: + # v1 parity via fix_phd: the split credential renders as one suffix + assert parse("John Ph. D.").suffix == "Ph. D." + assert parse("John Smith PhD MD").suffix == "PhD, MD" # unchanged + + +def test_phd_split_mid_name_is_a_suffix() -> None: + # v1 parity: fix_phd extracted the credential BEFORE parsing, so + # position never mattered; the merged piece is a suffix anywhere + pn = parse("Dr. John Ph. D. Smith") + assert pn.suffix == "Ph. D." + assert pn.family == "Smith" + assert pn.middle == "" From ec9e3a27f24a0f7d4284afbb403692f1bde50eeb Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 13 Jul 2026 02:58:40 -0700 Subject: [PATCH 85/88] Fold a leading never-given particle into the family name (I4) v1's handle_non_first_name_prefix was unported: 'de la Vega' produced given='de' with no ambiguity ('de' is not particles_ambiguous, so PARTICLE_OR_GIVEN never fires -- a silent misassignment). Port it as post_rules rule 1b keyed on the parsed given exactly like v1: a lone given token tagged particle-but-not-ambiguous with middles or family present folds into FAMILY; 'Jean de Mesnil' (non-leading) and bare 'de' (degenerate guard) stay untouched, and ambiguous 'van Gogh' keeps the given reading. v1-verified case row added. Found by cross-session PR review (I4). Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/_post_rules.py | 16 ++++++++++++++++ tests/v2/cases.py | 4 ++++ tests/v2/pipeline/test_post_rules.py | 25 +++++++++++++++++++++++++ 3 files changed, 45 insertions(+) diff --git a/nameparser/_pipeline/_post_rules.py b/nameparser/_pipeline/_post_rules.py index 4edfb39..a9fb8fe 100644 --- a/nameparser/_pipeline/_post_rules.py +++ b/nameparser/_pipeline/_post_rules.py @@ -71,6 +71,22 @@ def post_rules(state: ParseState) -> ParseState: for i in givens: _retag(tokens, i, Role.FAMILY) + # rule 1b: a leading particle that is NEVER a given name means the + # whole name is a surname -- fold given (and middles) into family + # (v1 handle_non_first_name_prefix; 'de la Vega' -> family, while + # ambiguous 'van Gogh' keeps the given reading). The middle/family + # guard leaves a degenerate bare 'de' as given rather than + # inventing a surname. + if len(givens) == 1 and (middles or families): + gtags = tokens[givens[0]].tags + if "particle" in gtags and "vocab:particle-ambiguous" not in gtags: + for i in givens + middles: + _retag(tokens, i, Role.FAMILY) + # downstream rules key on the role counts: recompute + givens = _idx(tokens, Role.GIVEN) + middles = _idx(tokens, Role.MIDDLE) + families = _idx(tokens, Role.FAMILY) + # v1 gates both rotations on `not self._had_comma` if state.structure is not Structure.NO_COMMA: return dataclasses.replace(state, tokens=tuple(tokens)) diff --git a/tests/v2/cases.py b/tests/v2/cases.py index b0c174d..92fdb9c 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -115,6 +115,10 @@ class Case: Case("phd_split_mid_name", "Dr. John Ph. D. Smith", {"title": "Dr.", "given": "John", "family": "Smith", "suffix": "Ph. D."}), + Case("leading_never_given_particle", "de la Vega", + {"family": "de la Vega"}, + notes="v1 handle_non_first_name_prefix: never-given leading " + "particle folds the whole name into family"), Case("unbalanced_quote", 'Jon "Nick Smith', {"given": "Jon", "middle": '"Nick', "family": "Smith"}, ambiguities=("unbalanced-delimiter",), diff --git a/tests/v2/pipeline/test_post_rules.py b/tests/v2/pipeline/test_post_rules.py index 12634ca..a08567c 100644 --- a/tests/v2/pipeline/test_post_rules.py +++ b/tests/v2/pipeline/test_post_rules.py @@ -7,6 +7,8 @@ _LEX = Lexicon( titles=frozenset({"mr", "sir"}), given_name_titles=frozenset({"sir"}), + particles=frozenset({"de", "la", "van"}), + particles_ambiguous=frozenset({"van"}), ) @@ -81,3 +83,26 @@ def test_turkic_rotation() -> None: assert _by_role(out, Role.GIVEN) == "Aygun" assert _by_role(out, Role.MIDDLE) == "Ali kizi" assert _by_role(out, Role.FAMILY) == "Mammadova" + + +def test_leading_never_given_particle_folds_into_family() -> None: + # v1 handle_non_first_name_prefix: a leading particle that is never + # a given name ('de') means the whole name is a surname + out = _parsed("de la Vega") + assert _by_role(out, Role.FAMILY) == "de la Vega" + assert not _by_role(out, Role.GIVEN) + + +def test_leading_ambiguous_particle_stays_given() -> None: + # 'van' is particles_ambiguous: the given reading stands (v1 parity) + out = _parsed("van Gogh") + assert _by_role(out, Role.GIVEN) == "van" + assert _by_role(out, Role.FAMILY) == "Gogh" + + +def test_degenerate_bare_particle_stays_given() -> None: + # v1's guard: with no middle or family, a bare 'de' keeps given='de' + # rather than inventing a surname + out = _parsed("de") + assert _by_role(out, Role.GIVEN) == "de" + assert not _by_role(out, Role.FAMILY) From 481f27afcf168139543782c438e54bb41b238ff4 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 13 Jul 2026 03:00:35 -0700 Subject: [PATCH 86/88] Wire the three dead Policy fields (I5) middle_as_family was validated but consumed by no stage -- ported v1's handle_middle_name_as_last as post_rules rule 4 (runs after the patronymic rotations, comma or not, matching v1's post_process order; span order reproduces v1's prepend). lenient_comma_suffixes now gates segment's post-comma predicate (strict mode vetoes initial-shaped suffix words). extra_suffix_delimiters cannot be honored until token re-splitting lands with the migration work, so setting it warns instead of silently doing nothing. Found by cross-session PR review (I5). Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/_post_rules.py | 13 ++++++++++--- nameparser/_pipeline/_segment.py | 12 +++++++++--- nameparser/_policy.py | 9 +++++++++ tests/v2/pipeline/test_post_rules.py | 14 ++++++++++++++ tests/v2/pipeline/test_segment.py | 13 +++++++++++++ tests/v2/test_policy.py | 10 ++++++++++ 6 files changed, 65 insertions(+), 6 deletions(-) diff --git a/nameparser/_pipeline/_post_rules.py b/nameparser/_pipeline/_post_rules.py index a9fb8fe..4b6dd1b 100644 --- a/nameparser/_pipeline/_post_rules.py +++ b/nameparser/_pipeline/_post_rules.py @@ -87,10 +87,12 @@ def post_rules(state: ParseState) -> ParseState: middles = _idx(tokens, Role.MIDDLE) families = _idx(tokens, Role.FAMILY) - # v1 gates both rotations on `not self._had_comma` - if state.structure is not Structure.NO_COMMA: - return dataclasses.replace(state, tokens=tuple(tokens)) + # v1 gates both rotations on `not self._had_comma`; the + # middle_as_family fold below runs comma or not (v1 order: + # patronymics first, then handle_middle_name_as_last) rules = state.policy.patronymic_rules + if state.structure is not Structure.NO_COMMA: + rules = frozenset() if PatronymicRule.EAST_SLAVIC in rules and \ len(givens) == 1 and len(middles) == 1 and len(families) == 1: tail = tokens[families[0]].text @@ -111,4 +113,9 @@ def post_rules(state: ParseState) -> ParseState: _retag(tokens, m2, Role.MIDDLE) _retag(tokens, f, Role.MIDDLE) _retag(tokens, g, Role.FAMILY) + # rule 4: opt-in fold of middles into family (v1 + # handle_middle_name_as_last; span order reproduces v1's prepend) + if state.policy.middle_as_family: + for i in _idx(tokens, Role.MIDDLE): + _retag(tokens, i, Role.FAMILY) return dataclasses.replace(state, tokens=tuple(tokens)) diff --git a/nameparser/_pipeline/_segment.py b/nameparser/_pipeline/_segment.py index 961f720..01dcbe2 100644 --- a/nameparser/_pipeline/_segment.py +++ b/nameparser/_pipeline/_segment.py @@ -5,7 +5,8 @@ COMMA_STRUCTURE ambiguities for unrecognized extra segments. Reads: Lexicon suffix vocabulary (via _vocab.is_suffix_lenient) -- the suffix-comma decision is definitionally vocabulary-dependent -(recorded plan deviation #3); Policy is not consulted here. +(recorded plan deviation #3); reads Policy.lenient_comma_suffixes +to pick the lenient or strict predicate. Decision (v1 parity): >=1 comma and every post-first segment entirely lenient-suffix AND >1 word before the first comma -> SUFFIX_COMMA; @@ -19,7 +20,7 @@ import dataclasses from nameparser._pipeline._state import ParseState, PendingAmbiguity, Structure -from nameparser._pipeline._vocab import is_suffix_lenient +from nameparser._pipeline._vocab import is_suffix_lenient, is_suffix_strict from nameparser._types import AmbiguityKind @@ -44,8 +45,13 @@ def segment(state: ParseState) -> ParseState: return dataclasses.replace(state, segments=segs, structure=Structure.NO_COMMA) + # lenient_comma_suffixes=False drops the post-comma test back to + # the strict predicate (initial-shaped suffix words stop qualifying) + predicate = (is_suffix_lenient if state.policy.lenient_comma_suffixes + else is_suffix_strict) + def suffixy(seg: tuple[int, ...]) -> bool: - return all(is_suffix_lenient(state.tokens[i].text, state.lexicon) + return all(predicate(state.tokens[i].text, state.lexicon) for i in seg) rest = groups[1:] diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 3ef4233..312c956 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -6,6 +6,7 @@ from __future__ import annotations import dataclasses +import warnings from dataclasses import dataclass, field from enum import Enum, StrEnum, auto @@ -125,6 +126,14 @@ def __post_init__(self) -> None: object.__setattr__( self, "extra_suffix_delimiters", frozenset(delimiters) ) + if delimiters: + # accepted-and-ignored would violate the fail-loud culture: + # the pipeline cannot honor these until token re-splitting + # lands with the migration work (v1 suffix_delimiter parity) + warnings.warn( + "extra_suffix_delimiters is not yet consumed by the " + "parse pipeline; it takes effect with the 2.0 migration " + "work", UserWarning, stacklevel=2) # Truthy strings ("no", "false") would silently invert the # caller's intent downstream; bools are the one field kind the # coercing checks above can't cover. diff --git a/tests/v2/pipeline/test_post_rules.py b/tests/v2/pipeline/test_post_rules.py index a08567c..ca7da24 100644 --- a/tests/v2/pipeline/test_post_rules.py +++ b/tests/v2/pipeline/test_post_rules.py @@ -106,3 +106,17 @@ def test_degenerate_bare_particle_stays_given() -> None: out = _parsed("de") assert _by_role(out, Role.GIVEN) == "de" assert not _by_role(out, Role.FAMILY) + + +def test_middle_as_family_folds_middles() -> None: + # v1 handle_middle_name_as_last, opt-in: middles prepend to family + out = _parsed("John Quincy Adams Smith", + Policy(middle_as_family=True)) + assert _by_role(out, Role.GIVEN) == "John" + assert not _by_role(out, Role.MIDDLE) + assert _by_role(out, Role.FAMILY) == "Quincy Adams Smith" + + +def test_middle_as_family_off_by_default() -> None: + out = _parsed("John Quincy Adams Smith") + assert _by_role(out, Role.MIDDLE) == "Quincy Adams" diff --git a/tests/v2/pipeline/test_segment.py b/tests/v2/pipeline/test_segment.py index aad0567..5aa5a9f 100644 --- a/tests/v2/pipeline/test_segment.py +++ b/tests/v2/pipeline/test_segment.py @@ -3,6 +3,8 @@ from nameparser._pipeline._segment import segment from nameparser._pipeline._state import ParseState, Structure from nameparser._pipeline._tokenize import tokenize +import dataclasses + from nameparser._policy import Policy from nameparser._types import AmbiguityKind @@ -74,3 +76,14 @@ def test_comma_only_input_is_no_comma_structure() -> None: out = _segmented(",,,") assert out.structure is Structure.NO_COMMA assert out.segments == () + + +def test_strict_comma_suffixes_veto_lenient_only_members() -> None: + # lenient_comma_suffixes=False: the post-comma test drops back to + # the strict predicate, so initial-shaped suffix words no longer + # qualify and the structure reads FAMILY_COMMA + state = ParseState( + original="John Ingram, V", lexicon=_LEX, + policy=dataclasses.replace(Policy(), lenient_comma_suffixes=False)) + out = segment(tokenize(extract_delimited(state))) + assert out.structure is Structure.FAMILY_COMMA diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index b84ecac..c8ff88f 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -75,6 +75,7 @@ def test_policy_delimiters_do_not_alias_caller_containers() -> None: assert ("'", "'") not in p.nickname_delimiters +@pytest.mark.filterwarnings("ignore:extra_suffix_delimiters is not yet") def test_extra_suffix_delimiters_validated_and_coerced() -> None: with pytest.raises(TypeError, match="bare string"): Policy(extra_suffix_delimiters="ab") # type: ignore[arg-type] @@ -99,6 +100,7 @@ def test_policy_patch_mirrors_policy_field_types() -> None: assert patch_annotation == f"{f.type} | _Unset" +@pytest.mark.filterwarnings("ignore:extra_suffix_delimiters is not yet") def test_policy_patch_canonicalizes_union_fields() -> None: p = PolicyPatch(extra_suffix_delimiters=frozenset({"-"})) assert isinstance(p.extra_suffix_delimiters, frozenset) @@ -223,3 +225,11 @@ def test_name_order_rejects_bare_string() -> None: Policy(name_order="gmf") # type: ignore[arg-type] with pytest.raises(TypeError, match="bare string"): PolicyPatch(name_order="gmf") # type: ignore[arg-type] + + +def test_extra_suffix_delimiters_warns_not_yet_consumed() -> None: + # accepted-and-ignored would violate the fail-loud culture: until + # the migration work wires v1's suffix_delimiter expansion, setting + # the field warns instead of silently doing nothing + with pytest.warns(UserWarning, match="not yet consumed"): + Policy(extra_suffix_delimiters=frozenset({"-"})) From e7ccd670ed5bee7b56c32a5fcf3dc408b978868c Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 13 Jul 2026 03:06:59 -0700 Subject: [PATCH 87/88] Restrict name_order to the exported orders; drop dangled ambiguities PR review I6/I7/I8/I9: - assemble omits an ambiguity whose referent tokens were ALL dropped (born-empty ambiguities like UNBALANCED_DELIMITER are kept -- they are token-independent by design) - Policy.name_order now accepts only GIVEN_FIRST / FAMILY_FIRST / FAMILY_FIRST_GIVEN_LAST: the unnamed permutations had no implemented assignment semantics and would silently misassign. Non-Role elements get the taxonomy's TypeError. - Span.__add__ comment no longer cites a nonexistent join stage; the real rationale is that no covering-span operation exists (anti-#100) - the stage ownership test now also pins token-level ownership: texts and spans fixed at tokenize, classify changes only tags, the role-assigning stages only roles (group also tags, for 'joined') - docstring polish: ParseState names the three ambiguity-recording stages; _types names the matches() call-time import; _assign's PARTICLE_OR_GIVEN wording is order-neutral Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/_assemble.py | 17 ++++++++++++----- nameparser/_pipeline/_assign.py | 5 +++-- nameparser/_pipeline/_state.py | 5 +++-- nameparser/_policy.py | 25 ++++++++++++++++++------- nameparser/_types.py | 12 ++++++------ tests/v2/pipeline/test_assemble.py | 24 ++++++++++++++++++++++++ tests/v2/pipeline/test_state.py | 23 +++++++++++++++++++++++ tests/v2/test_parser.py | 4 +++- tests/v2/test_policy.py | 17 ++++++++++++++++- 9 files changed, 108 insertions(+), 24 deletions(-) diff --git a/nameparser/_pipeline/_assemble.py b/nameparser/_pipeline/_assemble.py index 485f0ad..d3a6a16 100644 --- a/nameparser/_pipeline/_assemble.py +++ b/nameparser/_pipeline/_assemble.py @@ -25,10 +25,17 @@ def assemble(state: ParseState) -> ParsedName: continue role = t.role if t.role is not None else Role.GIVEN final[i] = Token(t.text, t.span, role, t.tags) - ambiguities = tuple( - Ambiguity(p.kind, p.detail, - tuple(final[i] for i in p.indices if i in final)) - for p in state.ambiguities) + ambiguities = [] + for pending in state.ambiguities: + materialized = tuple(final[i] for i in pending.indices + if i in final) + if pending.indices and not materialized: + # every referent was dropped: the ambiguity describes + # nothing that survives assembly. Born-empty ambiguities + # (unbalanced delimiters) are token-independent and kept. + continue + ambiguities.append( + Ambiguity(pending.kind, pending.detail, materialized)) return ParsedName(original=state.original, tokens=tuple(final.values()), - ambiguities=ambiguities) + ambiguities=tuple(ambiguities)) diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index e4a9aa2..d4b0e10 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -18,8 +18,9 @@ suffix; segments 2+ are suffixes (lenient -- segment already flagged non-suffixy ones COMMA_STRUCTURE). SUFFIX_COMMA: segment 0 as NO_COMMA; segments 1+ wholly SUFFIX. -Emits PARTICLE_OR_GIVEN when the given position consumed a leading -particles_ambiguous token with more pieces following ("Van Johnson"). +Emits PARTICLE_OR_GIVEN when the leading name piece is a lone +particles_ambiguous token with more pieces following ("Van Johnson") -- +whatever role name_order assigns that position. """ from __future__ import annotations diff --git a/nameparser/_pipeline/_state.py b/nameparser/_pipeline/_state.py index afa5092..56eb7a2 100644 --- a/nameparser/_pipeline/_state.py +++ b/nameparser/_pipeline/_state.py @@ -56,8 +56,9 @@ class ParseState: extract_delimited -> extracted/masked; tokenize -> tokens (span- sorted)/comma_offsets; segment -> segments/structure; classify -> token tags; group -> pieces/piece_tags/dropped AND maiden token - roles; assign/post_rules -> the remaining token roles; ambiguities - accumulate anywhere. Post-group, segments may retain indices of + roles; assign/post_rules -> the remaining token roles; ambiguities are + recorded by extract/segment/assign only (pinned by the ownership + test). Post-group, segments may retain indices of dropped tokens -- assign iterates pieces, never segments. This ownership map is pinned by tests/v2/pipeline/test_state.py.""" diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 312c956..0e76c7f 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -64,11 +64,21 @@ class Policy: def __post_init__(self) -> None: _reject_bare_string_order(self.name_order) order = tuple(self.name_order) - if len(order) != 3 or set(order) != _NAME_ROLES: + for element in order: + if not isinstance(element, Role): + raise TypeError( + f"name_order elements must be Role members, " + f"got {element!r}" + ) + # Only the three exported orders have implemented assignment + # semantics; the unnamed permutations would silently misassign. + # Pre-2.0 strictness is free -- relaxing later is compatible. + if order not in (GIVEN_FIRST, FAMILY_FIRST, + FAMILY_FIRST_GIVEN_LAST): raise ValueError( - f"name_order must be a permutation of (Role.GIVEN, " - f"Role.MIDDLE, Role.FAMILY), got {order!r}; use " - f"GIVEN_FIRST, FAMILY_FIRST, or FAMILY_FIRST_GIVEN_LAST" + f"name_order must be one of the exported orders, got " + f"{order!r}; use GIVEN_FIRST, FAMILY_FIRST, or " + f"FAMILY_FIRST_GIVEN_LAST" ) object.__setattr__(self, "name_order", order) if isinstance(self.patronymic_rules, str): @@ -159,9 +169,10 @@ def __repr__(self) -> str: if value == f.default: continue if f.name == "name_order": - # Only 3 of the 6 possible role permutations have named - # constants; fall back to a compact role-name tuple for - # the rest so this can't KeyError on an unnamed order. + # __post_init__ restricts to the three named orders, so + # the fallback is unreachable via the constructor; kept + # because repr must never raise (e.g. a smuggled + # __setstate__ value -- layout is validated, values not). order_repr = constant_names.get( value, "(" + ", ".join(r.name for r in value) + ")") parts.append(f"name_order={order_repr}") diff --git a/nameparser/_types.py b/nameparser/_types.py index b8065b6..a04aca4 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -3,8 +3,8 @@ Layering (enforced by tests/v2/test_layering.py): this module imports nothing from nameparser at module level -- it is the bottom of the module-import dependency graph. The rendering delegates import _render -at call time, and a TYPE_CHECKING-only _lexicon import supplies the -Lexicon annotation. +and matches() imports _parser at call time; TYPE_CHECKING-only imports +supply the Lexicon/Parser annotations. Repr policy (applies to every v2 type's __repr__, across this module and _lexicon.py/_policy.py/_locale.py): bounded output only. No repr may scale @@ -43,10 +43,10 @@ class Span(NamedTuple): end: int def __add__(self, other: object) -> NoReturn: # type: ignore[override] - # Inherited tuple + would concatenate two spans into a 4-tuple -- - # the natural but wrong spelling of "covering span". The real - # covering operation ships with its consumer, the pipeline's - # join stage. + # Inherited tuple + would concatenate two spans into a 4-tuple. + # There is deliberately NO covering-span operation: grouping is + # index-run based (spec §6, the anti-#100 invariant) and never + # merges spans. raise TypeError( "Span does not support +; tuple concatenation is not a " "covering span" diff --git a/tests/v2/pipeline/test_assemble.py b/tests/v2/pipeline/test_assemble.py index d1aaae9..ba66782 100644 --- a/tests/v2/pipeline/test_assemble.py +++ b/tests/v2/pipeline/test_assemble.py @@ -50,3 +50,27 @@ def test_assemble_drops_structural_marker_tokens() -> None: def test_empty_parse_is_falsy() -> None: assert not _parse("") assert not _parse(" ") + + +def test_ambiguity_with_all_indices_dropped_is_omitted() -> None: + # an ambiguity whose referent tokens were ALL dropped describes + # nothing; emitting it hollow would mislead consumers. Born-empty + # ambiguities (unbalanced delimiters) are kept -- they are + # token-independent by design. + from nameparser._pipeline._state import PendingAmbiguity + from nameparser._types import AmbiguityKind as AK + import dataclasses + state = run(ParseState(original="Jane Smith née Jones", lexicon=_LEX, + policy=Policy())) + née_idx = next(i for i, t in enumerate(state.tokens) + if t.text == "née") + poisoned = dataclasses.replace( + state, ambiguities=state.ambiguities + ( + PendingAmbiguity(AK.ORDER, "refers only to the marker", + (née_idx,)), + PendingAmbiguity(AK.UNBALANCED_DELIMITER, "born empty", ()), + )) + pn = assemble(poisoned) + kinds = [a.kind for a in pn.ambiguities] + assert AK.ORDER not in kinds # fully dangled: omitted + assert AK.UNBALANCED_DELIMITER in kinds # born empty: kept diff --git a/tests/v2/pipeline/test_state.py b/tests/v2/pipeline/test_state.py index b01433c..4ceff2b 100644 --- a/tests/v2/pipeline/test_state.py +++ b/tests/v2/pipeline/test_state.py @@ -54,6 +54,17 @@ def test_stage_field_ownership() -> None: "post_rules": {"tokens"}, } assert {s.__name__ for s in STAGES} == set(ownership) + # Within the tokens themselves the contract is finer: texts and + # spans are fixed at tokenize (the anti-#100 invariant -- tokens + # are never re-created), classify touches only tags, and the + # role-assigning stages touch only roles (group also tags, for the + # ph-d "joined" marker). + token_ownership = { + "classify": {"tags"}, + "group": {"tags", "role"}, + "assign": {"role"}, + "post_rules": {"role"}, + } for case in CASES: state = ParseState(original=case.text, lexicon=_Lexicon.default(), policy=case.policy or Policy()) @@ -65,3 +76,15 @@ def test_stage_field_ownership() -> None: if getattr(state, name) != value} assert changed <= ownership[stage.__name__], ( f"{case.id}: {stage.__name__} changed {changed - ownership[stage.__name__]}") + if stage.__name__ not in token_ownership: + continue + allowed = token_ownership[stage.__name__] + assert len(state.tokens) == len(before["tokens"]), ( + f"{case.id}: {stage.__name__} changed the token count") + for old, new in zip(before["tokens"], state.tokens): + token_changed = { + f.name for f in _dc.fields(old) + if getattr(old, f.name) != getattr(new, f.name)} + assert token_changed <= allowed, ( + f"{case.id}: {stage.__name__} changed token fields " + f"{token_changed - allowed} on {old.text!r}") diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index b215bd3..5467cf7 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -103,7 +103,9 @@ def test_parser_for_wraps_pack_errors_with_identity() -> None: # latent in a perfectly constructible Locale until apply time bad = Locale(code="xx", lexicon=Lexicon.empty(), policy=PolicyPatch(name_order=(1, 2, 3))) # type: ignore[arg-type] - with pytest.raises(ValueError, match="while applying locale 'xx'"): + # the rewrap preserves the taxonomy's exception type (here the + # non-Role element TypeError) while adding the pack identity + with pytest.raises(TypeError, match="while applying locale 'xx'"): parser_for(bad) diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index c8ff88f..902f917 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -37,6 +37,21 @@ def test_name_order_must_be_permutation_and_error_names_constants() -> None: Policy(name_order=(Role.GIVEN, Role.GIVEN, Role.FAMILY)) +def test_name_order_restricted_to_the_three_exported_orders() -> None: + # _name_positions only implements the three exported semantics; the + # unnamed permutations would silently misassign (PR review I7). + # Pre-2.0 strictness is free: relaxing later is compatible. + with pytest.raises(ValueError, match="GIVEN_FIRST"): + Policy(name_order=(Role.MIDDLE, Role.GIVEN, Role.FAMILY)) + + +def test_name_order_rejects_non_role_elements_with_type_error() -> None: + # taxonomy: wrong element type -> TypeError, not the permutation + # ValueError (PR review polish) + with pytest.raises(TypeError, match="Role"): + Policy(name_order=(1, 2, 3)) # type: ignore[arg-type] + + def test_patronymic_rules_coerce_and_reject() -> None: p = Policy(patronymic_rules=frozenset({"east-slavic"})) # type: ignore[arg-type] assert p.patronymic_rules == frozenset({PatronymicRule.EAST_SLAVIC}) @@ -175,7 +190,7 @@ def test_apply_patch_revalidates_deferred_values() -> None: # PolicyPatch documents lazy validation: invalid values sit latent in # the patch and must fail when applied, not silently flow into Policy. bad_order = PolicyPatch(name_order=(Role.TITLE, Role.GIVEN, Role.FAMILY)) - with pytest.raises(ValueError, match="permutation"): + with pytest.raises(ValueError, match="exported orders"): apply_patch(Policy(), bad_order) bad_rules = PolicyPatch(patronymic_rules=frozenset({"klingon"})) # type: ignore[arg-type] with pytest.raises(ValueError, match="valid rules"): From ddd04161bac52f3b26f7c631dcee0ed55a6b3fcc Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 13 Jul 2026 03:10:07 -0700 Subject: [PATCH 88/88] Add the review's test-coverage batch (T1-T4, T6) - T1: three-piece FAMILY_FIRST_GIVEN_LAST pins given-from-the-END semantics (not a rotation of FAMILY_FIRST) - T2: reverse-coverage property -- every input char lies in a token span, a masked delimited span, or is individually ignorable; no character silently vanishes - T3: case row for post-comma non-suffix extras ('Smith, John, Extra, Jr.' -> suffix 'Extra, Jr.' + COMMA_STRUCTURE; v1 parity pinned live) - T4: multiple unbalanced delimiters are each reported; the scan does not stop at the first unmatched opener - T6: digits join the Hypothesis stress alphabet Co-Authored-By: Claude Fable 5 --- tests/v2/cases.py | 6 ++++++ tests/v2/test_parser.py | 24 +++++++++++++++++++++++- tests/v2/test_properties.py | 30 ++++++++++++++++++++++++++++-- 3 files changed, 57 insertions(+), 3 deletions(-) diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 92fdb9c..72816df 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -40,6 +40,12 @@ class Case: {"given": "John", "family": "Smith"}), Case("suffix_comma", "John Smith, PhD", {"given": "John", "family": "Smith", "suffix": "PhD"}), + Case("comma_extras_become_suffixes", "Smith, John, Extra, Jr.", + {"given": "John", "family": "Smith", "suffix": "Extra, Jr."}, + ambiguities=("comma-structure",), + notes="post-comma segments land in suffix even when not " + "suffix-shaped; the ambiguity flags the guess (v1 " + "parity, pinned live 2026-07-13)"), Case("delavega", "Dr. Juan de la Vega III", {"title": "Dr.", "given": "Juan", "family": "de la Vega", "suffix": "III"}), diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index 5467cf7..2ed3477 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -3,7 +3,9 @@ import pytest from nameparser import Lexicon, Locale, Parser, Policy, PolicyPatch, parse, parser_for -from nameparser._policy import FAMILY_FIRST, PatronymicRule +from nameparser._policy import ( + FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, PatronymicRule, +) from nameparser._types import AmbiguityKind @@ -128,6 +130,26 @@ def test_matches_component_wise_case_insensitive() -> None: pn.matches(42) # type: ignore[arg-type] +def test_family_first_given_last_places_middle_between() -> None: + # T1: the three-piece FAMILY_FIRST_GIVEN_LAST assignment -- family + # from the front, given from the END, middle between (not a rotation + # of FAMILY_FIRST) + p = Parser(policy=Policy(name_order=FAMILY_FIRST_GIVEN_LAST)) + pn = p.parse("Zeng Xiao Long") + assert (pn.family, pn.middle, pn.given) == ("Zeng", "Xiao", "Long") + + +def test_multiple_unbalanced_delimiters_each_reported() -> None: + # T4: the extract scan continues past the first unmatched opener; + # each one is reported and treated as literal text + pn = parse('John "Jack (Smith') + unbalanced = [a for a in pn.ambiguities + if a.kind is AmbiguityKind.UNBALANCED_DELIMITER] + assert len(unbalanced) == 2 + assert pn.given == "John" and pn.family == "(Smith" + assert not pn.nickname + + def test_matches_accepts_explicit_parser() -> None: family_first = Parser(policy=Policy(name_order=FAMILY_FIRST)) pn = family_first.parse("Yamada Taro") diff --git a/tests/v2/test_properties.py b/tests/v2/test_properties.py index 157116d..0e4440b 100644 --- a/tests/v2/test_properties.py +++ b/tests/v2/test_properties.py @@ -9,10 +9,12 @@ from hypothesis import given, settings from hypothesis import strategies as st -from nameparser import parse +from nameparser import Lexicon, Policy, parse +from nameparser._pipeline import run +from nameparser._pipeline._state import ParseState _ALPHABET = st.sampled_from( - 'abcdefgh ABC .,،，\'"()«»‏‏\U0001f600éñßЖ-') + 'abcdefgh ABC 12 .,،，\'"()«»‏‏\U0001f600éñßЖ-') @given(st.text(alphabet=_ALPHABET, max_size=200)) @@ -50,3 +52,27 @@ def test_render_reparse_reaches_fixpoint(text: str) -> None: break s = nxt assert str(parse(s)) == s, f"no fixpoint within 10 rounds: {s!r}" + + +@given(st.text(alphabet=_ALPHABET, max_size=100)) +@settings(max_examples=300, deadline=None, derandomize=True) +def test_every_original_char_is_accounted_for(text: str) -> None: + # Reverse coverage (the dual of provenance): no character of the + # input silently vanishes. Every char lies in a token span, a + # masked delimited span, or is individually ignorable -- whitespace, + # a structural comma, or a char the strip options remove. Checked on + # the pre-assembly state because dropped/extracted tokens keep their + # spans there. + state = run(ParseState(original=text, lexicon=Lexicon.default(), + policy=Policy())) + covered: set[int] = set() + for tok in state.tokens: + covered.update(range(tok.span.start, tok.span.end)) + for span in state.masked: + covered.update(range(span.start, span.end)) + ignorable = {",", "،", "，", "\U0001f600", "‏"} + for i, ch in enumerate(text): + if i in covered or ch.isspace() or ch in ignorable: + continue + raise AssertionError( + f"char {ch!r} at {i} in {text!r} is unaccounted for")