diff --git a/HISTORY.md b/HISTORY.md index e56822da..23e4284d 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -11,11 +11,14 @@ The third number is for emergencies when we need to start branches for older rel Our backwards-compatibility policy can be found [here](https://github.com/python-attrs/cattrs/blob/main/.github/SECURITY.md). -## 25.4.0 (UNRELEASED) +## NEXT (UNRELEASED) - Add the {mod}`tomllib ` preconf converter. See [here](https://catt.rs/en/latest/preconf.html#tomllib) for details. ([#716](https://github.com/python-attrs/cattrs/pull/716)) +- Customizing un/structuring of _attrs_ classes, dataclasses, TypedDicts and dict NamedTuples is now possible by using `Annotated[T, override()]` on fields. + See [here](https://catt.rs/en/stable/customizing.html#using-typing-annotated-t-override) for more details. + ([#717](https://github.com/python-attrs/cattrs/pull/717)) - Fix structuring of nested generic classes with stringified annotations. ([#688](https://github.com/python-attrs/cattrs/pull/688)) - Python 3.9 is no longer supported, as it is end-of-life. Use previous versions on this Python version. diff --git a/docs/customizing.md b/docs/customizing.md index ba095c25..00ed377c 100644 --- a/docs/customizing.md +++ b/docs/customizing.md @@ -410,6 +410,42 @@ ClassWithInitFalse(number=2) ``` +## Using `typing.Annotated[T, override(...)]` + +The un/structuring process for _attrs_ classes, dataclasses, TypedDicts and dict NamedTuples can be customized by annotating the fields using `typing.Annotated[T, override()]`. + +```{doctest} +>>> from typing import Annotated + +>>> @define +... class ExampleClass: +... klass: Annotated[int, cattrs.override(rename="class")] + +>>> cattrs.unstructure(ExampleClass(1)) +{'class': 1} +>>> cattrs.structure({'class': 1}, ExampleClass) +ExampleClass(klass=1) +``` + +These customizations are automatically recognized by every {class}`Converter `. +They can still be overriden explicitly, see [](#custom-un-structuring-hooks). + +```{attention} +One of the fundamental [design decisions](why.md#design-decisions) of _cattrs_ is that serialization rules should be separate from the models themselves; +by using this feature you're going against the spirit of this design decision. + +However, software is written in many different context and with different constraints; and practicality _sometimes_ beats purity. +_Sometimes_, it's not worth introducing a mapping layer to rename one field. +_Sometimes_, you're busy prototyping and will clean up your code later. + +The danger is not any single compromise, but their accumulation. +One of the most important skills as a software engineer is knowing when the cost of a trade-off has crossed the line and it's time to do things properly. +``` + +```{versionadded} NEXT + +``` + ## Customizing Collections ```{currentmodule} cattrs.cols diff --git a/docs/defaulthooks.md b/docs/defaulthooks.md index 6585e7b9..c313807e 100644 --- a/docs/defaulthooks.md +++ b/docs/defaulthooks.md @@ -452,11 +452,11 @@ Tuples can be structured into classes using {meth}`structure_attrs_fromtuple() < A(a='string', b=2) ``` -Loading from tuples can be made the default by creating a new {class}`Converter ` with `unstruct_strat=cattr.UnstructureStrategy.AS_TUPLE`. +Loading from tuples can be made the default by creating a new {class}`Converter ` with `unstruct_strat=cattrs.UnstructureStrategy.AS_TUPLE`. ```{doctest} ->>> converter = cattrs.Converter(unstruct_strat=cattr.UnstructureStrategy.AS_TUPLE) +>>> converter = cattrs.Converter(unstruct_strat=cattrs.UnstructureStrategy.AS_TUPLE) >>> @define ... class A: ... a: str @@ -620,6 +620,12 @@ The {mod}`cattrs.cols` module contains hook factories for un/structuring named t [PEP 593](https://www.python.org/dev/peps/pep-0593/) annotations (`typing.Annotated[type, ...]`) are supported and are handled using the first type present in the annotated type. +Additionally, `typing.Annotated` types containing `cattrs.override()` are recognized and used by the _attrs_, dataclass, TypedDict and dict NamedTuple hook factories. + +```{versionchanged} NEXT +`Annotated[T, override()]` is now used by the _attrs_, dataclass, TypedDict and dict NamedTuple hook factories. +``` + ```{versionadded} 1.4.0 ``` diff --git a/src/cattrs/cols.py b/src/cattrs/cols.py index bd083d3b..64e4f2c7 100644 --- a/src/cattrs/cols.py +++ b/src/cattrs/cols.py @@ -5,15 +5,7 @@ from collections import defaultdict from collections.abc import Callable, Iterable from functools import partial -from typing import ( - TYPE_CHECKING, - Any, - DefaultDict, - Literal, - NamedTuple, - TypeVar, - get_type_hints, -) +from typing import TYPE_CHECKING, Any, DefaultDict, Literal, NamedTuple, TypeVar from attrs import NOTHING, Attribute, NothingType @@ -21,6 +13,7 @@ ANIES, AbcSet, get_args, + get_full_type_hints, get_origin, is_bare, is_frozenset, @@ -246,7 +239,7 @@ def _namedtuple_to_attrs(cl: type[tuple]) -> list[Attribute]: type=a, alias=name, ) - for name, a in get_type_hints(cl).items() + for name, a in get_full_type_hints(cl).items() ] diff --git a/src/cattrs/gen/__init__.py b/src/cattrs/gen/__init__.py index 792e7bf5..d2b424d6 100644 --- a/src/cattrs/gen/__init__.py +++ b/src/cattrs/gen/__init__.py @@ -33,7 +33,7 @@ from ._consts import AttributeOverride, already_generating, neutral from ._generics import generate_mapping from ._lc import generate_unique_filename -from ._shared import find_structure_handler +from ._shared import _annotated_override_or_default, find_structure_handler if TYPE_CHECKING: from ..converters import BaseConverter @@ -95,10 +95,13 @@ def make_dict_unstructure_fn_from_attrs( :param _cattrs_include_init_false: If true, _attrs_ fields marked as `init=False` will be included. - .. versionadded:: 24.1.0 - .. versionchanged:: 25.2.0 + .. versionadded:: 24.1.0 + .. versionchanged:: 25.2.0 The `_cattrs_use_alias` parameter takes its value from the given converter by default. + .. versionchanged:: NEXT + `typing.Annotated[T, override()]` is now recognized and can be used to customize + unstructuring. .. versionchanged:: NEXT When `_cattrs_omit_if_default` is true and the attribute has an attrs converter specified, the converter is applied to the default value before checking if it @@ -117,7 +120,13 @@ def make_dict_unstructure_fn_from_attrs( for a in attrs: attr_name = a.name - override = kwargs.get(attr_name, neutral) + if attr_name in kwargs: + override = kwargs[attr_name] + else: + override = _annotated_override_or_default(a.type, neutral) + if override != neutral: + kwargs[attr_name] = override + if override.omit: continue if override.omit is None and not a.init and not _cattrs_include_init_false: @@ -264,11 +273,14 @@ def make_dict_unstructure_fn( :param _cattrs_include_init_false: If true, _attrs_ fields marked as `init=False` will be included. - .. versionadded:: 23.2.0 *_cattrs_use_alias* - .. versionadded:: 23.2.0 *_cattrs_include_init_false* - .. versionchanged:: 25.2.0 + .. versionadded:: 23.2.0 *_cattrs_use_alias* + .. versionadded:: 23.2.0 *_cattrs_include_init_false* + .. versionchanged:: 25.2.0 The `_cattrs_use_alias` parameter takes its value from the given converter by default. + .. versionchanged:: NEXT + `typing.Annotated[T, override()]` is now recognized and can be used to customize + unstructuring. """ origin = get_origin(cl) attrs = adapted_fields(origin or cl) # type: ignore @@ -349,10 +361,13 @@ def make_dict_structure_fn_from_attrs( :param _cattrs_include_init_false: If true, _attrs_ fields marked as `init=False` will be included. - .. versionadded:: 24.1.0 - .. versionchanged:: 25.2.0 + .. versionadded:: 24.1.0 + .. versionchanged:: 25.2.0 The `_cattrs_use_alias` parameter takes its value from the given converter by default. + .. versionchanged:: NEXT + `typing.Annotated[T, override()]` is now recognized and can be used to customize + unstructuring. """ cl_name = cl.__name__ @@ -408,7 +423,13 @@ def make_dict_structure_fn_from_attrs( internal_arg_parts["__c_avn"] = AttributeValidationNote for a in attrs: an = a.name - override = kwargs.get(an, neutral) + if an in kwargs: + override = kwargs[an] + else: + override = _annotated_override_or_default(a.type, neutral) + if override != neutral: + kwargs[an] = override + if override.omit: continue if override.omit is None and not a.init and not _cattrs_include_init_false: @@ -539,14 +560,24 @@ def make_dict_structure_fn_from_attrs( # The first loop deals with required args. for a in attrs: an = a.name - override = kwargs.get(an, neutral) + + if an in kwargs: + override = kwargs[an] + else: + override = _annotated_override_or_default(a.type, neutral) + if override != neutral: + kwargs[an] = override + if override.omit: continue if override.omit is None and not a.init and not _cattrs_include_init_false: continue + if a.default is not NOTHING: non_required.append(a) + # The next loop will handle it. continue + t = a.type if isinstance(t, TypeVar): t = typevar_map.get(t.__name__, t) @@ -753,17 +784,20 @@ def make_dict_structure_fn( :param _cattrs_include_init_false: If true, _attrs_ fields marked as `init=False` will be included. - .. versionadded:: 23.2.0 *_cattrs_use_alias* - .. versionadded:: 23.2.0 *_cattrs_include_init_false* - .. versionchanged:: 23.2.0 + .. versionadded:: 23.2.0 *_cattrs_use_alias* + .. versionadded:: 23.2.0 *_cattrs_include_init_false* + .. versionchanged:: 23.2.0 The `_cattrs_forbid_extra_keys` and `_cattrs_detailed_validation` parameters take their values from the given converter by default. - .. versionchanged:: 24.1.0 + .. versionchanged:: 24.1.0 The `_cattrs_prefer_attrib_converters` parameter takes its value from the given converter by default. - .. versionchanged:: 25.2.0 + .. versionchanged:: 25.2.0 The `_cattrs_use_alias` parameter takes its value from the given converter by default. + .. versionchanged:: NEXT + `typing.Annotated[T, override()]` is now recognized and can be used to customize + unstructuring. """ mapping = {} diff --git a/src/cattrs/gen/_shared.py b/src/cattrs/gen/_shared.py index 904c7744..967661ee 100644 --- a/src/cattrs/gen/_shared.py +++ b/src/cattrs/gen/_shared.py @@ -4,15 +4,31 @@ from attrs import NOTHING, Attribute, Factory -from .._compat import is_bare_final +from .._compat import get_args, is_annotated, is_bare_final from ..dispatch import StructureHook from ..errors import StructureHandlerNotFoundError from ..fns import raise_error +from ._consts import AttributeOverride if TYPE_CHECKING: from ..converters import BaseConverter +def _annotated_override_or_default( + type: Any, default: AttributeOverride +) -> AttributeOverride: + """ + If the type is Annotated containing an AttributeOverride, return it. + Otherwise, return the default. + """ + if is_annotated(type): + for arg in get_args(type): + if isinstance(arg, AttributeOverride): + return arg + + return default + + def find_structure_handler( a: Attribute, type: Any, c: BaseConverter, prefer_attrs_converters: bool = False ) -> StructureHook | None: diff --git a/src/cattrs/gen/typeddicts.py b/src/cattrs/gen/typeddicts.py index e61488f4..fc6e173b 100644 --- a/src/cattrs/gen/typeddicts.py +++ b/src/cattrs/gen/typeddicts.py @@ -29,7 +29,7 @@ from ._consts import already_generating, neutral from ._generics import generate_mapping from ._lc import generate_unique_filename -from ._shared import find_structure_handler +from ._shared import _annotated_override_or_default, find_structure_handler if TYPE_CHECKING: from ..converters import BaseConverter @@ -102,11 +102,20 @@ def make_dict_unstructure_fn( # * all attributes resolve to `converter._unstructure_identity` for a in attrs: attr_name = a.name - override = kwargs.get(attr_name, neutral) + t = a.type + nrb = get_notrequired_base(t) + if nrb is not NOTHING: + t = nrb + + if attr_name in kwargs: + override = kwargs[attr_name] + else: + override = _annotated_override_or_default(t, neutral) + if override != neutral: + kwargs[attr_name] = override if override != neutral: break handler = None - t = a.type if isinstance(t, TypeVar): if t.__name__ in mapping: @@ -118,9 +127,6 @@ def make_dict_unstructure_fn( t = deep_copy_with(t, mapping, cl) if handler is None: - nrb = get_notrequired_base(t) - if nrb is not NOTHING: - t = nrb try: handler = converter.get_unstructure_hook(t) except RecursionError: @@ -135,10 +141,22 @@ def make_dict_unstructure_fn( for ix, a in enumerate(attrs): attr_name = a.name - override = kwargs.get(attr_name, neutral) + t = a.type + nrb = get_notrequired_base(t) + if nrb is not NOTHING: + t = nrb + + if attr_name in kwargs: + override = kwargs[attr_name] + else: + override = _annotated_override_or_default(t, neutral) + if override != neutral: + kwargs[attr_name] = override + if override.omit: lines.append(f" res.pop('{attr_name}', None)") continue + if override.rename is not None: # We also need to pop when renaming, since we're copying # the original. @@ -153,8 +171,6 @@ def make_dict_unstructure_fn( if override.unstruct_hook is not None: handler = override.unstruct_hook else: - t = a.type - if isinstance(t, TypeVar): if t.__name__ in mapping: t = mapping[t.__name__] @@ -164,9 +180,6 @@ def make_dict_unstructure_fn( t = deep_copy_with(t, mapping, cl) if handler is None: - nrb = get_notrequired_base(t) - if nrb is not NOTHING: - t = nrb try: handler = converter.get_unstructure_hook(t) except RecursionError: @@ -213,12 +226,15 @@ def make_dict_unstructure_fn( ) eval(compile(script, fname, "exec"), globs) + + res = globs[fn_name] + res.overrides = kwargs finally: working_set.remove(cl) if not working_set: del already_generating.working_set - return globs[fn_name] + return res def make_dict_structure_fn( @@ -319,20 +335,25 @@ def make_dict_structure_fn( for ix, a in enumerate(attrs): an = a.name attr_required = an in req_keys - override = kwargs.get(an, neutral) + t = a.type + nrb = get_notrequired_base(t) + if nrb is not NOTHING: + t = nrb + + if an in kwargs: + override = kwargs[an] + else: + override = _annotated_override_or_default(t, neutral) + if override != neutral: + kwargs[an] = override if override.omit: continue - t = a.type if isinstance(t, TypeVar): t = mapping.get(t.__name__, t) elif is_generic(t) and not is_bare(t) and not is_annotated(t): t = deep_copy_with(t, mapping, cl) - nrb = get_notrequired_base(t) - if nrb is not NOTHING: - t = nrb - if is_generic(t) and not is_bare(t) and not is_annotated(t): t = deep_copy_with(t, mapping, cl) @@ -392,7 +413,12 @@ def make_dict_structure_fn( for ix, a in enumerate(attrs): an = a.name attr_required = an in req_keys - override = kwargs.get(an, neutral) + if an in kwargs: + override = kwargs[an] + else: + override = _annotated_override_or_default(a.type, neutral) + if override != neutral: + kwargs[an] = override if override.omit: continue if not attr_required: @@ -441,13 +467,18 @@ def make_dict_structure_fn( if non_required: for ix, a in non_required: an = a.name - override = kwargs.get(an, neutral) t = a.type - nrb = get_notrequired_base(t) if nrb is not NOTHING: t = nrb + if an in kwargs: + override = kwargs[an] + else: + override = _annotated_override_or_default(t, neutral) + if override != neutral: + kwargs[an] = override + if isinstance(t, TypeVar): t = mapping.get(t.__name__, t) elif is_generic(t) and not is_bare(t) and not is_annotated(t): @@ -507,7 +538,9 @@ def make_dict_structure_fn( ) eval(compile(script, fname, "exec"), globs) - return globs[fn_name] + res = globs[fn_name] + res.overrides = kwargs + return res def _adapted_fields(cls: Any) -> list[Attribute]: diff --git a/tests/test_annotated_overrides.py b/tests/test_annotated_overrides.py new file mode 100644 index 00000000..eb537e9a --- /dev/null +++ b/tests/test_annotated_overrides.py @@ -0,0 +1,181 @@ +from dataclasses import dataclass +from typing import Annotated, NamedTuple, TypedDict + +from attrs import define + +from cattrs import Converter +from cattrs._compat import NotRequired +from cattrs.cols import ( + is_namedtuple, + namedtuple_dict_structure_factory, + namedtuple_dict_unstructure_factory, +) +from cattrs.gen import make_dict_structure_fn, make_dict_unstructure_fn, override +from cattrs.gen.typeddicts import make_dict_structure_fn as make_td_structure_fn +from cattrs.gen.typeddicts import make_dict_unstructure_fn as make_td_unstructure_fn + + +def test_annotated_override_attrs(genconverter: Converter): + """Annotated overrides work for attrs classes.""" + + @define + class A: + a: Annotated[int, override(rename="b")] + c: Annotated[int, override(omit=True)] = 1 + d: Annotated[int, override(rename="e")] = 2 + + instance = A(1) + # 'a' is renamed to 'b', 'c' is omitted. 'd' is default so present as 'e' + assert genconverter.unstructure(instance) == {"b": 1, "e": 2} + + assert genconverter.structure({"b": 1, "e": 2}, A) == A(1) + + +def test_annotated_override_dataclasses(genconverter: Converter): + """Annotated overrides work for dataclasses.""" + + @dataclass + class A: + a: Annotated[int, override(rename="b")] + c: Annotated[int, override(omit=True)] = 1 + + instance = A(1) + assert genconverter.unstructure(instance) == {"b": 1} + + assert genconverter.structure({"b": 1}, A) == A(1) + + +def test_annotated_override_typeddict(genconverter: Converter): + """Annotated overrides work for TypedDicts.""" + + class TD(TypedDict): + a: Annotated[int, override(rename="b")] + c: Annotated[int, override(omit=True)] + + instance: TD = {"a": 1, "c": 2} + + assert genconverter.unstructure(instance, TD) == {"b": 1} + + # Let's simplify and just test rename for now to avoid required field issues with omit. + class TD2(TypedDict): + a: Annotated[int, override(rename="b")] + + inst2: TD2 = {"a": 1} + assert genconverter.unstructure(inst2, TD2) == {"b": 1} + assert genconverter.structure({"b": 1}, TD2) == {"a": 1} + + +def test_annotated_override_namedtuple(genconverter: Converter): + """Annotated overrides work for NamedTuples using dict factories.""" + + # We need to register the dict factories for NamedTuples + genconverter.register_unstructure_hook_factory( + is_namedtuple, namedtuple_dict_unstructure_factory + ) + genconverter.register_structure_hook_factory( + is_namedtuple, namedtuple_dict_structure_factory + ) + + class NT(NamedTuple): + a: Annotated[int, override(rename="b")] + c: Annotated[int, override(omit=True)] = 1 + + instance = NT(1) + assert genconverter.unstructure(instance) == {"b": 1} + assert genconverter.structure({"b": 1}, NT) == NT(1) + + +def test_annotated_override_precedence(genconverter: Converter): + """Test that explicit kwargs override Annotated metadata.""" + + @define + class A: + a: Annotated[int, override(rename="b")] + + # Override the rename back to 'a' explicitly + unstructure_fn = make_dict_unstructure_fn(A, genconverter, a=override(rename="a")) + genconverter.register_unstructure_hook(A, unstructure_fn) + + assert genconverter.unstructure(A(1)) == {"a": 1} + + # # Structure override + structure_fn = make_dict_structure_fn(A, genconverter, a=override(rename="a")) + genconverter.register_structure_hook(A, structure_fn) + + assert genconverter.structure({"a": 1}, A) == A(1) + + +def test_annotated_override_hooks(genconverter: Converter): + """struct_hook and unstruct_hook work in Annotated.""" + + def double_hook(v): + return v * 2 + + def half_hook(v, _): + return v // 2 + + @define + class A: + a: Annotated[int, override(unstruct_hook=double_hook, struct_hook=half_hook)] + + assert genconverter.unstructure(A(10)) == {"a": 20} + assert genconverter.structure({"a": 20}, A) == A(10) + + +def test_annotated_override_omit_if_default(genconverter: Converter): + """omit_if_default works in Annotated.""" + + @define + class A: + a: Annotated[int, override(omit_if_default=True)] = 0 + b: int = 1 + + # a matches default, should be omitted. b matches default but no override, should stay (default behavior is to keep) + assert genconverter.unstructure(A()) == {"b": 1} + assert genconverter.unstructure(A(a=1)) == {"a": 1, "b": 1} + + +def test_overrides_attribute_populated(genconverter: Converter): + """The .overrides attribute is correctly populated.""" + + @dataclass + class A: + a: Annotated[int, override(rename="b")] + c: Annotated[int, override(omit=True)] = 1 + + # Test dataclasses (make_dict_unstructure_fn) + unstruct_hook = make_dict_unstructure_fn(A, genconverter) + assert unstruct_hook.overrides == { + "a": override(rename="b"), + "c": override(omit=True), + } + + struct_hook = make_dict_structure_fn(A, genconverter) + assert struct_hook.overrides == { + "a": override(rename="b"), + "c": override(omit=True), + } + + class TD(TypedDict): + a: Annotated[int, override(rename="b")] + c: NotRequired[Annotated[int, override(rename="d")]] + + td_unstruct_hook = make_td_unstructure_fn(TD, genconverter) + assert td_unstruct_hook.overrides == { + "a": override(rename="b"), + "c": override(rename="d"), + } + + td_struct_hook = make_td_structure_fn(TD, genconverter) + assert td_struct_hook.overrides == { + "a": override(rename="b"), + "c": override(rename="d"), + } + + # Test Precedence (explicit should win and be in overrides) + @dataclass + class B: + a: Annotated[int, override(rename="b")] + + hook_explicit = make_dict_unstructure_fn(B, genconverter, a=override(rename="c")) + assert hook_explicit.overrides["a"].rename == "c"