From 8caafe3bb35c8aec6ac86f02816bb9ed0bbda53e Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 21 Jan 2026 18:04:50 +0100 Subject: [PATCH 01/12] Start on enclosures Co-authored-by: Blaise Pabon --- Doc/reference/expressions.rst | 95 ++++++++++++++++++------- Doc/reference/introduction.rst | 2 + Doc/tools/extensions/grammar_snippet.py | 19 +++++ 3 files changed, 92 insertions(+), 24 deletions(-) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 165dfa69f880d0..30128c29d7d5e8 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -55,9 +55,16 @@ also categorized syntactically as atoms. The syntax for atoms is: .. productionlist:: python-grammar atom: `identifier` | `literal` | `enclosure` - enclosure: `parenth_form` | `list_display` | `dict_display` | `set_display` - : | `generator_expression` | `yield_atom` +.. grammar-snippet:: + :group: python-grammar + + enclosure: + | (`group` | `tuple` | `yield_atom` | `generator_expression`) # in (parentheses) + | `list_display` + | `dict_display` + | `set_display` + | `yield_atom` .. _atom-identifiers: @@ -211,36 +218,75 @@ string literals:: .. _parenthesized: -Parenthesized forms -------------------- +Parenthesized groups +-------------------- -.. index:: - single: parenthesized form - single: () (parentheses); tuple display +A :dfn:`group` is an expression enclosed in parentheses. +The parenthesized group evaluates to the same value as the expression inside. -A parenthesized form is an optional expression list enclosed in parentheses: +Groups are used to override or clarify +:ref:`operator precedence `, +in the same way as in math notation. +For example:: -.. productionlist:: python-grammar - parenth_form: "(" [`starred_expression`] ")" + >>> 3 + 2 * 4 + 11 + >>> (3 + 2) * 4 # Override precedence of the addition + 20 + >>> 3 + (2 * 4) # Same effect as without parentheses + 11 + + >>> 3 << 2 | 4 + 12 + >>> 3 << (2 | 4) # Override precedence of the bitwise OR + 192 + >>> (3 << 2) | 4 # Same as without parentheses, but much clearer + 12 -A parenthesized expression list yields whatever that expression list yields: if -the list contains at least one comma, it yields a tuple; otherwise, it yields -the single expression that makes up the expression list. +Formally, the syntax for groups is: -.. index:: pair: empty; tuple +.. grammar-snippet:: + :group: python-grammar -An empty pair of parentheses yields an empty tuple object. Since tuples are -immutable, the same rules as for literals apply (i.e., two occurrences of the empty -tuple may or may not yield the same object). + group: '(' `assignment_expression` ')' -.. index:: - single: comma - single: , (comma) -Note that tuples are not formed by the parentheses, but rather by use of the -comma. The exception is the empty tuple, for which parentheses *are* -required --- allowing unparenthesized "nothing" in expressions would cause -ambiguities and allow common typos to pass uncaught. + +Tuple displays +-------------- + +.. + + Parenthesized forms + ------------------- + + .. index:: + single: parenthesized form + single: () (parentheses); tuple display + + A parenthesized form is an optional expression list enclosed in parentheses: + + .. productionlist:: python-grammar + parenth_form: "(" [`starred_expression`] ")" + + A parenthesized expression list yields whatever that expression list yields: if + the list contains at least one comma, it yields a tuple; otherwise, it yields + the single expression that makes up the expression list. + + .. index:: pair: empty; tuple + + An empty pair of parentheses yields an empty tuple object. Since tuples are + immutable, the same rules as for literals apply (i.e., two occurrences of the empty + tuple may or may not yield the same object). + + .. index:: + single: comma + single: , (comma) + + Note that tuples are not formed by the parentheses, but rather by use of the + comma. The exception is the empty tuple, for which parentheses *are* + required --- allowing unparenthesized "nothing" in expressions would cause + ambiguities and allow common typos to pass uncaught. .. _comprehensions: @@ -2049,6 +2095,7 @@ their suffixes:: .. _operator-summary: +.. _operator-precedence: Operator precedence =================== diff --git a/Doc/reference/introduction.rst b/Doc/reference/introduction.rst index c62240b18cfe55..b48898bbdccd58 100644 --- a/Doc/reference/introduction.rst +++ b/Doc/reference/introduction.rst @@ -145,6 +145,8 @@ The definition to the right of the colon uses the following syntax elements: * ``e?``: A question mark has exactly the same meaning as square brackets: the preceding item is optional. * ``(e)``: Parentheses are used for grouping. +* ``# ...``: As in Python, ``#`` introduces a comment that continues until the + end of the line. The following notation is only used in :ref:`lexical definitions `. diff --git a/Doc/tools/extensions/grammar_snippet.py b/Doc/tools/extensions/grammar_snippet.py index 8078b7ebeb8076..3699c32a0de3a9 100644 --- a/Doc/tools/extensions/grammar_snippet.py +++ b/Doc/tools/extensions/grammar_snippet.py @@ -36,6 +36,21 @@ def __init__( self['classes'].append('sx') +class snippet_comment_node(nodes.inline): # noqa: N801 (snake_case is fine) + """Node for a comment in a grammar snippet.""" + + def __init__( + self, + rawsource: str = '', + text: str = '', + *children: Node, + **attributes: Any, + ) -> None: + super().__init__(rawsource, text, *children, **attributes) + # Use the Pygments highlight class for `Comment.Single` + self['classes'].append('c1') + + class GrammarSnippetBase(SphinxDirective): """Common functionality for GrammarSnippetDirective & CompatProductionList.""" @@ -51,6 +66,8 @@ class GrammarSnippetBase(SphinxDirective): (?P'[^']*') # string in 'quotes' | (?P"[^"]*") # string in "quotes" + | + (?P[#].*) # comment """, re.VERBOSE, ) @@ -147,6 +164,8 @@ def make_production( production_node += token_xrefs(content, group_name) case 'single_quoted' | 'double_quoted': production_node += snippet_string_node('', content) + case 'comment': + production_node += snippet_comment_node('', content) case 'text': production_node += nodes.Text(content) case _: From 801420612f38110a0910f0372beb23ceda4ac1be Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 11 Feb 2026 18:05:40 +0100 Subject: [PATCH 02/12] Work on tuple displays --- Doc/reference/expressions.rst | 125 ++++++++++++++++++++++++++------- Doc/reference/introduction.rst | 2 + 2 files changed, 101 insertions(+), 26 deletions(-) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 30128c29d7d5e8..c954b6ba87ca67 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -216,13 +216,17 @@ string literals:: Template(strings=('Hello', '!'), interpolations=(...)) +.. index:: + single: parenthesized form + single: () (parentheses) + .. _parenthesized: Parenthesized groups -------------------- -A :dfn:`group` is an expression enclosed in parentheses. -The parenthesized group evaluates to the same value as the expression inside. +A :dfn:`parenthesized group` is an expression enclosed in parentheses. +The group evaluates to the same value as the expression inside. Groups are used to override or clarify :ref:`operator precedence `, @@ -240,9 +244,15 @@ For example:: 12 >>> 3 << (2 | 4) # Override precedence of the bitwise OR 192 - >>> (3 << 2) | 4 # Same as without parentheses, but much clearer + >>> (3 << 2) | 4 # Same as without parentheses (but much clearer) 12 +Note that not everything in parentheses is a *group*. +Specifically, a parenthesized group must include exactly one expression, +and cannot end with a comma. +See :ref:`tuple displays ` and +:ref:`generator expressions ` for other parenthesized forms. + Formally, the syntax for groups is: .. grammar-snippet:: @@ -251,42 +261,105 @@ Formally, the syntax for groups is: group: '(' `assignment_expression` ')' +.. index:: + single: tuple display + +.. _tuple-display: Tuple displays -------------- -.. +A :dfn:`tuple display` is a parenthesized expression that evaluates to a +:class:`tuple` object. + +In the most common form, the parentheses contain two or more comma-separated +expressions:: + + >>> (1, 2) + (1, 2) + >>> ('one', 'two', 'thr' + 'ee') + ('one', 'two', 'three') + +The expressions may be followed by an additional comma, which has no effect. +(The trailing comma is often used for tuple displays that span multiple lines, +so when a new entry is later added at the end, the existing line does not +need to be modified):: + + >>> (1, 2,) + (1, 2) + >>> ( + ... 'one', + ... 'two', + ... 'thr' + 'ee', + ... ) + ('one', 'two', 'three') - Parenthesized forms - ------------------- +At runtime, evaluating a tuple display results in a tuple that contains +the results of the expressions, in order. +Since tuples are immutable, the same rules as for literals apply: two +occurrences of tuples with the `same values` may or may not yield the same object. - .. index:: - single: parenthesized form - single: () (parentheses); tuple display +... TODO:: Link `same values` to "Literals and object identity" from the previous PR - A parenthesized form is an optional expression list enclosed in parentheses: +A tuple display may also contain a *single* expression. +In this case, the trailing comma is mandatory -- without it, you get a +:ref:`parenthesized group `:: - .. productionlist:: python-grammar - parenth_form: "(" [`starred_expression`] ")" + >>> ('single',) + ('single',) - A parenthesized expression list yields whatever that expression list yields: if - the list contains at least one comma, it yields a tuple; otherwise, it yields - the single expression that makes up the expression list. +.. index:: pair: empty; tuple - .. index:: pair: empty; tuple +A tuple display may also contain *zero* expressions: +empty parentheses denote the empty tuple. +A trailing comma is *not* allowed in this case. - An empty pair of parentheses yields an empty tuple object. Since tuples are - immutable, the same rules as for literals apply (i.e., two occurrences of the empty - tuple may or may not yield the same object). +.. code-block:: - .. index:: - single: comma - single: , (comma) + >>> () + () - Note that tuples are not formed by the parentheses, but rather by use of the - comma. The exception is the empty tuple, for which parentheses *are* - required --- allowing unparenthesized "nothing" in expressions would cause - ambiguities and allow common typos to pass uncaught. +To put it in other words, a tuple display is a parenthesized list of either: + +- two or more comma-separated expressions, or +- zero or more expressions, each followed by a comma. + +The formal grammar for tuple expressions is: + +.. grammar-snippet:: + :group: python-grammar + + tuple: + | '(' `flexible_expression` (',' `flexible_expression`)+ [','] ')' + | '(' `flexible_expression` ',' ')' + | '(' ')' + +.. note:: + + .. index:: + single: comma + single: , (comma) + + Note that tuple displays are not the only way to form tuples. + In several places, Python's syntax allows forming a tuple without + parentheses, only with a comma-separated list of expressions. + The most prominent example is the ``return`` statement:: + + >>> def gimme_a_tuple(): + ... return 1, 2, 3 + ... + >>> gimme_a_tuple() + (1, 2, 3) + + .. note to contributors: + Another prominent example is the expression statement, + but as of this writing, its docs imply that you need parentheses there. + The example can be added after the documented grammar is fixed. + This is tracked, broadly, in gh-127833. + + These are not considered *tuple displays*, but follow similar rules. + The use of a comma forms a tuple; without a comma, these forms evaluate + to a single expression. .. _comprehensions: diff --git a/Doc/reference/introduction.rst b/Doc/reference/introduction.rst index b48898bbdccd58..a7e1ba292fa7b2 100644 --- a/Doc/reference/introduction.rst +++ b/Doc/reference/introduction.rst @@ -145,6 +145,8 @@ The definition to the right of the colon uses the following syntax elements: * ``e?``: A question mark has exactly the same meaning as square brackets: the preceding item is optional. * ``(e)``: Parentheses are used for grouping. +* ``s.e+``: Match one or more occurrences of ``e``, separated by ``s``. + This is identical to ``(e (s e)*)``. * ``# ...``: As in Python, ``#`` introduces a comment that continues until the end of the line. From a703113f7deb8724d1705bdbab0219c0d85e2425 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 18 Feb 2026 17:43:47 +0100 Subject: [PATCH 03/12] Polish groups & tuple displays --- Doc/reference/expressions.rst | 167 ++++++++++++++---------- Doc/reference/introduction.rst | 4 - Doc/tools/extensions/grammar_snippet.py | 19 --- 3 files changed, 98 insertions(+), 92 deletions(-) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 6468dcfd091205..a257945a51f3cf 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -48,9 +48,16 @@ Atoms .. index:: atom Atoms are the most basic elements of expressions. -The simplest atoms are :ref:`names ` or literals. -Forms enclosed in parentheses, brackets or braces are also categorized -syntactically as atoms. +The simplest atoms are :ref:`builtin constants `, +:ref:`names ` and :ref:`literals `. +More complex atoms are enclosed in paired delimiters: + +- ``()`` (parentheses): :ref:`groups `, + :ref:`tuple displays `, + :ref:`yield atoms `, and + :ref:`generator expressions `; +- ``[]`` (square brackets): :ref:`list displays `; +- ``{}`` (curly braces): :ref:`dictionary ` and :ref:`set ` displays. Formally, the syntax for atoms is: @@ -58,15 +65,20 @@ Formally, the syntax for atoms is: :group: python-grammar atom: + | `builtin_constant` + | `identifier` + | `literal` + | `enclosure` + builtin_constant: | 'True' | 'False' | 'None' | '...' - | `identifier` - | `literal` - | `enclosure` enclosure: - | (`group` | `tuple` | `yield_atom` | `generator_expression`) # in (parentheses) + | `group` + | `tuple` + | `yield_atom` + | `generator_expression` | `list_display` | `dict_display` | `set_display` @@ -199,6 +211,7 @@ The formal grammar for literals is: literal: `strings` | `NUMBER` +.. _literals-identity: .. index:: triple: immutable; data; type @@ -324,18 +337,11 @@ Groups are used to override or clarify in the same way as in math notation. For example:: - >>> 3 + 2 * 4 - 11 - >>> (3 + 2) * 4 # Override precedence of the addition - 20 - >>> 3 + (2 * 4) # Same effect as without parentheses - 11 - >>> 3 << 2 | 4 12 - >>> 3 << (2 | 4) # Override precedence of the bitwise OR + >>> 3 << (2 | 4) # Override precedence of the | (bitwise OR) 192 - >>> (3 << 2) | 4 # Same as without parentheses (but much clearer) + >>> (3 << 2) | 4 # Same as without parentheses (but more clear) 12 Note that not everything in parentheses is a *group*. @@ -354,6 +360,8 @@ Formally, the syntax for groups is: .. index:: single: tuple display + single: comma + single: , (comma) .. _tuple-display: @@ -371,33 +379,39 @@ expressions:: >>> ('one', 'two', 'thr' + 'ee') ('one', 'two', 'three') -The expressions may be followed by an additional comma, which has no effect. -(The trailing comma is often used for tuple displays that span multiple lines, -so when a new entry is later added at the end, the existing line does not -need to be modified):: +The expressions may be followed by an additional comma, which has no effect:: >>> (1, 2,) (1, 2) - >>> ( - ... 'one', - ... 'two', - ... 'thr' + 'ee', - ... ) - ('one', 'two', 'three') + +.. note:: + + The trailing comma is often used for tuple displays that span multiple lines + (using :ref:`implicit line joining `), + so when a new entry is later added at the end, the existing line does not + need to be modified:: + + >>> ( + ... 'one', + ... 'two', + ... 'three', + ... ) + ('one', 'two', 'three') At runtime, evaluating a tuple display results in a tuple that contains the results of the expressions, in order. -Since tuples are immutable, the same rules as for literals apply: two -occurrences of tuples with the `same values` may or may not yield the same object. - -... TODO:: Link `same values` to "Literals and object identity" from the previous PR +Since tuples are immutable, :ref:`object identity rules for literals ` +also apply to tuples: two occurrences of tuples with the same values may +or may not yield the same object. A tuple display may also contain a *single* expression. In this case, the trailing comma is mandatory -- without it, you get a :ref:`parenthesized group `:: - >>> ('single',) + >>> ('single',) # single-element tuple ('single',) + >>> ('single') # no comma: single string + 'single' .. index:: pair: empty; tuple @@ -415,6 +429,33 @@ To put it in other words, a tuple display is a parenthesized list of either: - two or more comma-separated expressions, or - zero or more expressions, each followed by a comma. +.. note:: + + Python's syntax also includes :ref:`expression lists `, + where a comma-separated list of expressions is *not* enclosed in parentheses + but evaluates to tuple. + + In other words, when it comes to tuple syntax, the comma is more important + that the use of parentheses. + Only the empty tuple is spelled without a comma. + +.. index:: + pair: iterable; unpacking + single: * (asterisk); in expression lists + +Any expression in a tuple display may be prefixed with an asterisk (``*``). +This denotes :ref:`iterable unpacking as in expression lists `: + + + >>> numbers = (1, 2) + >>> (*numbers, 'word', *numbers) + (1, 2, 'word', 1, 2) + +.. versionadded:: 3.5 + Iterable unpacking in tuple displays, originally proposed by :pep:`448`. + +.. index:: pair: trailing; comma + The formal grammar for tuple expressions is: .. grammar-snippet:: @@ -425,33 +466,6 @@ The formal grammar for tuple expressions is: | '(' `flexible_expression` ',' ')' | '(' ')' -.. note:: - - .. index:: - single: comma - single: , (comma) - - Note that tuple displays are not the only way to form tuples. - In several places, Python's syntax allows forming a tuple without - parentheses, only with a comma-separated list of expressions. - The most prominent example is the ``return`` statement:: - - >>> def gimme_a_tuple(): - ... return 1, 2, 3 - ... - >>> gimme_a_tuple() - (1, 2, 3) - - .. note to contributors: - Another prominent example is the expression statement, - but as of this writing, its docs imply that you need parentheses there. - The example can be added after the documented grammar is fixed. - This is tracked, broadly, in gh-127833. - - These are not considered *tuple displays*, but follow similar rules. - The use of a comma forms a tuple; without a comma, these forms evaluate - to a single expression. - .. _comprehensions: @@ -2288,6 +2302,10 @@ functions created with lambda expressions cannot contain statements or annotations. +.. index:: + single: comma + single: , (comma) + .. _exprlists: Expression lists @@ -2312,12 +2330,32 @@ containing at least one comma yields a tuple. The length of the tuple is the number of expressions in the list. The expressions are evaluated from left to right. +.. index:: pair: trailing; comma + +A trailing comma is required only to create a one-item tuple, +such as ``1,``; it is optional in all other cases. +A single expression without a +trailing comma doesn't create a tuple, but rather yields the value of that +expression. (To create an empty tuple, use an empty pair of parentheses: +``()``.) + + +.. _iterable-unpacking: + .. index:: pair: iterable; unpacking single: * (asterisk); in expression lists -An asterisk ``*`` denotes :dfn:`iterable unpacking`. Its operand must be -an :term:`iterable`. The iterable is expanded into a sequence of items, +Iterable unpacking +------------------ + +In an expression list or tuple, list or set display, any expression +may be prefixed with an asterisk (``*``). +This denotes :dfn:`iterable unpacking`. + +At runtime, the asterisk-prefixed expression must evaluate +to an :term:`iterable`. +The iterable is expanded into a sequence of items, which are included in the new tuple, list, or set, at the site of the unpacking. @@ -2327,15 +2365,6 @@ the unpacking. .. versionadded:: 3.11 Any item in an expression list may be starred. See :pep:`646`. -.. index:: pair: trailing; comma - -A trailing comma is required only to create a one-item tuple, -such as ``1,``; it is optional in all other cases. -A single expression without a -trailing comma doesn't create a tuple, but rather yields the value of that -expression. (To create an empty tuple, use an empty pair of parentheses: -``()``.) - .. _evalorder: diff --git a/Doc/reference/introduction.rst b/Doc/reference/introduction.rst index a7e1ba292fa7b2..c62240b18cfe55 100644 --- a/Doc/reference/introduction.rst +++ b/Doc/reference/introduction.rst @@ -145,10 +145,6 @@ The definition to the right of the colon uses the following syntax elements: * ``e?``: A question mark has exactly the same meaning as square brackets: the preceding item is optional. * ``(e)``: Parentheses are used for grouping. -* ``s.e+``: Match one or more occurrences of ``e``, separated by ``s``. - This is identical to ``(e (s e)*)``. -* ``# ...``: As in Python, ``#`` introduces a comment that continues until the - end of the line. The following notation is only used in :ref:`lexical definitions `. diff --git a/Doc/tools/extensions/grammar_snippet.py b/Doc/tools/extensions/grammar_snippet.py index 3699c32a0de3a9..8078b7ebeb8076 100644 --- a/Doc/tools/extensions/grammar_snippet.py +++ b/Doc/tools/extensions/grammar_snippet.py @@ -36,21 +36,6 @@ def __init__( self['classes'].append('sx') -class snippet_comment_node(nodes.inline): # noqa: N801 (snake_case is fine) - """Node for a comment in a grammar snippet.""" - - def __init__( - self, - rawsource: str = '', - text: str = '', - *children: Node, - **attributes: Any, - ) -> None: - super().__init__(rawsource, text, *children, **attributes) - # Use the Pygments highlight class for `Comment.Single` - self['classes'].append('c1') - - class GrammarSnippetBase(SphinxDirective): """Common functionality for GrammarSnippetDirective & CompatProductionList.""" @@ -66,8 +51,6 @@ class GrammarSnippetBase(SphinxDirective): (?P'[^']*') # string in 'quotes' | (?P"[^"]*") # string in "quotes" - | - (?P[#].*) # comment """, re.VERBOSE, ) @@ -164,8 +147,6 @@ def make_production( production_node += token_xrefs(content, group_name) case 'single_quoted' | 'double_quoted': production_node += snippet_string_node('', content) - case 'comment': - production_node += snippet_comment_node('', content) case 'text': production_node += nodes.Text(content) case _: From 8439f6a25a4ae03bf9186501d32d0222e773d3af Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 25 Feb 2026 17:53:56 +0100 Subject: [PATCH 04/12] Reorganize Displays Co-authored-by: Blaise Pabon --- Doc/reference/expressions.rst | 523 +++++++++++++++++++++------------- 1 file changed, 321 insertions(+), 202 deletions(-) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index a257945a51f3cf..ea24a87e8545b5 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -79,10 +79,12 @@ Formally, the syntax for atoms is: | `tuple` | `yield_atom` | `generator_expression` - | `list_display` - | `dict_display` - | `set_display` - + | `listcomp` + | `list` + | `dictcomp` + | `dict` + | `setcomp` + | `set` .. _atom-singletons: @@ -358,257 +360,224 @@ Formally, the syntax for groups is: group: '(' `assignment_expression` ')' -.. index:: - single: tuple display - single: comma - single: , (comma) +.. _comprehensions: +.. _displays: -.. _tuple-display: +Container displays +------------------ -Tuple displays --------------- +.. index:: single: comprehensions -A :dfn:`tuple display` is a parenthesized expression that evaluates to a -:class:`tuple` object. +For constructing builtin containers (lists, sets, tuples or dictionaries), +Python provides special syntax called :dfn:`displays`. +There are subtle differences between the four kinds of displays, +detailed in the following sections. +All displays, however, consist of comma-separated items enclosed in paired +delimiters. -In the most common form, the parentheses contain two or more comma-separated -expressions:: +For example, a *list display* is a series of expressions enclosed in +square brackets:: - >>> (1, 2) - (1, 2) - >>> ('one', 'two', 'thr' + 'ee') - ('one', 'two', 'three') + >>> ["one", "two", "three"] + ['one', 'two', 'three'] -The expressions may be followed by an additional comma, which has no effect:: +In list, tuple and dictionary displays, the series may be empty:: - >>> (1, 2,) - (1, 2) + >>> [] # empty list + [] + >>> () # empty tuple + [] + >>> {} # empty dictionary + {} + +.. index:: pair: trailing; comma + +If the series is not empty, the items may be followed by an additional comma, +which has no effect:: + + >>> ["one", "two", "three",] # note comma after "three" + ['one', 'two', 'three'] .. note:: - The trailing comma is often used for tuple displays that span multiple lines + The trailing comma is often used for displays that span multiple lines (using :ref:`implicit line joining `), - so when a new entry is later added at the end, the existing line does not - need to be modified:: + so when a future programmer adds a new entry at the end, they do not + need to modify an existing line:: - >>> ( + >>> [ ... 'one', ... 'two', ... 'three', - ... ) - ('one', 'two', 'three') + ... ] + ['one', 'two', 'three'] -At runtime, evaluating a tuple display results in a tuple that contains -the results of the expressions, in order. -Since tuples are immutable, :ref:`object identity rules for literals ` -also apply to tuples: two occurrences of tuples with the same values may -or may not yield the same object. - -A tuple display may also contain a *single* expression. -In this case, the trailing comma is mandatory -- without it, you get a -:ref:`parenthesized group `:: +At runtime, when a display is evaluated, the listed items are evaluated from +left to right and placed into a new container of the appropriate type. - >>> ('single',) # single-element tuple - ('single',) - >>> ('single') # no comma: single string - 'single' +.. index:: + pair: iterable; unpacking + single: * (asterisk); in expression lists -.. index:: pair: empty; tuple +For tuple, list and set displays, any item in the display may be prefixed +with an asterisk (``*``). +This denotes :ref:`iterable unpacking `. +At runtime, the asterisk-prefixed expression must evaluate to an iterable, +whose contents are inserted into the container at the location of +the unpacking. For example:: -A tuple display may also contain *zero* expressions: -empty parentheses denote the empty tuple. -A trailing comma is *not* allowed in this case. + >>> numbers = (1, 2) + >>> [*numbers, 'word', *numbers] + (1, 2, 'word', 1, 2) -.. code-block:: +Dictionary displays use a similar mechanism called +:ref:`dictionary unpacking `, denoted with a double +asterisk (``**``). - >>> () - () +A more advanced form of displays are :dfn:`comprehensions`, where items are +computed via a set of looping and filtering instructions. +See the :ref:`comprehensions` section for details. -To put it in other words, a tuple display is a parenthesized list of either: +.. versionadded:: 3.5 + Iterable and dictionary unpacking in displays, originally proposed + by :pep:`448`. -- two or more comma-separated expressions, or -- zero or more expressions, each followed by a comma. -.. note:: - - Python's syntax also includes :ref:`expression lists `, - where a comma-separated list of expressions is *not* enclosed in parentheses - but evaluates to tuple. +.. _lists: - In other words, when it comes to tuple syntax, the comma is more important - that the use of parentheses. - Only the empty tuple is spelled without a comma. +List displays +^^^^^^^^^^^^^ .. index:: - pair: iterable; unpacking - single: * (asterisk); in expression lists - -Any expression in a tuple display may be prefixed with an asterisk (``*``). -This denotes :ref:`iterable unpacking as in expression lists `: + pair: list; display + pair: list; comprehensions + pair: empty; list + pair: object; list + single: [] (square brackets); list expression + single: , (comma); expression list +A :dfn:`list display` is a possibly empty series of expressions enclosed in +square brackets. For example:: - >>> numbers = (1, 2) - >>> (*numbers, 'word', *numbers) - (1, 2, 'word', 1, 2) - -.. versionadded:: 3.5 - Iterable unpacking in tuple displays, originally proposed by :pep:`448`. + >>> ["one", "two", "three"] + ['one', 'two', 'three'] + >>> ["one"] # One-element list + ['one'] + >>> [] # empty list + [] -.. index:: pair: trailing; comma +See :ref:`displays` for general information on displays. -The formal grammar for tuple expressions is: +The formal grammar for list displays is: .. grammar-snippet:: :group: python-grammar - tuple: - | '(' `flexible_expression` (',' `flexible_expression`)+ [','] ')' - | '(' `flexible_expression` ',' ')' - | '(' ')' + list: '[' [`flexible_expression_list`] ']' -.. _comprehensions: +.. _set: -Displays for lists, sets and dictionaries ------------------------------------------ +Set displays +^^^^^^^^^^^^ -.. index:: single: comprehensions +.. index:: + pair: set; display + pair: set; comprehensions + pair: object; set + single: {} (curly brackets); set expression + single: , (comma); expression list -For constructing a list, a set or a dictionary Python provides special syntax -called "displays", each of them in two flavors: +A :dfn:`set display` is a *non-empty* series of expressions enclosed in +curly braces. For example:: -* either the container contents are listed explicitly, or + >>> {"one", "two", "three"} + {'one', 'three', 'two'} + >>> {"one"} # One-element set + {'one'} -* they are computed via a set of looping and filtering instructions, called a - :dfn:`comprehension`. +See :ref:`displays` for general information on displays. -.. index:: - single: for; in comprehensions - single: if; in comprehensions - single: async for; in comprehensions +There is no special syntax for the empty set. +The ``{}`` literal is a :ref:`dictionary display ` that constructs an +empty dictionary. -Common syntax elements for comprehensions are: +The formal grammar for set displays is: -.. productionlist:: python-grammar - comprehension: `flexible_expression` `comp_for` - comp_for: ["async"] "for" `target_list` "in" `or_test` [`comp_iter`] - comp_iter: `comp_for` | `comp_if` - comp_if: "if" `or_test` [`comp_iter`] - -The comprehension consists of a single expression followed by at least one -:keyword:`!for` clause and zero or more :keyword:`!for` or :keyword:`!if` -clauses. In this case, the elements of the new container are those that would -be produced by considering each of the :keyword:`!for` or :keyword:`!if` -clauses a block, nesting from left to right, and evaluating the expression to -produce an element each time the innermost block is reached. If the expression -is starred, the result will instead be unpacked to produce zero or more -elements. - -However, aside from the iterable expression in the leftmost :keyword:`!for` clause, -the comprehension is executed in a separate implicitly nested scope. This ensures -that names assigned to in the target list don't "leak" into the enclosing scope. - -The iterable expression in the leftmost :keyword:`!for` clause is evaluated -directly in the enclosing scope and then passed as an argument to the implicitly -nested scope. Subsequent :keyword:`!for` clauses and any filter condition in the -leftmost :keyword:`!for` clause cannot be evaluated in the enclosing scope as -they may depend on the values obtained from the leftmost iterable. For example: -``[x*y for x in range(10) for y in range(x, x+10)]``. - -To ensure the comprehension always results in a container of the appropriate -type, ``yield`` and ``yield from`` expressions are prohibited in the implicitly -nested scope. +.. grammar-snippet:: + :group: python-grammar -.. index:: - single: await; in comprehensions - -Since Python 3.6, in an :keyword:`async def` function, an :keyword:`!async for` -clause may be used to iterate over a :term:`asynchronous iterator`. -A comprehension in an :keyword:`!async def` function may consist of either a -:keyword:`!for` or :keyword:`!async for` clause following the leading -expression, may contain additional :keyword:`!for` or :keyword:`!async for` -clauses, and may also use :keyword:`await` expressions. - -If a comprehension contains :keyword:`!async for` clauses, or if it contains -:keyword:`!await` expressions or other asynchronous comprehensions anywhere except -the iterable expression in the leftmost :keyword:`!for` clause, it is called an -:dfn:`asynchronous comprehension`. An asynchronous comprehension may suspend the -execution of the coroutine function in which it appears. -See also :pep:`530`. + set: '{' `flexible_expression_list` '}' -.. versionadded:: 3.6 - Asynchronous comprehensions were introduced. -.. versionchanged:: 3.8 - ``yield`` and ``yield from`` prohibited in the implicitly nested scope. +.. index:: + single: tuple display + single: comma + single: , (comma) -.. versionchanged:: 3.11 - Asynchronous comprehensions are now allowed inside comprehensions in - asynchronous functions. Outer comprehensions implicitly become - asynchronous. +.. _tuple-display: -.. versionchanged:: 3.15 - Unpacking with the ``*`` operator is now allowed in the expression. +.. index:: pair: empty; tuple +Tuple displays +^^^^^^^^^^^^^^ -.. _lists: +A :dfn:`tuple display` is a series of expressions enclosed in +parentheses. For example:: -List displays -------------- + >>> (1, 2) + (1, 2) + >>> ('one', 'two', 'thr' + 'ee') + ('one', 'two', 'three') -.. index:: - pair: list; display - pair: list; comprehensions - pair: empty; list - pair: object; list - single: [] (square brackets); list expression - single: , (comma); expression list + >>> () # an empty tuple + () -A list display is a possibly empty series of expressions enclosed in square -brackets: +See :ref:`displays` for general information on displays. -.. productionlist:: python-grammar - list_display: "[" [`flexible_expression_list` | `comprehension`] "]" +To avoid ambiguity, if a tuple display has exactly one element, +it requires a trailing comma. +Without it, you get a :ref:`parenthesized group `:: -A list display yields a new list object, the contents being specified by either -a list of expressions or a comprehension. When a comma-separated list of -expressions is supplied, its elements are evaluated from left to right and -placed into the list object in that order. When a comprehension is supplied, -the list is constructed from the elements resulting from the comprehension. + >>> ('single',) # single-element tuple + ('single',) + >>> ('single') # no comma: single string + 'single' +To put it in other words, a tuple display is a parenthesized list of either: -.. _set: +- two or more comma-separated expressions, or +- zero or more expressions, each followed by a comma. -Set displays ------------- +Since tuples are immutable, :ref:`object identity rules for literals ` +also apply to tuples: at runtime, two occurrences of tuples with the same +values may or may not yield the same object. -.. index:: - pair: set; display - pair: set; comprehensions - pair: object; set - single: {} (curly brackets); set expression - single: , (comma); expression list +.. note:: -A set display is denoted by curly braces and distinguishable from dictionary -displays by the lack of colons separating keys and values: + Python's syntax also includes :ref:`expression lists `, + where a comma-separated list of expressions is *not* enclosed in parentheses + but evaluates to tuple. -.. productionlist:: python-grammar - set_display: "{" (`flexible_expression_list` | `comprehension`) "}" + In other words, when it comes to tuple syntax, the comma is more important + that the use of parentheses. + Only the empty tuple is spelled without a comma. -A set display yields a new mutable set object, the contents being specified by -either a sequence of expressions or a comprehension. When a comma-separated -list of expressions is supplied, its elements are evaluated from left to right -and added to the set object. When a comprehension is supplied, the set is -constructed from the elements resulting from the comprehension. -An empty set cannot be constructed with ``{}``; this literal constructs an empty -dictionary. +The formal grammar for tuple displays is: +.. grammar-snippet:: + :group: python-grammar + + tuple: + | '(' `flexible_expression` (',' `flexible_expression`)+ [','] ')' + | '(' `flexible_expression` ',' ')' + | '(' ')' .. _dict: Dictionary displays -------------------- +^^^^^^^^^^^^^^^^^^^ .. index:: pair: dictionary; display @@ -619,35 +588,66 @@ Dictionary displays single: : (colon); in dictionary expressions single: , (comma); in dictionary displays -A dictionary display is a possibly empty series of dict items (key/value pairs) -enclosed in curly braces: +A :dfn:`dictionary display` is a possibly empty series of :dfn:`dict items` +enclosed in curly braces. +Each dict item is a colon-separated pair of expressions: the :dfn:`key` +and its associated :dfn:`value`. +For example:: -.. productionlist:: python-grammar - dict_display: "{" [`dict_item_list` | `dict_comprehension`] "}" - dict_item_list: `dict_item` ("," `dict_item`)* [","] - dict_comprehension: `dict_item` `comp_for` - dict_item: `expression` ":" `expression` | "**" `or_expr` + >>> {1: 'one', 2: 'two'} + {1: 'one', 2: 'two'} -A dictionary display yields a new dictionary object. +At runtime, when a dictionary comprehension is evaluated, the expressions +are evaluated from left to right. +Each key object is used as a key into the dictionary to store the +corresponding value. +This means that you can specify the same key multiple times in the +comprehension, and the final dictionary's value for a given key will be the +last one given. +For example:: -If a comma-separated sequence of dict items is given, they are evaluated -from left to right to define the entries of the dictionary: each key object is -used as a key into the dictionary to store the corresponding value. This means -that you can specify the same key multiple times in the dict item list, and the -final dictionary's value for that key will be the last one given. + >>> {1: 'this will be overridden', 2: 'two', 1: 'also overridden', 1: 'one'} + {1: 'one', 2: 'two'} .. index:: unpacking; dictionary single: **; in dictionary displays -A double asterisk ``**`` denotes :dfn:`dictionary unpacking`. -Its operand must be a :term:`mapping`. Each mapping item is added -to the new dictionary. Later values replace values already set by -earlier dict items and earlier dictionary unpackings. +.. _dict-unpacking: + +Instead of a key-value pair, a dict item may be an expression prefixed by +a double asterisk ``**``. This denotes :dfn:`dictionary unpacking`. +At runtime, the expression must evaluate to a :term:`mapping`; +each item of the mapping is added to the new dictionary. +As with key-value pairs, later values replace values already set by +earlier items and unpackings. +This may be used to override a set of defaults:: + + >>> defaults = {'color': 'blue', 'count': 8} + >>> overrides = {'color': 'yellow'} + >>> {**defaults, **overrides} + {'color': 'yellow', 'count': 8} + +.. productionlist:: python-grammar + dict_display: "{" [`dict_item_list` | `dict_comprehension`] "}" + dict_item_list: `dict_item` ("," `dict_item`)* [","] + dict_comprehension: `dict_item` `comp_for` + dict_item: `expression` ":" `expression` | "**" `or_expr` .. versionadded:: 3.5 Unpacking into dictionary displays, originally proposed by :pep:`448`. + + +Comprehensions +-------------- + +TODO + + + + + A dict comprehension may take one of two forms: - The first form uses two expressions separated with a colon followed by the @@ -682,6 +682,125 @@ prevails. .. versionchanged:: 3.15 Unpacking with the ``**`` operator is now allowed in dictionary comprehensions. + + + + + +.. index:: single: comprehensions + + + + + + + +There are no *tuple comprehensions*; a similar syntax is instead used +for :ref:`generator expressions `. + + +.. + + , each of them in two flavors: + + * either the container contents are listed explicitly, or + + * they are computed via a set of looping and filtering instructions, called a + :dfn:`comprehension`. + + .. index:: + single: for; in comprehensions + single: if; in comprehensions + single: async for; in comprehensions + + Common syntax elements for comprehensions are: + + .. productionlist:: python-grammar + comprehension: `flexible_expression` `comp_for` + comp_for: ["async"] "for" `target_list` "in" `or_test` [`comp_iter`] + comp_iter: `comp_for` | `comp_if` + comp_if: "if" `or_test` [`comp_iter`] + + The comprehension consists of a single expression followed by at least one + :keyword:`!for` clause and zero or more :keyword:`!for` or :keyword:`!if` + clauses. In this case, the elements of the new container are those that would + be produced by considering each of the :keyword:`!for` or :keyword:`!if` + clauses a block, nesting from left to right, and evaluating the expression to + produce an element each time the innermost block is reached. If the expression + is starred, the result will instead be unpacked to produce zero or more + elements. + + However, aside from the iterable expression in the leftmost :keyword:`!for` clause, + the comprehension is executed in a separate implicitly nested scope. This ensures + that names assigned to in the target list don't "leak" into the enclosing scope. + + The iterable expression in the leftmost :keyword:`!for` clause is evaluated + directly in the enclosing scope and then passed as an argument to the implicitly + nested scope. Subsequent :keyword:`!for` clauses and any filter condition in the + leftmost :keyword:`!for` clause cannot be evaluated in the enclosing scope as + they may depend on the values obtained from the leftmost iterable. For example: + ``[x*y for x in range(10) for y in range(x, x+10)]``. + + To ensure the comprehension always results in a container of the appropriate + type, ``yield`` and ``yield from`` expressions are prohibited in the implicitly + nested scope. + + .. index:: + single: await; in comprehensions + + Since Python 3.6, in an :keyword:`async def` function, an :keyword:`!async for` + clause may be used to iterate over a :term:`asynchronous iterator`. + A comprehension in an :keyword:`!async def` function may consist of either a + :keyword:`!for` or :keyword:`!async for` clause following the leading + expression, may contain additional :keyword:`!for` or :keyword:`!async for` + clauses, and may also use :keyword:`await` expressions. + + If a comprehension contains :keyword:`!async for` clauses, or if it contains + :keyword:`!await` expressions or other asynchronous comprehensions anywhere except + the iterable expression in the leftmost :keyword:`!for` clause, it is called an + :dfn:`asynchronous comprehension`. An asynchronous comprehension may suspend the + execution of the coroutine function in which it appears. + See also :pep:`530`. + + .. versionadded:: 3.6 + Asynchronous comprehensions were introduced. + + .. versionchanged:: 3.8 + ``yield`` and ``yield from`` prohibited in the implicitly nested scope. + + .. versionchanged:: 3.11 + Asynchronous comprehensions are now allowed inside comprehensions in + asynchronous functions. Outer comprehensions implicitly become + asynchronous. + + .. versionchanged:: 3.15 + Unpacking with the ``*`` operator is now allowed in the expression. + + +.. + + + + A list display yields a new list object, the contents being specified by either + a list of expressions or a comprehension. When a comma-separated list of + expressions is supplied, its elements are evaluated from left to right and + placed into the list object in that order. When a comprehension is supplied, + the list is constructed from the elements resulting from the comprehension. + +.. + + + A set display is denoted by curly braces and distinguishable from dictionary + displays by the lack of colons separating keys and values: + + A set display yields a new mutable set object, the contents being specified by + either a sequence of expressions or a comprehension. When a comma-separated + list of expressions is supplied, its elements are evaluated from left to right + and added to the set object. When a comprehension is supplied, the set is + constructed from the elements resulting from the comprehension. + + + .. _genexpr: Generator expressions From 30228c5e1790a7bb54a7d17b5649211d4e611386 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 25 Mar 2026 17:46:46 +0100 Subject: [PATCH 05/12] Work on comprehensions Co-authored-by: Blaise Pabon Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/reference/expressions.rst | 155 ++++++++++++++++++++++++++++++---- 1 file changed, 138 insertions(+), 17 deletions(-) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index ea24a87e8545b5..3fcadf4f3f8842 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -360,7 +360,6 @@ Formally, the syntax for groups is: group: '(' `assignment_expression` ')' -.. _comprehensions: .. _displays: Container displays @@ -380,8 +379,10 @@ square brackets:: >>> ["one", "two", "three"] ['one', 'two', 'three'] + >>> [1 + 2, 2 + 3] + [3, 5] -In list, tuple and dictionary displays, the series may be empty:: +In list, tuple and dictionary (but not set) displays, the series may be empty:: >>> [] # empty list [] @@ -419,8 +420,8 @@ left to right and placed into a new container of the appropriate type. pair: iterable; unpacking single: * (asterisk); in expression lists -For tuple, list and set displays, any item in the display may be prefixed -with an asterisk (``*``). +For tuple, list and set (but not dict) displays, any item in the display may +be prefixed with an asterisk (``*``). This denotes :ref:`iterable unpacking `. At runtime, the asterisk-prefixed expression must evaluate to an iterable, whose contents are inserted into the container at the location of @@ -428,11 +429,12 @@ the unpacking. For example:: >>> numbers = (1, 2) >>> [*numbers, 'word', *numbers] - (1, 2, 'word', 1, 2) + [1, 2, 'word', 1, 2] Dictionary displays use a similar mechanism called -:ref:`dictionary unpacking `, denoted with a double +*dictionary unpacking*, denoted with a double asterisk (``**``). +See :ref:`dict` for details. A more advanced form of displays are :dfn:`comprehensions`, where items are computed via a set of looping and filtering instructions. @@ -501,6 +503,7 @@ See :ref:`displays` for general information on displays. There is no special syntax for the empty set. The ``{}`` literal is a :ref:`dictionary display ` that constructs an empty dictionary. +Call :class:`set() ` with no arguments to get an empty set. The formal grammar for set displays is: @@ -527,9 +530,6 @@ parentheses. For example:: >>> (1, 2) (1, 2) - >>> ('one', 'two', 'thr' + 'ee') - ('one', 'two', 'three') - >>> () # an empty tuple () @@ -606,7 +606,12 @@ comprehension, and the final dictionary's value for a given key will be the last one given. For example:: - >>> {1: 'this will be overridden', 2: 'two', 1: 'also overridden', 1: 'one'} + >>> { + ... 1: 'this will be overridden', + ... 2: 'two', + ... 1: 'also overridden', + ... 1: 'one', + ... } {1: 'one', 2: 'two'} .. index:: @@ -638,10 +643,133 @@ This may be used to override a set of defaults:: Unpacking into dictionary displays, originally proposed by :pep:`448`. +.. index:: + single: for; in comprehensions + single: if; in comprehensions + single: async for; in comprehensions + +.. _comprehensions: Comprehensions -------------- +List, set and dictionary :dfn:`comprehensions` are a form of :ref:`displays`, +where items are computed via a set of looping and filtering instructions +rather than listed explicitly. + +In its simplest form, a comprehension consists of a single expression +followed by a :keyword:`!for` clause, all enclosed in paired delimiters. +For example, a list of the first ten squares is:: + + >>> [x**2 for x in range(10)] + [0, 1, 4, 9, 16, 25, 36, 49, 64, 81] + +A set of lowercase letters:: + + >>> {x.lower() for x in ('a', 'A', 'b', 'C')} + {'c', 'a', 'b'} + +A dict mapping some function names to functions:: + + >>> {f.__name__: f for f in (print, hex, any)} + {'print': , + 'hex': , + 'any': } + +There are no *tuple comprehensions*; a similar syntax is instead used +for :ref:`generator expressions `. + +Unpacking in comprehensions +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Filtering in comprehensions +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The :keyword:`!for` clause may be followed by an :keyword:`!if` clause +with an expression. + +For example, a list of names from the :mod:`math` module +that start with `f` is:: + + >>> [name for name in vars(math) if name.startswith('f')] + ['fabs', 'factorial', 'floor', 'fma', 'fmod', 'frexp', 'fsum'] + +Complex comprehensions +^^^^^^^^^^^^^^^^^^^^^^ + +The :keyword:`!for` clause may be followed zero or more additional +:keyword:`!for` or :keyword:`!if` clauses. +For example, here is a list of names exposed by two Python modules +that start with ``a``:: + + >>> import math + >>> import array + >>> [ + ... name + ... for module in (array, math) + ... for name in vars(module) + ... if name.startswith('a') + ... ] + ['array', 'acos', 'acosh', 'asin', 'asinh', 'atan', 'atan2', 'atanh'] + + +Asynchronous comprehensions +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +TODO: When a comprehension is evaluated, +TODO: - scopes +TODO: async for (& await) + + >>> system_defaults = {'color': 'blue', 'count': 8} + >>> user_defaults = {'color': 'yellow'} + >>> overrides = {'count': 5} + + >>> {**d for d in (system_defaults, user_defaults, overrides)} + {'color': 'yellow', 'count': 5} + + +at least one +:keyword:`!for` clause and zero or more :keyword:`!for` or :keyword:`!if` +clauses. In this case, the elements of the new container are those that would +be produced by considering each of the :keyword:`!for` or :keyword:`!if` +clauses a block, nesting from left to right, and evaluating the expression to +produce an element each time the innermost block is reached. If the expression +is starred, the result will instead be unpacked to produce zero or more +elements. + +However, aside from the iterable expression in the leftmost :keyword:`!for` clause, +the comprehension is executed in a separate implicitly nested scope. This ensures +that names assigned to in the target list don't "leak" into the enclosing scope. + +The iterable expression in the leftmost :keyword:`!for` clause is evaluated +directly in the enclosing scope and then passed as an argument to the implicitly +nested scope. Subsequent :keyword:`!for` clauses and any filter condition in the +leftmost :keyword:`!for` clause cannot be evaluated in the enclosing scope as +they may depend on the values obtained from the leftmost iterable. For example: +``[x*y for x in range(10) for y in range(x, x+10)]``. + +To ensure the comprehension always results in a container of the appropriate +type, ``yield`` and ``yield from`` expressions are prohibited in the implicitly +nested scope. + +.. index:: + single: await; in comprehensions + +Since Python 3.6, in an :keyword:`async def` function, an :keyword:`!async for` +clause may be used to iterate over a :term:`asynchronous iterator`. +A comprehension in an :keyword:`!async def` function may consist of either a +:keyword:`!for` or :keyword:`!async for` clause following the leading +expression, may contain additional :keyword:`!for` or :keyword:`!async for` +clauses, and may also use :keyword:`await` expressions. + +If a comprehension contains :keyword:`!async for` clauses, or if it contains +:keyword:`!await` expressions or other asynchronous comprehensions anywhere except +the iterable expression in the leftmost :keyword:`!for` clause, it is called an +:dfn:`asynchronous comprehension`. An asynchronous comprehension may suspend the +execution of the coroutine function in which it appears. +See also :pep:`530`. + + TODO @@ -695,8 +823,6 @@ prevails. -There are no *tuple comprehensions*; a similar syntax is instead used -for :ref:`generator expressions `. .. @@ -708,11 +834,6 @@ for :ref:`generator expressions `. * they are computed via a set of looping and filtering instructions, called a :dfn:`comprehension`. - .. index:: - single: for; in comprehensions - single: if; in comprehensions - single: async for; in comprehensions - Common syntax elements for comprehensions are: .. productionlist:: python-grammar From 74ee03b7bd521f3d79f5b8b7cbe10c21519994fc Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 1 Apr 2026 17:59:54 +0200 Subject: [PATCH 06/12] Work on comprehensions Co-authored-by: Blaise Pabon --- Doc/reference/expressions.rst | 355 +++++++++++++++++----------------- 1 file changed, 175 insertions(+), 180 deletions(-) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 3fcadf4f3f8842..310de28b67ed5f 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -633,6 +633,8 @@ This may be used to override a set of defaults:: >>> {**defaults, **overrides} {'color': 'yellow', 'count': 8} +TODO: + .. productionlist:: python-grammar dict_display: "{" [`dict_item_list` | `dict_comprehension`] "}" dict_item_list: `dict_item` ("," `dict_item`)* [","] @@ -644,16 +646,15 @@ This may be used to override a set of defaults:: .. index:: + single: comprehensions single: for; in comprehensions - single: if; in comprehensions - single: async for; in comprehensions .. _comprehensions: Comprehensions -------------- -List, set and dictionary :dfn:`comprehensions` are a form of :ref:`displays`, +List, set and dictionary :dfn:`comprehensions` are a form of :ref:`displays` where items are computed via a set of looping and filtering instructions rather than listed explicitly. @@ -664,23 +665,70 @@ For example, a list of the first ten squares is:: >>> [x**2 for x in range(10)] [0, 1, 4, 9, 16, 25, 36, 49, 64, 81] -A set of lowercase letters:: +The comprehension is roughly equivalent to defining and calling the following +function:: + + def make_list_of_squares(iterable): + result = [] + for x in iterable: + result.append(x**2) + return result + + make_list_of_squares(range(10)) - >>> {x.lower() for x in ('a', 'A', 'b', 'C')} +Set comprehensions work similarly. +For example, here is a set of lowercase letters:: + + >>> {x.lower() for x in ['a', 'A', 'b', 'C']} {'c', 'a', 'b'} -A dict mapping some function names to functions:: +This corresponds roughly to calling this function:: + + def make_lowercase_set(iterable): + result = set(iterable) + for x in : + result.append(x.lower()) + return result + + make_lowercase_set(['a', 'A', 'b', 'C']) + +Dictionary comprehensions start with a colon-separated key-value pair instead +of an expression. For example:: - >>> {f.__name__: f for f in (print, hex, any)} + >>> {f.__name__: f for f in [print, hex, any]} {'print': , 'hex': , 'any': } -There are no *tuple comprehensions*; a similar syntax is instead used -for :ref:`generator expressions `. +This corresponds roughly to:: -Unpacking in comprehensions -^^^^^^^^^^^^^^^^^^^^^^^^^^^ + def make_dict_mapping_names_to_functions(iterable): + result = {} + for f in iterable: + result[f.__name__] = f + return result + + iterable([print, hex, any]) + +As in other kinds of dictionary displays, the same key may be specified +multiple times. +Earlier values are overwritten by ones that are evaluated later. + +There are no *tuple comprehensions*, but a similar syntax is instead used +for :ref:`generator expressions `, from which you can construct +a tuple like this:: + + >>> tuple(x**2 for x in range(10)) + (0, 1, 4, 9, 16, 25, 36, 49, 64, 81) + +.. versionchanged:: 3.8 + Prior to Python 3.8, in dict comprehensions, the evaluation order of key + and value was not well-defined. In CPython, the value was evaluated before + the key. Starting with 3.8, the key is evaluated before the value, as + proposed by :pep:`572`. + + +.. index:: single: if; in comprehensions Filtering in comprehensions ^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -694,11 +742,28 @@ that start with `f` is:: >>> [name for name in vars(math) if name.startswith('f')] ['fabs', 'factorial', 'floor', 'fma', 'fmod', 'frexp', 'fsum'] +This roughly corresponds to defining and calling the following function:: + + def get_math_f_names(iterable): + result = [] + for name in iterable: + if name.startswith('f'): + result.append(name) + return result + + get_math_f_names(vars(math)) + +Filtering is a special case of more complex comprehensions. +See the next section for a more formal description. + + +.. _complex-comprehensions: + Complex comprehensions ^^^^^^^^^^^^^^^^^^^^^^ -The :keyword:`!for` clause may be followed zero or more additional -:keyword:`!for` or :keyword:`!if` clauses. +Generally, a comprehension's initial :keyword:`!for` clause may be followed +zero or more additional :keyword:`!for` or :keyword:`!if` clauses. For example, here is a list of names exposed by two Python modules that start with ``a``:: @@ -706,220 +771,150 @@ that start with ``a``:: >>> import array >>> [ ... name - ... for module in (array, math) + ... for module in [array, math] ... for name in vars(module) ... if name.startswith('a') ... ] ['array', 'acos', 'acosh', 'asin', 'asinh', 'atan', 'atan2', 'atanh'] +This roughly corresponds to defining and calling:: -Asynchronous comprehensions -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -TODO: When a comprehension is evaluated, -TODO: - scopes -TODO: async for (& await) - - >>> system_defaults = {'color': 'blue', 'count': 8} - >>> user_defaults = {'color': 'yellow'} - >>> overrides = {'count': 5} + def get_a_names(iterable): + result = [] + for module in iterable: + for name in vars(module): + if name.startswith('a'): + result.append(name) + return result - >>> {**d for d in (system_defaults, user_defaults, overrides)} - {'color': 'yellow', 'count': 5} + get_a_names([array, math]) +In this case, and in the simpler cases in the previous sections, +the elements of the new container are those that would be produced by +considering each of the :keyword:`!for` or :keyword:`!if` clauses a block, +nesting from left to right, and evaluating the expression to produce an +element (or dictionary entry) each time the innermost block is reached. -at least one -:keyword:`!for` clause and zero or more :keyword:`!for` or :keyword:`!if` -clauses. In this case, the elements of the new container are those that would -be produced by considering each of the :keyword:`!for` or :keyword:`!if` -clauses a block, nesting from left to right, and evaluating the expression to -produce an element each time the innermost block is reached. If the expression -is starred, the result will instead be unpacked to produce zero or more -elements. +Aside from the iterable expression in the leftmost :keyword:`!for` clause, +the comprehension is executed in a separate implicitly nested scope. +This ensures that names assigned to in the target list don't "leak" into +the enclosing scope. +For example:: -However, aside from the iterable expression in the leftmost :keyword:`!for` clause, -the comprehension is executed in a separate implicitly nested scope. This ensures -that names assigned to in the target list don't "leak" into the enclosing scope. + >>> x = 'old value' + >>> [x**2 for x in range(10)] + >>> x + 'old value' The iterable expression in the leftmost :keyword:`!for` clause is evaluated directly in the enclosing scope and then passed as an argument to the implicitly -nested scope. Subsequent :keyword:`!for` clauses and any filter condition in the +nested scope. + +Subsequent :keyword:`!for` clauses and any filter condition in the leftmost :keyword:`!for` clause cannot be evaluated in the enclosing scope as -they may depend on the values obtained from the leftmost iterable. For example: -``[x*y for x in range(10) for y in range(x, x+10)]``. +they may depend on the values obtained from the leftmost iterable. To ensure the comprehension always results in a container of the appropriate type, ``yield`` and ``yield from`` expressions are prohibited in the implicitly nested scope. -.. index:: - single: await; in comprehensions - -Since Python 3.6, in an :keyword:`async def` function, an :keyword:`!async for` -clause may be used to iterate over a :term:`asynchronous iterator`. -A comprehension in an :keyword:`!async def` function may consist of either a -:keyword:`!for` or :keyword:`!async for` clause following the leading -expression, may contain additional :keyword:`!for` or :keyword:`!async for` -clauses, and may also use :keyword:`await` expressions. - -If a comprehension contains :keyword:`!async for` clauses, or if it contains -:keyword:`!await` expressions or other asynchronous comprehensions anywhere except -the iterable expression in the leftmost :keyword:`!for` clause, it is called an -:dfn:`asynchronous comprehension`. An asynchronous comprehension may suspend the -execution of the coroutine function in which it appears. -See also :pep:`530`. - - -TODO - - - - - -A dict comprehension may take one of two forms: - -- The first form uses two expressions separated with a colon followed by the - usual "for" and "if" clauses. When the comprehension is run, the resulting - key and value elements are inserted in the new dictionary in the order they - are produced. - -- The second form uses a single expression prefixed by the ``**`` dictionary - unpacking operator followed by the usual "for" and "if" clauses. When the - comprehension is evaluated, the expression is evaluated and then unpacked, - inserting zero or more key/value pairs into the new dictionary. - -Both forms of dictionary comprehension retain the property that if the same key -is specified multiple times, the associated value in the resulting dictionary -will be the last one specified. - -.. index:: pair: immutable; object - hashable - -Restrictions on the types of the key values are listed earlier in section -:ref:`types`. (To summarize, the key type should be :term:`hashable`, which excludes -all mutable objects.) Clashes between duplicate keys are not detected; the last -value (textually rightmost in the display) stored for a given key value -prevails. - .. versionchanged:: 3.8 - Prior to Python 3.8, in dict comprehensions, the evaluation order of key - and value was not well-defined. In CPython, the value was evaluated before - the key. Starting with 3.8, the key is evaluated before the value, as - proposed by :pep:`572`. - -.. versionchanged:: 3.15 - Unpacking with the ``**`` operator is now allowed in dictionary comprehensions. - - - - - - -.. index:: single: comprehensions - - - - - - - - + ``yield`` and ``yield from`` prohibited in the implicitly nested scope. -.. - , each of them in two flavors: +Unpacking in comprehensions +^^^^^^^^^^^^^^^^^^^^^^^^^^^ - * either the container contents are listed explicitly, or +If the expression of a list or set comprehension is starred, the result will +be :ref:`unpacked ` to produce +zero or more elements. - * they are computed via a set of looping and filtering instructions, called a - :dfn:`comprehension`. +This is often used for "flattening" lists, for example:: - Common syntax elements for comprehensions are: + >>> students = ['Petr', 'Blaise', 'Jarka'] + >>> teachers = ['Salim', 'Bartosz'] + >>> lists_of_people = [students, teachers] + >>> [*people for people in lists_of_people] + ['Petr', 'Blaise', 'Jarka', 'Salim', 'Bartosz'] - .. productionlist:: python-grammar - comprehension: `flexible_expression` `comp_for` - comp_for: ["async"] "for" `target_list` "in" `or_test` [`comp_iter`] - comp_iter: `comp_for` | `comp_if` - comp_if: "if" `or_test` [`comp_iter`] +This comprehension roughly corresponds to:: - The comprehension consists of a single expression followed by at least one - :keyword:`!for` clause and zero or more :keyword:`!for` or :keyword:`!if` - clauses. In this case, the elements of the new container are those that would - be produced by considering each of the :keyword:`!for` or :keyword:`!if` - clauses a block, nesting from left to right, and evaluating the expression to - produce an element each time the innermost block is reached. If the expression - is starred, the result will instead be unpacked to produce zero or more - elements. + def flatten_names(lists_of_people): + result = [] + for people in lists_of_people: + result.extend(people) + return result - However, aside from the iterable expression in the leftmost :keyword:`!for` clause, - the comprehension is executed in a separate implicitly nested scope. This ensures - that names assigned to in the target list don't "leak" into the enclosing scope. +In dict comprehensions, a double-starred expression will be evaluated and +then unpacked using :ref:`dictionary unpacking `, +inserting zero or more key/value pairs into the new dictionary. +As in other kinds of dictionary displays, if the same key is specified +multiple times, the associated value in the resulting dictionary +will be the last one specified. - The iterable expression in the leftmost :keyword:`!for` clause is evaluated - directly in the enclosing scope and then passed as an argument to the implicitly - nested scope. Subsequent :keyword:`!for` clauses and any filter condition in the - leftmost :keyword:`!for` clause cannot be evaluated in the enclosing scope as - they may depend on the values obtained from the leftmost iterable. For example: - ``[x*y for x in range(10) for y in range(x, x+10)]``. +For example:: - To ensure the comprehension always results in a container of the appropriate - type, ``yield`` and ``yield from`` expressions are prohibited in the implicitly - nested scope. + >>> system_defaults = {'color': 'blue', 'count': 8} + >>> user_defaults = {'color': 'yellow'} + >>> overrides = {'count': 5} - .. index:: - single: await; in comprehensions + >>> configuration_sets = [system_defaults, user_defaults, overrides] - Since Python 3.6, in an :keyword:`async def` function, an :keyword:`!async for` - clause may be used to iterate over a :term:`asynchronous iterator`. - A comprehension in an :keyword:`!async def` function may consist of either a - :keyword:`!for` or :keyword:`!async for` clause following the leading - expression, may contain additional :keyword:`!for` or :keyword:`!async for` - clauses, and may also use :keyword:`await` expressions. + >>> {**d for d in configuration_sets} + {'color': 'yellow', 'count': 5} - If a comprehension contains :keyword:`!async for` clauses, or if it contains - :keyword:`!await` expressions or other asynchronous comprehensions anywhere except - the iterable expression in the leftmost :keyword:`!for` clause, it is called an - :dfn:`asynchronous comprehension`. An asynchronous comprehension may suspend the - execution of the coroutine function in which it appears. - See also :pep:`530`. +.. versionadded:: 3.15 - .. versionadded:: 3.6 - Asynchronous comprehensions were introduced. + Unpacking in comprehensions using the ``*`` and ``**`` operators + was introduced in :pep:`798`. - .. versionchanged:: 3.8 - ``yield`` and ``yield from`` prohibited in the implicitly nested scope. - .. versionchanged:: 3.11 - Asynchronous comprehensions are now allowed inside comprehensions in - asynchronous functions. Outer comprehensions implicitly become - asynchronous. +.. index:: + single: async for; in comprehensions + single: await; in comprehensions - .. versionchanged:: 3.15 - Unpacking with the ``*`` operator is now allowed in the expression. +Asynchronous comprehensions +^^^^^^^^^^^^^^^^^^^^^^^^^^^ +In an :keyword:`async def` function, an :keyword:`!async for` +clause may be used to iterate over a :term:`asynchronous iterator`. +A comprehension in an :keyword:`!async def` function may consist of either a +:keyword:`!for` or :keyword:`!async for` clause following the leading +expression, may contain additional :keyword:`!for` or :keyword:`!async for` +clauses, and may also use :keyword:`await` expressions. -.. +If a comprehension contains :keyword:`!async for` clauses, or if it contains +:keyword:`!await` expressions or other asynchronous comprehensions anywhere except +the iterable expression in the leftmost :keyword:`!for` clause, it is called an +:dfn:`asynchronous comprehension`. An asynchronous comprehension may suspend the +execution of the coroutine function in which it appears. +.. versionadded:: 3.6 + Asynchronous comprehensions were introduced in :pep:`530`. - A list display yields a new list object, the contents being specified by either - a list of expressions or a comprehension. When a comma-separated list of - expressions is supplied, its elements are evaluated from left to right and - placed into the list object in that order. When a comprehension is supplied, - the list is constructed from the elements resulting from the comprehension. +.. versionchanged:: 3.11 + Asynchronous comprehensions are now allowed inside comprehensions in + asynchronous functions. Outer comprehensions implicitly become + asynchronous. -.. +Formal grammar for comprehensions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - A set display is denoted by curly braces and distinguishable from dictionary - displays by the lack of colons separating keys and values: +.. grammar-snippet:: + :group: python-grammar - A set display yields a new mutable set object, the contents being specified by - either a sequence of expressions or a comprehension. When a comma-separated - list of expressions is supplied, its elements are evaluated from left to right - and added to the set object. When a comprehension is supplied, the set is - constructed from the elements resulting from the comprehension. + listcomp: + | '[' `flexible_expression` `for_if_clause`+ ']' + setcomp: + | '{' `flexible_expression` `for_if_clause`+ '}' + dictcomp: + | '{' `dict_item` `for_if_clause`+ '}' + | '{' '**' `expression` `for_if_clause`+ '}' + for_if_clause: + | ['async'] 'for' `target_list` 'in' `or_test` ('if' `or_test`)* .. _genexpr: From f6cf7261a4b9cd37d0e496950a67ba57d0f5f40b Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 8 Apr 2026 18:02:56 +0200 Subject: [PATCH 07/12] Comtinue --- Doc/reference/expressions.rst | 78 ++++++++++++++++++++++------------- 1 file changed, 49 insertions(+), 29 deletions(-) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 310de28b67ed5f..6e0bd7e81698cb 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -633,18 +633,21 @@ This may be used to override a set of defaults:: >>> {**defaults, **overrides} {'color': 'yellow', 'count': 8} -TODO: - -.. productionlist:: python-grammar - dict_display: "{" [`dict_item_list` | `dict_comprehension`] "}" - dict_item_list: `dict_item` ("," `dict_item`)* [","] - dict_comprehension: `dict_item` `comp_for` - dict_item: `expression` ":" `expression` | "**" `or_expr` - .. versionadded:: 3.5 Unpacking into dictionary displays, originally proposed by :pep:`448`. +The formal grammar for dict displays is: + +.. grammar-snippet:: + :group: python-grammar + + dict: '{' [`double_starred_kvpairs`] '}' + double_starred_kvpairs: ','.`double_starred_kvpair`+ [','] + double_starred_kvpair: '**' `or_expr` | `kvpair` + kvpair: `expression` ':' `expression` + + .. index:: single: comprehensions single: for; in comprehensions @@ -659,14 +662,23 @@ where items are computed via a set of looping and filtering instructions rather than listed explicitly. In its simplest form, a comprehension consists of a single expression -followed by a :keyword:`!for` clause, all enclosed in paired delimiters. +followed by a :keyword:`!for` clause. +The :keyword:`!for` clause has the same syntax as the header of a +:ref:`for statement `, without a trailing colon. + For example, a list of the first ten squares is:: >>> [x**2 for x in range(10)] [0, 1, 4, 9, 16, 25, 36, 49, 64, 81] -The comprehension is roughly equivalent to defining and calling the following -function:: +At run time, a list comprehension creates a new list. +The expression after :keyword:`!in` must evaluate to an :term:`iterable`. +For each element of this iterable, the element is bound to the :keyword:`!for` +clause's target as in a :keyword:`!for` statement, then the expression +before :keyword:`!for` is evaluated with the target in scope and the result +is added to the new list. +Thus, the example above is roughly equivalent to defining and calling +the following function:: def make_list_of_squares(iterable): result = [] @@ -682,11 +694,11 @@ For example, here is a set of lowercase letters:: >>> {x.lower() for x in ['a', 'A', 'b', 'C']} {'c', 'a', 'b'} -This corresponds roughly to calling this function:: +At run time, this corresponds roughly to calling this function:: def make_lowercase_set(iterable): result = set(iterable) - for x in : + for x in iterable: result.append(x.lower()) return result @@ -695,17 +707,17 @@ This corresponds roughly to calling this function:: Dictionary comprehensions start with a colon-separated key-value pair instead of an expression. For example:: - >>> {f.__name__: f for f in [print, hex, any]} + >>> {func.__name__: func for func in [print, hex, any]} {'print': , 'hex': , 'any': } -This corresponds roughly to:: +At run time, this corresponds roughly to:: def make_dict_mapping_names_to_functions(iterable): result = {} - for f in iterable: - result[f.__name__] = f + for func in iterable: + result[func.__name__] = func return result iterable([print, hex, any]) @@ -714,9 +726,9 @@ As in other kinds of dictionary displays, the same key may be specified multiple times. Earlier values are overwritten by ones that are evaluated later. -There are no *tuple comprehensions*, but a similar syntax is instead used -for :ref:`generator expressions `, from which you can construct -a tuple like this:: +There are no *tuple comprehensions*. +A similar syntax is instead used for :ref:`generator expressions `, +from which you can construct a tuple like this:: >>> tuple(x**2 for x in range(10)) (0, 1, 4, 9, 16, 25, 36, 49, 64, 81) @@ -742,7 +754,11 @@ that start with `f` is:: >>> [name for name in vars(math) if name.startswith('f')] ['fabs', 'factorial', 'floor', 'fma', 'fmod', 'frexp', 'fsum'] -This roughly corresponds to defining and calling the following function:: +At run time, the expression after :keyword:`!if` is evaluated before +each element is added to the resulting container. +If the expression evaluates to false, the element is skipped. +Thus, the above example roughly corresponds to defining and calling the +following function:: def get_math_f_names(iterable): result = [] @@ -764,11 +780,11 @@ Complex comprehensions Generally, a comprehension's initial :keyword:`!for` clause may be followed zero or more additional :keyword:`!for` or :keyword:`!if` clauses. -For example, here is a list of names exposed by two Python modules -that start with ``a``:: +For example, here is a list of names exposed by two Python modules, +filtered to only include names that start with ``a``:: - >>> import math >>> import array + >>> import math >>> [ ... name ... for module in [array, math] @@ -777,7 +793,7 @@ that start with ``a``:: ... ] ['array', 'acos', 'acosh', 'asin', 'asinh', 'atan', 'atan2', 'atanh'] -This roughly corresponds to defining and calling:: +At run time, this roughly corresponds to defining and calling:: def get_a_names(iterable): result = [] @@ -789,8 +805,7 @@ This roughly corresponds to defining and calling:: get_a_names([array, math]) -In this case, and in the simpler cases in the previous sections, -the elements of the new container are those that would be produced by +The elements of the new container are those that would be produced by considering each of the :keyword:`!for` or :keyword:`!if` clauses a block, nesting from left to right, and evaluating the expression to produce an element (or dictionary entry) each time the innermost block is reached. @@ -802,7 +817,7 @@ the enclosing scope. For example:: >>> x = 'old value' - >>> [x**2 for x in range(10)] + >>> [x**2 for x in range(10)] # this `x` is local to the comprehension >>> x 'old value' @@ -814,6 +829,9 @@ Subsequent :keyword:`!for` clauses and any filter condition in the leftmost :keyword:`!for` clause cannot be evaluated in the enclosing scope as they may depend on the values obtained from the leftmost iterable. +TODO: PEP 572: + ..due to a limitation in CPython’s symbol table analysis process, the reference implementation raises SyntaxError for all uses of named expressions inside comprehension iterable expressions, rather than only raising them when the named expression target conflicts with one of the iteration variables in the comprehension. This could be revisited given sufficiently compelling examples, but the extra complexity needed to implement the more selective restriction doesn’t seem worthwhile for purely hypothetical use cases. + To ensure the comprehension always results in a container of the appropriate type, ``yield`` and ``yield from`` expressions are prohibited in the implicitly nested scope. @@ -837,7 +855,7 @@ This is often used for "flattening" lists, for example:: >>> [*people for people in lists_of_people] ['Petr', 'Blaise', 'Jarka', 'Salim', 'Bartosz'] -This comprehension roughly corresponds to:: +At run time, this comprehension roughly corresponds to:: def flatten_names(lists_of_people): result = [] @@ -902,6 +920,8 @@ execution of the coroutine function in which it appears. Formal grammar for comprehensions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The formal grammar for comprehensions is: + .. grammar-snippet:: :group: python-grammar From 041d821917e5dd1ff97b562cc57f24b2de728b79 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 15 Apr 2026 17:01:46 +0200 Subject: [PATCH 08/12] Add a note on assignment expressions --- Doc/reference/expressions.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 6e0bd7e81698cb..b5ff904a8fa879 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -829,13 +829,15 @@ Subsequent :keyword:`!for` clauses and any filter condition in the leftmost :keyword:`!for` clause cannot be evaluated in the enclosing scope as they may depend on the values obtained from the leftmost iterable. -TODO: PEP 572: - ..due to a limitation in CPython’s symbol table analysis process, the reference implementation raises SyntaxError for all uses of named expressions inside comprehension iterable expressions, rather than only raising them when the named expression target conflicts with one of the iteration variables in the comprehension. This could be revisited given sufficiently compelling examples, but the extra complexity needed to implement the more selective restriction doesn’t seem worthwhile for purely hypothetical use cases. - To ensure the comprehension always results in a container of the appropriate type, ``yield`` and ``yield from`` expressions are prohibited in the implicitly nested scope. +:ref:`Assignment expressions ` are not allowed +inside comprehension iterable expressions (that is, the expressions after +the :keyword:`!in` keyword), nor anywhere within comprehensions that +appear directly in a class definition. + .. versionchanged:: 3.8 ``yield`` and ``yield from`` prohibited in the implicitly nested scope. From ebe2dde4a9511816c06ae4698b47063f3f2d00e0 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 15 Apr 2026 17:16:38 +0200 Subject: [PATCH 09/12] Touch ups --- Doc/reference/expressions.rst | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index b5ff904a8fa879..0738e46981d556 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -68,19 +68,18 @@ Formally, the syntax for atoms is: | `builtin_constant` | `identifier` | `literal` - | `enclosure` - builtin_constant: - | 'True' - | 'False' - | 'None' - | '...' - enclosure: + | `parenthesized_enclosure` + | `bracketed_enclosure` + | `braced_enclosure` + parenthesized_enclosure: | `group` | `tuple` | `yield_atom` | `generator_expression` + bracketed_enclosure: | `listcomp` | `list` + braced_enclosure: | `dictcomp` | `dict` | `setcomp` @@ -111,6 +110,13 @@ Evaluation of these atoms yields the corresponding value. ^^^^^ SyntaxError: cannot assign to False +Formally, the syntax for built-in constants is: + +.. grammar-snippet:: + :group: python-grammar + + builtin_constant: 'True' | 'False' | 'None' | '...' + .. _atom-identifiers: Identifiers (Names) @@ -657,9 +663,9 @@ The formal grammar for dict displays is: Comprehensions -------------- -List, set and dictionary :dfn:`comprehensions` are a form of :ref:`displays` -where items are computed via a set of looping and filtering instructions -rather than listed explicitly. +List, set and dictionary :dfn:`comprehensions` are a form of +:ref:`container displays ` where items are computed via a set of +looping and filtering instructions rather than listed explicitly. In its simplest form, a comprehension consists of a single expression followed by a :keyword:`!for` clause. @@ -755,8 +761,8 @@ that start with `f` is:: ['fabs', 'factorial', 'floor', 'fma', 'fmod', 'frexp', 'fsum'] At run time, the expression after :keyword:`!if` is evaluated before -each element is added to the resulting container. -If the expression evaluates to false, the element is skipped. +each element is added to the resulting container, and if it is false, +the element is skipped. Thus, the above example roughly corresponds to defining and calling the following function:: @@ -778,7 +784,7 @@ See the next section for a more formal description. Complex comprehensions ^^^^^^^^^^^^^^^^^^^^^^ -Generally, a comprehension's initial :keyword:`!for` clause may be followed +Generally, a comprehension's initial :keyword:`!for` clause may be followed by zero or more additional :keyword:`!for` or :keyword:`!if` clauses. For example, here is a list of names exposed by two Python modules, filtered to only include names that start with ``a``:: From 52d92ee7f9a7105275d9a93d54c3308eb894174c Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 15 Apr 2026 17:31:42 +0200 Subject: [PATCH 10/12] Lint fix --- Doc/reference/expressions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 0738e46981d556..f1d9bec897834d 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -755,7 +755,7 @@ The :keyword:`!for` clause may be followed by an :keyword:`!if` clause with an expression. For example, a list of names from the :mod:`math` module -that start with `f` is:: +that start with ``f`` is:: >>> [name for name in vars(math) if name.startswith('f')] ['fabs', 'factorial', 'floor', 'fma', 'fmod', 'frexp', 'fsum'] From 336ac3d3a3131b7e12871b1aa9b9f7ff6ea331c0 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 15 Apr 2026 17:39:39 +0200 Subject: [PATCH 11/12] Grammar fixes --- Doc/reference/expressions.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index f1d9bec897834d..3e5da8029c8a00 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -933,12 +933,12 @@ The formal grammar for comprehensions is: .. grammar-snippet:: :group: python-grammar - listcomp: - | '[' `flexible_expression` `for_if_clause`+ ']' - setcomp: - | '{' `flexible_expression` `for_if_clause`+ '}' + listcomp: '[' `comprehension` ']' + setcomp: '{' `comprehension` '}' + comprehension: `flexible_expression` `for_if_clause`+ + dictcomp: - | '{' `dict_item` `for_if_clause`+ '}' + | '{' `kvpair` `for_if_clause`+ '}' | '{' '**' `expression` `for_if_clause`+ '}' for_if_clause: @@ -958,7 +958,7 @@ Generator expressions A generator expression is a compact generator notation in parentheses: .. productionlist:: python-grammar - generator_expression: "(" `flexible_expression` `comp_for` ")" + generator_expression: "(" `comprehension` ")" A generator expression yields a new generator object. Its syntax is the same as for comprehensions, except that it is enclosed in parentheses instead of From da26496971ecf232059259a18025fe9efb183bf5 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 15 Apr 2026 17:50:10 +0200 Subject: [PATCH 12/12] Add back old HTML IDs; exclude old grammar tokens --- Doc/reference/expressions.rst | 5 +++-- Doc/tools/removed-ids.txt | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 3e5da8029c8a00..85a05cf150bca3 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -332,6 +332,7 @@ Formally: single: parenthesized form single: () (parentheses) +.. _parenthesized-forms: .. _parenthesized: Parenthesized groups @@ -365,7 +366,7 @@ Formally, the syntax for groups is: group: '(' `assignment_expression` ')' - +.. _displays-for-lists-sets-and-dictionaries: .. _displays: Container displays @@ -642,7 +643,6 @@ This may be used to override a set of defaults:: .. versionadded:: 3.5 Unpacking into dictionary displays, originally proposed by :pep:`448`. - The formal grammar for dict displays is: .. grammar-snippet:: @@ -924,6 +924,7 @@ execution of the coroutine function in which it appears. asynchronous functions. Outer comprehensions implicitly become asynchronous. +.. _comprehension-grammar: Formal grammar for comprehensions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Doc/tools/removed-ids.txt b/Doc/tools/removed-ids.txt index f3cd8bf0ef5bb9..c833bd408455d6 100644 --- a/Doc/tools/removed-ids.txt +++ b/Doc/tools/removed-ids.txt @@ -1 +1,15 @@ # HTML IDs excluded from the check-html-ids.py check. + + +## Old names for grammar tokens +reference/expressions.html: grammar-token-python-grammar-comp_for +reference/expressions.html: grammar-token-python-grammar-comp_if +reference/expressions.html: grammar-token-python-grammar-comp_iter +reference/expressions.html: grammar-token-python-grammar-dict_comprehension +reference/expressions.html: grammar-token-python-grammar-dict_display +reference/expressions.html: grammar-token-python-grammar-dict_item +reference/expressions.html: grammar-token-python-grammar-dict_item_list +reference/expressions.html: grammar-token-python-grammar-enclosure +reference/expressions.html: grammar-token-python-grammar-list_display +reference/expressions.html: grammar-token-python-grammar-parenth_form +reference/expressions.html: grammar-token-python-grammar-set_display