Merge branch '3.11' into backport-acfe02f-3.11

Author: encukou

.readthedocs.yml

@@ -8,11 +8,53 @@ sphinx:
    configuration: Doc/conf.py
 
 build:
-  os: ubuntu-22.04
+  os: ubuntu-24.04
   tools:
     python: "3"
+  apt_packages:
+    - jq
 
-  commands:
-    - make -C Doc venv html
-    - mkdir _readthedocs
-    - mv Doc/build/html _readthedocs/html
+  jobs:
+    post_system_dependencies:
+      # https://docs.readthedocs.com/platform/stable/guides/build/skip-build.html#skip-builds-based-on-conditions
+      #
+      # Cancel building pull requests when there are no changes in the Doc
+      # directory or RTD configuration, or if we can't cleanly merge the base
+      # branch.
+      - |
+        set -eEux;
+        if [ "$READTHEDOCS_VERSION_TYPE" = "external" ];
+        then
+          base_branch=$(wget -qO- "https://api.github.com/repos/python/cpython/pulls/$READTHEDOCS_VERSION" | jq -er ".base.ref");
+          git fetch --depth=50 origin $base_branch:origin-$base_branch;
+          for attempt in $(seq 10);
+          do
+            if ! git merge-base HEAD origin-$base_branch;
+            then
+              git fetch --deepen=50 origin $base_branch;
+            else
+              break;
+            fi;
+          done;
+          if ! git -c "user.name=rtd" -c "user.email=no-reply@readthedocs.org" merge --no-stat --no-edit origin-$base_branch;
+          then
+            echo "Unsuccessful merge with '$base_branch' branch, skipping the build";
+            exit 183;
+          fi;
+          if git diff --exit-code --stat origin-$base_branch -- Doc/ .readthedocs.yml;
+          then
+            echo "No changes to Doc/ - skipping the build.";
+            exit 183;
+          fi;
+        fi;
+    create_environment:
+      - echo "Skipping default environment creation"
+    install:
+      - asdf plugin add uv
+      - asdf install uv latest
+      - asdf global uv latest
+    build:
+      html:
+        - make -C Doc venv html
+        - mkdir -p "$READTHEDOCS_OUTPUT"
+        - mv Doc/build/html "$READTHEDOCS_OUTPUT/"

Doc/library/pyexpat.rst

@@ -239,16 +239,71 @@ XMLParser Objects
    .. versionadded:: 3.11.9
 
 
-:class:`!xmlparser` objects have the following methods to mitigate some
-common XML vulnerabilities.
+:class:`!xmlparser` objects have the following methods to tune protections
+against some common XML vulnerabilities.
+
+.. method:: xmlparser.SetBillionLaughsAttackProtectionActivationThreshold(threshold, /)
+
+   Sets the number of output bytes needed to activate protection against
+   `billion laughs`_ attacks.
+
+   The number of output bytes includes amplification from entity expansion
+   and reading DTD files.
+
+   Parser objects usually have a protection activation threshold of 8 MiB,
+   but the actual default value depends on the underlying Expat library.
+
+   An :exc:`ExpatError` is raised if this method is called on a
+   |xml-non-root-parser| parser.
+   The corresponding :attr:`~ExpatError.lineno` and :attr:`~ExpatError.offset`
+   should not be used as they may have no special meaning.
+
+   .. note::
+
+      Activation thresholds below 4 MiB are known to break support for DITA 1.3
+      payload and are hence not recommended.
+
+   .. versionadded:: next
+
+.. method:: xmlparser.SetBillionLaughsAttackProtectionMaximumAmplification(max_factor, /)
+
+   Sets the maximum tolerated amplification factor for protection against
+   `billion laughs`_ attacks.
+
+   The amplification factor is calculated as ``(direct + indirect) / direct``
+   while parsing, where ``direct`` is the number of bytes read from
+   the primary document in parsing and ``indirect`` is the number of
+   bytes added by expanding entities and reading of external DTD files.
+
+   The *max_factor* value must be a non-NaN :class:`float` value greater than
+   or equal to 1.0. Peak amplifications of factor 15,000 for the entire payload
+   and of factor 30,000 in the middle of parsing have been observed with small
+   benign files in practice. In particular, the activation threshold should be
+   carefully chosen to avoid false positives.
+
+   Parser objects usually have a maximum amplification factor of 100,
+   but the actual default value depends on the underlying Expat library.
+
+   An :exc:`ExpatError` is raised if this method is called on a
+   |xml-non-root-parser| parser or if *max_factor* is outside the valid range.
+   The corresponding :attr:`~ExpatError.lineno` and :attr:`~ExpatError.offset`
+   should not be used as they may have no special meaning.
+
+   .. note::
+
+      The maximum amplification factor is only considered if the threshold
+      that can be adjusted by :meth:`.SetBillionLaughsAttackProtectionActivationThreshold`
+      is exceeded.
+
+   .. versionadded:: next
 
 .. method:: xmlparser.SetAllocTrackerActivationThreshold(threshold, /)
 
    Sets the number of allocated bytes of dynamic memory needed to activate
    protection against disproportionate use of RAM.
 
-   By default, parser objects have an allocation activation threshold of 64 MiB,
-   or equivalently 67,108,864 bytes.
+   Parser objects usually have an allocation activation threshold of 64 MiB,
+   but the actual default value depends on the underlying Expat library.
 
    An :exc:`ExpatError` is raised if this method is called on a
    |xml-non-root-parser| parser.
@@ -272,7 +327,8 @@ common XML vulnerabilities.
    near the start of parsing even with benign files in practice. In particular,
    the activation threshold should be carefully chosen to avoid false positives.
 
-   By default, parser objects have a maximum amplification factor of 100.0.
+   Parser objects usually have a maximum amplification factor of 100,
+   but the actual default value depends on the underlying Expat library.
 
    An :exc:`ExpatError` is raised if this method is called on a
    |xml-non-root-parser| parser or if *max_factor* is outside the valid range.
@@ -1004,4 +1060,6 @@ The ``errors`` module has the following attributes:
    not. See https://www.w3.org/TR/2006/REC-xml11-20060816/#NT-EncodingDecl
    and https://www.iana.org/assignments/character-sets/character-sets.xhtml.
 
+
+.. _billion laughs: https://en.wikipedia.org/wiki/Billion_laughs_attack
 .. |xml-non-root-parser| replace:: :ref:`non-root <xmlparser-non-root>`

Doc/using/windows.rst

@@ -401,9 +401,9 @@ on using nuget. What follows is a summary that is sufficient for Python
 developers.
 
 The ``nuget.exe`` command line tool may be downloaded directly from
-``https://aka.ms/nugetclidl``, for example, using curl or PowerShell. With the
-tool, the latest version of Python for 64-bit or 32-bit machines is installed
-using::
+``https://dist.nuget.org/win-x86-commandline/latest/nuget.exe``, for example,
+using curl or PowerShell. With the tool, the latest version of Python for
+64-bit or 32-bit machines is installed using::
 
    nuget.exe install python -ExcludeVersion -OutputDirectory .
    nuget.exe install pythonx86 -ExcludeVersion -OutputDirectory .

Include/pyexpat.h

@@ -57,6 +57,11 @@ struct PyExpat_CAPI
         XML_Parser parser, unsigned long long activationThresholdBytes);
     XML_Bool (*SetAllocTrackerMaximumAmplification)(
         XML_Parser parser, float maxAmplificationFactor);
+    /* might be NULL for expat < 2.4.0 */
+    XML_Bool (*SetBillionLaughsAttackProtectionActivationThreshold)(
+        XML_Parser parser, unsigned long long activationThresholdBytes);
+    XML_Bool (*SetBillionLaughsAttackProtectionMaximumAmplification)(
+        XML_Parser parser, float maxAmplificationFactor);
     /* always add new stuff to the end! */
 };
 

Lib/shutil.py

@@ -1241,27 +1241,9 @@ def _unpack_zipfile(filename, extract_dir):
     if not zipfile.is_zipfile(filename):
         raise ReadError("%s is not a zip file" % filename)
 
-    zip = zipfile.ZipFile(filename)
-    try:
-        for info in zip.infolist():
-            name = info.filename
-
-            # don't extract absolute paths or ones with .. in them
-            if name.startswith('/') or '..' in name:
-                continue
-
-            targetpath = os.path.join(extract_dir, *name.split('/'))
-            if not targetpath:
-                continue
-
-            _ensure_directory(targetpath)
-            if not name.endswith('/'):
-                # file
-                with zip.open(name, 'r') as source, \
-                        open(targetpath, 'wb') as target:
-                    copyfileobj(source, target)
-    finally:
-        zip.close()
+    with zipfile.ZipFile(filename) as zip:
+        zip._ignore_invalid_names = True
+        zip.extractall(extract_dir)
 
 def _unpack_tarfile(filename, extract_dir, *, filter=None):
     """Unpack tar/tar.gz/tar.bz2/tar.xz `filename` to `extract_dir`

Lib/test/test_asyncio/test_sock_lowlevel.py

@@ -427,6 +427,27 @@ def test_recvfrom_into(self):
             self.loop.run_until_complete(
                 self._basetest_datagram_recvfrom_into(server_address))
 
+    async def _basetest_datagram_recvfrom_into_wrong_size(self, server_address):
+        # Call sock_sendto() with a size larger than the buffer
+        with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock:
+            sock.setblocking(False)
+
+            buf = bytearray(5000)
+            data = b'\x01' * 4096
+            wrong_size = len(buf) + 1
+            await self.loop.sock_sendto(sock, data, server_address)
+            with self.assertRaises(ValueError):
+                await self.loop.sock_recvfrom_into(
+                    sock, buf, wrong_size)
+
+            size, addr = await self.loop.sock_recvfrom_into(sock, buf)
+            self.assertEqual(buf[:size], data)
+
+    def test_recvfrom_into_wrong_size(self):
+        with test_utils.run_udp_echo_server() as server_address:
+            self.loop.run_until_complete(
+                self._basetest_datagram_recvfrom_into_wrong_size(server_address))
+
     async def _basetest_datagram_sendto_blocking(self, server_address):
         # Sad path, sock.sendto() raises BlockingIOError
         # This involves patching sock.sendto() to raise BlockingIOError but

Lib/test/test_pyexpat.py

@@ -1003,6 +1003,64 @@ def test_set_maximum_amplification__fail_for_subparser(self):
         self.assert_root_parser_failure(setter, 123.45)
 
 
+@unittest.skipIf(expat.version_info < (2, 4, 0), "requires Expat >= 2.4.0")
+class ExpansionProtectionTest(AttackProtectionTestBase, unittest.TestCase):
+
+    def assert_rejected(self, func, /, *args, **kwargs):
+        """Check that func(*args, **kwargs) hits the allocation limit."""
+        msg = (
+            r"limit on input amplification factor \(from DTD and entities\) "
+            r"breached: line \d+, column \d+"
+        )
+        self.assertRaisesRegex(expat.ExpatError, msg, func, *args, **kwargs)
+
+    def set_activation_threshold(self, parser, threshold):
+        return parser.SetBillionLaughsAttackProtectionActivationThreshold(threshold)
+
+    def set_maximum_amplification(self, parser, max_factor):
+        return parser.SetBillionLaughsAttackProtectionMaximumAmplification(max_factor)
+
+    def test_set_activation_threshold__threshold_reached(self):
+        parser = expat.ParserCreate()
+        # Choose a threshold expected to be always reached.
+        self.set_activation_threshold(parser, 3)
+        # Check that the threshold is reached by choosing a small factor
+        # and a payload whose peak amplification factor exceeds it.
+        self.assertIsNone(self.set_maximum_amplification(parser, 1.0))
+        payload = self.exponential_expansion_payload(ncols=10, nrows=4)
+        self.assert_rejected(parser.Parse, payload, True)
+
+    def test_set_activation_threshold__threshold_not_reached(self):
+        parser = expat.ParserCreate()
+        # Choose a threshold expected to be never reached.
+        self.set_activation_threshold(parser, pow(10, 5))
+        # Check that the threshold is reached by choosing a small factor
+        # and a payload whose peak amplification factor exceeds it.
+        self.assertIsNone(self.set_maximum_amplification(parser, 1.0))
+        payload = self.exponential_expansion_payload(ncols=10, nrows=4)
+        self.assertIsNotNone(parser.Parse(payload, True))
+
+    def test_set_maximum_amplification__amplification_exceeded(self):
+        parser = expat.ParserCreate()
+        # Unconditionally enable maximum activation factor.
+        self.set_activation_threshold(parser, 0)
+        # Choose a max amplification factor expected to always be exceeded.
+        self.assertIsNone(self.set_maximum_amplification(parser, 1.0))
+        # Craft a payload for which the peak amplification factor is > 1.0.
+        payload = self.exponential_expansion_payload(ncols=1, nrows=2)
+        self.assert_rejected(parser.Parse, payload, True)
+
+    def test_set_maximum_amplification__amplification_not_exceeded(self):
+        parser = expat.ParserCreate()
+        # Unconditionally enable maximum activation factor.
+        self.set_activation_threshold(parser, 0)
+        # Choose a max amplification factor expected to never be exceeded.
+        self.assertIsNone(self.set_maximum_amplification(parser, 1e4))
+        # Craft a payload for which the peak amplification factor is < 1e4.
+        payload = self.exponential_expansion_payload(ncols=1, nrows=2)
+        self.assertIsNotNone(parser.Parse(payload, True))
+
+
 @unittest.skipIf(expat.version_info < (2, 7, 2), "requires Expat >= 2.7.2")
 class MemoryProtectionTest(AttackProtectionTestBase, unittest.TestCase):
 

Lib/test/test_shutil.py

@@ -1799,6 +1799,71 @@ def test_unpack_archive_zip(self):
         with self.assertRaises(TypeError):
             self.check_unpack_archive('zip', filter='data')
 
+    def test_unpack_archive_zip_badpaths(self):
+        srcdir = self.mkdtemp()
+        zipname = os.path.join(srcdir, 'test.zip')
+        abspath = os.path.join(srcdir, 'abspath')
+        with zipfile.ZipFile(zipname, 'w') as zf:
+            zf.writestr(abspath, 'badfile')
+            zf.writestr(os.sep + abspath, 'badfile')
+            zf.writestr('/abspath', 'badfile')
+            zf.writestr('C:/abspath', 'badfile')
+            zf.writestr('D:\\abspath', 'badfile')
+            zf.writestr('E:abspath', 'badfile')
+            zf.writestr('F:/G:/abspath', 'badfile')
+            zf.writestr('//server/share/abspath', 'badfile')
+            zf.writestr('\\\\server2\\share\\abspath', 'badfile')
+            zf.writestr('../relpath', 'badfile')
+            zf.writestr(os.pardir + os.sep + 'relpath2', 'badfile')
+            zf.writestr('good/file', 'goodfile')
+            zf.writestr('good..file', 'goodfile')
+
+        dstdir = os.path.join(self.mkdtemp(), 'dst')
+        unpack_archive(zipname, dstdir)
+        self.assertTrue(os.path.isfile(os.path.join(dstdir, 'good', 'file')))
+        self.assertTrue(os.path.isfile(os.path.join(dstdir, 'good..file')))
+        self.assertFalse(os.path.exists(abspath))
+        self.assertFalse(os.path.exists(os.path.join(dstdir, 'abspath')))
+        self.assertFalse(os.path.exists(os.path.join(dstdir, 'G_')))
+        self.assertFalse(os.path.exists(os.path.join(dstdir, 'server')))
+        if os.name != 'nt':
+            self.assertTrue(os.path.isfile(os.path.join(dstdir, 'C:', 'abspath')))
+            self.assertTrue(os.path.isfile(os.path.join(dstdir, 'D:\\abspath')))
+            self.assertTrue(os.path.isfile(os.path.join(dstdir, 'E:abspath')))
+            self.assertTrue(os.path.isfile(os.path.join(dstdir, 'F:', 'G:', 'abspath')))
+            self.assertTrue(os.path.isfile(os.path.join(dstdir, '\\\\server2\\share\\abspath')))
+        if os.pardir == '..':
+            self.assertFalse(os.path.exists(os.path.join(dstdir, '..', 'relpath')))
+            self.assertFalse(os.path.exists(os.path.join(dstdir, 'relpath')))
+        else:
+            self.assertTrue(os.path.isfile(os.path.join(dstdir, '..', 'relpath')))
+        self.assertFalse(os.path.exists(os.path.join(dstdir, os.pardir, 'relpath2')))
+        self.assertFalse(os.path.exists(os.path.join(dstdir, 'relpath2')))
+
+        dstdir2 = os.path.join(self.mkdtemp(), 'dst')
+        os.mkdir(dstdir2)
+        with os_helper.change_cwd(dstdir2):
+            unpack_archive(zipname, '')
+            self.assertTrue(os.path.isfile(os.path.join('good', 'file')))
+            self.assertTrue(os.path.isfile('good..file'))
+            self.assertFalse(os.path.exists(abspath))
+            self.assertFalse(os.path.exists('abspath'))
+            self.assertFalse(os.path.exists('C_'))
+            self.assertFalse(os.path.exists('server'))
+            if os.name != 'nt':
+                self.assertTrue(os.path.isfile(os.path.join('C:', 'abspath')))
+                self.assertTrue(os.path.isfile('D:\\abspath'))
+                self.assertTrue(os.path.isfile('E:abspath'))
+                self.assertTrue(os.path.isfile(os.path.join('F:', 'G:', 'abspath')))
+                self.assertTrue(os.path.isfile('\\\\server2\\share\\abspath'))
+            if os.pardir == '..':
+                self.assertFalse(os.path.exists(os.path.join('..', 'relpath')))
+                self.assertFalse(os.path.exists('relpath'))
+            else:
+                self.assertTrue(os.path.isfile(os.path.join('..', 'relpath')))
+            self.assertFalse(os.path.exists(os.path.join(os.pardir, 'relpath2')))
+            self.assertFalse(os.path.exists('relpath2'))
+
     def test_unpack_registry(self):
 
         formats = get_unpack_formats()