From 4b79eb0fbaecd4806730ebcb44f175ec796bbf12 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 17 Aug 2026 19:11:46 +0200 Subject: [PATCH 1/3] chunkers/reader: memcpy instead of memoryview slice assignment pypy's cpyext memoryview does not support slice assignment (mv[a:b] = src): the statement works in interpreted code, but raises TypeError when it is compiled into a C extension, like in this .pyx module. That broke about 500 tests on pypy - basically every operation that writes an archive. Copying via memcpy through Cython typed memoryviews avoids creating the slice objects and is a wash on CPython (1.689 vs 1.680 GB/s, best of 6, chunking a 2 GiB file through the block reader path). Co-Authored-By: Claude Opus 5 --- src/borg/chunkers/reader.pyx | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/borg/chunkers/reader.pyx b/src/borg/chunkers/reader.pyx index dccfbc9129..943f066a9f 100644 --- a/src/borg/chunkers/reader.pyx +++ b/src/borg/chunkers/reader.pyx @@ -8,6 +8,8 @@ import stat import time from collections import namedtuple +from libc.string cimport memcpy + from ..platform import safe_fadvise from ..constants import CH_DATA, CH_ALLOC, CH_HOLE, zeros @@ -59,6 +61,18 @@ def release_chunk_data(data): data.release() +cdef _copy_into(target, Py_ssize_t t_offset, source, Py_ssize_t s_offset, Py_ssize_t count): + """memcpy count bytes from source[s_offset:] into target[t_offset:]. + + We do not use memoryview slice assignment here: pypy's cpyext memoryview does not + support it, and one memcpy is cheaper than creating the slice objects anyway. + """ + cdef unsigned char[::1] dst = target + cdef const unsigned char[::1] src = source + if count: + memcpy(&dst[t_offset], &src[s_offset], count) + + def dread(offset, size, fd=None, fh=-1): use_fh = fh >= 0 if use_fh: @@ -393,7 +407,7 @@ class FileReader: else: data = os.read(self.fh, size - pos) got = len(data) - tv[pos:pos + got] = data + _copy_into(tv, pos, data, 0, got) if got > 0: safe_fadvise(self.fh, self.direct_offset, got, "DONTNEED") else: @@ -403,7 +417,7 @@ class FileReader: # file-like object without readinto: fall back to read + copy data = self.fd.read(size - pos) got = len(data) - tv[pos:pos + got] = data + _copy_into(tv, pos, data, 0, got) if not got: break # EOF pos += got @@ -470,12 +484,11 @@ class FileReader: if allocation == CH_DATA: assert data is not None - # one memcpy: block -> target (the source slice is a view, not a copy) - with memoryview(data) as dv: - tv[bytes_read:bytes_read + to_read] = dv[self.offset:self.offset + to_read] + # one memcpy: block -> target + _copy_into(tv, bytes_read, data, self.offset, to_read) else: # holes / all-zero blocks: write zeros (target may contain stale data) - tv[bytes_read:bytes_read + to_read] = zeros[:to_read] + _copy_into(tv, bytes_read, zeros, 0, to_read) bytes_read += to_read From 6cb72057c2dc2cbea2d402098abfcc3b6021a0c4 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 17 Aug 2026 19:12:08 +0200 Subject: [PATCH 2/3] pypy support: add is_pypy, adjust test expectations, see #1755 borg works on pypy3.11: tested with a py3.11 nightly build (8ed7cc691813, PyPy 8.0.0-alpha0) on macOS/arm64, where the full test suite passes (2534 passed, 1 xfailed) and CPython is unaffected. The pypy incompatibilities found in the first round (see #1755 and the previous version of this branch) are all fixed in pypy now, so their workarounds are gone again: fcntl.F_FULLFSYNC (pypy/pypy#5543), hmac.digest with memoryview/bytearray (pypy/pypy#5544), os.link(follow_symlinks=False) (pypy/pypy#5545). The chunk-data memoryview leak (pypy/pypy#5546) is fixed, too: chunking 21.5 GB without releasing the memoryviews now peaks at 252 MB instead of growing linearly with the data volume. What remains is not pypy bugs: - platformflags: add is_pypy. - item_test: xfail test_unknown_property - setting undeclared attributes on cdef class instances is not blocked under cpyext. - msgpack_test: pypy only has the pure-python msgpack, so expect it to be slow there. - lock_cmds_test: tolerate the pure-python msgpack warning on stderr, and use sys.executable instead of "python3" (robustness, not pypy-specific). Co-Authored-By: Claude Opus 5 --- src/borg/platformflags.py | 3 +++ src/borg/testsuite/archiver/lock_cmds_test.py | 12 +++++++----- src/borg/testsuite/helpers/msgpack_test.py | 4 ++++ src/borg/testsuite/item_test.py | 2 ++ 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/borg/platformflags.py b/src/borg/platformflags.py index c6f98270b7..92cc83f7b2 100644 --- a/src/borg/platformflags.py +++ b/src/borg/platformflags.py @@ -20,3 +20,6 @@ # MSYS2 (on Windows) is_msystem = is_win32 and "MSYSTEM" in os.environ + +# Python implementation +is_pypy = sys.implementation.name == "pypy" diff --git a/src/borg/testsuite/archiver/lock_cmds_test.py b/src/borg/testsuite/archiver/lock_cmds_test.py index 68a3e1f03d..935b9cc8ea 100644 --- a/src/borg/testsuite/archiver/lock_cmds_test.py +++ b/src/borg/testsuite/archiver/lock_cmds_test.py @@ -28,13 +28,14 @@ def test_with_lock(tmp_path): print("sys.path: %r" % sys.path) print("PYTHONPATH: %s" % env.get("PYTHONPATH", "")) print("PATH: %s" % env.get("PATH", "")) - command0 = "python3", "-m", "borg", "repo-create", "--encryption=none-sha256" + python = sys.executable or "python3" + command0 = python, "-m", "borg", "repo-create", "--encryption=none-sha256" # Timings must be adjusted so that command1 keeps running while command2 tries to get the lock, # so that lock acquisition for command2 fails as the test expects it. lock_wait = 2 - command1 = ("python3", "-c", 'import sys; print("first command - acquires the lock", flush=True); sys.stdin.read()') - command2 = "python3", "-c", 'print("second command - should never get executed")' - borgwl = "python3", "-m", "borg", "with-lock", f"--lock-wait={lock_wait}" + command1 = (python, "-c", 'import sys; print("first command - acquires the lock", flush=True); sys.stdin.read()') + command2 = python, "-c", 'print("second command - should never get executed")' + borgwl = python, "-m", "borg", "with-lock", f"--lock-wait={lock_wait}" popen_options = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=env) subprocess.run(command0, env=env, check=True, text=True, capture_output=True) assert repo_path.exists() @@ -50,7 +51,8 @@ def test_with_lock(tmp_path): assert "Failed to create/acquire the lock" in err_out assert p2.returncode == 73 # LockTimeout: could not acquire the lock, p1 already has it out, err_out = p1.communicate(input="") # Unblock command1 and read output - assert not err_out + # ignore the pure-python msgpack warning borg emits on pypy + assert not [line for line in err_out.splitlines() if "pure-python msgpack" not in line] assert p1.returncode == 0 diff --git a/src/borg/testsuite/helpers/msgpack_test.py b/src/borg/testsuite/helpers/msgpack_test.py index 41960da784..37a64895d5 100644 --- a/src/borg/testsuite/helpers/msgpack_test.py +++ b/src/borg/testsuite/helpers/msgpack_test.py @@ -3,6 +3,7 @@ from ...helpers.msgpack import is_slow_msgpack from ...platform import is_cygwin +from ...platformflags import is_pypy def expected_py_mp_slow_combination(): @@ -13,6 +14,9 @@ def expected_py_mp_slow_combination(): # msgpack is slow on Cygwin if is_cygwin: return True + # pypy only has the pure-python msgpack (which pypy's jit hopefully makes fast enough) + if is_pypy: + return True # msgpack < 1.0.6 did not have Python 3.12 wheels if sys.version_info[:2] == (3, 12) and msgpack.version < (1, 0, 6): return True diff --git a/src/borg/testsuite/item_test.py b/src/borg/testsuite/item_test.py index c795430fb3..e220eb9b27 100644 --- a/src/borg/testsuite/item_test.py +++ b/src/borg/testsuite/item_test.py @@ -4,6 +4,7 @@ from ..item import Item, chunks_contents_equal from ..helpers import StableDict from ..helpers.msgpack import Timestamp +from ..platformflags import is_pypy def test_item_empty(): @@ -131,6 +132,7 @@ def test_item_dict_property(): assert item.as_dict() == {"xattrs": {"foo": "bar", "bar": "baz"}} +@pytest.mark.xfail(is_pypy, reason="setting undeclared attributes on cdef class instances is not blocked on pypy") def test_unknown_property(): # We do not want the user to be able to set unknown attributes — # they will not appear in the .as_dict() result dictionary. From 4be21707ef66031f4cfdd9de5956ca338d156239 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 18 Aug 2026 06:31:48 +0200 Subject: [PATCH 3/3] chunkers/reader: limit the read size in the no-readv fallback, see #1755 Without os.readv (win32, pypy) we read via os.read and copy into the scan buffer. os.read allocates a buffer of the requested size, no matter how few bytes it then returns, so requesting the whole free scan buffer (up to 8MiB for the default max chunk size) is expensive for small files: one allocation of that size per read call, several calls per file. Backing up 20000 small files (82.6MiB of data) requested 468GiB in 60002 os.read calls. Capping the request at 256KiB - same call count, same data - cuts the time spent in os.read from 15.1s to 1.3s and the total runtime from 27.6s to 12.3s on pypy (which also zeroes the allocation). On CPython without readv it is 4.04s -> 2.42s, close to the 2.36s of the readv path. Chunking a 2GiB file is unaffected (the big reads there return what they ask for). Co-Authored-By: Claude Opus 5 --- src/borg/chunkers/reader.pyx | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/borg/chunkers/reader.pyx b/src/borg/chunkers/reader.pyx index 943f066a9f..78d4f19186 100644 --- a/src/borg/chunkers/reader.pyx +++ b/src/borg/chunkers/reader.pyx @@ -18,9 +18,14 @@ from ..constants import CH_DATA, CH_ALLOC, CH_HOLE, zeros # because the FS also needs to support this. has_seek_hole = hasattr(os, 'SEEK_DATA') and hasattr(os, 'SEEK_HOLE') -# os.readv is POSIX; on platforms without it (win32) we fall back to os.read + copy. +# os.readv is POSIX; on platforms without it (win32, pypy) we fall back to os.read + copy. has_readv = hasattr(os, 'readv') +# Upper bound for a single os.read in that fallback: os.read allocates a buffer of the +# requested size (pypy also zeroes it), no matter how few bytes it then returns. Asking for +# the whole free scan buffer would thus cost MBs of allocation per small file, see #1755. +READ_FALLBACK_SIZE = 256 * 1024 + _Chunk = namedtuple('_Chunk', 'meta data') _Chunk.__doc__ = """\ Chunk namedtuple @@ -405,7 +410,9 @@ class FileReader: if has_readv: got = os.readv(self.fh, [tv[pos:size]]) else: - data = os.read(self.fh, size - pos) + # os.read allocates a buffer of the requested size, so ask for the + # block size rather than for the whole free scan buffer, see #1755. + data = os.read(self.fh, min(size - pos, READ_FALLBACK_SIZE)) got = len(data) _copy_into(tv, pos, data, 0, got) if got > 0: