From a10cb9211b19a7316f421cb36596dd11d6655cae Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 17 Aug 2026 02:51:18 +0200 Subject: [PATCH 1/3] create: add --map to give an input map, see #4363 An input map describes the content ranges of the single input file: data ranges are read and stored, zero ranges are stored as holes without reading them. Primary use case: backing up snapshots of large (esp. thin-provisioned) block devices, where the storage layer knows which ranges are in use. Co-Authored-By: Claude Fable 5 --- docs/man/borg-create.1 | 31 +++++++++- docs/usage/create.rst.inc | 28 ++++++++++ src/borg/archive.py | 20 ++++++- src/borg/archiver/create_cmd.py | 56 +++++++++++++++++++ src/borg/helpers/__init__.py | 1 + src/borg/helpers/fs.py | 53 ++++++++++++++++++ .../testsuite/archiver/create_cmd_test.py | 54 +++++++++++++++++- src/borg/testsuite/helpers/fs_test.py | 51 +++++++++++++++++ 8 files changed, 290 insertions(+), 4 deletions(-) diff --git a/docs/man/borg-create.1 b/docs/man/borg-create.1 index 7729fbbfad..c2b7407384 100644 --- a/docs/man/borg-create.1 +++ b/docs/man/borg-create.1 @@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "borg-create" "1" "2026-08-15" "" "borg backup tool" +.TH "borg-create" "1" "2026-08-17" "" "borg backup tool" .SH Name borg-create \- Creates a new archive. .SH SYNOPSIS @@ -320,6 +320,9 @@ open and read block and char device files as well as FIFOs as if they were regul .TP .BI \-\-read\-special\-timeout \ SECONDS when reading from FIFOs or character devices (see \-\-read\-special): skip the file with an error if no data arrives for more than SECONDS (this includes waiting for a FIFO\(aqs writer to connect). Give 0 to wait forever. default: 1800 seconds. +.TP +.BI \-\-map \ MAPFILE +give a map file describing the content ranges of the (single) input file, so borg does not need to read all of it. See the \fIInput maps\fP section below. .UNINDENT .SS Archive options .INDENT 0.0 @@ -581,6 +584,32 @@ creation a bit. .sp By default, the content read from stdin is stored in a file called \(aqstdin\(aq. Use \fB\-\-stdin\-name\fP to change the name. +.SS Input maps +.sp +Usually, borg reads the complete input to determine its contents. If you already +know the contents of parts of the input from an external source of truth, you can +give that information via \fB\-\-map MAPFILE\fP and borg will not read the known parts. +The primary use case is backing up snapshots of large block devices (e.g. LVM thin +volumes), where the storage layer knows which ranges are in use. +.sp +\fB\-\-map\fP requires giving exactly one input path, which must be a regular file or +(with \fB\-\-read\-special\fP) a block device. +.sp +The map file must describe the whole input: one range per line, in the form +\fBSTART LENGTH STATE\fP (byte values, decimal or 0x\-prefixed hexadecimal). The +ranges must be sorted, non\-overlapping and contiguous, starting at offset 0 and +covering the exact input size. \fB#\fP starts a comment, empty lines are ignored. +STATE is one of: +.INDENT 0.0 +.IP \(bu 2 +\fBdata\fP: the range\(aqs contents are read and backed up. +.IP \(bu 2 +\fBzero\fP: the range is known to read as all\-zero bytes. borg stores a hole +(all\-zero range) of that size without reading the range. +.UNINDENT +.sp +\fBThe map is trusted\fP: if it is wrong (e.g. a range marked \fBzero\fP actually +contains data), the archive will not match the input and borg cannot detect that. .SS Feeding all file paths from externally .sp Usually, you give a starting path (recursion root) to borg and then borg diff --git a/docs/usage/create.rst.inc b/docs/usage/create.rst.inc index 268b6387f1..bc705c63a7 100644 --- a/docs/usage/create.rst.inc +++ b/docs/usage/create.rst.inc @@ -99,6 +99,8 @@ borg create +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | | ``--read-special-timeout SECONDS`` | when reading from FIFOs or character devices (see --read-special): skip the file with an error if no data arrives for more than SECONDS (this includes waiting for a FIFO's writer to connect). Give 0 to wait forever. default: 1800 seconds. | +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``--map MAPFILE`` | give a map file describing the content ranges of the (single) input file, so borg does not need to read all of it. See the *Input maps* section below. | + +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | **Archive options** | +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | | ``--comment COMMENT`` | add a comment text to the archive | @@ -173,6 +175,7 @@ borg create --files-changed MODE specify how to detect if a file has changed during backup (ctime, mtime, disabled). default: ctime (on Windows: mtime, because ctime is file creation time there). --read-special open and read block and char device files as well as FIFOs as if they were regular files. Also follows symlinks pointing to these kinds of files. --read-special-timeout SECONDS when reading from FIFOs or character devices (see --read-special): skip the file with an error if no data arrives for more than SECONDS (this includes waiting for a FIFO's writer to connect). Give 0 to wait forever. default: 1800 seconds. + --map MAPFILE give a map file describing the content ranges of the (single) input file, so borg does not need to read all of it. See the *Input maps* section below. Archive options @@ -445,6 +448,31 @@ creation a bit. By default, the content read from stdin is stored in a file called 'stdin'. Use ``--stdin-name`` to change the name. +Input maps +++++++++++ + +Usually, borg reads the complete input to determine its contents. If you already +know the contents of parts of the input from an external source of truth, you can +give that information via ``--map MAPFILE`` and borg will not read the known parts. +The primary use case is backing up snapshots of large block devices (e.g. LVM thin +volumes), where the storage layer knows which ranges are in use. + +``--map`` requires giving exactly one input path, which must be a regular file or +(with ``--read-special``) a block device. + +The map file must describe the whole input: one range per line, in the form +``START LENGTH STATE`` (byte values, decimal or 0x-prefixed hexadecimal). The +ranges must be sorted, non-overlapping and contiguous, starting at offset 0 and +covering the exact input size. ``#`` starts a comment, empty lines are ignored. +STATE is one of: + +- ``data``: the range's contents are read and backed up. +- ``zero``: the range is known to read as all-zero bytes. borg stores a hole + (all-zero range) of that size without reading the range. + +**The map is trusted**: if it is wrong (e.g. a range marked ``zero`` actually +contains data), the archive will not match the input and borg cannot detect that. + Feeding all file paths from externally ++++++++++++++++++++++++++++++++++++++ diff --git a/src/borg/archive.py b/src/borg/archive.py index 23dc784323..078bcf9ccb 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -43,6 +43,7 @@ from .helpers import safe_ns from .helpers import ellipsis_truncate, ProgressIndicatorPercent, log_multi, get_progress_dt from .helpers import os_open, flags_normal, flags_dir, O_, SpecialFileReader +from .helpers import MAP_DATA, input_map_check_size from .helpers import os_stat from .helpers import msgpack from .helpers.lrucache import LRUCache @@ -1351,6 +1352,7 @@ def __init__( file_status_printer=None, files_changed="mtime" if is_win32 else "ctime", read_special_timeout=None, + input_map=None, ): self.metadata_collector = metadata_collector self.cache = cache @@ -1361,6 +1363,7 @@ def __init__( self.print_file_status = file_status_printer or (lambda *args: None) self.files_changed = files_changed self.read_special_timeout = read_special_timeout + self.input_map = input_map # --map: content range info for the single input file, see #4363 self.hlm = HardLinkManager(id_type=tuple, info_type=(list, type(None))) # (dev, ino) -> chunks or None self.stats = Statistics(output_json=log_json) # threading: done by cache (including progress) @@ -1552,7 +1555,11 @@ def process_file(self, *, path, parent_fd, name, st, cache, flags=flags_normal, chunk_entry = cache.reuse_chunk(chunk_id, chunk_size, self.stats) item.chunks.append(chunk_entry) else: # normal case, no "2nd+" hard link - if not is_special_file: + if self.input_map is not None: + # --map: the given map replaces the files cache as content/change information, see #4363. + hashed_path = path_hash = None + known, chunks = False, None + elif not is_special_file: hashed_path = safe_encode(item.path) # path as in archive item! started_hashing = time.monotonic() path_hash = self.key.id_hash(hashed_path) @@ -1593,7 +1600,16 @@ def process_file(self, *, path, parent_fd, name, st, cache, flags=flags_normal, # and still commit the archive -- referencing chunks that were never durably # stored. An unwrapped repository OSError is critical and aborts create before # archive.save() runs (see the BackupOSError docstring). - if read_special_timeout is not None: + if self.input_map is not None: + # --map: read only the "data" ranges, store "zero" ranges as holes without + # reading them, see #4363. Non-seekable inputs (fifo/chr) are rejected earlier. + with backup_io("seek"): + input_size = st.st_size if stat.S_ISREG(st.st_mode) else os.lseek(fd, 0, os.SEEK_END) + os.lseek(fd, 0, os.SEEK_SET) + input_map_check_size(self.input_map, input_size) + fmap = [(start, length, state == MAP_DATA) for start, length, state in self.input_map] + chunk_iter = self.chunker.chunkify(None, fd, fmap=fmap, st=st) + elif read_special_timeout is not None: # all reads go through the timeout-enforcing wrapper (fh stays unused). chunk_iter = self.chunker.chunkify(SpecialFileReader(fd, read_special_timeout), st=st) else: diff --git a/src/borg/archiver/create_cmd.py b/src/borg/archiver/create_cmd.py index 5dbc2ec455..4270523bd8 100644 --- a/src/borg/archiver/create_cmd.py +++ b/src/borg/archiver/create_cmd.py @@ -18,6 +18,7 @@ from ..helpers import comment_validator, ChunkerParams, FilesystemPathSpec, CompressionSpec from ..helpers import archivename_validator, FilesCacheMode, files_cache_mode_no_ctime from ..helpers import octal_int, nonnegative_seconds +from ..helpers import read_input_map from ..helpers import eval_escapes from ..helpers import timestamp, archive_ts_now from ..helpers import get_cache_dir, os_stat, get_strip_prefix, slashify @@ -71,6 +72,27 @@ def do_create(self, args, repository, manifest): read_special_timeout = READ_SPECIAL_TIMEOUT_DEFAULT if read_special_timeout == 0: read_special_timeout = None # wait forever + input_map = None + if args.input_map is not None: + # --map only makes sense for a single, seekable input file, see #4363. + if args.paths_from_stdin or args.paths_from_command or args.paths_from_shell_command: + raise CommandError("--map cannot be used with --paths-from-*.") + if args.content_from_command: + raise CommandError("--map cannot be used with --content-from-command.") + if len(args.paths) != 1: + raise CommandError("--map requires exactly one input path.") + if args.paths[0] == "-": + raise CommandError("--map cannot be used with stdin input.") + try: + st_map = os.stat(args.paths[0], follow_symlinks=True) + except OSError as e: + raise CommandError(f"--map input: {args.paths[0]}: {e}") + if stat.S_ISBLK(st_map.st_mode): + if not args.read_special: + raise CommandError("--map with a block device requires --read-special.") + elif not stat.S_ISREG(st_map.st_mode): + raise CommandError("--map input must be a regular file or a block device.") + input_map = read_input_map(args.input_map) if is_win32: # st_ctime is the file *creation* time on Windows, not the "metadata change time", # so a ctime based files cache mode would not detect content changes of a file that @@ -315,6 +337,7 @@ def create_inner(archive, cache, fso): file_status_printer=self.print_file_status, files_changed=args.files_changed, read_special_timeout=read_special_timeout, + input_map=input_map, ) create_inner(archive, cache, fso) args.stats |= args.json @@ -955,6 +978,31 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser): By default, the content read from stdin is stored in a file called 'stdin'. Use ``--stdin-name`` to change the name. + Input maps + ++++++++++ + + Usually, borg reads the complete input to determine its contents. If you already + know the contents of parts of the input from an external source of truth, you can + give that information via ``--map MAPFILE`` and borg will not read the known parts. + The primary use case is backing up snapshots of large block devices (e.g. LVM thin + volumes), where the storage layer knows which ranges are in use. + + ``--map`` requires giving exactly one input path, which must be a regular file or + (with ``--read-special``) a block device. + + The map file must describe the whole input: one range per line, in the form + ``START LENGTH STATE`` (byte values, decimal or 0x-prefixed hexadecimal). The + ranges must be sorted, non-overlapping and contiguous, starting at offset 0 and + covering the exact input size. ``#`` starts a comment, empty lines are ignored. + STATE is one of: + + - ``data``: the range's contents are read and backed up. + - ``zero``: the range is known to read as all-zero bytes. borg stores a hole + (all-zero range) of that size without reading the range. + + **The map is trusted**: if it is wrong (e.g. a range marked ``zero`` actually + contains data), the archive will not match the input and borg cannot detect that. + Feeding all file paths from externally ++++++++++++++++++++++++++++++++++++++ @@ -1146,6 +1194,14 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser): "for a FIFO's writer to connect). Give 0 to wait forever. default: %d seconds." % READ_SPECIAL_TIMEOUT_DEFAULT, ) + fs_group.add_argument( + "--map", + metavar="MAPFILE", + dest="input_map", + action=Highlander, + help="give a map file describing the content ranges of the (single) input file, " + "so borg does not need to read all of it. See the *Input maps* section below.", + ) archive_group = subparser.add_argument_group("Archive options") archive_group.add_argument( diff --git a/src/borg/helpers/__init__.py b/src/borg/helpers/__init__.py index eb2a976098..2f7420eca1 100644 --- a/src/borg/helpers/__init__.py +++ b/src/borg/helpers/__init__.py @@ -24,6 +24,7 @@ from .fs import dir_is_tagged, dir_is_cachedir, remove_dotdot_prefixes, make_path_safe, scandir_inorder from .fs import secure_erase, safe_unlink, dash_open, os_open, os_stat, get_strip_prefix, umount, slashify from .fs import SpecialFileReader +from .fs import MAP_DATA, MAP_ZERO, MAP_SAME, read_input_map, input_map_check_size from .fs import O_, flags_dir, flags_dir_follow, flags_special_follow, flags_special from .fs import flags_base, flags_normal, flags_normal_follow, flags_noatime from .fs import HardLinkManager diff --git a/src/borg/helpers/fs.py b/src/borg/helpers/fs.py index 653371a1b5..e7a3d44ebf 100644 --- a/src/borg/helpers/fs.py +++ b/src/borg/helpers/fs.py @@ -640,6 +640,59 @@ def seek(self, pos, whence=os.SEEK_SET): raise OSError(errno.ESPIPE, os.strerror(errno.ESPIPE)) +# input map range states (borg create --map), see #4363. +MAP_DATA = "data" # range contains data, read and store it +MAP_ZERO = "zero" # range is known to read as all-zero, store a hole without reading it +MAP_SAME = "same" # range is known unchanged vs. a reference archive, reuse its chunks (--reuse-from) + + +def read_input_map(path, *, allow_same=False): + """ + Read and validate an input map file (borg create --map), see #4363. + + Format: one range per line: "START LENGTH STATE" (decimal or 0x-prefixed hex), + STATE being data, zero or same. '#' starts a comment, empty lines are ignored. + The ranges must be sorted, non-overlapping and contiguous, starting at offset 0. + + Returns a list of (start, length, state) tuples. + """ + states = {MAP_DATA, MAP_ZERO} | ({MAP_SAME} if allow_same else set()) + input_map = [] + expected_start = 0 + with open(path) as f: + for lineno, line in enumerate(f, start=1): + line = line.split("#", 1)[0].strip() + if not line: + continue + err = f"{path}:{lineno}: invalid input map line" + try: + start_s, length_s, state = line.split() + start, length = int(start_s, 0), int(length_s, 0) + except ValueError: + raise Error(f"{err}: expected 'START LENGTH STATE', got: {line!r}") from None + if state not in states: + raise Error(f"{err}: invalid state {state!r} (expected: {', '.join(sorted(states))})") + if length <= 0: + raise Error(f"{err}: LENGTH must be positive") + if start != expected_start: + raise Error( + f"{err}: ranges must be sorted, contiguous and start at offset 0 " + f"(expected START {expected_start}, got {start})" + ) + input_map.append((start, length, state)) + expected_start = start + length + if not input_map: + raise Error(f"{path}: empty input map") + return input_map + + +def input_map_check_size(input_map, size): + """Check that *input_map* covers [0, size) exactly - a mismatch means the map does not belong to this input.""" + covered = input_map[-1][0] + input_map[-1][1] + if covered != size: + raise Error(f"input map covers {covered} bytes, but the input has {size} bytes") + + def umount(mountpoint): from . import set_ec diff --git a/src/borg/testsuite/archiver/create_cmd_test.py b/src/borg/testsuite/archiver/create_cmd_test.py index c8d8077f22..cfc18e7016 100644 --- a/src/borg/testsuite/archiver/create_cmd_test.py +++ b/src/borg/testsuite/archiver/create_cmd_test.py @@ -17,7 +17,7 @@ from ...platformflags import is_msystem from ...repository import Repository from ...helpers import CommandError, BackupPermissionError, BackupTimeoutError, BackupBrokenSymlinkError -from ...helpers import BackupWarning +from ...helpers import BackupWarning, Error from .. import has_lchflags, has_mknod from .. import changedir from .. import ( @@ -1552,3 +1552,55 @@ def test_exclude_nodump_dir_with_file(archivers, request): list_output = cmd(archiver, "list", "test", "--short") assert "input/nd\n" not in list_output assert "input/nd/file_in_ndir\n" not in list_output + + +def test_create_map(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + block = 4096 + data_a, data_b, data_c = os.urandom(block), os.urandom(block), os.urandom(block) + # data_b is real data, but the map claims that range reads as zero: + # borg must not read "zero" ranges, so the archive must contain zeros there. + create_regular_file(archiver.input_path, "file", contents=data_a + data_b + data_c + b"\0" * block) + map_path = os.fspath(archiver.tmpdir / "input.map") + with open(map_path, "w") as f: + f.write("# test input map\n") + f.write(f"0 {block} data\n") + f.write(f"0x1000 {block} zero\n") + f.write(f"{2 * block} {block} data\n") + f.write(f"{3 * block} {block} zero\n") + expected = data_a + b"\0" * block + data_c + b"\0" * block + for name, chunker_args in [("test-fixed", ("--chunker-params", "fixed,4096")), ("test-default", ())]: + cmd(archiver, "create", *chunker_args, "--map", map_path, name, "input/file") + with changedir("output"): + cmd(archiver, "extract", name) + with open(os.path.join("output", "input", "file"), "rb") as f: + assert f.read() == expected + shutil.rmtree("output") + os.mkdir("output") + + +def test_create_map_errors(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + create_regular_file(archiver.input_path, "file", size=2 * 4096) + create_regular_file(archiver.input_path, "file2", size=100) + map_path = os.fspath(archiver.tmpdir / "input.map") + with open(map_path, "w") as f: + f.write("0 4096 data\n") + + def expect_error(exc_class, *args): + if archiver.FORK_DEFAULT: + cmd(archiver, *args, exit_code=exc_class().exit_code) + else: + with pytest.raises(exc_class): + cmd(archiver, *args) + + # --map requires exactly one input path + expect_error(CommandError, "create", "--map", map_path, "test", "input/file", "input/file2") + # --map input must not be a directory + expect_error(CommandError, "create", "--map", map_path, "test", "input") + # --map cannot be used with stdin input + expect_error(CommandError, "create", "--map", map_path, "test", "-") + # the map covers 4096 bytes, but the input file has 8192 bytes + expect_error(Error, "create", "--map", map_path, "test", "input/file") diff --git a/src/borg/testsuite/helpers/fs_test.py b/src/borg/testsuite/helpers/fs_test.py index e67a230013..d93ddb1c6c 100644 --- a/src/borg/testsuite/helpers/fs_test.py +++ b/src/borg/testsuite/helpers/fs_test.py @@ -24,7 +24,13 @@ make_path_safe, map_chars, SpecialFileReader, + MAP_DATA, + MAP_ZERO, + MAP_SAME, + read_input_map, + input_map_check_size, ) +from ...helpers.errors import Error from ...platform import is_win32, is_darwin from .. import are_fifos_supported, are_hardlinks_supported from .. import rejected_dotdot_paths @@ -606,3 +612,48 @@ def test_special_file_reader_not_seekable(): with pytest.raises(OSError) as exc_info: reader.seek(0) assert exc_info.value.errno == errno.ESPIPE + + +def test_read_input_map(tmp_path): + p = tmp_path / "input.map" + p.write_text("# a comment\n\n0 0x1000 data\n0x1000 4096 zero # trailing comment\n8192 100 data\n") + assert read_input_map(os.fspath(p)) == [(0, 4096, MAP_DATA), (4096, 4096, MAP_ZERO), (8192, 100, MAP_DATA)] + + +def test_read_input_map_same_state(tmp_path): + p = tmp_path / "input.map" + p.write_text("0 100 same\n") + with pytest.raises(Error, match="invalid state"): + read_input_map(os.fspath(p)) + assert read_input_map(os.fspath(p), allow_same=True) == [(0, 100, MAP_SAME)] + + +@pytest.mark.parametrize( + "content, match", + [ + ("", "empty input map"), + ("0 100\n", "expected 'START LENGTH STATE'"), + ("0 100 data extra\n", "expected 'START LENGTH STATE'"), + ("x 100 data\n", "expected 'START LENGTH STATE'"), + ("0 100 nodata\n", "invalid state"), + ("0 0 data\n", "LENGTH must be positive"), + ("0 -5 data\n", "LENGTH must be positive"), + ("100 100 data\n", "expected START 0"), + ("0 100 data\n50 100 zero\n", "expected START 100"), + ("0 100 data\n200 100 zero\n", "expected START 100"), + ], +) +def test_read_input_map_invalid(tmp_path, content, match): + p = tmp_path / "input.map" + p.write_text(content) + with pytest.raises(Error, match=match): + read_input_map(os.fspath(p)) + + +def test_input_map_check_size(): + input_map = [(0, 100, MAP_DATA), (100, 50, MAP_ZERO)] + input_map_check_size(input_map, 150) # exact coverage - ok + with pytest.raises(Error, match="covers 150 bytes, but the input has 151"): + input_map_check_size(input_map, 151) + with pytest.raises(Error, match="covers 150 bytes, but the input has 149"): + input_map_check_size(input_map, 149) From 65f13da3aae3f3496d94fd1d3f7615d379c861d2 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 17 Aug 2026 02:59:44 +0200 Subject: [PATCH 2/3] create: add --reuse-from to reuse a reference archive's chunks, see #4363 For input map ranges marked "same", reuse the chunks of the reference archive's item instead of reading the input. Reference chunks that only partially overlap "same" ranges are re-read completely, so the result is correct with any chunker; with the fixed chunker, read windows align with the reference chunk grid. Reference chunks missing from the repo are read again (like the files cache does on a lost chunk). Together with --map, this implements efficient incremental backups of block device snapshots: only changed ranges need to be read, e.g. as reported by thin_delta for LVM thin volume snapshots. Co-Authored-By: Claude Fable 5 --- docs/man/borg-create.1 | 25 +- docs/usage/create.rst.inc | 24 +- src/borg/archive.py | 113 ++++++++- src/borg/archiver/create_cmd.py | 58 ++++- src/borg/testsuite/archive_test.py | 71 ++++++ .../testsuite/archiver/create_cmd_test.py | 218 ++++++++++++++++++ 6 files changed, 497 insertions(+), 12 deletions(-) diff --git a/docs/man/borg-create.1 b/docs/man/borg-create.1 index c2b7407384..f25b819f37 100644 --- a/docs/man/borg-create.1 +++ b/docs/man/borg-create.1 @@ -323,6 +323,12 @@ when reading from FIFOs or character devices (see \-\-read\-special): skip the f .TP .BI \-\-map \ MAPFILE give a map file describing the content ranges of the (single) input file, so borg does not need to read all of it. See the \fIInput maps\fP section below. +.TP +.BI \-\-reuse\-from \ ARCHIVE +reuse the chunks of this reference archive for the input map\(aqs \fBsame\fP ranges (requires \-\-map). See the \fIInput maps\fP section below. +.TP +.BI \-\-reuse\-path \ PATH +archive\-internal path of the reference item in the \-\-reuse\-from archive (only needed if that archive contains more than one file item). .UNINDENT .SS Archive options .INDENT 0.0 @@ -606,10 +612,27 @@ STATE is one of: .IP \(bu 2 \fBzero\fP: the range is known to read as all\-zero bytes. borg stores a hole (all\-zero range) of that size without reading the range. +.IP \(bu 2 +\fBsame\fP: the range is known to be identical to the same range of the input +backed up in the \fB\-\-reuse\-from REFARCHIVE\fP reference archive (usually: the +previous backup of an earlier snapshot of the same device). borg reuses the +reference archive\(aqs chunks for such ranges without reading them. This state +requires \fB\-\-reuse\-from\fP\&. .UNINDENT .sp +The reference archive must contain exactly one file item; if it contains more, +select the reference item with \fB\-\-reuse\-path PATH\fP (its archive\-internal path). +Reference chunks that only partially overlap \fBsame\fP ranges are re\-read from +the input, so any chunker gives correct results \- but a fixed block size chunker +(e.g. \fB\-\-chunker\-params fixed,4194304\fP, same parameters as used for the +reference archive) avoids re\-reading at the edges of changed ranges and gives +stable chunk boundaries across backups. +.sp \fBThe map is trusted\fP: if it is wrong (e.g. a range marked \fBzero\fP actually -contains data), the archive will not match the input and borg cannot detect that. +contains data, or a range marked \fBsame\fP actually changed), the archive will +not match the input and borg cannot detect that. Independently verify the +source producing the maps, and consider doing a periodic full read backup +(without \fB\-\-map\fP). .SS Feeding all file paths from externally .sp Usually, you give a starting path (recursion root) to borg and then borg diff --git a/docs/usage/create.rst.inc b/docs/usage/create.rst.inc index bc705c63a7..2e4a3e533b 100644 --- a/docs/usage/create.rst.inc +++ b/docs/usage/create.rst.inc @@ -101,6 +101,10 @@ borg create +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | | ``--map MAPFILE`` | give a map file describing the content ranges of the (single) input file, so borg does not need to read all of it. See the *Input maps* section below. | +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``--reuse-from ARCHIVE`` | reuse the chunks of this reference archive for the input map's ``same`` ranges (requires --map). See the *Input maps* section below. | + +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``--reuse-path PATH`` | archive-internal path of the reference item in the --reuse-from archive (only needed if that archive contains more than one file item). | + +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | **Archive options** | +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | | ``--comment COMMENT`` | add a comment text to the archive | @@ -176,6 +180,8 @@ borg create --read-special open and read block and char device files as well as FIFOs as if they were regular files. Also follows symlinks pointing to these kinds of files. --read-special-timeout SECONDS when reading from FIFOs or character devices (see --read-special): skip the file with an error if no data arrives for more than SECONDS (this includes waiting for a FIFO's writer to connect). Give 0 to wait forever. default: 1800 seconds. --map MAPFILE give a map file describing the content ranges of the (single) input file, so borg does not need to read all of it. See the *Input maps* section below. + --reuse-from ARCHIVE reuse the chunks of this reference archive for the input map's ``same`` ranges (requires --map). See the *Input maps* section below. + --reuse-path PATH archive-internal path of the reference item in the --reuse-from archive (only needed if that archive contains more than one file item). Archive options @@ -469,9 +475,25 @@ STATE is one of: - ``data``: the range's contents are read and backed up. - ``zero``: the range is known to read as all-zero bytes. borg stores a hole (all-zero range) of that size without reading the range. +- ``same``: the range is known to be identical to the same range of the input + backed up in the ``--reuse-from REFARCHIVE`` reference archive (usually: the + previous backup of an earlier snapshot of the same device). borg reuses the + reference archive's chunks for such ranges without reading them. This state + requires ``--reuse-from``. + +The reference archive must contain exactly one file item; if it contains more, +select the reference item with ``--reuse-path PATH`` (its archive-internal path). +Reference chunks that only partially overlap ``same`` ranges are re-read from +the input, so any chunker gives correct results - but a fixed block size chunker +(e.g. ``--chunker-params fixed,4194304``, same parameters as used for the +reference archive) avoids re-reading at the edges of changed ranges and gives +stable chunk boundaries across backups. **The map is trusted**: if it is wrong (e.g. a range marked ``zero`` actually -contains data), the archive will not match the input and borg cannot detect that. +contains data, or a range marked ``same`` actually changed), the archive will +not match the input and borg cannot detect that. Independently verify the +source producing the maps, and consider doing a periodic full read backup +(without ``--map``). Feeding all file paths from externally ++++++++++++++++++++++++++++++++++++++ diff --git a/src/borg/archive.py b/src/borg/archive.py index 078bcf9ccb..80063ee178 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -43,7 +43,7 @@ from .helpers import safe_ns from .helpers import ellipsis_truncate, ProgressIndicatorPercent, log_multi, get_progress_dt from .helpers import os_open, flags_normal, flags_dir, O_, SpecialFileReader -from .helpers import MAP_DATA, input_map_check_size +from .helpers import MAP_DATA, MAP_ZERO, MAP_SAME, input_map_check_size from .helpers import os_stat from .helpers import msgpack from .helpers.lrucache import LRUCache @@ -1300,7 +1300,7 @@ def __init__(self, *, key, cache, add_item, rechunkify): self.add_item = add_item self.rechunkify = rechunkify - def process_file_chunks(self, item, cache, stats, show_progress, chunk_iter, chunk_processor=None): + def process_file_chunks(self, item, cache, stats, show_progress, chunk_iter, chunk_processor=None, append=False): if not chunk_processor: def chunk_processor(chunk): @@ -1313,7 +1313,8 @@ def chunk_processor(chunk): release_chunk_data(data) return chunk_entry - item.chunks = [] + if not append: + item.chunks = [] # a --reuse-from caller calls this repeatedly, appending to existing chunks. for chunk in chunk_iter: chunk_entry = chunk_processor(chunk) item.chunks.append(chunk_entry) @@ -1333,6 +1334,71 @@ def maybe_exclude_by_attr(item): raise BackupItemExcluded +def build_reuse_plan(input_map, ref_chunks, size, seen_chunk): + """ + Build a processing plan for --map with --reuse-from, see #4363. + + A reference chunk is reused iff its whole extent lies within "same" map ranges and it + still exists in the repo (seen_chunk). Everything else is read from the input (with + "zero" ranges stored as holes without reading). Reading whole reference chunks that + intersect changed ranges keeps the result correct for any chunker; with the fixed + chunker, reference chunk boundaries and read windows align exactly. + + Returns a list of parts covering [0, size) in order: + ("reuse", [ChunkListEntry, ...]) or ("read", [(start, length, is_data), ...]). + """ + # merge adjacent "same" ranges, so a reference chunk spanning two of them is still reusable. + same = [] + for start, length, state in input_map: + if state != MAP_SAME: + continue + if same and same[-1][1] == start: + same[-1][1] = start + length + else: + same.append([start, start + length]) + + def read_part(a, b): + # intersect extent [a, b) with the map's ranges: "zero" ranges become holes (not read), + # everything else (data, or same parts of non-reusable reference chunks) is read. + fmap = [] + for start, length, state in input_map: + sub_start, sub_end = max(start, a), min(start + length, b) + if sub_start < sub_end: + fmap.append((sub_start, sub_end - sub_start, state != MAP_ZERO)) + return "read", fmap + + parts = [] + read_start = None # start of the current not-yet-flushed read extent + offset = 0 + si = 0 # index into same[], both same[] and the chunks are sorted by offset + for entry in ref_chunks: + start, end = offset, offset + entry.size + offset = end + if start >= size: + break + while si < len(same) and same[si][1] <= start: + si += 1 + reusable = ( + end <= size and si < len(same) and same[si][0] <= start and end <= same[si][1] and seen_chunk(entry.id) + ) + if reusable: + if read_start is not None: + parts.append(read_part(read_start, start)) + read_start = None + if parts and parts[-1][0] == "reuse": + parts[-1][1].append(entry) + else: + parts.append(("reuse", [entry])) + elif read_start is None: + read_start = start + if read_start is not None: + # covers non-reusable chunks at the end and any input tail beyond the reference chunks. + parts.append(read_part(read_start, size)) + elif offset < size: + parts.append(read_part(offset, size)) + return parts + + class FilesystemObjectProcessors: # When ported to threading, then this doesn't need chunker, cache, key any more. # process_file becomes a callback passed to __init__. @@ -1353,6 +1419,7 @@ def __init__( files_changed="mtime" if is_win32 else "ctime", read_special_timeout=None, input_map=None, + reuse_chunks=None, ): self.metadata_collector = metadata_collector self.cache = cache @@ -1364,6 +1431,7 @@ def __init__( self.files_changed = files_changed self.read_special_timeout = read_special_timeout self.input_map = input_map # --map: content range info for the single input file, see #4363 + self.reuse_chunks = reuse_chunks # --reuse-from: the reference archive item's chunk list, see #4363 self.hlm = HardLinkManager(id_type=tuple, info_type=(list, type(None))) # (dev, ino) -> chunks or None self.stats = Statistics(output_json=log_json) # threading: done by cache (including progress) @@ -1607,18 +1675,47 @@ def process_file(self, *, path, parent_fd, name, st, cache, flags=flags_normal, input_size = st.st_size if stat.S_ISREG(st.st_mode) else os.lseek(fd, 0, os.SEEK_END) os.lseek(fd, 0, os.SEEK_SET) input_map_check_size(self.input_map, input_size) - fmap = [(start, length, state == MAP_DATA) for start, length, state in self.input_map] - chunk_iter = self.chunker.chunkify(None, fd, fmap=fmap, st=st) + if self.reuse_chunks is not None: + # --reuse-from: reuse the reference archive's chunks for "same" ranges + # without reading them. Each "read" part gets its own chunkify call, so + # chunks never span the gap left by reused parts. + plan = build_reuse_plan(self.input_map, self.reuse_chunks, input_size, cache.seen_chunk) + item.chunks = [] + for kind, payload in plan: + if kind == "reuse": + for entry in payload: + item.chunks.append(cache.reuse_chunk(entry.id, entry.size, self.stats)) + if self.show_progress: + self.stats.show_progress(item=item) + else: + chunk_iter = self.chunker.chunkify(None, fd, fmap=payload, st=st) + self.process_file_chunks( + item, + cache, + self.stats, + self.show_progress, + backup_io_iter(chunk_iter), + append=True, + ) + else: + fmap = [(start, length, state == MAP_DATA) for start, length, state in self.input_map] + chunk_iter = self.chunker.chunkify(None, fd, fmap=fmap, st=st) + self.process_file_chunks( + item, cache, self.stats, self.show_progress, backup_io_iter(chunk_iter) + ) elif read_special_timeout is not None: # all reads go through the timeout-enforcing wrapper (fh stays unused). chunk_iter = self.chunker.chunkify(SpecialFileReader(fd, read_special_timeout), st=st) + self.process_file_chunks( + item, cache, self.stats, self.show_progress, backup_io_iter(chunk_iter) + ) else: # passing st saves FileReader a stat call; regular files take the # direct read path, special files (--read-special) the buffered one. chunk_iter = self.chunker.chunkify(None, fd, st=st) - self.process_file_chunks( - item, cache, self.stats, self.show_progress, backup_io_iter(chunk_iter) - ) + self.process_file_chunks( + item, cache, self.stats, self.show_progress, backup_io_iter(chunk_iter) + ) self.stats.chunking_time = self.chunker.chunking_time end_reading = time.time_ns() with backup_io("fstat2"): diff --git a/src/borg/archiver/create_cmd.py b/src/borg/archiver/create_cmd.py index 4270523bd8..767d0f6b4a 100644 --- a/src/borg/archiver/create_cmd.py +++ b/src/borg/archiver/create_cmd.py @@ -72,6 +72,10 @@ def do_create(self, args, repository, manifest): read_special_timeout = READ_SPECIAL_TIMEOUT_DEFAULT if read_special_timeout == 0: read_special_timeout = None # wait forever + if args.reuse_from is not None and args.input_map is None: + raise CommandError("--reuse-from requires --map.") + if args.reuse_path is not None and args.reuse_from is None: + raise CommandError("--reuse-path requires --reuse-from.") input_map = None if args.input_map is not None: # --map only makes sense for a single, seekable input file, see #4363. @@ -92,7 +96,24 @@ def do_create(self, args, repository, manifest): raise CommandError("--map with a block device requires --read-special.") elif not stat.S_ISREG(st_map.st_mode): raise CommandError("--map input must be a regular file or a block device.") - input_map = read_input_map(args.input_map) + input_map = read_input_map(args.input_map, allow_same=args.reuse_from is not None) + reuse_chunks = None + if args.reuse_from is not None: + ref_info = manifest.archives.get_one([args.reuse_from]) + ref_archive = Archive(manifest, ref_info.id) + ref_items = [item for item in ref_archive.iter_items() if "chunks" in item] + if args.reuse_path is not None: + ref_items = [item for item in ref_items if item.path == args.reuse_path] + if not ref_items: + raise CommandError( + f"--reuse-from: no file item with path {args.reuse_path!r} in reference archive." + ) + if len(ref_items) != 1: + raise CommandError( + f"--reuse-from: reference archive has {len(ref_items)} file items, " + f"use --reuse-path to select the reference item." + ) + reuse_chunks = ref_items[0].chunks if is_win32: # st_ctime is the file *creation* time on Windows, not the "metadata change time", # so a ctime based files cache mode would not detect content changes of a file that @@ -338,6 +359,7 @@ def create_inner(archive, cache, fso): files_changed=args.files_changed, read_special_timeout=read_special_timeout, input_map=input_map, + reuse_chunks=reuse_chunks, ) create_inner(archive, cache, fso) args.stats |= args.json @@ -999,9 +1021,25 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser): - ``data``: the range's contents are read and backed up. - ``zero``: the range is known to read as all-zero bytes. borg stores a hole (all-zero range) of that size without reading the range. + - ``same``: the range is known to be identical to the same range of the input + backed up in the ``--reuse-from REFARCHIVE`` reference archive (usually: the + previous backup of an earlier snapshot of the same device). borg reuses the + reference archive's chunks for such ranges without reading them. This state + requires ``--reuse-from``. + + The reference archive must contain exactly one file item; if it contains more, + select the reference item with ``--reuse-path PATH`` (its archive-internal path). + Reference chunks that only partially overlap ``same`` ranges are re-read from + the input, so any chunker gives correct results - but a fixed block size chunker + (e.g. ``--chunker-params fixed,4194304``, same parameters as used for the + reference archive) avoids re-reading at the edges of changed ranges and gives + stable chunk boundaries across backups. **The map is trusted**: if it is wrong (e.g. a range marked ``zero`` actually - contains data), the archive will not match the input and borg cannot detect that. + contains data, or a range marked ``same`` actually changed), the archive will + not match the input and borg cannot detect that. Independently verify the + source producing the maps, and consider doing a periodic full read backup + (without ``--map``). Feeding all file paths from externally ++++++++++++++++++++++++++++++++++++++ @@ -1202,6 +1240,22 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser): help="give a map file describing the content ranges of the (single) input file, " "so borg does not need to read all of it. See the *Input maps* section below.", ) + fs_group.add_argument( + "--reuse-from", + metavar="ARCHIVE", + dest="reuse_from", + action=Highlander, + help="reuse the chunks of this reference archive for the input map's ``same`` " + "ranges (requires --map). See the *Input maps* section below.", + ) + fs_group.add_argument( + "--reuse-path", + metavar="PATH", + dest="reuse_path", + action=Highlander, + help="archive-internal path of the reference item in the --reuse-from archive " + "(only needed if that archive contains more than one file item).", + ) archive_group = subparser.add_argument_group("Archive options") archive_group.add_argument( diff --git a/src/borg/testsuite/archive_test.py b/src/borg/testsuite/archive_test.py index cb2aee520e..d1c5e53b96 100644 --- a/src/borg/testsuite/archive_test.py +++ b/src/borg/testsuite/archive_test.py @@ -579,3 +579,74 @@ def test_archives_get_by_id_missing_returns_none(): manifest = Mock() archives = Archives(repo, manifest) assert archives.get_by_id(b"\x01" * 32) is None + + +class TestBuildReusePlan: + """Tests for the --map/--reuse-from processing plan, see #4363.""" + + @staticmethod + def plan(input_map, chunk_sizes, size, missing=()): + from ..archive import build_reuse_plan + + ref_chunks = [ChunkListEntry(i.to_bytes(4, "big"), chunk_size) for i, chunk_size in enumerate(chunk_sizes)] + return ref_chunks, build_reuse_plan(input_map, ref_chunks, size, lambda id: id not in missing) + + def test_all_same(self): + ref, plan = self.plan([(0, 8192, "same")], [4096, 4096], 8192) + assert plan == [("reuse", ref)] + + def test_all_data_and_zero(self): + _, plan = self.plan([(0, 4096, "data"), (4096, 4096, "zero")], [4096, 4096], 8192) + assert plan == [("read", [(0, 4096, True), (4096, 4096, False)])] + + def test_mixed(self): + # chunks 0 and 2 reusable, chunk 1 changed. + input_map = [(0, 4096, "same"), (4096, 4096, "data"), (8192, 4096, "same")] + ref, plan = self.plan(input_map, [4096, 4096, 4096], 12288) + assert plan == [("reuse", ref[0:1]), ("read", [(4096, 4096, True)]), ("reuse", ref[2:3])] + + def test_partial_overlap_reads_whole_chunk(self): + # a reference chunk that only partially lies in a "same" range is re-read completely, + # including its "same" part. + input_map = [(0, 6000, "same"), (6000, 2192, "data")] + ref, plan = self.plan(input_map, [4096, 4096], 8192) + assert plan == [("reuse", ref[0:1]), ("read", [(4096, 1904, True), (6000, 2192, True)])] + + def test_adjacent_same_ranges_merge(self): + # a chunk spanning two adjacent "same" map ranges is still reusable. + input_map = [(0, 2048, "same"), (2048, 6144, "same")] + ref, plan = self.plan(input_map, [4096, 4096], 8192) + assert plan == [("reuse", ref)] + + def test_zero_inside_read_part(self): + # a "zero" range makes its chunk non-reusable, but is stored as a hole, not read. + input_map = [(0, 4096, "same"), (4096, 4096, "zero")] + ref, plan = self.plan(input_map, [4096, 4096], 8192) + assert plan == [("reuse", ref[0:1]), ("read", [(4096, 4096, False)])] + + def test_grown_input(self): + # the tail beyond the reference chunks must be read. + input_map = [(0, 8192, "same"), (8192, 4096, "data")] + ref, plan = self.plan(input_map, [4096, 4096], 12288) + assert plan == [("reuse", ref), ("read", [(8192, 4096, True)])] + + def test_shrunk_input(self): + # reference chunks beyond the new input size are dropped. + input_map = [(0, 4096, "same")] + ref, plan = self.plan(input_map, [4096, 4096, 4096], 4096) + assert plan == [("reuse", ref[0:1])] + + def test_shrunk_input_straddling_chunk(self): + # a reference chunk straddling the new end of the input cannot be reused. + input_map = [(0, 6000, "same")] + ref, plan = self.plan(input_map, [4096, 4096], 6000) + assert plan == [("reuse", ref[0:1]), ("read", [(4096, 1904, True)])] + + def test_missing_chunk_is_read(self): + input_map = [(0, 8192, "same")] + ref, plan = self.plan(input_map, [4096, 4096], 8192, missing={ref_id(1)}) + assert plan == [("reuse", ref[0:1]), ("read", [(4096, 4096, True)])] + + +def ref_id(i): + return i.to_bytes(4, "big") diff --git a/src/borg/testsuite/archiver/create_cmd_test.py b/src/borg/testsuite/archiver/create_cmd_test.py index cfc18e7016..f668b11bb8 100644 --- a/src/borg/testsuite/archiver/create_cmd_test.py +++ b/src/borg/testsuite/archiver/create_cmd_test.py @@ -1604,3 +1604,221 @@ def expect_error(exc_class, *args): expect_error(CommandError, "create", "--map", map_path, "test", "-") # the map covers 4096 bytes, but the input file has 8192 bytes expect_error(Error, "create", "--map", map_path, "test", "input/file") + + +def test_create_map_reuse_from_fixed(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + block = 4096 + blocks = [os.urandom(block) for _ in range(4)] + fname = os.path.join(archiver.input_path, "file") + create_regular_file(archiver.input_path, "file", contents=b"".join(blocks)) + map_path = os.fspath(archiver.tmpdir / "input.map") + with open(map_path, "w") as f: + f.write(f"0 {4 * block} data\n") + cmd(archiver, "create", "--chunker-params", "fixed,4096", "--map", map_path, "ref", "input/file") + # modify block 1 (declared "data" below) and block 2 (declared "same"!): the new archive + # must contain the new block 1, but the OLD block 2 - proving that borg reused the + # reference archive's chunks instead of reading the "same" ranges. + new_block1, sneaky_block2 = os.urandom(block), os.urandom(block) + with open(fname, "r+b") as f: + f.seek(block) + f.write(new_block1) + f.write(sneaky_block2) + with open(map_path, "w") as f: + f.write(f"0 {block} same\n") + f.write(f"{block} {block} data\n") + f.write(f"{2 * block} {block} same\n") + f.write(f"{3 * block} {block} zero\n") + cmd( + archiver, + "create", + "--chunker-params", + "fixed,4096", + "--map", + map_path, + "--reuse-from", + "ref", + "test", + "input/file", + ) + with changedir("output"): + cmd(archiver, "extract", "test") + with open(os.path.join("output", "input", "file"), "rb") as f: + assert f.read() == blocks[0] + new_block1 + blocks[2] + b"\0" * block + + +def test_create_map_reuse_from_cdc(archivers, request): + # the reuse plan re-reads reference chunks that only partially overlap changed ranges, + # so it must also be correct with content-defined chunking (default chunker). + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + size = 1 << 20 + content = bytearray(os.urandom(size)) + fname = os.path.join(archiver.input_path, "file") + create_regular_file(archiver.input_path, "file", contents=bytes(content)) + map_path = os.fspath(archiver.tmpdir / "input.map") + with open(map_path, "w") as f: + f.write(f"0 {size} data\n") + cmd(archiver, "create", "ref", "input/file") + # change an unaligned range in the middle, keep everything else really unchanged. + start, length = 400000, 5000 + content[start : start + length] = os.urandom(length) + with open(fname, "wb") as f: + f.write(content) + with open(map_path, "w") as f: + f.write(f"0 {start} same\n") + f.write(f"{start} {length} data\n") + f.write(f"{start + length} {size - start - length} same\n") + cmd(archiver, "create", "--map", map_path, "--reuse-from", "ref", "test", "input/file") + with changedir("output"): + cmd(archiver, "extract", "test") + with open(os.path.join("output", "input", "file"), "rb") as f: + assert f.read() == bytes(content) + + +def test_create_map_reuse_from_resize(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + block = 4096 + blocks = [os.urandom(block) for _ in range(3)] + fname = os.path.join(archiver.input_path, "file") + create_regular_file(archiver.input_path, "file", contents=blocks[0] + blocks[1]) + map_path = os.fspath(archiver.tmpdir / "input.map") + with open(map_path, "w") as f: + f.write(f"0 {2 * block} data\n") + cmd(archiver, "create", "--chunker-params", "fixed,4096", "ref", "input/file") + # input grew: the tail beyond the reference chunks must be read. + with open(fname, "ab") as f: + f.write(blocks[2]) + with open(map_path, "w") as f: + f.write(f"0 {2 * block} same\n") + f.write(f"{2 * block} {block} data\n") + cmd( + archiver, + "create", + "--chunker-params", + "fixed,4096", + "--map", + map_path, + "--reuse-from", + "ref", + "grown", + "input/file", + ) + # input shrank: reference chunks beyond the new size must be dropped. + with open(fname, "r+b") as f: + f.truncate(block) + with open(map_path, "w") as f: + f.write(f"0 {block} same\n") + cmd( + archiver, + "create", + "--chunker-params", + "fixed,4096", + "--map", + map_path, + "--reuse-from", + "ref", + "shrunk", + "input/file", + ) + with changedir("output"): + cmd(archiver, "extract", "grown") + with open(os.path.join("output", "input", "file"), "rb") as f: + assert f.read() == blocks[0] + blocks[1] + blocks[2] + shutil.rmtree("output") + os.mkdir("output") + with changedir("output"): + cmd(archiver, "extract", "shrunk") + with open(os.path.join("output", "input", "file"), "rb") as f: + assert f.read() == blocks[0] + + +def test_create_map_reuse_from_missing_chunk(archivers, request, monkeypatch): + # when a reference chunk is not in the repo (any more), borg must fall back to reading. + archiver = request.getfixturevalue(archivers) + if archiver.FORK_DEFAULT: + pytest.skip("needs in-process monkeypatching") + cmd(archiver, "repo-create", RK_ENCRYPTION) + block = 4096 + content = os.urandom(2 * block) + create_regular_file(archiver.input_path, "file", contents=content) + map_path = os.fspath(archiver.tmpdir / "input.map") + with open(map_path, "w") as f: + f.write(f"0 {2 * block} data\n") + cmd(archiver, "create", "--chunker-params", "fixed,4096", "ref", "input/file") + with open(map_path, "w") as f: + f.write(f"0 {2 * block} same\n") + from ...cache import AdHocWithFilesCache + + monkeypatch.setattr(AdHocWithFilesCache, "seen_chunk", lambda self, id, size=None: False) + cmd( + archiver, + "create", + "--chunker-params", + "fixed,4096", + "--map", + map_path, + "--reuse-from", + "ref", + "test", + "input/file", + ) + with changedir("output"): + cmd(archiver, "extract", "test") + with open(os.path.join("output", "input", "file"), "rb") as f: + assert f.read() == content + + +def test_create_map_reuse_from_errors(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + create_regular_file(archiver.input_path, "file", size=4096) + create_regular_file(archiver.input_path, "file2", size=4096) + map_path = os.fspath(archiver.tmpdir / "input.map") + with open(map_path, "w") as f: + f.write("0 4096 same\n") + + def expect_error(exc_class, *args): + if archiver.FORK_DEFAULT: + cmd(archiver, *args, exit_code=exc_class().exit_code) + else: + with pytest.raises(exc_class): + cmd(archiver, *args) + + cmd(archiver, "create", "ref2", "input") # two file items + # --reuse-from requires --map + expect_error(CommandError, "create", "--reuse-from", "ref2", "test", "input/file") + # --reuse-path requires --reuse-from + expect_error(CommandError, "create", "--map", map_path, "--reuse-path", "input/file", "test", "input/file") + # "same" state requires --reuse-from + expect_error(Error, "create", "--map", map_path, "test", "input/file") + # ambiguous reference archive (two file items, no --reuse-path) + expect_error(CommandError, "create", "--map", map_path, "--reuse-from", "ref2", "test", "input/file") + # --reuse-path selecting a non-existing item + expect_error( + CommandError, + "create", + "--map", + map_path, + "--reuse-from", + "ref2", + "--reuse-path", + "nosuch", + "test", + "input/file", + ) + # disambiguated via --reuse-path: works + cmd( + archiver, + "create", + "--map", + map_path, + "--reuse-from", + "ref2", + "--reuse-path", + "input/file", + "test", + "input/file", + ) From b35d627747e9e7ef1b4606158de7e18ac7894599 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 17 Aug 2026 03:03:15 +0200 Subject: [PATCH 3/3] scripts: add lvm-thin-map.py to generate input maps from thin metadata, see #4363 Converts thin_dump XML (full mode: allocation map of one thin LV) and thin_delta XML (delta mode: differences between two thin snapshots) into the borg create --map format. The docstring documents the full workflow including the reserve/release_metadata_snap steps and the snapshot discipline needed for --reuse-from. Also add a docs section pointing from the --read-special LVM example to the input map based approach. Co-Authored-By: Claude Fable 5 --- docs/man/borg-create.1 | 4 + docs/usage/create.rst.inc | 4 + docs/usage/notes.rst | 20 ++++ scripts/lvm-thin-map.py | 166 ++++++++++++++++++++++++++++++++ src/borg/archiver/create_cmd.py | 4 + 5 files changed, 198 insertions(+) create mode 100755 scripts/lvm-thin-map.py diff --git a/docs/man/borg-create.1 b/docs/man/borg-create.1 index f25b819f37..26f9aa43c3 100644 --- a/docs/man/borg-create.1 +++ b/docs/man/borg-create.1 @@ -633,6 +633,10 @@ contains data, or a range marked \fBsame\fP actually changed), the archive will not match the input and borg cannot detect that. Independently verify the source producing the maps, and consider doing a periodic full read backup (without \fB\-\-map\fP). +.sp +For LVM thin volume snapshots, maps can be generated from \fBthin_dump\fP / +\fBthin_delta\fP XML with the \fBscripts/lvm\-thin\-map.py\fP converter from the +borg sources; its docstring shows the complete workflow. .SS Feeding all file paths from externally .sp Usually, you give a starting path (recursion root) to borg and then borg diff --git a/docs/usage/create.rst.inc b/docs/usage/create.rst.inc index 2e4a3e533b..5380031c6f 100644 --- a/docs/usage/create.rst.inc +++ b/docs/usage/create.rst.inc @@ -495,6 +495,10 @@ not match the input and borg cannot detect that. Independently verify the source producing the maps, and consider doing a periodic full read backup (without ``--map``). +For LVM thin volume snapshots, maps can be generated from ``thin_dump`` / +``thin_delta`` XML with the ``scripts/lvm-thin-map.py`` converter from the +borg sources; its docstring shows the complete workflow. + Feeding all file paths from externally ++++++++++++++++++++++++++++++++++++++ diff --git a/docs/usage/notes.rst b/docs/usage/notes.rst index d6f846ff3c..622b5888a2 100644 --- a/docs/usage/notes.rst +++ b/docs/usage/notes.rst @@ -180,6 +180,26 @@ Now, let's see how to restore some LVs from such a backup. $ borg extract --stdout arch dev/vg0/root-snapshot > /dev/vg0/root $ borg extract --stdout arch dev/vg0/home-snapshot > /dev/vg0/home +Efficient backups of LVM thin volume snapshots +++++++++++++++++++++++++++++++++++++++++++++++ + +Backing up a block device as shown above reads the whole device every time. +For snapshots of LVM *thin* volumes, the thin pool's metadata already knows +which ranges are allocated and which ranges changed between two snapshots, +so most of the reading can be skipped: see the ``--map`` and ``--reuse-from`` +options of ``borg create`` (section *Input maps* in ``borg create --help``) +and the ``scripts/lvm-thin-map.py`` converter in the borg sources, which +turns ``thin_dump`` / ``thin_delta`` XML into borg input maps. The script's +docstring shows the complete workflow: an initial full backup using the +allocation map (unallocated ranges are stored as holes without reading them), +then incremental backups that only read the ranges that changed since the +previous snapshot, while reusing the previous archive's chunks for everything +else. Use ``--chunker-params fixed,4194304`` (or similar) for such backups. + +Note that borg trusts these maps - it cannot detect a wrong or stale map, so +keep the snapshot discipline described in the script's docstring and consider +doing a periodic full read backup (without ``--map``). + .. _separate_compaction: diff --git a/scripts/lvm-thin-map.py b/scripts/lvm-thin-map.py new file mode 100755 index 0000000000..b97d83dd89 --- /dev/null +++ b/scripts/lvm-thin-map.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +""" +Generate a borg input map (borg create --map) from LVM thin pool metadata, see #4363. + +This converts XML produced by the thin-provisioning-tools (thin_dump / thin_delta, +version >= 0.7.4 required - older thin_delta versions had bugs) into the map format +expected by ``borg create --map``, so borg only reads the parts of a thin LV snapshot +that actually contain (changed) data. + +full mode - initial (or periodic full-read) backup of a thin LV snapshot: + + lvcreate -s -n snap1 vg/lv # snapshot to back up (keep it for delta mode!) + lvchange -ay -Ky vg/snap1 + dmsetup message vg-pool-tpool 0 reserve_metadata_snap + thin_dump -m --dev-id $(lvs --noheadings -o thin_id vg/snap1) /dev/mapper/vg-pool_tmeta \ + | lvm-thin-map.py full --device /dev/vg/snap1 > snap1.map + dmsetup message vg-pool-tpool 0 release_metadata_snap + borg create --read-special --chunker-params fixed,4194304 \ + --map snap1.map lv-backup /dev/vg/snap1 + + Unallocated ranges read as zeros, so borg stores them as holes without reading them. + +delta mode - incremental backup against the previous snapshot's archive: + + lvcreate -s -n snap2 vg/lv + lvchange -ay -Ky vg/snap2 + dmsetup message vg-pool-tpool 0 reserve_metadata_snap + thin_delta -m --snap1 $(lvs --noheadings -o thin_id vg/snap1) \ + --snap2 $(lvs --noheadings -o thin_id vg/snap2) /dev/mapper/vg-pool_tmeta \ + | lvm-thin-map.py delta --device /dev/vg/snap2 > snap2.map + dmsetup message vg-pool-tpool 0 release_metadata_snap + borg create --read-special --chunker-params fixed,4194304 \ + --map snap2.map --reuse-from lv-backup lv-backup /dev/vg/snap2 + lvremove vg/snap1 # snap2 is the reference for the next delta + + Ranges that are identical in both snapshots become "same" ranges: borg reuses the + chunks of the --reuse-from reference archive for them, without reading the device. + +Notes: + +- The reference snapshot (--snap1) must be the snapshot that was backed up into the + --reuse-from archive - THIS IS NOT CHECKED and cannot be. A wrong pairing silently + produces an archive that does not match the device. +- Remember to release_metadata_snap; a leftover metadata snapshot blocks future reserves. +- The pool's chunk size (data_block_size) defines the map granularity. borg's fixed + chunker block size does not need to match it; ranges are given in exact bytes. +""" + +import argparse +import os +import sys +import xml.etree.ElementTree as ET + +DELTA_STATES = { + "same": "same", # unchanged between the two snapshots -> reuse reference chunks + "different": "data", # changed -> read + "right_only": "data", # newly allocated -> read + "left_only": "zero", # deallocated (discarded) -> reads as zeros now +} +SECTOR = 512 + + +def die(msg): + print(f"lvm-thin-map: error: {msg}", file=sys.stderr) + sys.exit(2) + + +def device_size(args): + if args.size is not None: + return args.size + fd = os.open(args.device, os.O_RDONLY) + try: + return os.lseek(fd, 0, os.SEEK_END) + finally: + os.close(fd) + + +def parse_ranges(xml_file, mode, meta): + """Yield (start_block, length_blocks, state) from thin_dump / thin_delta XML; fill *meta* in place.""" + inside_device = False + context = ET.iterparse(xml_file, events=("start", "end")) + for event, elem in context: + if event == "start": + if elem.tag == "superblock": + meta["data_block_size"] = int(elem.get("data_block_size")) * SECTOR + elif elem.tag == "diff": + meta["left"], meta["right"] = elem.get("left"), elem.get("right") + elif elem.tag == "device": + if "dev_id" in meta: + die("XML contains more than one , re-run thin_dump with --dev-id") + meta["dev_id"] = elem.get("dev_id") + inside_device = True + elif elem.tag == "range": + die("nested elements found - run thin_delta without --verbose") + continue + # end events + if elem.tag == "device": + inside_device = False + elif mode == "delta" and elem.tag in DELTA_STATES: + yield int(elem.get("begin")), int(elem.get("length")), DELTA_STATES[elem.tag] + elif mode == "full" and elem.tag in ("single_mapping", "range_mapping"): + if not inside_device: + die(f"<{elem.tag}> outside of a element - unsupported thin_dump output") + if elem.tag == "single_mapping": + yield int(elem.get("origin_block")), 1, "data" + else: + yield int(elem.get("origin_begin")), int(elem.get("length")), "data" + elem.clear() + + +def main(): + parser = argparse.ArgumentParser( + description="convert thin_dump/thin_delta XML into a borg input map (borg create --map)" + ) + parser.add_argument("mode", choices=("full", "delta"), help="full: thin_dump XML, delta: thin_delta XML") + parser.add_argument("xml", nargs="?", default="-", help="XML input file (default: stdin)") + size_group = parser.add_mutually_exclusive_group(required=True) + size_group.add_argument("--device", help="get the map's total size from this block device") + size_group.add_argument("--size", type=int, help="give the map's total size in bytes") + args = parser.parse_intermixed_args() + + size = device_size(args) + xml_file = sys.stdin.buffer if args.xml == "-" else args.xml + + out = [] # coalesced [start, end, state] ranges, in bytes + offset = 0 # next expected byte offset + meta = {} + + def emit(start, end, state): + nonlocal offset + if start < offset: + die(f"XML ranges overlap or are not sorted (at byte offset {start})") + if start > offset: + emit_range(offset, start, "zero") # gap: unallocated in all snapshots, reads as zeros + emit_range(start, end, state) + offset = end + + def emit_range(start, end, state): + if out and out[-1][2] == state and out[-1][1] == start: + out[-1][1] = end + else: + out.append([start, end, state]) + + for begin, length, state in parse_ranges(xml_file, args.mode, meta): + block_size = meta["data_block_size"] + emit(begin * block_size, (begin + length) * block_size, state) + + if "data_block_size" not in meta: + die("no found in the XML input") + if offset > size: + die(f"XML mappings end at {offset}, beyond the given size {size} - wrong device or size?") + if offset < size: + emit_range(offset, size, "zero") + + print(f"# borg input map generated by lvm-thin-map.py ({args.mode} mode)") + print(f"# size: {size} bytes, thin pool chunk size: {meta['data_block_size']} bytes") + if args.mode == "delta": + print(f"# thin_delta left (reference) dev_id: {meta.get('left')}, right dev_id: {meta.get('right')}") + else: + print(f"# thin_dump dev_id: {meta.get('dev_id')}") + for start, end, state in out: + print(f"{start} {end - start} {state}") + + +if __name__ == "__main__": + main() diff --git a/src/borg/archiver/create_cmd.py b/src/borg/archiver/create_cmd.py index 767d0f6b4a..6181177d45 100644 --- a/src/borg/archiver/create_cmd.py +++ b/src/borg/archiver/create_cmd.py @@ -1041,6 +1041,10 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser): source producing the maps, and consider doing a periodic full read backup (without ``--map``). + For LVM thin volume snapshots, maps can be generated from ``thin_dump`` / + ``thin_delta`` XML with the ``scripts/lvm-thin-map.py`` converter from the + borg sources; its docstring shows the complete workflow. + Feeding all file paths from externally ++++++++++++++++++++++++++++++++++++++