From 39bdfaaa2eb8fc04e7cca262883ec3a071906ed1 Mon Sep 17 00:00:00 2001 From: Hugo Wallenburg Date: Sat, 19 Apr 2025 20:43:10 +0200 Subject: [PATCH 1/6] Adds int_or_interval format parser Accepts either int or interval, first tries parsing int then tries parsing as interval if that fails. Returns a timedelta for easy date math later. Now allows intervals of length 0 as a 0-length timedelta is perfectly fine to work with. --- src/borg/helpers/__init__.py | 2 +- src/borg/helpers/parseformat.py | 18 +++++- .../testsuite/helpers/parseformat_test.py | 56 ++++++++++++++++--- 3 files changed, 65 insertions(+), 11 deletions(-) diff --git a/src/borg/helpers/__init__.py b/src/borg/helpers/__init__.py index ca19f5c890..751d79497a 100644 --- a/src/borg/helpers/__init__.py +++ b/src/borg/helpers/__init__.py @@ -27,7 +27,7 @@ from .misc import ChunkIteratorFileWrapper, open_item, chunkit, iter_separated, ErrorIgnoringTextIOWrapper from .parseformat import bin_to_hex, hex_to_bin, safe_encode, safe_decode from .parseformat import text_to_json, binary_to_json, remove_surrogates, join_cmd -from .parseformat import eval_escapes, decode_dict, positive_int_validator, interval +from .parseformat import eval_escapes, decode_dict, positive_int_validator, interval, int_or_interval from .parseformat import PathSpec, SortBySpec, ChunkerParams, FilesCacheMode, partial_format, DatetimeWrapper from .parseformat import format_file_size, parse_file_size, FileSize from .parseformat import sizeof_fmt, sizeof_fmt_iec, sizeof_fmt_decimal, Location, text_validator diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py index 193023df51..e7cc8d653f 100644 --- a/src/borg/helpers/parseformat.py +++ b/src/borg/helpers/parseformat.py @@ -13,7 +13,7 @@ from pathlib import Path from typing import ClassVar, Any, TYPE_CHECKING, Literal from collections import OrderedDict -from datetime import datetime, timezone +from datetime import datetime, timezone, timedelta from functools import partial from string import Formatter @@ -155,12 +155,24 @@ def interval(s): except ValueError: seconds = -1 - if seconds <= 0: - raise argparse.ArgumentTypeError(f'Invalid number "{number}": expected positive integer') + if seconds < 0: + raise argparse.ArgumentTypeError(f'Invalid number "{number}": expected nonnegative integer') return seconds +def int_or_interval(s): + try: + return int(s) + except ValueError: + pass + + try: + return timedelta(seconds=interval(s)) + except argparse.ArgumentTypeError as e: + raise argparse.ArgumentTypeError(f"Value is neither an integer nor an interval: {e}") + + def ChunkerParams(s): params = s.strip().split(",") count = len(params) diff --git a/src/borg/testsuite/helpers/parseformat_test.py b/src/borg/testsuite/helpers/parseformat_test.py index 85600c819e..0948151c78 100644 --- a/src/borg/testsuite/helpers/parseformat_test.py +++ b/src/borg/testsuite/helpers/parseformat_test.py @@ -1,7 +1,8 @@ import base64 import os +import re from argparse import ArgumentTypeError -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone import pytest @@ -16,6 +17,7 @@ format_file_size, parse_file_size, interval, + int_or_interval, partial_format, clean_lines, format_line, @@ -376,6 +378,7 @@ def test_format_timedelta(): @pytest.mark.parametrize( "timeframe, num_secs", [ + ("0S", 0), ("5S", 5), ("2M", 2 * 60), ("1H", 60 * 60), @@ -392,9 +395,9 @@ def test_interval(timeframe, num_secs): @pytest.mark.parametrize( "invalid_interval, error_tuple", [ - ("H", ('Invalid number "": expected positive integer',)), - ("-1d", ('Invalid number "-1": expected positive integer',)), - ("food", ('Invalid number "foo": expected positive integer',)), + ("H", ('Invalid number "": expected nonnegative integer',)), + ("-1d", ('Invalid number "-1": expected nonnegative integer',)), + ("food", ('Invalid number "foo": expected nonnegative integer',)), ], ) def test_interval_time_unit(invalid_interval, error_tuple): @@ -403,10 +406,49 @@ def test_interval_time_unit(invalid_interval, error_tuple): assert exc.value.args == error_tuple -def test_interval_number(): +@pytest.mark.parametrize( + "invalid_input, error_regex", + [ + ("x", r'^Unexpected time unit "x": choose from'), + ("-1t", r'^Unexpected time unit "t": choose from'), + ("fool", r'^Unexpected time unit "l": choose from'), + ("abc", r'^Unexpected time unit "c": choose from'), + (" abc ", r'^Unexpected time unit " ": choose from'), + ], +) +def test_interval_invalid_time_format(invalid_input, error_regex): + with pytest.raises(ArgumentTypeError) as exc: + interval(invalid_input) + assert re.search(error_regex, exc.value.args[0]) + + +@pytest.mark.parametrize( + "input, result", + [ + ("0", 0), + ("5", 5), + (" 999 ", 999), + ("0S", timedelta(seconds=0)), + ("5S", timedelta(seconds=5)), + ("1m", timedelta(days=31)), + ], +) +def test_int_or_interval(input, result): + assert int_or_interval(input) == result + + +@pytest.mark.parametrize( + "invalid_input, error_regex", + [ + ("H", r"Value is neither an integer nor an interval:"), + ("-1d", r"Value is neither an integer nor an interval:"), + ("food", r"Value is neither an integer nor an interval:"), + ], +) +def test_int_or_interval_time_unit(invalid_input, error_regex): with pytest.raises(ArgumentTypeError) as exc: - interval("5") - assert exc.value.args == ('Unexpected time unit "5": choose from y, m, w, d, H, M, S',) + int_or_interval(invalid_input) + assert re.search(error_regex, exc.value.args[0]) def test_parse_timestamp(): From 65864e9ce939614483772c6ece708773805284bb Mon Sep 17 00:00:00 2001 From: Hugo Wallenburg Date: Sat, 19 Apr 2025 21:11:05 +0200 Subject: [PATCH 2/6] Adds optional interval support for all prune retention flags Support is added for setting prune retention with either an int (keep n archives) or an interval (keep within). This works much like --keep-within currently does, but extends support to all retention filters. Additionally adds a generic --keep flag to take over (or live alongside) both --keep-last and --keep-within. --keep-last is no longer an alias of --keep-secondly, now keeps archives made on the same second. Comparisons against archive timestamp are made to use local timezone instead of UTC. Should be equal result in practice, but allows for easier testing with frozen local time. --- requirements.d/development.txt | 1 + src/borg/archiver/prune_cmd.py | 169 ++++++----- src/borg/constants.py | 2 + src/borg/testsuite/archiver/prune_cmd_test.py | 271 +++++++++++++++--- 4 files changed, 332 insertions(+), 111 deletions(-) diff --git a/requirements.d/development.txt b/requirements.d/development.txt index 96c20f0f18..c65071e0d3 100644 --- a/requirements.d/development.txt +++ b/requirements.d/development.txt @@ -11,6 +11,7 @@ pytest-xdist coverage[toml] pytest-cov pytest-benchmark +freezegun Cython pre-commit bandit[toml] diff --git a/src/borg/archiver/prune_cmd.py b/src/borg/archiver/prune_cmd.py index 2f18b485b8..f5e7abf468 100644 --- a/src/borg/archiver/prune_cmd.py +++ b/src/borg/archiver/prune_cmd.py @@ -1,16 +1,17 @@ import argparse from collections import OrderedDict -from datetime import datetime, timezone, timedelta +from datetime import datetime, timezone import logging from operator import attrgetter import os +import itertools from ._common import with_repository, Highlander from ..archive import Archive from ..cache import Cache from ..constants import * # NOQA -from ..helpers import ArchiveFormatter, interval, sig_int, ProgressIndicatorPercent, CommandError, Error -from ..helpers import archivename_validator +from ..helpers import interval, int_or_interval, sig_int, archivename_validator +from ..helpers import ArchiveFormatter, ProgressIndicatorPercent, CommandError, Error from ..manifest import Manifest from ..logger import create_logger @@ -18,20 +19,23 @@ logger = create_logger() -def prune_within(archives, seconds, kept_because): - target = datetime.now(timezone.utc) - timedelta(seconds=seconds) - kept_counter = 0 - result = [] - for a in archives: - if a.ts > target: - kept_counter += 1 - kept_because[a.id] = ("within", kept_counter) - result.append(a) - return result +# The *_period_func group of functions create period grouping keys to group together archives falling within a certain +# period. Among archives in each of these groups, only the latest (by creation timestamp) is kept. -def default_period_func(pattern): +def unique_period_func(): + counter = itertools.count() + + def unique_values(_a): + """Group archives by an incrementing counter, practically making each archive a group of 1""" + return next(counter) + + return unique_values + + +def pattern_period_func(pattern): def inner(a): + """Group archives by extracting given strftime-pattern from their creation timestamp""" # compute in local timezone return a.ts.astimezone().strftime(pattern) @@ -39,6 +43,7 @@ def inner(a): def quarterly_13weekly_period_func(a): + """Group archives by extracting the ISO-8601 13-week quarter from their creation timestamp""" (year, week, _) = a.ts.astimezone().isocalendar() # local time if week <= 13: # Weeks containing Jan 4th to Mar 28th (leap year) or 29th- 91 (13*7) @@ -60,6 +65,7 @@ def quarterly_13weekly_period_func(a): def quarterly_3monthly_period_func(a): + """Group archives by extracting the 3-month quarter from their creation timestamp""" lt = a.ts.astimezone() # local time if lt.month <= 3: # 1-1 to 3-31 @@ -77,42 +83,64 @@ def quarterly_3monthly_period_func(a): PRUNING_PATTERNS = OrderedDict( [ - ("secondly", default_period_func("%Y-%m-%d %H:%M:%S")), - ("minutely", default_period_func("%Y-%m-%d %H:%M")), - ("hourly", default_period_func("%Y-%m-%d %H")), - ("daily", default_period_func("%Y-%m-%d")), - ("weekly", default_period_func("%G-%V")), - ("monthly", default_period_func("%Y-%m")), + # Each archive is considered for keeping + ("within", unique_period_func()), + ("last", unique_period_func()), + ("keep", unique_period_func()), + # Last archive (by creation timestamp) within period group is consiedered for keeping + ("secondly", pattern_period_func("%Y-%m-%d %H:%M:%S")), + ("minutely", pattern_period_func("%Y-%m-%d %H:%M")), + ("hourly", pattern_period_func("%Y-%m-%d %H")), + ("daily", pattern_period_func("%Y-%m-%d")), + ("weekly", pattern_period_func("%G-%V")), + ("monthly", pattern_period_func("%Y-%m")), ("quarterly_13weekly", quarterly_13weekly_period_func), ("quarterly_3monthly", quarterly_3monthly_period_func), - ("yearly", default_period_func("%Y")), + ("yearly", pattern_period_func("%Y")), ] ) -def prune_split(archives, rule, n, kept_because=None): - last = None +# Datetime cannot represent times before datetime.min, so a day is added to allow for time zone offset. +DATETIME_MIN_WITH_ZONE = datetime.min.replace(tzinfo=timezone.utc) + + +def prune_split(archives, rule, n_or_interval, base_timestamp, kept_because={}): + if isinstance(n_or_interval, int): + n, earliest_timestamp = n_or_interval, None + else: + n, earliest_timestamp = None, base_timestamp - n_or_interval + + def can_retain(a, keep): + if n is not None: + return len(keep) < n + else: + return a.ts > earliest_timestamp + keep = [] - period_func = PRUNING_PATTERNS[rule] - if kept_because is None: - kept_because = {} - if n == 0: + if n == 0 or len(archives) == 0: return keep a = None - for a in sorted(archives, key=attrgetter("ts"), reverse=True): + last = None + period_func = PRUNING_PATTERNS[rule] + sorted_archives = sorted(archives, key=attrgetter("ts"), reverse=True) + for a in sorted_archives: + if not can_retain(a, keep): + break period = period_func(a) if period != last: last = period if a.id not in kept_because: keep.append(a) kept_because[a.id] = (rule, len(keep)) - if len(keep) == n: - break + # Keep oldest archive if we didn't reach the target retention count - if a is not None and len(keep) < n and a.id not in kept_because: + a = sorted_archives[-1] + if a is not None and a.id not in kept_because and can_retain(a, keep): keep.append(a) kept_because[a.id] = (rule + "[oldest]", len(keep)) + return keep @@ -120,8 +148,13 @@ class PruneMixIn: @with_repository(compatibility=(Manifest.Operation.DELETE,)) def do_prune(self, args, repository, manifest): """Prune archives according to specified rules.""" - if not any( - ( + if all( + # Needs explicit None-check as interval arg may be 0 (Falsey) + e is None + for e in ( + args.keep, + args.within, + args.last, args.secondly, args.minutely, args.hourly, @@ -131,11 +164,10 @@ def do_prune(self, args, repository, manifest): args.quarterly_13weekly, args.quarterly_3monthly, args.yearly, - args.within, ) ): raise CommandError( - 'At least one of the "keep-within", "keep-last", ' + 'At least one of the "keep", "keep-within", "keep-last", ' '"keep-secondly", "keep-minutely", "keep-hourly", "keep-daily", ' '"keep-weekly", "keep-monthly", "keep-13weekly", "keep-3monthly", ' 'or "keep-yearly" settings must be specified.' @@ -159,15 +191,12 @@ def do_prune(self, args, repository, manifest): # (, ) kept_because = {} - # find archives which need to be kept because of the keep-within rule - if args.within: - keep += prune_within(archives, args.within, kept_because) - + base_timestamp = datetime.now().astimezone() # find archives which need to be kept because of the various time period rules for rule in PRUNING_PATTERNS.keys(): - num = getattr(args, rule, None) - if num is not None: - keep += prune_split(archives, rule, num, kept_because) + num_or_interval = getattr(args, rule, None) + if num_or_interval is not None: + keep += prune_split(archives, rule, num_or_interval, base_timestamp, kept_because) to_delete = set(archives) - set(keep) with Cache(repository, manifest, iec=args.iec) as cache: @@ -312,81 +341,81 @@ def build_parser_prune(self, subparsers, common_parser, mid_common_parser): help="keep all archives within this time interval", ) subparser.add_argument( - "--keep-last", + "--keep-last", dest="last", type=int, action=Highlander, help="number of archives to keep" + ) + subparser.add_argument( + "--keep", + dest="keep", + type=int_or_interval, + action=Highlander, + help="number or time interval of archives to keep", + ) + subparser.add_argument( "--keep-secondly", dest="secondly", - type=int, - default=0, + type=int_or_interval, action=Highlander, - help="number of secondly archives to keep", + help="number or time interval of secondly archives to keep", ) subparser.add_argument( "--keep-minutely", dest="minutely", - type=int, - default=0, + type=int_or_interval, action=Highlander, - help="number of minutely archives to keep", + help="number or time interval of minutely archives to keep", ) subparser.add_argument( "-H", "--keep-hourly", dest="hourly", - type=int, - default=0, + type=int_or_interval, action=Highlander, - help="number of hourly archives to keep", + help="number or time interval of hourly archives to keep", ) subparser.add_argument( "-d", "--keep-daily", dest="daily", - type=int, - default=0, + type=int_or_interval, action=Highlander, - help="number of daily archives to keep", + help="number or time interval of daily archives to keep", ) subparser.add_argument( "-w", "--keep-weekly", dest="weekly", - type=int, - default=0, + type=int_or_interval, action=Highlander, - help="number of weekly archives to keep", + help="number or time interval of weekly archives to keep", ) subparser.add_argument( "-m", "--keep-monthly", dest="monthly", - type=int, - default=0, + type=int_or_interval, action=Highlander, - help="number of monthly archives to keep", + help="number or time interval of monthly archives to keep", ) quarterly_group = subparser.add_mutually_exclusive_group() quarterly_group.add_argument( "--keep-13weekly", dest="quarterly_13weekly", - type=int, - default=0, - help="number of quarterly archives to keep (13 week strategy)", + type=int_or_interval, + help="number or time interval of quarterly archives to keep (13 week strategy)", ) quarterly_group.add_argument( "--keep-3monthly", dest="quarterly_3monthly", - type=int, - default=0, - help="number of quarterly archives to keep (3 month strategy)", + type=int_or_interval, + help="number or time interval of quarterly archives to keep (3 month strategy)", ) subparser.add_argument( "-y", "--keep-yearly", dest="yearly", - type=int, - default=0, + type=int_or_interval, action=Highlander, - help="number of yearly archives to keep", + help="number or time interval of yearly archives to keep", ) define_archive_filters_group(subparser, sort_by=False, first_last=False) subparser.add_argument( diff --git a/src/borg/constants.py b/src/borg/constants.py index bbbbc56103..79ed33724b 100644 --- a/src/borg/constants.py +++ b/src/borg/constants.py @@ -142,7 +142,9 @@ EXIT_SIGNAL_BASE = 128 # terminated due to signal, rc = 128 + sig_no ISO_FORMAT_NO_USECS = "%Y-%m-%dT%H:%M:%S" +ISO_FORMAT_NO_USECS_ZONE = ISO_FORMAT_NO_USECS + "%z" ISO_FORMAT = ISO_FORMAT_NO_USECS + ".%f" +ISO_FORMAT_ZONE = ISO_FORMAT + "%z" DASHES = "-" * 78 diff --git a/src/borg/testsuite/archiver/prune_cmd_test.py b/src/borg/testsuite/archiver/prune_cmd_test.py index a18212c85e..0d0eb4389d 100644 --- a/src/borg/testsuite/archiver/prune_cmd_test.py +++ b/src/borg/testsuite/archiver/prune_cmd_test.py @@ -1,22 +1,22 @@ -import re -from datetime import datetime, timezone, timedelta - import pytest +import re +from datetime import datetime, timezone +from freezegun import freeze_time from ...constants import * # NOQA -from ...archiver.prune_cmd import prune_split, prune_within +from ...archiver.prune_cmd import prune_split +from ...helpers import CommandError from . import cmd, RK_ENCRYPTION, src_dir, generate_archiver_tests -from ...helpers import interval pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local,remote,binary") # NOQA -def _create_archive_ts(archiver, name, y, m, d, H=0, M=0, S=0): +def _create_archive_ts(archiver, name, y, m, d, H=0, M=0, S=0, us=0, tzinfo=None): cmd( archiver, "create", "--timestamp", - datetime(y, m, d, H, M, S, 0).strftime(ISO_FORMAT_NO_USECS), # naive == local time / local tz + datetime(y, m, d, H, M, S, us, tzinfo=tzinfo).strftime(ISO_FORMAT_ZONE), name, src_dir, ) @@ -256,7 +256,7 @@ def test_prune_ignore_protected(archivers, request): cmd(archiver, "create", "archive3", archiver.input_path) output = cmd(archiver, "prune", "--list", "--keep-last=1", "--match-archives=sh:archive*") assert "archive1" not in output # @PROT archives are completely ignored. - assert re.search(r"Keeping archive \(rule: secondly #1\):\s+archive3", output) + assert re.search(r"Keeping archive \(rule: last #1\):\s+archive3", output) assert re.search(r"Pruning archive \(.*?\):\s+archive2", output) output = cmd(archiver, "repo-list") assert "archive1" in output # @PROT protected archive1 from deletion @@ -283,38 +283,6 @@ def __repr__(self): local_tz = datetime.now(tz=timezone.utc).astimezone(tz=None).tzinfo -def test_prune_within(): - def subset(lst, indices): - return {lst[i] for i in indices} - - def dotest(test_archives, within, indices): - for ta in test_archives, reversed(test_archives): - kept_because = {} - keep = prune_within(ta, interval(within), kept_because) - assert set(keep) == subset(test_archives, indices) - assert all("within" == kept_because[a.id][0] for a in keep) - - # 1 minute, 1.5 hours, 2.5 hours, 3.5 hours, 25 hours, 49 hours - test_offsets = [60, 90 * 60, 150 * 60, 210 * 60, 25 * 60 * 60, 49 * 60 * 60] - now = datetime.now(timezone.utc) - test_dates = [now - timedelta(seconds=s) for s in test_offsets] - test_archives = [MockArchive(date, i) for i, date in enumerate(test_dates)] - - dotest(test_archives, "15S", []) - dotest(test_archives, "2M", [0]) - dotest(test_archives, "1H", [0]) - dotest(test_archives, "2H", [0, 1]) - dotest(test_archives, "3H", [0, 1, 2]) - dotest(test_archives, "24H", [0, 1, 2, 3]) - dotest(test_archives, "26H", [0, 1, 2, 3, 4]) - dotest(test_archives, "2d", [0, 1, 2, 3, 4]) - dotest(test_archives, "50H", [0, 1, 2, 3, 4, 5]) - dotest(test_archives, "3d", [0, 1, 2, 3, 4, 5]) - dotest(test_archives, "1w", [0, 1, 2, 3, 4, 5]) - dotest(test_archives, "1m", [0, 1, 2, 3, 4, 5]) - dotest(test_archives, "1y", [0, 1, 2, 3, 4, 5]) - - @pytest.mark.parametrize( "rule,num_to_keep,expected_ids", [ @@ -353,7 +321,7 @@ def subset(lst, ids): MockArchive(datetime(2017, 10, 1, 10, 0, 5, tzinfo=local_tz), 13), ] kept_because = {} - keep = prune_split(archives, rule, num_to_keep, kept_because) + keep = prune_split(archives, rule, num_to_keep, None, kept_because) assert set(keep) == subset(archives, expected_ids) for item in keep: @@ -400,3 +368,224 @@ def test_prune_split_no_archives(): assert keep == [] assert kept_because == {} + +def test_prune_keep_last_same_second(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", "test1", src_dir) + cmd(archiver, "create", "test2", src_dir) + output = cmd(archiver, "prune", "--list", "--dry-run", "--keep-last=2") + # Both archives are kept even though they have the same timestamp to the second. Would previously have failed with + # old behavior of --keep-last. Archives sorted on seconds, order is undefined. + assert re.search(r"Keeping archive \(rule: last #\d\):\s+test1", output) + assert re.search(r"Keeping archive \(rule: last #\d\):\s+test2", output) + + +@freeze_time(datetime(2023, 12, 31, 23, 59, 59, tzinfo=None)) # Non-leap year ending on a Sunday +def test_prune_keep_int_or_interval(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_archive_ts(archiver, "test-1", 2023, 12, 31, 23, 59, 59) + _create_archive_ts(archiver, "test-2", 2023, 12, 31, 23, 59, 59) + _create_archive_ts(archiver, "test-3", 2023, 12, 31, 23, 59, 58) + for keep_arg in ["--keep=2", "--keep=1S"]: + output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg) + assert re.search(r"Keeping archive \(rule: keep #\d\):\s+test-1", output) + assert re.search(r"Keeping archive \(rule: keep #\d\):\s+test-2", output) + assert re.search(r"Would prune:\s+test-3", output) + + +@freeze_time(datetime(2023, 12, 31, 23, 59, 59, tzinfo=None)) +def test_prune_keep_int_or_interval_zero(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_archive_ts(archiver, "test", 2023, 12, 31, 23, 59, 59) + for keep_arg in ["--keep=0", "--keep=0S"]: + output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg) + assert re.search(r"Would prune:\s+test", output) + + +@freeze_time(datetime(2023, 12, 31, 23, 59, 59, tzinfo=None)) +def test_prune_keep_secondly_int_or_interval(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_archive_ts(archiver, "test-1", 2023, 12, 31, 23, 59, 58) + _create_archive_ts(archiver, "test-2", 2023, 12, 31, 23, 59, 57, 1) + _create_archive_ts(archiver, "test-3", 2023, 12, 31, 23, 59, 57) + _create_archive_ts(archiver, "test-4", 2023, 12, 31, 23, 59, 56, 999999) + for keep_arg in ["--keep-secondly=2", "--keep-secondly=2S"]: + output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg) + assert re.search(r"Keeping archive \(rule: secondly #1\):\s+test-1", output) + assert re.search(r"Keeping archive \(rule: secondly #2\):\s+test-2", output) + assert re.search(r"Would prune:\s+test-3", output) + assert re.search(r"Would prune:\s+test-4", output) + + +@freeze_time(datetime(2023, 12, 31, 23, 59, 0, tzinfo=None)) +def test_prune_keep_minutely_int_or_interval(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_archive_ts(archiver, "test-1", 2023, 12, 31, 23, 58) + _create_archive_ts(archiver, "test-2", 2023, 12, 31, 23, 57, 1) + _create_archive_ts(archiver, "test-3", 2023, 12, 31, 23, 57) + _create_archive_ts(archiver, "test-4", 2023, 12, 31, 23, 56, 0, 1) # Last possible microsecond + _create_archive_ts(archiver, "test-5", 2023, 12, 31, 23, 56) + for keep_arg in ["--keep-minutely=3", "--keep-minutely=3M"]: + output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg) + assert re.search(r"Keeping archive \(rule: minutely #1\):\s+test-1", output) + assert re.search(r"Keeping archive \(rule: minutely #2\):\s+test-2", output) + assert re.search(r"Would prune:\s+test-3", output) + assert re.search(r"Keeping archive \(rule: minutely #3\):\s+test-4", output) + assert re.search(r"Would prune:\s+test-5", output) + + +@freeze_time(datetime(2023, 12, 31, 23, 0, 0, tzinfo=None)) +def test_prune_keep_hourly_int_or_interval(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_archive_ts(archiver, "test-1", 2023, 12, 31, 22) + _create_archive_ts(archiver, "test-2", 2023, 12, 31, 21, us=1) + _create_archive_ts(archiver, "test-3", 2023, 12, 31, 21) + _create_archive_ts(archiver, "test-4", 2023, 12, 31, 20, us=1) # Last possible microsecond + _create_archive_ts(archiver, "test-5", 2023, 12, 31, 20) + for keep_arg in ["--keep-hourly=3", "--keep-hourly=3H"]: + output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg) + assert re.search(r"Keeping archive \(rule: hourly #1\):\s+test-1", output) + assert re.search(r"Keeping archive \(rule: hourly #2\):\s+test-2", output) + assert re.search(r"Would prune:\s+test-3", output) + assert re.search(r"Keeping archive \(rule: hourly #3\):\s+test-4", output) + assert re.search(r"Would prune:\s+test-5", output) + + +@freeze_time(datetime(2023, 12, 31, 0, 0, 0, tzinfo=None)) +def test_prune_keep_daily_int_or_interval(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_archive_ts(archiver, "test-1", 2023, 12, 30) + _create_archive_ts(archiver, "test-2", 2023, 12, 29, S=1) + _create_archive_ts(archiver, "test-3", 2023, 12, 29) + _create_archive_ts(archiver, "test-4", 2023, 12, 28, us=1) # Last possible microsecond + _create_archive_ts(archiver, "test-5", 2023, 12, 28) + for keep_arg in ["--keep-daily=3", "--keep-daily=3d"]: + output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg) + assert re.search(r"Keeping archive \(rule: daily #1\):\s+test-1", output) + assert re.search(r"Keeping archive \(rule: daily #2\):\s+test-2", output) + assert re.search(r"Would prune:\s+test-3", output) + assert re.search(r"Keeping archive \(rule: daily #3\):\s+test-4", output) + assert re.search(r"Would prune:\s+test-5", output) + + +@freeze_time(datetime(2023, 12, 31, 0, 0, 0, tzinfo=None)) +def test_prune_keep_weekly_int_or_interval(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_archive_ts(archiver, "test-1", 2023, 12, 24) + _create_archive_ts(archiver, "test-2", 2023, 12, 17, us=1) + _create_archive_ts(archiver, "test-3", 2023, 12, 17) + _create_archive_ts(archiver, "test-4", 2023, 12, 10, us=1) # Last possible microsecond + _create_archive_ts(archiver, "test-5", 2023, 12, 10) + for keep_arg in ["--keep-weekly=3", "--keep-weekly=3w"]: + output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg) + assert re.search(r"Keeping archive \(rule: weekly #1\):\s+test-1", output) + assert re.search(r"Keeping archive \(rule: weekly #2\):\s+test-2", output) + assert re.search(r"Would prune:\s+test-3", output) + assert re.search(r"Keeping archive \(rule: weekly #3\):\s+test-4", output) + assert re.search(r"Would prune:\s+test-5", output) + + +@freeze_time(datetime(2023, 12, 31, 0, 0, 0, tzinfo=None)) +def test_prune_keep_monthly_int_or_interval(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_archive_ts(archiver, "test-1", 2023, 11, 30) + _create_archive_ts(archiver, "test-2", 2023, 10, 30, us=1) # Month defined as 31 days, so not Oct 31st + _create_archive_ts(archiver, "test-3", 2023, 10, 30) + _create_archive_ts(archiver, "test-4", 2023, 9, 29, us=1) # Last possible microsecond + _create_archive_ts(archiver, "test-5", 2023, 9, 29) + for keep_arg in ["--keep-monthly=3", "--keep-monthly=3m"]: + output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg) + assert re.search(r"Keeping archive \(rule: monthly #1\):\s+test-1", output) + assert re.search(r"Keeping archive \(rule: monthly #2\):\s+test-2", output) + assert re.search(r"Would prune:\s+test-3", output) + assert re.search(r"Keeping archive \(rule: monthly #3\):\s+test-4", output) + assert re.search(r"Would prune:\s+test-5", output) + + +# 2023-12-31 is Sunday, week 52. Makes these week calculations a little easier. +@freeze_time(datetime(2023, 12, 31, 0, 0, 0, tzinfo=None)) +def test_prune_keep_13weekly_int_or_interval(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_archive_ts(archiver, "test-1", 2023, 10, 1) + _create_archive_ts(archiver, "test-2", 2023, 7, 2, us=1) + _create_archive_ts(archiver, "test-3", 2023, 7, 2) + _create_archive_ts(archiver, "test-4", 2023, 4, 2, us=1) # Last possible microsecond + _create_archive_ts(archiver, "test-5", 2023, 4, 2) + for keep_arg in ["--keep-13weekly=3", "--keep-13weekly=39w"]: + output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg) + assert re.search(r"Keeping archive \(rule: quarterly_13weekly #1\):\s+test-1", output) + assert re.search(r"Keeping archive \(rule: quarterly_13weekly #2\):\s+test-2", output) + assert re.search(r"Would prune:\s+test-3", output) + assert re.search(r"Keeping archive \(rule: quarterly_13weekly #3\):\s+test-4", output) + assert re.search(r"Would prune:\s+test-5", output) + + +@freeze_time(datetime(2023, 12, 31, 0, 0, 0, tzinfo=None)) +def test_prune_keep_3monthly_int_or_interval(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_archive_ts(archiver, "test-1", 2023, 9, 30) + _create_archive_ts(archiver, "test-2", 2023, 6, 30, us=1) + _create_archive_ts(archiver, "test-3", 2023, 6, 30) + _create_archive_ts(archiver, "test-4", 2023, 3, 31, us=1) # Last possible microsecond + _create_archive_ts(archiver, "test-5", 2023, 3, 31) + for keep_arg in ["--keep-3monthly=3", f"--keep-3monthly={(datetime.now()-datetime(2023, 3, 31)).days}d"]: + output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg) + assert re.search(r"Keeping archive \(rule: quarterly_3monthly #1\):\s+test-1", output) + assert re.search(r"Keeping archive \(rule: quarterly_3monthly #2\):\s+test-2", output) + assert re.search(r"Would prune:\s+test-3", output) + assert re.search(r"Keeping archive \(rule: quarterly_3monthly #3\):\s+test-4", output) + assert re.search(r"Would prune:\s+test-5", output) + + +@freeze_time(datetime(2023, 12, 31, 0, 0, 0, tzinfo=None)) +def test_prune_keep_yearly_int_or_interval(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_archive_ts(archiver, "test-1", 2022, 12, 31) + _create_archive_ts(archiver, "test-2", 2021, 12, 31, us=1) + _create_archive_ts(archiver, "test-3", 2021, 12, 31) + _create_archive_ts(archiver, "test-4", 2020, 12, 31, us=1) # Last possible microsecond + _create_archive_ts(archiver, "test-5", 2020, 12, 31) + for keep_arg in ["--keep-yearly=3", "--keep-yearly=3y"]: + output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg) + assert re.search(r"Keeping archive \(rule: yearly #1\):\s+test-1", output) + assert re.search(r"Keeping archive \(rule: yearly #2\):\s+test-2", output) + assert re.search(r"Would prune:\s+test-3", output) + assert re.search(r"Keeping archive \(rule: yearly #3\):\s+test-4", output) + assert re.search(r"Would prune:\s+test-5", output) + + +def test_prune_no_args(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + with pytest.raises(CommandError) as error: + cmd(archiver, "prune") + output = str(error.value) + assert re.search(r"At least one of the .* settings must be specified.", output) + assert re.search(r"keep(?!-)", output) + flags = [ + "last", + "within", + "secondly", + "minutely", + "hourly", + "daily", + "weekly", + "monthly", + "yearly", + "13weekly", + "3monthly", + ] + for flag in flags: + assert f"keep-{flag}" in output From 073f3e3acd4aedc509afbe860d0b53c02ade8deb Mon Sep 17 00:00:00 2001 From: Hugo Wallenburg Date: Tue, 3 Jun 2025 23:11:55 +0200 Subject: [PATCH 3/6] Removes unnecessarily complicated local timezone in test Default with tzinfo=None is local timezone anyway, no need to set it manually. --- src/borg/testsuite/archiver/prune_cmd_test.py | 55 ++++++++----------- 1 file changed, 23 insertions(+), 32 deletions(-) diff --git a/src/borg/testsuite/archiver/prune_cmd_test.py b/src/borg/testsuite/archiver/prune_cmd_test.py index 0d0eb4389d..675d9bc362 100644 --- a/src/borg/testsuite/archiver/prune_cmd_test.py +++ b/src/borg/testsuite/archiver/prune_cmd_test.py @@ -265,24 +265,14 @@ def test_prune_ignore_protected(archivers, request): class MockArchive: def __init__(self, ts, id): - self.ts = ts + # Real archive objects have UTC zoned timestamps + self.ts = ts.replace(tzinfo=timezone.utc) self.id = id def __repr__(self): return f"{self.id}: {self.ts.isoformat()}" -# This is the local timezone of the system running the tests. -# We need this e.g. to construct archive timestamps for the prune tests, -# because borg prune operates in the local timezone (it first converts the -# archive timestamp to the local timezone). So, if we want the y/m/d/h/m/s -# values which prune uses to be exactly the ones we give [and NOT shift them -# by tzoffset], we need to give the timestamps in the same local timezone. -# Please note that the timestamps in a real borg archive or manifest are -# stored in UTC timezone. -local_tz = datetime.now(tz=timezone.utc).astimezone(tz=None).tzinfo - - @pytest.mark.parametrize( "rule,num_to_keep,expected_ids", [ @@ -302,23 +292,23 @@ def subset(lst, ids): archives = [ # years apart - MockArchive(datetime(2015, 1, 1, 10, 0, 0, tzinfo=local_tz), 1), - MockArchive(datetime(2016, 1, 1, 10, 0, 0, tzinfo=local_tz), 2), - MockArchive(datetime(2017, 1, 1, 10, 0, 0, tzinfo=local_tz), 3), + MockArchive(datetime(2015, 1, 1, 10, 0, 0), 1), + MockArchive(datetime(2016, 1, 1, 10, 0, 0), 2), + MockArchive(datetime(2017, 1, 1, 10, 0, 0), 3), # months apart - MockArchive(datetime(2017, 2, 1, 10, 0, 0, tzinfo=local_tz), 4), - MockArchive(datetime(2017, 3, 1, 10, 0, 0, tzinfo=local_tz), 5), + MockArchive(datetime(2017, 2, 1, 10, 0, 0), 4), + MockArchive(datetime(2017, 3, 1, 10, 0, 0), 5), # days apart - MockArchive(datetime(2017, 3, 2, 10, 0, 0, tzinfo=local_tz), 6), - MockArchive(datetime(2017, 3, 3, 10, 0, 0, tzinfo=local_tz), 7), - MockArchive(datetime(2017, 3, 4, 10, 0, 0, tzinfo=local_tz), 8), + MockArchive(datetime(2017, 3, 2, 10, 0, 0), 6), + MockArchive(datetime(2017, 3, 3, 10, 0, 0), 7), + MockArchive(datetime(2017, 3, 4, 10, 0, 0), 8), # minutes apart - MockArchive(datetime(2017, 10, 1, 9, 45, 0, tzinfo=local_tz), 9), - MockArchive(datetime(2017, 10, 1, 9, 55, 0, tzinfo=local_tz), 10), + MockArchive(datetime(2017, 10, 1, 9, 45, 0), 9), + MockArchive(datetime(2017, 10, 1, 9, 55, 0), 10), # seconds apart - MockArchive(datetime(2017, 10, 1, 10, 0, 1, tzinfo=local_tz), 11), - MockArchive(datetime(2017, 10, 1, 10, 0, 3, tzinfo=local_tz), 12), - MockArchive(datetime(2017, 10, 1, 10, 0, 5, tzinfo=local_tz), 13), + MockArchive(datetime(2017, 10, 1, 10, 0, 1), 11), + MockArchive(datetime(2017, 10, 1, 10, 0, 3), 12), + MockArchive(datetime(2017, 10, 1, 10, 0, 5), 13), ] kept_because = {} keep = prune_split(archives, rule, num_to_keep, None, kept_because) @@ -334,17 +324,17 @@ def subset(lst, ids): archives = [ # oldest backup, but not last in its year - MockArchive(datetime(2018, 1, 1, 10, 0, 0, tzinfo=local_tz), 1), + MockArchive(datetime(2018, 1, 1, 10, 0, 0), 1), # an interim backup - MockArchive(datetime(2018, 12, 30, 10, 0, 0, tzinfo=local_tz), 2), + MockArchive(datetime(2018, 12, 30, 10, 0, 0), 2), # year-end backups - MockArchive(datetime(2018, 12, 31, 10, 0, 0, tzinfo=local_tz), 3), - MockArchive(datetime(2019, 12, 31, 10, 0, 0, tzinfo=local_tz), 4), + MockArchive(datetime(2018, 12, 31, 10, 0, 0), 3), + MockArchive(datetime(2019, 12, 31, 10, 0, 0), 4), ] # Keep oldest when retention target can't otherwise be met kept_because = {} - keep = prune_split(archives, "yearly", 3, kept_because) + keep = prune_split(archives, "yearly", 3, None, kept_because) assert set(keep) == subset(archives, [1, 3, 4]) assert kept_because[1][0] == "yearly[oldest]" @@ -353,7 +343,7 @@ def subset(lst, ids): # Otherwise, prune it kept_because = {} - keep = prune_split(archives, "yearly", 2, kept_because) + keep = prune_split(archives, "yearly", 2, None, kept_because) assert set(keep) == subset(archives, [3, 4]) assert kept_because[3][0] == "yearly" @@ -364,11 +354,12 @@ def test_prune_split_no_archives(): archives = [] kept_because = {} - keep = prune_split(archives, "yearly", 3, kept_because) + keep = prune_split(archives, "yearly", 3, None, kept_because) assert keep == [] assert kept_because == {} + def test_prune_keep_last_same_second(archivers, request): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) From c57cecd2ac4af500e34ff3031095d0aa9b0a78a2 Mon Sep 17 00:00:00 2001 From: Hugo Wallenburg Date: Sun, 28 Dec 2025 18:56:55 +0100 Subject: [PATCH 4/6] fixup! Adds optional interval support for all prune retention flags --- src/borg/archiver/prune_cmd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/borg/archiver/prune_cmd.py b/src/borg/archiver/prune_cmd.py index f5e7abf468..aa8f864751 100644 --- a/src/borg/archiver/prune_cmd.py +++ b/src/borg/archiver/prune_cmd.py @@ -149,7 +149,7 @@ class PruneMixIn: def do_prune(self, args, repository, manifest): """Prune archives according to specified rules.""" if all( - # Needs explicit None-check as interval arg may be 0 (Falsey) + # Needs explicit None-check to cover Falsey timedelta(0) e is None for e in ( args.keep, From b60ebb5ff5017bd97f6d6c45a2847166521232cc Mon Sep 17 00:00:00 2001 From: Hugo Wallenburg Date: Sun, 28 Dec 2025 23:45:21 +0100 Subject: [PATCH 5/6] Adds FlexibleDelta to optionally use calendar deltas --- pyproject.toml | 1 + src/borg/helpers/time.py | 126 +++++++++++++++++++----- src/borg/testsuite/helpers/time_test.py | 75 +++++++++++++- 3 files changed, 176 insertions(+), 26 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7d8f608bc5..ddde2db4c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ dependencies = [ "platformdirs >=2.6.0, <5.0.0; sys_platform != 'darwin'", # for others: 2.6+ works consistently. "argon2-cffi", "shtab>=1.8.0", + "dateutils" ] [project.optional-dependencies] diff --git a/src/borg/helpers/time.py b/src/borg/helpers/time.py index 7dfe0d38c4..fa8d6d6679 100644 --- a/src/borg/helpers/time.py +++ b/src/borg/helpers/time.py @@ -1,6 +1,7 @@ import os import re from datetime import datetime, timezone, timedelta +from dateutil.relativedelta import relativedelta, MO def parse_timestamp(timestamp, tzinfo=timezone.utc): @@ -110,6 +111,105 @@ def format_timedelta(td): return txt +class FlexibleDelta: + """ + Represents an interval that _may_ respect the calendar with relation to week boundaries and exact month lengths. + """ + + _unit_relativedelta_map = { + "y": lambda count: relativedelta(years=count), + "m": lambda count: relativedelta(months=count), + "w": lambda count: relativedelta(weeks=count), + "d": lambda count: relativedelta(days=count), + "H": lambda count: relativedelta(hours=count), + "M": lambda count: relativedelta(minutes=count), + "S": lambda count: relativedelta(seconds=count), + } + + _unit_timedelta_map = { + "y": lambda count: timedelta(days=count * 365), + "m": lambda count: timedelta(days=count * 31), + "w": lambda count: timedelta(weeks=count), + "d": lambda count: timedelta(days=count), + "H": lambda count: timedelta(hours=count), + "M": lambda count: timedelta(minutes=count), + "S": lambda count: timedelta(seconds=count), + } + + _unit_fuzzy_round_func_map = { + "y": lambda earlier: relativedelta( + years=0 if earlier else 1, month=1, day=1, hour=0, minute=0, second=0, microsecond=0 + ), + "m": lambda earlier: relativedelta( + months=0 if earlier else 1, day=1, hour=0, minute=0, second=0, microsecond=0 + ), + "w": lambda earlier: relativedelta( + weekday=MO(-1), weeks=(0 if earlier else 1), hour=0, minute=0, second=0, microsecond=0 + ), + "d": lambda earlier: relativedelta(days=0 if earlier else 1, hour=0, minute=0, second=0, microsecond=0), + "H": lambda earlier: relativedelta(hours=0 if earlier else 1, minute=0, second=0, microsecond=0), + "M": lambda earlier: relativedelta(minutes=0 if earlier else 1, second=0, microsecond=0), + "S": lambda earlier: relativedelta(seconds=0 if earlier else 1, microsecond=0), + } + + def __init__(self, count, unit, fuzzy): + self.relativedelta = self._unit_relativedelta_map[unit](count) + self.timedelta = self._unit_timedelta_map[unit](count) + self.fuzzy_round_func = self._unit_fuzzy_round_func_map[unit] + self.fuzzy = fuzzy + + # For repr + self.count = count + self.unit = unit + + def __repr__(self): + return f'{self.__class__.__name__}(count={self.count}, unit="{self.unit}", fuzzy={self.fuzzy})' + + _interval_regex = re.compile(r"^(?P\d+)(?P[ymwdHMS])(?Pz)?$") + + @classmethod + def parse(cls, interval_string, fuzzyable=False): + """ + Parse interval string into + """ + match = cls._interval_regex.search(interval_string) + + if not match: + raise ValueError(f"Invalid interval format: {interval_string}") + + count = int(match.group("count")) + unit = match.group("unit") + fuzzy = fuzzyable and match.group("fuzzy") is not None + + return cls(count, unit, fuzzy) + + @classmethod + def parse_fuzzy(cls, interval_string): + """ + Convenience fuzzy parser for easy use with argparse + """ + return cls.parse(interval_string, fuzzyable=True) + + def apply(self, base_ts, earlier=False, calendar=False): + scale = -1 if earlier else 1 + delta = self.relativedelta if calendar else self.timedelta + + offset_ts = base_ts + delta * scale + + if self.fuzzy: + # Offset further so that timestamp represents the start/end of its unit. e.g. "1yz" rounds result either up + # or down to nearest full year after applying initial offset (2025-07-31 - "1yz" = 2024-01-01). + offset_ts += self.fuzzy_round_func(earlier) + + return offset_ts + + def add_to(self, base_ts, calendar=False): + return self.apply(base_ts, earlier=False, calendar=calendar) + + def subtract_from(self, base_ts, calendar=False): + return self.apply(base_ts, earlier=True, calendar=calendar) + + def calculate_relative_offset(format_string, from_ts, earlier=False): """ Calculate an offset based on a relative marker (e.g., 7d for 7 days, 8m for 8 months). @@ -119,31 +219,7 @@ def calculate_relative_offset(format_string, from_ts, earlier=False): if from_ts is None: from_ts = archive_ts_now() - if format_string is not None: - offset_regex = re.compile(r"(?P\d+)(?P[ymwdHMS])") - match = offset_regex.search(format_string) - - if match: - unit = match.group("unit") - offset = int(match.group("offset")) - offset *= -1 if earlier else 1 - - if unit == "y": - return from_ts.replace(year=from_ts.year + offset) - elif unit == "m": - return offset_n_months(from_ts, offset) - elif unit == "w": - return from_ts + timedelta(days=offset * 7) - elif unit == "d": - return from_ts + timedelta(days=offset) - elif unit == "H": - return from_ts + timedelta(seconds=offset * 60 * 60) - elif unit == "M": - return from_ts + timedelta(seconds=offset * 60) - elif unit == "S": - return from_ts + timedelta(seconds=offset) - - raise ValueError(f"Invalid relative ts offset format: {format_string}") + return FlexibleDelta.parse(format_string).apply(from_ts, earlier=earlier, calendar=True) def offset_n_months(from_ts, n_months): diff --git a/src/borg/testsuite/helpers/time_test.py b/src/borg/testsuite/helpers/time_test.py index 7dcffc8278..edd96d4563 100644 --- a/src/borg/testsuite/helpers/time_test.py +++ b/src/borg/testsuite/helpers/time_test.py @@ -1,7 +1,8 @@ import pytest from datetime import datetime, timezone +from zoneinfo import ZoneInfo -from ...helpers.time import safe_ns, safe_s, SUPPORT_32BIT_PLATFORMS +from ...helpers.time import safe_ns, safe_s, SUPPORT_32BIT_PLATFORMS, FlexibleDelta def utcfromtimestamp(timestamp): @@ -36,3 +37,75 @@ def test_safe_timestamps(): utcfromtimestamp(beyond_y10k) assert utcfromtimestamp(safe_s(beyond_y10k)) > datetime(2262, 1, 1) assert utcfromtimestamp(safe_ns(beyond_y10k) / 1000000000) > datetime(2262, 1, 1) + + +def test_flexible_delta(): + # Year delta across leap year + delta_1year = FlexibleDelta.parse("1y") + dt_2023 = datetime(year=2023, month=12, day=24) + assert delta_1year.add_to(dt_2023) == datetime(year=2024, month=12, day=23) + assert delta_1year.add_to(dt_2023, calendar=True) == datetime(year=2024, month=12, day=24) + assert delta_1year.subtract_from(dt_2023) == datetime(year=2022, month=12, day=24) + assert delta_1year.subtract_from(dt_2023, calendar=True) == datetime(year=2022, month=12, day=24) + + delta_1month = FlexibleDelta.parse("1m") + + # Month delta across leap day + dt_leap_february = datetime(year=2024, month=2, day=20) + assert delta_1month.add_to(dt_leap_february) == datetime(year=2024, month=3, day=22) + assert delta_1month.add_to(dt_leap_february, calendar=True) == datetime(year=2024, month=3, day=20) + assert delta_1month.subtract_from(dt_leap_february) == datetime(year=2024, month=1, day=20) + assert delta_1month.subtract_from(dt_leap_february, calendar=True) == datetime(year=2024, month=1, day=20) + + # Month delta across non-leap day February + dt_nonleap_february = datetime(year=2025, month=2, day=20) + assert delta_1month.add_to(dt_nonleap_february) == datetime(year=2025, month=3, day=23) + assert delta_1month.add_to(dt_nonleap_february, calendar=True) == datetime(year=2025, month=3, day=20) + + # Month delta across 31-day month boundary + dt_july = datetime(year=2025, month=7, day=20) + assert delta_1month.add_to(dt_july) == datetime(year=2025, month=8, day=20) + assert delta_1month.add_to(dt_july, calendar=True) == datetime(year=2025, month=8, day=20) + + # Month delta across 30-day month boundary + dt_july = datetime(year=2025, month=6, day=20) + assert delta_1month.add_to(dt_july) == datetime(year=2025, month=7, day=21) + assert delta_1month.add_to(dt_july, calendar=True) == datetime(year=2025, month=7, day=20) + + # Day delta across summe/winter time change + delta_1day = FlexibleDelta.parse("1d") + dt_oslo_wintertime = datetime(year=2026, month=3, day=28, hour=10, tzinfo=ZoneInfo("Europe/Oslo")) + assert delta_1day.add_to(dt_oslo_wintertime) == datetime( + year=2026, month=3, day=29, hour=10, tzinfo=ZoneInfo("Europe/Oslo") + ) + assert delta_1day.add_to(dt_oslo_wintertime, calendar=True) == datetime( + year=2026, month=3, day=29, hour=10, tzinfo=ZoneInfo("Europe/Oslo") + ) + + # Fuzzy day delta + delta_fuzzy_1day = FlexibleDelta.parse("1dz", fuzzyable=True) + dt = datetime(year=2026, month=1, day=1, hour=12) + assert delta_fuzzy_1day.add_to(dt) == datetime(year=2026, month=1, day=3, hour=0) # +1day -> end of day + assert delta_fuzzy_1day.subtract_from(dt) == datetime(year=2025, month=12, day=31, hour=0) # -1day -> start of day + + # Fuzzy month delta + delta_fuzzy_1month = FlexibleDelta.parse("1mz", fuzzyable=True) + dt = datetime(year=2026, month=1, day=30) + assert delta_fuzzy_1month.add_to(dt) == datetime( + year=2026, month=4, day=1 + ) # End of next month (31 days) (for inclusive check, ie. +1μs) + assert delta_fuzzy_1month.add_to(dt, calendar=True) == datetime( + year=2026, month=3, day=1 + ) # End of next month (for inclusive check, ie. +1μs) + assert delta_fuzzy_1month.subtract_from(dt) == datetime(year=2025, month=12, day=1) # Start of previous month + + # Fuzzy week delta + delta_fuzzy_1week = FlexibleDelta.parse("1wz", fuzzyable=True) + dt = datetime(year=2024, month=2, day=28) # A Wednesday, leap year + assert delta_fuzzy_1week.add_to(dt) == datetime( + year=2024, month=3, day=11 + ) # End of next week (for inclusive check, ie. +1μs) + assert delta_fuzzy_1week.add_to(dt, calendar=True) == datetime( + year=2024, month=3, day=11 + ) # End of next week (for inclusive check, ie. +1μs) + assert delta_fuzzy_1week.subtract_from(dt) == datetime(year=2024, month=2, day=19) # Start of previous week From 66b105a57b388b1cb74470640d472d73882e4c2a Mon Sep 17 00:00:00 2001 From: Hugo Wallenburg Date: Mon, 29 Dec 2025 16:50:10 +0100 Subject: [PATCH 6/6] Adds fuzzy prune intervals and comprehensive interval test --- src/borg/archiver/prune_cmd.py | 36 +- src/borg/helpers/__init__.py | 4 +- src/borg/helpers/parseformat.py | 10 +- src/borg/helpers/time.py | 11 +- src/borg/testsuite/archiver/prune_cmd_test.py | 475 +++++++++++++++--- .../testsuite/helpers/parseformat_test.py | 21 +- 6 files changed, 442 insertions(+), 115 deletions(-) diff --git a/src/borg/archiver/prune_cmd.py b/src/borg/archiver/prune_cmd.py index aa8f864751..da57ce7255 100644 --- a/src/borg/archiver/prune_cmd.py +++ b/src/borg/archiver/prune_cmd.py @@ -10,7 +10,7 @@ from ..archive import Archive from ..cache import Cache from ..constants import * # NOQA -from ..helpers import interval, int_or_interval, sig_int, archivename_validator +from ..helpers import interval, int_or_flexibledelta, sig_int, archivename_validator from ..helpers import ArchiveFormatter, ProgressIndicatorPercent, CommandError, Error from ..manifest import Manifest @@ -105,11 +105,11 @@ def quarterly_3monthly_period_func(a): DATETIME_MIN_WITH_ZONE = datetime.min.replace(tzinfo=timezone.utc) -def prune_split(archives, rule, n_or_interval, base_timestamp, kept_because={}): - if isinstance(n_or_interval, int): - n, earliest_timestamp = n_or_interval, None +def prune_split(archives, rule, n_or_flexibledelta, base_timestamp, kept_because={}): + if isinstance(n_or_flexibledelta, int): + n, earliest_timestamp = n_or_flexibledelta, None else: - n, earliest_timestamp = None, base_timestamp - n_or_interval + n, earliest_timestamp = None, n_or_flexibledelta.subtract_from(base_timestamp, calendar=True) def can_retain(a, keep): if n is not None: @@ -194,9 +194,9 @@ def do_prune(self, args, repository, manifest): base_timestamp = datetime.now().astimezone() # find archives which need to be kept because of the various time period rules for rule in PRUNING_PATTERNS.keys(): - num_or_interval = getattr(args, rule, None) - if num_or_interval is not None: - keep += prune_split(archives, rule, num_or_interval, base_timestamp, kept_because) + n_or_flexibledelta = getattr(args, rule, None) + if n_or_flexibledelta is not None: + keep += prune_split(archives, rule, n_or_flexibledelta, base_timestamp, kept_because) to_delete = set(archives) - set(keep) with Cache(repository, manifest, iec=args.iec) as cache: @@ -346,21 +346,21 @@ def build_parser_prune(self, subparsers, common_parser, mid_common_parser): subparser.add_argument( "--keep", dest="keep", - type=int_or_interval, + type=int_or_flexibledelta, action=Highlander, help="number or time interval of archives to keep", ) subparser.add_argument( "--keep-secondly", dest="secondly", - type=int_or_interval, + type=int_or_flexibledelta, action=Highlander, help="number or time interval of secondly archives to keep", ) subparser.add_argument( "--keep-minutely", dest="minutely", - type=int_or_interval, + type=int_or_flexibledelta, action=Highlander, help="number or time interval of minutely archives to keep", ) @@ -368,7 +368,7 @@ def build_parser_prune(self, subparsers, common_parser, mid_common_parser): "-H", "--keep-hourly", dest="hourly", - type=int_or_interval, + type=int_or_flexibledelta, action=Highlander, help="number or time interval of hourly archives to keep", ) @@ -376,7 +376,7 @@ def build_parser_prune(self, subparsers, common_parser, mid_common_parser): "-d", "--keep-daily", dest="daily", - type=int_or_interval, + type=int_or_flexibledelta, action=Highlander, help="number or time interval of daily archives to keep", ) @@ -384,7 +384,7 @@ def build_parser_prune(self, subparsers, common_parser, mid_common_parser): "-w", "--keep-weekly", dest="weekly", - type=int_or_interval, + type=int_or_flexibledelta, action=Highlander, help="number or time interval of weekly archives to keep", ) @@ -392,7 +392,7 @@ def build_parser_prune(self, subparsers, common_parser, mid_common_parser): "-m", "--keep-monthly", dest="monthly", - type=int_or_interval, + type=int_or_flexibledelta, action=Highlander, help="number or time interval of monthly archives to keep", ) @@ -400,20 +400,20 @@ def build_parser_prune(self, subparsers, common_parser, mid_common_parser): quarterly_group.add_argument( "--keep-13weekly", dest="quarterly_13weekly", - type=int_or_interval, + type=int_or_flexibledelta, help="number or time interval of quarterly archives to keep (13 week strategy)", ) quarterly_group.add_argument( "--keep-3monthly", dest="quarterly_3monthly", - type=int_or_interval, + type=int_or_flexibledelta, help="number or time interval of quarterly archives to keep (3 month strategy)", ) subparser.add_argument( "-y", "--keep-yearly", dest="yearly", - type=int_or_interval, + type=int_or_flexibledelta, action=Highlander, help="number or time interval of yearly archives to keep", ) diff --git a/src/borg/helpers/__init__.py b/src/borg/helpers/__init__.py index 751d79497a..8afe551b9f 100644 --- a/src/borg/helpers/__init__.py +++ b/src/borg/helpers/__init__.py @@ -27,7 +27,7 @@ from .misc import ChunkIteratorFileWrapper, open_item, chunkit, iter_separated, ErrorIgnoringTextIOWrapper from .parseformat import bin_to_hex, hex_to_bin, safe_encode, safe_decode from .parseformat import text_to_json, binary_to_json, remove_surrogates, join_cmd -from .parseformat import eval_escapes, decode_dict, positive_int_validator, interval, int_or_interval +from .parseformat import eval_escapes, decode_dict, positive_int_validator, interval, int_or_flexibledelta from .parseformat import PathSpec, SortBySpec, ChunkerParams, FilesCacheMode, partial_format, DatetimeWrapper from .parseformat import format_file_size, parse_file_size, FileSize from .parseformat import sizeof_fmt, sizeof_fmt_iec, sizeof_fmt_decimal, Location, text_validator @@ -43,7 +43,7 @@ from .process import popen_with_error_handling, is_terminal, prepare_subprocess_env, create_filter_process from .progress import ProgressIndicatorPercent, ProgressIndicatorMessage from .time import parse_timestamp, timestamp, safe_timestamp, safe_s, safe_ns, MAX_S, SUPPORT_32BIT_PLATFORMS -from .time import format_time, format_timedelta, OutputTimestamp, archive_ts_now +from .time import format_time, format_timedelta, OutputTimestamp, archive_ts_now, FlexibleDelta from .yes_no import yes, TRUISH, FALSISH, DEFAULTISH from .msgpack import is_slow_msgpack, is_supported_msgpack, get_limited_unpacker diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py index e7cc8d653f..3d8977f621 100644 --- a/src/borg/helpers/parseformat.py +++ b/src/borg/helpers/parseformat.py @@ -13,7 +13,7 @@ from pathlib import Path from typing import ClassVar, Any, TYPE_CHECKING, Literal from collections import OrderedDict -from datetime import datetime, timezone, timedelta +from datetime import datetime, timezone from functools import partial from string import Formatter @@ -24,7 +24,7 @@ from .errors import Error from .fs import get_keys_dir, make_path_safe from .msgpack import Timestamp -from .time import OutputTimestamp, format_time, safe_timestamp +from .time import OutputTimestamp, format_time, safe_timestamp, FlexibleDelta from .. import __version__ as borg_version from .. import __version_tuple__ as borg_version_tuple from ..constants import * # NOQA @@ -161,15 +161,15 @@ def interval(s): return seconds -def int_or_interval(s): +def int_or_flexibledelta(s): try: return int(s) except ValueError: pass try: - return timedelta(seconds=interval(s)) - except argparse.ArgumentTypeError as e: + return FlexibleDelta.parse(s, fuzzyable=True) + except ValueError as e: raise argparse.ArgumentTypeError(f"Value is neither an integer nor an interval: {e}") diff --git a/src/borg/helpers/time.py b/src/borg/helpers/time.py index fa8d6d6679..c58d901fad 100644 --- a/src/borg/helpers/time.py +++ b/src/borg/helpers/time.py @@ -165,12 +165,21 @@ def __init__(self, count, unit, fuzzy): def __repr__(self): return f'{self.__class__.__name__}(count={self.count}, unit="{self.unit}", fuzzy={self.fuzzy})' + def __eq__(self, other): + if not isinstance(other, FlexibleDelta): + return NotImplemented + return (self.count, self.unit, self.fuzzy) == (other.count, other.unit, other.fuzzy) + _interval_regex = re.compile(r"^(?P\d+)(?P[ymwdHMS])(?Pz)?$") @classmethod def parse(cls, interval_string, fuzzyable=False): """ - Parse interval string into + Parse interval string into FlexibleDelta on a form like "1w", "25d", or "6Hz" where the unit is one of "y" + (years), "m" (months), "w" (weeks), "d" (days), "H" (hours), "M" (minutes), "S" (seconds). A trailing "z" + indicates a fuzzy delta (if fuzzyable is True) which is calculated as "apply delta and then take either the + start or end of the current unit's interval" -- subtracting "1dz" means subtracting 1 day and then adjusting to + the start of that day. """ match = cls._interval_regex.search(interval_string) diff --git a/src/borg/testsuite/archiver/prune_cmd_test.py b/src/borg/testsuite/archiver/prune_cmd_test.py index 675d9bc362..efb2dc7f96 100644 --- a/src/borg/testsuite/archiver/prune_cmd_test.py +++ b/src/borg/testsuite/archiver/prune_cmd_test.py @@ -11,15 +11,12 @@ pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local,remote,binary") # NOQA +def _create_archive_dt(archiver, name, dt, tzinfo=None): + cmd(archiver, "create", "--timestamp", dt.replace(tzinfo=tzinfo).strftime(ISO_FORMAT_ZONE), name, src_dir) + + def _create_archive_ts(archiver, name, y, m, d, H=0, M=0, S=0, us=0, tzinfo=None): - cmd( - archiver, - "create", - "--timestamp", - datetime(y, m, d, H, M, S, us, tzinfo=tzinfo).strftime(ISO_FORMAT_ZONE), - name, - src_dir, - ) + _create_archive_dt(archiver, name, datetime(y, m, d, H, M, S, us), tzinfo=tzinfo) def test_prune_repository(archivers, request): @@ -373,7 +370,7 @@ def test_prune_keep_last_same_second(archivers, request): @freeze_time(datetime(2023, 12, 31, 23, 59, 59, tzinfo=None)) # Non-leap year ending on a Sunday -def test_prune_keep_int_or_interval(archivers, request): +def test_prune_keep_int_or_flexibledelta(archivers, request): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) _create_archive_ts(archiver, "test-1", 2023, 12, 31, 23, 59, 59) @@ -387,7 +384,7 @@ def test_prune_keep_int_or_interval(archivers, request): @freeze_time(datetime(2023, 12, 31, 23, 59, 59, tzinfo=None)) -def test_prune_keep_int_or_interval_zero(archivers, request): +def test_prune_keep_int_or_flexibledelta_zero(archivers, request): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) _create_archive_ts(archiver, "test", 2023, 12, 31, 23, 59, 59) @@ -397,7 +394,7 @@ def test_prune_keep_int_or_interval_zero(archivers, request): @freeze_time(datetime(2023, 12, 31, 23, 59, 59, tzinfo=None)) -def test_prune_keep_secondly_int_or_interval(archivers, request): +def test_prune_keep_secondly_int_or_flexibledelta(archivers, request): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) _create_archive_ts(archiver, "test-1", 2023, 12, 31, 23, 59, 58) @@ -405,15 +402,15 @@ def test_prune_keep_secondly_int_or_interval(archivers, request): _create_archive_ts(archiver, "test-3", 2023, 12, 31, 23, 59, 57) _create_archive_ts(archiver, "test-4", 2023, 12, 31, 23, 59, 56, 999999) for keep_arg in ["--keep-secondly=2", "--keep-secondly=2S"]: - output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg) - assert re.search(r"Keeping archive \(rule: secondly #1\):\s+test-1", output) - assert re.search(r"Keeping archive \(rule: secondly #2\):\s+test-2", output) - assert re.search(r"Would prune:\s+test-3", output) - assert re.search(r"Would prune:\s+test-4", output) + output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg).splitlines() + assert re.search(r"Keeping archive \(rule: secondly #1\):\s+test-1", output.pop(0)) + assert re.search(r"Keeping archive \(rule: secondly #2\):\s+test-2", output.pop(0)) + assert re.search(r"Would prune:\s+test-3", output.pop(0)) + assert re.search(r"Would prune:\s+test-4", output.pop(0)) @freeze_time(datetime(2023, 12, 31, 23, 59, 0, tzinfo=None)) -def test_prune_keep_minutely_int_or_interval(archivers, request): +def test_prune_keep_minutely_int_or_flexibledelta(archivers, request): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) _create_archive_ts(archiver, "test-1", 2023, 12, 31, 23, 58) @@ -422,16 +419,16 @@ def test_prune_keep_minutely_int_or_interval(archivers, request): _create_archive_ts(archiver, "test-4", 2023, 12, 31, 23, 56, 0, 1) # Last possible microsecond _create_archive_ts(archiver, "test-5", 2023, 12, 31, 23, 56) for keep_arg in ["--keep-minutely=3", "--keep-minutely=3M"]: - output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg) - assert re.search(r"Keeping archive \(rule: minutely #1\):\s+test-1", output) - assert re.search(r"Keeping archive \(rule: minutely #2\):\s+test-2", output) - assert re.search(r"Would prune:\s+test-3", output) - assert re.search(r"Keeping archive \(rule: minutely #3\):\s+test-4", output) - assert re.search(r"Would prune:\s+test-5", output) + output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg).splitlines() + assert re.search(r"Keeping archive \(rule: minutely #1\):\s+test-1", output.pop(0)) + assert re.search(r"Keeping archive \(rule: minutely #2\):\s+test-2", output.pop(0)) + assert re.search(r"Would prune:\s+test-3", output.pop(0)) + assert re.search(r"Keeping archive \(rule: minutely #3\):\s+test-4", output.pop(0)) + assert re.search(r"Would prune:\s+test-5", output.pop(0)) @freeze_time(datetime(2023, 12, 31, 23, 0, 0, tzinfo=None)) -def test_prune_keep_hourly_int_or_interval(archivers, request): +def test_prune_keep_hourly_int_or_flexibledelta(archivers, request): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) _create_archive_ts(archiver, "test-1", 2023, 12, 31, 22) @@ -440,34 +437,34 @@ def test_prune_keep_hourly_int_or_interval(archivers, request): _create_archive_ts(archiver, "test-4", 2023, 12, 31, 20, us=1) # Last possible microsecond _create_archive_ts(archiver, "test-5", 2023, 12, 31, 20) for keep_arg in ["--keep-hourly=3", "--keep-hourly=3H"]: - output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg) - assert re.search(r"Keeping archive \(rule: hourly #1\):\s+test-1", output) - assert re.search(r"Keeping archive \(rule: hourly #2\):\s+test-2", output) - assert re.search(r"Would prune:\s+test-3", output) - assert re.search(r"Keeping archive \(rule: hourly #3\):\s+test-4", output) - assert re.search(r"Would prune:\s+test-5", output) + output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg).splitlines() + assert re.search(r"Keeping archive \(rule: hourly #1\):\s+test-1", output.pop(0)) + assert re.search(r"Keeping archive \(rule: hourly #2\):\s+test-2", output.pop(0)) + assert re.search(r"Would prune:\s+test-3", output.pop(0)) + assert re.search(r"Keeping archive \(rule: hourly #3\):\s+test-4", output.pop(0)) + assert re.search(r"Would prune:\s+test-5", output.pop(0)) -@freeze_time(datetime(2023, 12, 31, 0, 0, 0, tzinfo=None)) -def test_prune_keep_daily_int_or_interval(archivers, request): +@freeze_time(datetime(2023, 12, 31, 12, 0, 0, tzinfo=None)) +def test_prune_keep_daily_int_or_flexibledelta(archivers, request): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) _create_archive_ts(archiver, "test-1", 2023, 12, 30) _create_archive_ts(archiver, "test-2", 2023, 12, 29, S=1) _create_archive_ts(archiver, "test-3", 2023, 12, 29) - _create_archive_ts(archiver, "test-4", 2023, 12, 28, us=1) # Last possible microsecond - _create_archive_ts(archiver, "test-5", 2023, 12, 28) + _create_archive_ts(archiver, "test-4", 2023, 12, 28, 12, us=1) # Last possible microsecond + _create_archive_ts(archiver, "test-5", 2023, 12, 28, 12) for keep_arg in ["--keep-daily=3", "--keep-daily=3d"]: - output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg) - assert re.search(r"Keeping archive \(rule: daily #1\):\s+test-1", output) - assert re.search(r"Keeping archive \(rule: daily #2\):\s+test-2", output) - assert re.search(r"Would prune:\s+test-3", output) - assert re.search(r"Keeping archive \(rule: daily #3\):\s+test-4", output) - assert re.search(r"Would prune:\s+test-5", output) + output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg).splitlines() + assert re.search(r"Keeping archive \(rule: daily #1\):\s+test-1", output.pop(0)) + assert re.search(r"Keeping archive \(rule: daily #2\):\s+test-2", output.pop(0)) + assert re.search(r"Would prune:\s+test-3", output.pop(0)) + assert re.search(r"Keeping archive \(rule: daily #3\):\s+test-4", output.pop(0)) + assert re.search(r"Would prune:\s+test-5", output.pop(0)) @freeze_time(datetime(2023, 12, 31, 0, 0, 0, tzinfo=None)) -def test_prune_keep_weekly_int_or_interval(archivers, request): +def test_prune_keep_weekly_int_or_flexibledelta(archivers, request): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) _create_archive_ts(archiver, "test-1", 2023, 12, 24) @@ -476,35 +473,35 @@ def test_prune_keep_weekly_int_or_interval(archivers, request): _create_archive_ts(archiver, "test-4", 2023, 12, 10, us=1) # Last possible microsecond _create_archive_ts(archiver, "test-5", 2023, 12, 10) for keep_arg in ["--keep-weekly=3", "--keep-weekly=3w"]: - output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg) - assert re.search(r"Keeping archive \(rule: weekly #1\):\s+test-1", output) - assert re.search(r"Keeping archive \(rule: weekly #2\):\s+test-2", output) - assert re.search(r"Would prune:\s+test-3", output) - assert re.search(r"Keeping archive \(rule: weekly #3\):\s+test-4", output) - assert re.search(r"Would prune:\s+test-5", output) + output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg).splitlines() + assert re.search(r"Keeping archive \(rule: weekly #1\):\s+test-1", output.pop(0)) + assert re.search(r"Keeping archive \(rule: weekly #2\):\s+test-2", output.pop(0)) + assert re.search(r"Would prune:\s+test-3", output.pop(0)) + assert re.search(r"Keeping archive \(rule: weekly #3\):\s+test-4", output.pop(0)) + assert re.search(r"Would prune:\s+test-5", output.pop(0)) @freeze_time(datetime(2023, 12, 31, 0, 0, 0, tzinfo=None)) -def test_prune_keep_monthly_int_or_interval(archivers, request): +def test_prune_keep_monthly_int_or_flexibledelta(archivers, request): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) _create_archive_ts(archiver, "test-1", 2023, 11, 30) - _create_archive_ts(archiver, "test-2", 2023, 10, 30, us=1) # Month defined as 31 days, so not Oct 31st + _create_archive_ts(archiver, "test-2", 2023, 10, 30, us=1) _create_archive_ts(archiver, "test-3", 2023, 10, 30) - _create_archive_ts(archiver, "test-4", 2023, 9, 29, us=1) # Last possible microsecond - _create_archive_ts(archiver, "test-5", 2023, 9, 29) + _create_archive_ts(archiver, "test-4", 2023, 9, 30, us=1) # Last possible microsecond + _create_archive_ts(archiver, "test-5", 2023, 9, 30) for keep_arg in ["--keep-monthly=3", "--keep-monthly=3m"]: - output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg) - assert re.search(r"Keeping archive \(rule: monthly #1\):\s+test-1", output) - assert re.search(r"Keeping archive \(rule: monthly #2\):\s+test-2", output) - assert re.search(r"Would prune:\s+test-3", output) - assert re.search(r"Keeping archive \(rule: monthly #3\):\s+test-4", output) - assert re.search(r"Would prune:\s+test-5", output) + output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg).splitlines() + assert re.search(r"Keeping archive \(rule: monthly #1\):\s+test-1", output.pop(0)) + assert re.search(r"Keeping archive \(rule: monthly #2\):\s+test-2", output.pop(0)) + assert re.search(r"Would prune:\s+test-3", output.pop(0)) + assert re.search(r"Keeping archive \(rule: monthly #3\):\s+test-4", output.pop(0)) + assert re.search(r"Would prune:\s+test-5", output.pop(0)) # 2023-12-31 is Sunday, week 52. Makes these week calculations a little easier. @freeze_time(datetime(2023, 12, 31, 0, 0, 0, tzinfo=None)) -def test_prune_keep_13weekly_int_or_interval(archivers, request): +def test_prune_keep_13weekly_int_or_flexibledelta(archivers, request): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) _create_archive_ts(archiver, "test-1", 2023, 10, 1) @@ -513,34 +510,34 @@ def test_prune_keep_13weekly_int_or_interval(archivers, request): _create_archive_ts(archiver, "test-4", 2023, 4, 2, us=1) # Last possible microsecond _create_archive_ts(archiver, "test-5", 2023, 4, 2) for keep_arg in ["--keep-13weekly=3", "--keep-13weekly=39w"]: - output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg) - assert re.search(r"Keeping archive \(rule: quarterly_13weekly #1\):\s+test-1", output) - assert re.search(r"Keeping archive \(rule: quarterly_13weekly #2\):\s+test-2", output) - assert re.search(r"Would prune:\s+test-3", output) - assert re.search(r"Keeping archive \(rule: quarterly_13weekly #3\):\s+test-4", output) - assert re.search(r"Would prune:\s+test-5", output) + output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg).splitlines() + assert re.search(r"Keeping archive \(rule: quarterly_13weekly #1\):\s+test-1", output.pop(0)) + assert re.search(r"Keeping archive \(rule: quarterly_13weekly #2\):\s+test-2", output.pop(0)) + assert re.search(r"Would prune:\s+test-3", output.pop(0)) + assert re.search(r"Keeping archive \(rule: quarterly_13weekly #3\):\s+test-4", output.pop(0)) + assert re.search(r"Would prune:\s+test-5", output.pop(0)) @freeze_time(datetime(2023, 12, 31, 0, 0, 0, tzinfo=None)) -def test_prune_keep_3monthly_int_or_interval(archivers, request): +def test_prune_keep_3monthly_int_or_flexibledelta(archivers, request): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) - _create_archive_ts(archiver, "test-1", 2023, 9, 30) + _create_archive_ts(archiver, "test-1", 2023, 9, 30) # 31st December - 3 calendar months _create_archive_ts(archiver, "test-2", 2023, 6, 30, us=1) _create_archive_ts(archiver, "test-3", 2023, 6, 30) _create_archive_ts(archiver, "test-4", 2023, 3, 31, us=1) # Last possible microsecond _create_archive_ts(archiver, "test-5", 2023, 3, 31) for keep_arg in ["--keep-3monthly=3", f"--keep-3monthly={(datetime.now()-datetime(2023, 3, 31)).days}d"]: - output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg) - assert re.search(r"Keeping archive \(rule: quarterly_3monthly #1\):\s+test-1", output) - assert re.search(r"Keeping archive \(rule: quarterly_3monthly #2\):\s+test-2", output) - assert re.search(r"Would prune:\s+test-3", output) - assert re.search(r"Keeping archive \(rule: quarterly_3monthly #3\):\s+test-4", output) - assert re.search(r"Would prune:\s+test-5", output) + output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg).splitlines() + assert re.search(r"Keeping archive \(rule: quarterly_3monthly #1\):\s+test-1", output.pop(0)) + assert re.search(r"Keeping archive \(rule: quarterly_3monthly #2\):\s+test-2", output.pop(0)) + assert re.search(r"Would prune:\s+test-3", output.pop(0)) + assert re.search(r"Keeping archive \(rule: quarterly_3monthly #3\):\s+test-4", output.pop(0)) + assert re.search(r"Would prune:\s+test-5", output.pop(0)) @freeze_time(datetime(2023, 12, 31, 0, 0, 0, tzinfo=None)) -def test_prune_keep_yearly_int_or_interval(archivers, request): +def test_prune_keep_yearly_int_or_flexibledelta(archivers, request): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) _create_archive_ts(archiver, "test-1", 2022, 12, 31) @@ -549,12 +546,332 @@ def test_prune_keep_yearly_int_or_interval(archivers, request): _create_archive_ts(archiver, "test-4", 2020, 12, 31, us=1) # Last possible microsecond _create_archive_ts(archiver, "test-5", 2020, 12, 31) for keep_arg in ["--keep-yearly=3", "--keep-yearly=3y"]: - output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg) - assert re.search(r"Keeping archive \(rule: yearly #1\):\s+test-1", output) - assert re.search(r"Keeping archive \(rule: yearly #2\):\s+test-2", output) - assert re.search(r"Would prune:\s+test-3", output) - assert re.search(r"Keeping archive \(rule: yearly #3\):\s+test-4", output) - assert re.search(r"Would prune:\s+test-5", output) + output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg).splitlines() + assert re.search(r"Keeping archive \(rule: yearly #1\):\s+test-1", output.pop(0)) + assert re.search(r"Keeping archive \(rule: yearly #2\):\s+test-2", output.pop(0)) + assert re.search(r"Would prune:\s+test-3", output.pop(0)) + assert re.search(r"Keeping archive \(rule: yearly #3\):\s+test-4", output.pop(0)) + assert re.search(r"Would prune:\s+test-5", output.pop(0)) + + +@freeze_time(datetime(2025, 12, 24, 12, 0, 0, tzinfo=None)) +def test_prune_fuzzy_days(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_archive_ts(archiver, "test-1", 2025, 12, 23, 12, us=1) + _create_archive_ts(archiver, "test-2", 2025, 12, 23, 10) + output_nonfuzzy = cmd(archiver, "prune", "--list", "--dry-run", "--keep=1d").splitlines() + assert re.search(r"Keeping archive \(rule: keep #1\):\s+test-1", output_nonfuzzy.pop(0)) + assert re.search(r"Would prune:\s+test-2", output_nonfuzzy.pop(0)) + output_fuzzy = cmd(archiver, "prune", "--list", "--dry-run", "--keep=1dz").splitlines() + assert re.search(r"Keeping archive \(rule: keep #1\):\s+test-1", output_fuzzy.pop(0)) + + +@freeze_time(datetime(2025, 12, 24, 12, 0, 0, tzinfo=None)) # A Wednesday +def test_prune_fuzzy_weeks(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_archive_ts(archiver, "test-1", 2025, 12, 17, 12, us=1) # The previous Wednesday + _create_archive_ts(archiver, "test-2", 2025, 12, 15, 10) # Monday, start of previous week + output_nonfuzzy = cmd(archiver, "prune", "--list", "--dry-run", "--keep=1w").splitlines() + assert re.search(r"Keeping archive \(rule: keep #1\):\s+test-1", output_nonfuzzy.pop(0)) + assert re.search(r"Would prune:\s+test-2", output_nonfuzzy.pop(0)) + output_fuzzy = cmd(archiver, "prune", "--list", "--dry-run", "--keep=1wz").splitlines() + assert re.search(r"Keeping archive \(rule: keep #1\):\s+test-1", output_fuzzy.pop(0)) + assert re.search(r"Keeping archive \(rule: keep #2\):\s+test-2", output_fuzzy.pop(0)) + + +@freeze_time(datetime(2025, 12, 29, 12, 0, 0, tzinfo=None)) # Wednesday, end of year +def test_prune_repository_timedelta_everything_exact_vs_fuzzy(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + + # Archiving strategy: One archive is made at 12:00 every day and assumed to take 0 seconds. Pruning is done + # immediately afterwards. It is now 2025-12-29 12:00, the last Monday of the year, the 12:00 archive has been + # created and we are now running prune maintenance. + + # ---- daily: last 2 weeks ---- + # Assuming every archive is created at exactly 12:00:00 and we run prune also at exactly 12:00:00, + # "--keep-daily=2w" will keep THIRTEEN archives (assuming one every day) as the one from exactly 2 weeks ago falls + # one microsecond outside of range. + _create_archive_ts(archiver, "2025-12-29", 2025, 12, 29, 12) # daily 1 + _create_archive_ts(archiver, "2025-12-28", 2025, 12, 28, 12) # daily 2 + _create_archive_ts(archiver, "2025-12-27", 2025, 12, 27, 12) # daily 3 + _create_archive_ts(archiver, "2025-12-26", 2025, 12, 26, 12) # daily 4 + _create_archive_ts(archiver, "2025-12-25", 2025, 12, 25, 12) # daily 5 + _create_archive_ts(archiver, "2025-12-24", 2025, 12, 24, 12) # daily 6 + _create_archive_ts(archiver, "2025-12-23", 2025, 12, 23, 12) # daily 7 + _create_archive_ts(archiver, "2025-12-22", 2025, 12, 22, 12) # daily 8 + # Archiver not run on 2025-12-21 (daily 10) + _create_archive_ts(archiver, "2025-12-20", 2025, 12, 20, 12) # daily 10 + _create_archive_ts(archiver, "2025-12-19", 2025, 12, 19, 12) # daily 11 + _create_archive_ts(archiver, "2025-12-18", 2025, 12, 18, 12) # daily 12 + # Fuzzy by week includes the Tuesday and Wednesday 12:00 archives, as well as the one from the preceding Monday + _create_archive_ts(archiver, "2025-12-17", 2025, 12, 17, 12) # daily fuzzy 1 + _create_archive_ts(archiver, "2025-12-16", 2025, 12, 16, 12) # daily fuzzy 2 + _create_archive_ts(archiver, "2025-12-15", 2025, 12, 15, 12) # daily fuzzy 3 + # Fuzzy cutoff on crossing Sunday-Monday boundary. These would not exist in the exact case as stated in the example. + _create_archive_ts(archiver, "2025-12-14", 2025, 12, 14, 12) # daily fuzzy 4 + _create_archive_ts(archiver, "2025-12-13", 2025, 12, 13, 12) # daily fuzzy 5 + _create_archive_ts(archiver, "2025-12-12", 2025, 12, 12, 12) # daily fuzzy 6 + _create_archive_ts(archiver, "2025-12-11", 2025, 12, 11, 12) # daily fuzzy 7 + _create_archive_ts(archiver, "2025-12-10", 2025, 12, 10, 12) # daily fuzzy 8 + _create_archive_ts(archiver, "2025-12-09", 2025, 12, 9, 12) # daily fuzzy 9 + _create_archive_ts(archiver, "2025-12-08", 2025, 12, 8, 12) # daily fuzzy 10 + + # ---- weekly: last 6 months --- + # _create_archive_ts(archiver, "2025-12-14", 2025, 12, 14, 12) # weekly 1 (Duplicate with daily fuzzy 1) + _create_archive_ts(archiver, "2025-12-07", 2025, 12, 7, 12) # weekly 2 + _create_archive_ts(archiver, "2025-11-30", 2025, 11, 30, 12) # weekly 3 + _create_archive_ts(archiver, "2025-11-23", 2025, 11, 23, 12) # weekly 4 + _create_archive_ts(archiver, "2025-11-16", 2025, 11, 16, 12) # weekly 5 + _create_archive_ts(archiver, "2025-11-09", 2025, 11, 9, 12) # weekly 6 + _create_archive_ts(archiver, "2025-11-02", 2025, 11, 2, 12) # weekly 7 + _create_archive_ts(archiver, "2025-10-26", 2025, 10, 26, 12) # weekly 8 + _create_archive_ts(archiver, "2025-10-19", 2025, 10, 19, 12) # weekly 9 + _create_archive_ts(archiver, "2025-10-12", 2025, 10, 12, 12) # weekly 10 + _create_archive_ts(archiver, "2025-10-05", 2025, 10, 5, 12) # weekly 11 + _create_archive_ts( + archiver, "2025-09-27", 2025, 9, 27, 12 + ) # weekly 12 (archiver not run on Sunday 09-28, so 09-27 is kept) + _create_archive_ts(archiver, "2025-09-21", 2025, 9, 21, 12) # weekly 13 + _create_archive_ts(archiver, "2025-09-14", 2025, 9, 14, 12) # weekly 14 + _create_archive_ts(archiver, "2025-09-07", 2025, 9, 7, 12) # weekly 15 + _create_archive_ts(archiver, "2025-08-31", 2025, 8, 31, 12) # weekly 16 + _create_archive_ts(archiver, "2025-08-24", 2025, 8, 24, 12) # weekly 17 + _create_archive_ts(archiver, "2025-08-17", 2025, 8, 17, 12) # weekly 18 + _create_archive_ts(archiver, "2025-08-10", 2025, 8, 10, 12) # weekly 19 + _create_archive_ts(archiver, "2025-08-03", 2025, 8, 3, 12) # weekly 20 + _create_archive_ts(archiver, "2025-07-27", 2025, 7, 27, 12) # weekly 21 + _create_archive_ts(archiver, "2025-07-20", 2025, 7, 20, 12) # weekly 22 + _create_archive_ts(archiver, "2025-07-13", 2025, 7, 13, 12) # weekly 23 + _create_archive_ts(archiver, "2025-07-06", 2025, 7, 6, 12) # weekly 24 + # 12-31 minus 6 months is 06-30, meaning fuzzy month catches all backups in June + _create_archive_ts(archiver, "2025-06-29", 2025, 6, 29, 12) # weekly fuzzy 1 + _create_archive_ts(archiver, "2025-06-22", 2025, 6, 22, 12) # weekly fuzzy 2 + _create_archive_ts(archiver, "2025-06-15", 2025, 6, 15, 12) # weekly fuzzy 3 + _create_archive_ts(archiver, "2025-06-08", 2025, 6, 8, 12) # weekly fuzzy 4 + _create_archive_ts(archiver, "2025-06-01", 2025, 6, 1, 12) # weekly fuzzy 5 + + # ---- monthly: last year ---- + # Last Sunday each month (lowest granularity kept when monthly retention kick in) for the rest of the year + # _create_archive_ts(archiver, "2025-06-29", 2025, 6, 29, 12) # monthly 1 (Duplicate with weekly fuzzy 1) + _create_archive_ts(archiver, "2025-05-25", 2025, 5, 25, 12) # monhtly 2 + _create_archive_ts(archiver, "2025-04-27", 2025, 4, 27, 12) # monhtly 3 + _create_archive_ts(archiver, "2025-03-30", 2025, 3, 30, 12) # monhtly 4 + _create_archive_ts(archiver, "2025-02-23", 2025, 2, 23, 12) # monhtly 5 + _create_archive_ts(archiver, "2025-01-26", 2025, 1, 26, 12) # monhtly 6 + # Last Sunday each month through 2024 as per fuzzy year delta + _create_archive_ts(archiver, "2024-12-29", 2024, 12, 29, 12) # monthly fuzzy 1 + _create_archive_ts(archiver, "2024-11-24", 2024, 11, 24, 12) # monthly fuzzy 2 + _create_archive_ts(archiver, "2024-10-27", 2024, 10, 27, 12) # monthly fuzzy 3 + _create_archive_ts(archiver, "2024-09-29", 2024, 9, 29, 12) # monthly fuzzy 4 + _create_archive_ts(archiver, "2024-08-25", 2024, 8, 25, 12) # monthly fuzzy 5 + _create_archive_ts(archiver, "2024-07-28", 2024, 7, 28, 12) # monthly fuzzy 6 + _create_archive_ts(archiver, "2024-06-30", 2024, 6, 30, 12) # monthly fuzzy 7 + _create_archive_ts(archiver, "2024-05-26", 2024, 5, 26, 12) # monthly fuzzy 8 + _create_archive_ts(archiver, "2024-04-28", 2024, 4, 28, 12) # monthly fuzzy 9 + _create_archive_ts(archiver, "2024-03-31", 2024, 3, 31, 12) # monthly fuzzy 10 + _create_archive_ts(archiver, "2024-02-25", 2024, 2, 25, 12) # monthly fuzzy 11 + _create_archive_ts(archiver, "2024-01-28", 2024, 1, 28, 12) # monthly fuzzy 12 + + # ---- yearly: exactly 3 ----- + # _create_archive_ts(archiver, "2024-12-29", 2024, 12, 29, 12) # yearly 1 (Duplicate with monthly fuzzy 1) + _create_archive_ts(archiver, "2023-12-31", 2023, 12, 31, 12) # yearly 2 + _create_archive_ts(archiver, "2022-12-25", 2022, 12, 25, 12) # yearly 3 + _create_archive_ts(archiver, "2021-12-26", 2021, 12, 26, 12) # yearly 4 + + # ---- Exact deltas ------------------------------------------------ + output_exact = cmd( + archiver, + "prune", + "--list", + "--dry-run", + "--keep-daily=2w", + "--keep-weekly=6m", + "--keep-monthly=1y", + "--keep-yearly=3", + ) + print("Prune output (exact):") + print(output_exact) + output_exact = list(reversed(output_exact.splitlines())) + + # Daily within 2 weeks + assert re.search(r"Keeping archive \(rule: daily #1\):\s+2025-12-29", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: daily #2\):\s+2025-12-28", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: daily #3\):\s+2025-12-27", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: daily #4\):\s+2025-12-26", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: daily #5\):\s+2025-12-25", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: daily #6\):\s+2025-12-24", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: daily #7\):\s+2025-12-23", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: daily #8\):\s+2025-12-22", output_exact.pop()) + # Nothing on 2025-12-21 + assert re.search(r"Keeping archive \(rule: daily #9\):\s+2025-12-20", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: daily #10\):\s+2025-12-19", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: daily #11\):\s+2025-12-18", output_exact.pop()) + # Would match with daily by fuzzy week + assert re.search(r"Keeping archive \(rule: daily #12\):\s+2025-12-17", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: daily #13\):\s+2025-12-16", output_exact.pop()) + assert re.search(r"Would prune:\s+2025-12-15", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #1\):\s+2025-12-14", output_exact.pop()) + assert re.search(r"Would prune:\s+2025-12-13", output_exact.pop()) + assert re.search(r"Would prune:\s+2025-12-12", output_exact.pop()) + assert re.search(r"Would prune:\s+2025-12-11", output_exact.pop()) + assert re.search(r"Would prune:\s+2025-12-10", output_exact.pop()) + assert re.search(r"Would prune:\s+2025-12-09", output_exact.pop()) + assert re.search(r"Would prune:\s+2025-12-08", output_exact.pop()) + + # Weekly within 6 months + assert re.search(r"Keeping archive \(rule: weekly #2\):\s+2025-12-07", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #3\):\s+2025-11-30", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #4\):\s+2025-11-23", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #5\):\s+2025-11-16", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #6\):\s+2025-11-09", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #7\):\s+2025-11-02", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #8\):\s+2025-10-26", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #9\):\s+2025-10-19", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #10\):\s+2025-10-12", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #11\):\s+2025-10-05", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #12\):\s+2025-09-27", output_exact.pop()) # (Not 09-28) + assert re.search(r"Keeping archive \(rule: weekly #13\):\s+2025-09-21", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #14\):\s+2025-09-14", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #15\):\s+2025-09-07", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #16\):\s+2025-08-31", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #17\):\s+2025-08-24", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #18\):\s+2025-08-17", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #19\):\s+2025-08-10", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #20\):\s+2025-08-03", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #21\):\s+2025-07-27", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #22\):\s+2025-07-20", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #23\):\s+2025-07-13", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #24\):\s+2025-07-06", output_exact.pop()) + # Would match with weekly by fuzzy month + assert re.search(r"Keeping archive \(rule: monthly #1\):\s+2025-06-29", output_exact.pop()) + assert re.search(r"Would prune:\s+2025-06-22", output_exact.pop()) + assert re.search(r"Would prune:\s+2025-06-15", output_exact.pop()) + assert re.search(r"Would prune:\s+2025-06-08", output_exact.pop()) + assert re.search(r"Would prune:\s+2025-06-01", output_exact.pop()) + + # Monthly within 1 year + assert re.search(r"Keeping archive \(rule: monthly #2\):\s+2025-05-25", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: monthly #3\):\s+2025-04-27", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: monthly #4\):\s+2025-03-30", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: monthly #5\):\s+2025-02-23", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: monthly #6\):\s+2025-01-26", output_exact.pop()) + # Would match with monthly by fuzzy year + assert re.search(r"Keeping archive \(rule: yearly #1\):\s+2024-12-29", output_exact.pop()) + assert re.search(r"Would prune:\s+2024-11-24", output_exact.pop()) + assert re.search(r"Would prune:\s+2024-10-27", output_exact.pop()) + assert re.search(r"Would prune:\s+2024-09-29", output_exact.pop()) + assert re.search(r"Would prune:\s+2024-08-25", output_exact.pop()) + assert re.search(r"Would prune:\s+2024-07-28", output_exact.pop()) + assert re.search(r"Would prune:\s+2024-06-30", output_exact.pop()) + assert re.search(r"Would prune:\s+2024-05-26", output_exact.pop()) + assert re.search(r"Would prune:\s+2024-04-28", output_exact.pop()) + assert re.search(r"Would prune:\s+2024-03-31", output_exact.pop()) + assert re.search(r"Would prune:\s+2024-02-25", output_exact.pop()) + assert re.search(r"Would prune:\s+2024-01-28", output_exact.pop()) + + # Yearly x3 + assert re.search(r"Keeping archive \(rule: yearly #2\):\s+2023-12-31", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: yearly #3\):\s+2022-12-25", output_exact.pop()) + # Would match if yearly #1 matched with fuzzy by month instead + assert re.search(r"Would prune:\s+2021-12-26", output_exact.pop()) + assert len(output_exact) == 0 + + # ---- Exact deltas ------------------------------------------------ + output_fuzzy = cmd( + archiver, + "prune", + "--list", + "--dry-run", + "--keep-daily=2wz", + "--keep-weekly=6mz", + "--keep-monthly=1yz", + "--keep-yearly=3", + ) + print("Prune output (fuzzy):") + print(output_fuzzy) + output_fuzzy = list(reversed(output_fuzzy.splitlines())) + + # Daily within 2 weeks + assert re.search(r"Keeping archive \(rule: daily #1\):\s+2025-12-29", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: daily #2\):\s+2025-12-28", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: daily #3\):\s+2025-12-27", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: daily #4\):\s+2025-12-26", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: daily #5\):\s+2025-12-25", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: daily #6\):\s+2025-12-24", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: daily #7\):\s+2025-12-23", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: daily #8\):\s+2025-12-22", output_fuzzy.pop()) + # Nothing on 2025-12-21 + assert re.search(r"Keeping archive \(rule: daily #9\):\s+2025-12-20", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: daily #10\):\s+2025-12-19", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: daily #11\):\s+2025-12-18", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: daily #12\):\s+2025-12-17", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: daily #13\):\s+2025-12-16", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: daily #14\):\s+2025-12-15", output_fuzzy.pop()) + + # Weekly within 6 months + assert re.search(r"Keeping archive \(rule: weekly #1\):\s+2025-12-14", output_fuzzy.pop()) + assert re.search(r"Would prune:\s+2025-12-13", output_fuzzy.pop()) + assert re.search(r"Would prune:\s+2025-12-12", output_fuzzy.pop()) + assert re.search(r"Would prune:\s+2025-12-11", output_fuzzy.pop()) + assert re.search(r"Would prune:\s+2025-12-10", output_fuzzy.pop()) + assert re.search(r"Would prune:\s+2025-12-09", output_fuzzy.pop()) + assert re.search(r"Would prune:\s+2025-12-08", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #2\):\s+2025-12-07", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #3\):\s+2025-11-30", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #4\):\s+2025-11-23", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #5\):\s+2025-11-16", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #6\):\s+2025-11-09", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #7\):\s+2025-11-02", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #8\):\s+2025-10-26", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #9\):\s+2025-10-19", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #10\):\s+2025-10-12", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #11\):\s+2025-10-05", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #12\):\s+2025-09-27", output_fuzzy.pop()) # (Not 09-28) + assert re.search(r"Keeping archive \(rule: weekly #13\):\s+2025-09-21", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #14\):\s+2025-09-14", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #15\):\s+2025-09-07", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #16\):\s+2025-08-31", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #17\):\s+2025-08-24", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #18\):\s+2025-08-17", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #19\):\s+2025-08-10", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #20\):\s+2025-08-03", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #21\):\s+2025-07-27", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #22\):\s+2025-07-20", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #23\):\s+2025-07-13", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #24\):\s+2025-07-06", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #25\):\s+2025-06-29", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #26\):\s+2025-06-22", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #27\):\s+2025-06-15", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #28\):\s+2025-06-08", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #29\):\s+2025-06-01", output_fuzzy.pop()) + + # Monthly within 1 year + assert re.search(r"Keeping archive \(rule: monthly #1\):\s+2025-05-25", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: monthly #2\):\s+2025-04-27", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: monthly #3\):\s+2025-03-30", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: monthly #4\):\s+2025-02-23", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: monthly #5\):\s+2025-01-26", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: monthly #6\):\s+2024-12-29", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: monthly #7\):\s+2024-11-24", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: monthly #8\):\s+2024-10-27", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: monthly #9\):\s+2024-09-29", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: monthly #10\):\s+2024-08-25", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: monthly #11\):\s+2024-07-28", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: monthly #12\):\s+2024-06-30", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: monthly #13\):\s+2024-05-26", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: monthly #14\):\s+2024-04-28", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: monthly #15\):\s+2024-03-31", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: monthly #16\):\s+2024-02-25", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: monthly #17\):\s+2024-01-28", output_fuzzy.pop()) + + # Yearly x3 + assert re.search(r"Keeping archive \(rule: yearly #1\):\s+2023-12-31", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: yearly #2\):\s+2022-12-25", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: yearly #3\):\s+2021-12-26", output_fuzzy.pop()) + assert len(output_fuzzy) == 0 def test_prune_no_args(archivers, request): diff --git a/src/borg/testsuite/helpers/parseformat_test.py b/src/borg/testsuite/helpers/parseformat_test.py index 0948151c78..a0637fc474 100644 --- a/src/borg/testsuite/helpers/parseformat_test.py +++ b/src/borg/testsuite/helpers/parseformat_test.py @@ -2,7 +2,7 @@ import os import re from argparse import ArgumentTypeError -from datetime import datetime, timedelta, timezone +from datetime import datetime, timezone import pytest @@ -17,7 +17,7 @@ format_file_size, parse_file_size, interval, - int_or_interval, + int_or_flexibledelta, partial_format, clean_lines, format_line, @@ -27,7 +27,7 @@ eval_escapes, ChunkerParams, ) -from ...helpers.time import format_timedelta, parse_timestamp +from ...helpers.time import format_timedelta, parse_timestamp, FlexibleDelta def test_bin_to_hex(): @@ -428,13 +428,14 @@ def test_interval_invalid_time_format(invalid_input, error_regex): ("0", 0), ("5", 5), (" 999 ", 999), - ("0S", timedelta(seconds=0)), - ("5S", timedelta(seconds=5)), - ("1m", timedelta(days=31)), + ("0S", FlexibleDelta(count=0, unit="S", fuzzy=False)), + ("5S", FlexibleDelta(count=5, unit="S", fuzzy=False)), + ("1m", FlexibleDelta(count=1, unit="m", fuzzy=False)), + ("1mz", FlexibleDelta(count=1, unit="m", fuzzy=True)), ], ) -def test_int_or_interval(input, result): - assert int_or_interval(input) == result +def test_int_or_flexibledelta(input, result): + assert int_or_flexibledelta(input) == result @pytest.mark.parametrize( @@ -445,9 +446,9 @@ def test_int_or_interval(input, result): ("food", r"Value is neither an integer nor an interval:"), ], ) -def test_int_or_interval_time_unit(invalid_input, error_regex): +def test_int_or_flexibledelta_time_unit(invalid_input, error_regex): with pytest.raises(ArgumentTypeError) as exc: - int_or_interval(invalid_input) + int_or_flexibledelta(invalid_input) assert re.search(error_regex, exc.value.args[0])