diff --git a/pyproject.toml b/pyproject.toml index 7d8f608bc5..ddde2db4c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ dependencies = [ "platformdirs >=2.6.0, <5.0.0; sys_platform != 'darwin'", # for others: 2.6+ works consistently. "argon2-cffi", "shtab>=1.8.0", + "dateutils" ] [project.optional-dependencies] diff --git a/requirements.d/development.txt b/requirements.d/development.txt index 96c20f0f18..c65071e0d3 100644 --- a/requirements.d/development.txt +++ b/requirements.d/development.txt @@ -11,6 +11,7 @@ pytest-xdist coverage[toml] pytest-cov pytest-benchmark +freezegun Cython pre-commit bandit[toml] diff --git a/src/borg/archiver/prune_cmd.py b/src/borg/archiver/prune_cmd.py index 2f18b485b8..da57ce7255 100644 --- a/src/borg/archiver/prune_cmd.py +++ b/src/borg/archiver/prune_cmd.py @@ -1,16 +1,17 @@ import argparse from collections import OrderedDict -from datetime import datetime, timezone, timedelta +from datetime import datetime, timezone import logging from operator import attrgetter import os +import itertools from ._common import with_repository, Highlander from ..archive import Archive from ..cache import Cache from ..constants import * # NOQA -from ..helpers import ArchiveFormatter, interval, sig_int, ProgressIndicatorPercent, CommandError, Error -from ..helpers import archivename_validator +from ..helpers import interval, int_or_flexibledelta, sig_int, archivename_validator +from ..helpers import ArchiveFormatter, ProgressIndicatorPercent, CommandError, Error from ..manifest import Manifest from ..logger import create_logger @@ -18,20 +19,23 @@ logger = create_logger() -def prune_within(archives, seconds, kept_because): - target = datetime.now(timezone.utc) - timedelta(seconds=seconds) - kept_counter = 0 - result = [] - for a in archives: - if a.ts > target: - kept_counter += 1 - kept_because[a.id] = ("within", kept_counter) - result.append(a) - return result +# The *_period_func group of functions create period grouping keys to group together archives falling within a certain +# period. Among archives in each of these groups, only the latest (by creation timestamp) is kept. -def default_period_func(pattern): +def unique_period_func(): + counter = itertools.count() + + def unique_values(_a): + """Group archives by an incrementing counter, practically making each archive a group of 1""" + return next(counter) + + return unique_values + + +def pattern_period_func(pattern): def inner(a): + """Group archives by extracting given strftime-pattern from their creation timestamp""" # compute in local timezone return a.ts.astimezone().strftime(pattern) @@ -39,6 +43,7 @@ def inner(a): def quarterly_13weekly_period_func(a): + """Group archives by extracting the ISO-8601 13-week quarter from their creation timestamp""" (year, week, _) = a.ts.astimezone().isocalendar() # local time if week <= 13: # Weeks containing Jan 4th to Mar 28th (leap year) or 29th- 91 (13*7) @@ -60,6 +65,7 @@ def quarterly_13weekly_period_func(a): def quarterly_3monthly_period_func(a): + """Group archives by extracting the 3-month quarter from their creation timestamp""" lt = a.ts.astimezone() # local time if lt.month <= 3: # 1-1 to 3-31 @@ -77,42 +83,64 @@ def quarterly_3monthly_period_func(a): PRUNING_PATTERNS = OrderedDict( [ - ("secondly", default_period_func("%Y-%m-%d %H:%M:%S")), - ("minutely", default_period_func("%Y-%m-%d %H:%M")), - ("hourly", default_period_func("%Y-%m-%d %H")), - ("daily", default_period_func("%Y-%m-%d")), - ("weekly", default_period_func("%G-%V")), - ("monthly", default_period_func("%Y-%m")), + # Each archive is considered for keeping + ("within", unique_period_func()), + ("last", unique_period_func()), + ("keep", unique_period_func()), + # Last archive (by creation timestamp) within period group is consiedered for keeping + ("secondly", pattern_period_func("%Y-%m-%d %H:%M:%S")), + ("minutely", pattern_period_func("%Y-%m-%d %H:%M")), + ("hourly", pattern_period_func("%Y-%m-%d %H")), + ("daily", pattern_period_func("%Y-%m-%d")), + ("weekly", pattern_period_func("%G-%V")), + ("monthly", pattern_period_func("%Y-%m")), ("quarterly_13weekly", quarterly_13weekly_period_func), ("quarterly_3monthly", quarterly_3monthly_period_func), - ("yearly", default_period_func("%Y")), + ("yearly", pattern_period_func("%Y")), ] ) -def prune_split(archives, rule, n, kept_because=None): - last = None +# Datetime cannot represent times before datetime.min, so a day is added to allow for time zone offset. +DATETIME_MIN_WITH_ZONE = datetime.min.replace(tzinfo=timezone.utc) + + +def prune_split(archives, rule, n_or_flexibledelta, base_timestamp, kept_because={}): + if isinstance(n_or_flexibledelta, int): + n, earliest_timestamp = n_or_flexibledelta, None + else: + n, earliest_timestamp = None, n_or_flexibledelta.subtract_from(base_timestamp, calendar=True) + + def can_retain(a, keep): + if n is not None: + return len(keep) < n + else: + return a.ts > earliest_timestamp + keep = [] - period_func = PRUNING_PATTERNS[rule] - if kept_because is None: - kept_because = {} - if n == 0: + if n == 0 or len(archives) == 0: return keep a = None - for a in sorted(archives, key=attrgetter("ts"), reverse=True): + last = None + period_func = PRUNING_PATTERNS[rule] + sorted_archives = sorted(archives, key=attrgetter("ts"), reverse=True) + for a in sorted_archives: + if not can_retain(a, keep): + break period = period_func(a) if period != last: last = period if a.id not in kept_because: keep.append(a) kept_because[a.id] = (rule, len(keep)) - if len(keep) == n: - break + # Keep oldest archive if we didn't reach the target retention count - if a is not None and len(keep) < n and a.id not in kept_because: + a = sorted_archives[-1] + if a is not None and a.id not in kept_because and can_retain(a, keep): keep.append(a) kept_because[a.id] = (rule + "[oldest]", len(keep)) + return keep @@ -120,8 +148,13 @@ class PruneMixIn: @with_repository(compatibility=(Manifest.Operation.DELETE,)) def do_prune(self, args, repository, manifest): """Prune archives according to specified rules.""" - if not any( - ( + if all( + # Needs explicit None-check to cover Falsey timedelta(0) + e is None + for e in ( + args.keep, + args.within, + args.last, args.secondly, args.minutely, args.hourly, @@ -131,11 +164,10 @@ def do_prune(self, args, repository, manifest): args.quarterly_13weekly, args.quarterly_3monthly, args.yearly, - args.within, ) ): raise CommandError( - 'At least one of the "keep-within", "keep-last", ' + 'At least one of the "keep", "keep-within", "keep-last", ' '"keep-secondly", "keep-minutely", "keep-hourly", "keep-daily", ' '"keep-weekly", "keep-monthly", "keep-13weekly", "keep-3monthly", ' 'or "keep-yearly" settings must be specified.' @@ -159,15 +191,12 @@ def do_prune(self, args, repository, manifest): # (, ) kept_because = {} - # find archives which need to be kept because of the keep-within rule - if args.within: - keep += prune_within(archives, args.within, kept_because) - + base_timestamp = datetime.now().astimezone() # find archives which need to be kept because of the various time period rules for rule in PRUNING_PATTERNS.keys(): - num = getattr(args, rule, None) - if num is not None: - keep += prune_split(archives, rule, num, kept_because) + n_or_flexibledelta = getattr(args, rule, None) + if n_or_flexibledelta is not None: + keep += prune_split(archives, rule, n_or_flexibledelta, base_timestamp, kept_because) to_delete = set(archives) - set(keep) with Cache(repository, manifest, iec=args.iec) as cache: @@ -312,81 +341,81 @@ def build_parser_prune(self, subparsers, common_parser, mid_common_parser): help="keep all archives within this time interval", ) subparser.add_argument( - "--keep-last", + "--keep-last", dest="last", type=int, action=Highlander, help="number of archives to keep" + ) + subparser.add_argument( + "--keep", + dest="keep", + type=int_or_flexibledelta, + action=Highlander, + help="number or time interval of archives to keep", + ) + subparser.add_argument( "--keep-secondly", dest="secondly", - type=int, - default=0, + type=int_or_flexibledelta, action=Highlander, - help="number of secondly archives to keep", + help="number or time interval of secondly archives to keep", ) subparser.add_argument( "--keep-minutely", dest="minutely", - type=int, - default=0, + type=int_or_flexibledelta, action=Highlander, - help="number of minutely archives to keep", + help="number or time interval of minutely archives to keep", ) subparser.add_argument( "-H", "--keep-hourly", dest="hourly", - type=int, - default=0, + type=int_or_flexibledelta, action=Highlander, - help="number of hourly archives to keep", + help="number or time interval of hourly archives to keep", ) subparser.add_argument( "-d", "--keep-daily", dest="daily", - type=int, - default=0, + type=int_or_flexibledelta, action=Highlander, - help="number of daily archives to keep", + help="number or time interval of daily archives to keep", ) subparser.add_argument( "-w", "--keep-weekly", dest="weekly", - type=int, - default=0, + type=int_or_flexibledelta, action=Highlander, - help="number of weekly archives to keep", + help="number or time interval of weekly archives to keep", ) subparser.add_argument( "-m", "--keep-monthly", dest="monthly", - type=int, - default=0, + type=int_or_flexibledelta, action=Highlander, - help="number of monthly archives to keep", + help="number or time interval of monthly archives to keep", ) quarterly_group = subparser.add_mutually_exclusive_group() quarterly_group.add_argument( "--keep-13weekly", dest="quarterly_13weekly", - type=int, - default=0, - help="number of quarterly archives to keep (13 week strategy)", + type=int_or_flexibledelta, + help="number or time interval of quarterly archives to keep (13 week strategy)", ) quarterly_group.add_argument( "--keep-3monthly", dest="quarterly_3monthly", - type=int, - default=0, - help="number of quarterly archives to keep (3 month strategy)", + type=int_or_flexibledelta, + help="number or time interval of quarterly archives to keep (3 month strategy)", ) subparser.add_argument( "-y", "--keep-yearly", dest="yearly", - type=int, - default=0, + type=int_or_flexibledelta, action=Highlander, - help="number of yearly archives to keep", + help="number or time interval of yearly archives to keep", ) define_archive_filters_group(subparser, sort_by=False, first_last=False) subparser.add_argument( diff --git a/src/borg/constants.py b/src/borg/constants.py index bbbbc56103..79ed33724b 100644 --- a/src/borg/constants.py +++ b/src/borg/constants.py @@ -142,7 +142,9 @@ EXIT_SIGNAL_BASE = 128 # terminated due to signal, rc = 128 + sig_no ISO_FORMAT_NO_USECS = "%Y-%m-%dT%H:%M:%S" +ISO_FORMAT_NO_USECS_ZONE = ISO_FORMAT_NO_USECS + "%z" ISO_FORMAT = ISO_FORMAT_NO_USECS + ".%f" +ISO_FORMAT_ZONE = ISO_FORMAT + "%z" DASHES = "-" * 78 diff --git a/src/borg/helpers/__init__.py b/src/borg/helpers/__init__.py index ca19f5c890..8afe551b9f 100644 --- a/src/borg/helpers/__init__.py +++ b/src/borg/helpers/__init__.py @@ -27,7 +27,7 @@ from .misc import ChunkIteratorFileWrapper, open_item, chunkit, iter_separated, ErrorIgnoringTextIOWrapper from .parseformat import bin_to_hex, hex_to_bin, safe_encode, safe_decode from .parseformat import text_to_json, binary_to_json, remove_surrogates, join_cmd -from .parseformat import eval_escapes, decode_dict, positive_int_validator, interval +from .parseformat import eval_escapes, decode_dict, positive_int_validator, interval, int_or_flexibledelta from .parseformat import PathSpec, SortBySpec, ChunkerParams, FilesCacheMode, partial_format, DatetimeWrapper from .parseformat import format_file_size, parse_file_size, FileSize from .parseformat import sizeof_fmt, sizeof_fmt_iec, sizeof_fmt_decimal, Location, text_validator @@ -43,7 +43,7 @@ from .process import popen_with_error_handling, is_terminal, prepare_subprocess_env, create_filter_process from .progress import ProgressIndicatorPercent, ProgressIndicatorMessage from .time import parse_timestamp, timestamp, safe_timestamp, safe_s, safe_ns, MAX_S, SUPPORT_32BIT_PLATFORMS -from .time import format_time, format_timedelta, OutputTimestamp, archive_ts_now +from .time import format_time, format_timedelta, OutputTimestamp, archive_ts_now, FlexibleDelta from .yes_no import yes, TRUISH, FALSISH, DEFAULTISH from .msgpack import is_slow_msgpack, is_supported_msgpack, get_limited_unpacker diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py index 193023df51..3d8977f621 100644 --- a/src/borg/helpers/parseformat.py +++ b/src/borg/helpers/parseformat.py @@ -24,7 +24,7 @@ from .errors import Error from .fs import get_keys_dir, make_path_safe from .msgpack import Timestamp -from .time import OutputTimestamp, format_time, safe_timestamp +from .time import OutputTimestamp, format_time, safe_timestamp, FlexibleDelta from .. import __version__ as borg_version from .. import __version_tuple__ as borg_version_tuple from ..constants import * # NOQA @@ -155,12 +155,24 @@ def interval(s): except ValueError: seconds = -1 - if seconds <= 0: - raise argparse.ArgumentTypeError(f'Invalid number "{number}": expected positive integer') + if seconds < 0: + raise argparse.ArgumentTypeError(f'Invalid number "{number}": expected nonnegative integer') return seconds +def int_or_flexibledelta(s): + try: + return int(s) + except ValueError: + pass + + try: + return FlexibleDelta.parse(s, fuzzyable=True) + except ValueError as e: + raise argparse.ArgumentTypeError(f"Value is neither an integer nor an interval: {e}") + + def ChunkerParams(s): params = s.strip().split(",") count = len(params) diff --git a/src/borg/helpers/time.py b/src/borg/helpers/time.py index 7dfe0d38c4..c58d901fad 100644 --- a/src/borg/helpers/time.py +++ b/src/borg/helpers/time.py @@ -1,6 +1,7 @@ import os import re from datetime import datetime, timezone, timedelta +from dateutil.relativedelta import relativedelta, MO def parse_timestamp(timestamp, tzinfo=timezone.utc): @@ -110,6 +111,114 @@ def format_timedelta(td): return txt +class FlexibleDelta: + """ + Represents an interval that _may_ respect the calendar with relation to week boundaries and exact month lengths. + """ + + _unit_relativedelta_map = { + "y": lambda count: relativedelta(years=count), + "m": lambda count: relativedelta(months=count), + "w": lambda count: relativedelta(weeks=count), + "d": lambda count: relativedelta(days=count), + "H": lambda count: relativedelta(hours=count), + "M": lambda count: relativedelta(minutes=count), + "S": lambda count: relativedelta(seconds=count), + } + + _unit_timedelta_map = { + "y": lambda count: timedelta(days=count * 365), + "m": lambda count: timedelta(days=count * 31), + "w": lambda count: timedelta(weeks=count), + "d": lambda count: timedelta(days=count), + "H": lambda count: timedelta(hours=count), + "M": lambda count: timedelta(minutes=count), + "S": lambda count: timedelta(seconds=count), + } + + _unit_fuzzy_round_func_map = { + "y": lambda earlier: relativedelta( + years=0 if earlier else 1, month=1, day=1, hour=0, minute=0, second=0, microsecond=0 + ), + "m": lambda earlier: relativedelta( + months=0 if earlier else 1, day=1, hour=0, minute=0, second=0, microsecond=0 + ), + "w": lambda earlier: relativedelta( + weekday=MO(-1), weeks=(0 if earlier else 1), hour=0, minute=0, second=0, microsecond=0 + ), + "d": lambda earlier: relativedelta(days=0 if earlier else 1, hour=0, minute=0, second=0, microsecond=0), + "H": lambda earlier: relativedelta(hours=0 if earlier else 1, minute=0, second=0, microsecond=0), + "M": lambda earlier: relativedelta(minutes=0 if earlier else 1, second=0, microsecond=0), + "S": lambda earlier: relativedelta(seconds=0 if earlier else 1, microsecond=0), + } + + def __init__(self, count, unit, fuzzy): + self.relativedelta = self._unit_relativedelta_map[unit](count) + self.timedelta = self._unit_timedelta_map[unit](count) + self.fuzzy_round_func = self._unit_fuzzy_round_func_map[unit] + self.fuzzy = fuzzy + + # For repr + self.count = count + self.unit = unit + + def __repr__(self): + return f'{self.__class__.__name__}(count={self.count}, unit="{self.unit}", fuzzy={self.fuzzy})' + + def __eq__(self, other): + if not isinstance(other, FlexibleDelta): + return NotImplemented + return (self.count, self.unit, self.fuzzy) == (other.count, other.unit, other.fuzzy) + + _interval_regex = re.compile(r"^(?P\d+)(?P[ymwdHMS])(?Pz)?$") + + @classmethod + def parse(cls, interval_string, fuzzyable=False): + """ + Parse interval string into FlexibleDelta on a form like "1w", "25d", or "6Hz" where the unit is one of "y" + (years), "m" (months), "w" (weeks), "d" (days), "H" (hours), "M" (minutes), "S" (seconds). A trailing "z" + indicates a fuzzy delta (if fuzzyable is True) which is calculated as "apply delta and then take either the + start or end of the current unit's interval" -- subtracting "1dz" means subtracting 1 day and then adjusting to + the start of that day. + """ + match = cls._interval_regex.search(interval_string) + + if not match: + raise ValueError(f"Invalid interval format: {interval_string}") + + count = int(match.group("count")) + unit = match.group("unit") + fuzzy = fuzzyable and match.group("fuzzy") is not None + + return cls(count, unit, fuzzy) + + @classmethod + def parse_fuzzy(cls, interval_string): + """ + Convenience fuzzy parser for easy use with argparse + """ + return cls.parse(interval_string, fuzzyable=True) + + def apply(self, base_ts, earlier=False, calendar=False): + scale = -1 if earlier else 1 + delta = self.relativedelta if calendar else self.timedelta + + offset_ts = base_ts + delta * scale + + if self.fuzzy: + # Offset further so that timestamp represents the start/end of its unit. e.g. "1yz" rounds result either up + # or down to nearest full year after applying initial offset (2025-07-31 - "1yz" = 2024-01-01). + offset_ts += self.fuzzy_round_func(earlier) + + return offset_ts + + def add_to(self, base_ts, calendar=False): + return self.apply(base_ts, earlier=False, calendar=calendar) + + def subtract_from(self, base_ts, calendar=False): + return self.apply(base_ts, earlier=True, calendar=calendar) + + def calculate_relative_offset(format_string, from_ts, earlier=False): """ Calculate an offset based on a relative marker (e.g., 7d for 7 days, 8m for 8 months). @@ -119,31 +228,7 @@ def calculate_relative_offset(format_string, from_ts, earlier=False): if from_ts is None: from_ts = archive_ts_now() - if format_string is not None: - offset_regex = re.compile(r"(?P\d+)(?P[ymwdHMS])") - match = offset_regex.search(format_string) - - if match: - unit = match.group("unit") - offset = int(match.group("offset")) - offset *= -1 if earlier else 1 - - if unit == "y": - return from_ts.replace(year=from_ts.year + offset) - elif unit == "m": - return offset_n_months(from_ts, offset) - elif unit == "w": - return from_ts + timedelta(days=offset * 7) - elif unit == "d": - return from_ts + timedelta(days=offset) - elif unit == "H": - return from_ts + timedelta(seconds=offset * 60 * 60) - elif unit == "M": - return from_ts + timedelta(seconds=offset * 60) - elif unit == "S": - return from_ts + timedelta(seconds=offset) - - raise ValueError(f"Invalid relative ts offset format: {format_string}") + return FlexibleDelta.parse(format_string).apply(from_ts, earlier=earlier, calendar=True) def offset_n_months(from_ts, n_months): diff --git a/src/borg/testsuite/archiver/prune_cmd_test.py b/src/borg/testsuite/archiver/prune_cmd_test.py index a18212c85e..efb2dc7f96 100644 --- a/src/borg/testsuite/archiver/prune_cmd_test.py +++ b/src/borg/testsuite/archiver/prune_cmd_test.py @@ -1,25 +1,22 @@ -import re -from datetime import datetime, timezone, timedelta - import pytest +import re +from datetime import datetime, timezone +from freezegun import freeze_time from ...constants import * # NOQA -from ...archiver.prune_cmd import prune_split, prune_within +from ...archiver.prune_cmd import prune_split +from ...helpers import CommandError from . import cmd, RK_ENCRYPTION, src_dir, generate_archiver_tests -from ...helpers import interval pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local,remote,binary") # NOQA -def _create_archive_ts(archiver, name, y, m, d, H=0, M=0, S=0): - cmd( - archiver, - "create", - "--timestamp", - datetime(y, m, d, H, M, S, 0).strftime(ISO_FORMAT_NO_USECS), # naive == local time / local tz - name, - src_dir, - ) +def _create_archive_dt(archiver, name, dt, tzinfo=None): + cmd(archiver, "create", "--timestamp", dt.replace(tzinfo=tzinfo).strftime(ISO_FORMAT_ZONE), name, src_dir) + + +def _create_archive_ts(archiver, name, y, m, d, H=0, M=0, S=0, us=0, tzinfo=None): + _create_archive_dt(archiver, name, datetime(y, m, d, H, M, S, us), tzinfo=tzinfo) def test_prune_repository(archivers, request): @@ -256,7 +253,7 @@ def test_prune_ignore_protected(archivers, request): cmd(archiver, "create", "archive3", archiver.input_path) output = cmd(archiver, "prune", "--list", "--keep-last=1", "--match-archives=sh:archive*") assert "archive1" not in output # @PROT archives are completely ignored. - assert re.search(r"Keeping archive \(rule: secondly #1\):\s+archive3", output) + assert re.search(r"Keeping archive \(rule: last #1\):\s+archive3", output) assert re.search(r"Pruning archive \(.*?\):\s+archive2", output) output = cmd(archiver, "repo-list") assert "archive1" in output # @PROT protected archive1 from deletion @@ -265,56 +262,14 @@ def test_prune_ignore_protected(archivers, request): class MockArchive: def __init__(self, ts, id): - self.ts = ts + # Real archive objects have UTC zoned timestamps + self.ts = ts.replace(tzinfo=timezone.utc) self.id = id def __repr__(self): return f"{self.id}: {self.ts.isoformat()}" -# This is the local timezone of the system running the tests. -# We need this e.g. to construct archive timestamps for the prune tests, -# because borg prune operates in the local timezone (it first converts the -# archive timestamp to the local timezone). So, if we want the y/m/d/h/m/s -# values which prune uses to be exactly the ones we give [and NOT shift them -# by tzoffset], we need to give the timestamps in the same local timezone. -# Please note that the timestamps in a real borg archive or manifest are -# stored in UTC timezone. -local_tz = datetime.now(tz=timezone.utc).astimezone(tz=None).tzinfo - - -def test_prune_within(): - def subset(lst, indices): - return {lst[i] for i in indices} - - def dotest(test_archives, within, indices): - for ta in test_archives, reversed(test_archives): - kept_because = {} - keep = prune_within(ta, interval(within), kept_because) - assert set(keep) == subset(test_archives, indices) - assert all("within" == kept_because[a.id][0] for a in keep) - - # 1 minute, 1.5 hours, 2.5 hours, 3.5 hours, 25 hours, 49 hours - test_offsets = [60, 90 * 60, 150 * 60, 210 * 60, 25 * 60 * 60, 49 * 60 * 60] - now = datetime.now(timezone.utc) - test_dates = [now - timedelta(seconds=s) for s in test_offsets] - test_archives = [MockArchive(date, i) for i, date in enumerate(test_dates)] - - dotest(test_archives, "15S", []) - dotest(test_archives, "2M", [0]) - dotest(test_archives, "1H", [0]) - dotest(test_archives, "2H", [0, 1]) - dotest(test_archives, "3H", [0, 1, 2]) - dotest(test_archives, "24H", [0, 1, 2, 3]) - dotest(test_archives, "26H", [0, 1, 2, 3, 4]) - dotest(test_archives, "2d", [0, 1, 2, 3, 4]) - dotest(test_archives, "50H", [0, 1, 2, 3, 4, 5]) - dotest(test_archives, "3d", [0, 1, 2, 3, 4, 5]) - dotest(test_archives, "1w", [0, 1, 2, 3, 4, 5]) - dotest(test_archives, "1m", [0, 1, 2, 3, 4, 5]) - dotest(test_archives, "1y", [0, 1, 2, 3, 4, 5]) - - @pytest.mark.parametrize( "rule,num_to_keep,expected_ids", [ @@ -334,26 +289,26 @@ def subset(lst, ids): archives = [ # years apart - MockArchive(datetime(2015, 1, 1, 10, 0, 0, tzinfo=local_tz), 1), - MockArchive(datetime(2016, 1, 1, 10, 0, 0, tzinfo=local_tz), 2), - MockArchive(datetime(2017, 1, 1, 10, 0, 0, tzinfo=local_tz), 3), + MockArchive(datetime(2015, 1, 1, 10, 0, 0), 1), + MockArchive(datetime(2016, 1, 1, 10, 0, 0), 2), + MockArchive(datetime(2017, 1, 1, 10, 0, 0), 3), # months apart - MockArchive(datetime(2017, 2, 1, 10, 0, 0, tzinfo=local_tz), 4), - MockArchive(datetime(2017, 3, 1, 10, 0, 0, tzinfo=local_tz), 5), + MockArchive(datetime(2017, 2, 1, 10, 0, 0), 4), + MockArchive(datetime(2017, 3, 1, 10, 0, 0), 5), # days apart - MockArchive(datetime(2017, 3, 2, 10, 0, 0, tzinfo=local_tz), 6), - MockArchive(datetime(2017, 3, 3, 10, 0, 0, tzinfo=local_tz), 7), - MockArchive(datetime(2017, 3, 4, 10, 0, 0, tzinfo=local_tz), 8), + MockArchive(datetime(2017, 3, 2, 10, 0, 0), 6), + MockArchive(datetime(2017, 3, 3, 10, 0, 0), 7), + MockArchive(datetime(2017, 3, 4, 10, 0, 0), 8), # minutes apart - MockArchive(datetime(2017, 10, 1, 9, 45, 0, tzinfo=local_tz), 9), - MockArchive(datetime(2017, 10, 1, 9, 55, 0, tzinfo=local_tz), 10), + MockArchive(datetime(2017, 10, 1, 9, 45, 0), 9), + MockArchive(datetime(2017, 10, 1, 9, 55, 0), 10), # seconds apart - MockArchive(datetime(2017, 10, 1, 10, 0, 1, tzinfo=local_tz), 11), - MockArchive(datetime(2017, 10, 1, 10, 0, 3, tzinfo=local_tz), 12), - MockArchive(datetime(2017, 10, 1, 10, 0, 5, tzinfo=local_tz), 13), + MockArchive(datetime(2017, 10, 1, 10, 0, 1), 11), + MockArchive(datetime(2017, 10, 1, 10, 0, 3), 12), + MockArchive(datetime(2017, 10, 1, 10, 0, 5), 13), ] kept_because = {} - keep = prune_split(archives, rule, num_to_keep, kept_because) + keep = prune_split(archives, rule, num_to_keep, None, kept_because) assert set(keep) == subset(archives, expected_ids) for item in keep: @@ -366,17 +321,17 @@ def subset(lst, ids): archives = [ # oldest backup, but not last in its year - MockArchive(datetime(2018, 1, 1, 10, 0, 0, tzinfo=local_tz), 1), + MockArchive(datetime(2018, 1, 1, 10, 0, 0), 1), # an interim backup - MockArchive(datetime(2018, 12, 30, 10, 0, 0, tzinfo=local_tz), 2), + MockArchive(datetime(2018, 12, 30, 10, 0, 0), 2), # year-end backups - MockArchive(datetime(2018, 12, 31, 10, 0, 0, tzinfo=local_tz), 3), - MockArchive(datetime(2019, 12, 31, 10, 0, 0, tzinfo=local_tz), 4), + MockArchive(datetime(2018, 12, 31, 10, 0, 0), 3), + MockArchive(datetime(2019, 12, 31, 10, 0, 0), 4), ] # Keep oldest when retention target can't otherwise be met kept_because = {} - keep = prune_split(archives, "yearly", 3, kept_because) + keep = prune_split(archives, "yearly", 3, None, kept_because) assert set(keep) == subset(archives, [1, 3, 4]) assert kept_because[1][0] == "yearly[oldest]" @@ -385,7 +340,7 @@ def subset(lst, ids): # Otherwise, prune it kept_because = {} - keep = prune_split(archives, "yearly", 2, kept_because) + keep = prune_split(archives, "yearly", 2, None, kept_because) assert set(keep) == subset(archives, [3, 4]) assert kept_because[3][0] == "yearly" @@ -396,7 +351,549 @@ def test_prune_split_no_archives(): archives = [] kept_because = {} - keep = prune_split(archives, "yearly", 3, kept_because) + keep = prune_split(archives, "yearly", 3, None, kept_because) assert keep == [] assert kept_because == {} + + +def test_prune_keep_last_same_second(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", "test1", src_dir) + cmd(archiver, "create", "test2", src_dir) + output = cmd(archiver, "prune", "--list", "--dry-run", "--keep-last=2") + # Both archives are kept even though they have the same timestamp to the second. Would previously have failed with + # old behavior of --keep-last. Archives sorted on seconds, order is undefined. + assert re.search(r"Keeping archive \(rule: last #\d\):\s+test1", output) + assert re.search(r"Keeping archive \(rule: last #\d\):\s+test2", output) + + +@freeze_time(datetime(2023, 12, 31, 23, 59, 59, tzinfo=None)) # Non-leap year ending on a Sunday +def test_prune_keep_int_or_flexibledelta(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_archive_ts(archiver, "test-1", 2023, 12, 31, 23, 59, 59) + _create_archive_ts(archiver, "test-2", 2023, 12, 31, 23, 59, 59) + _create_archive_ts(archiver, "test-3", 2023, 12, 31, 23, 59, 58) + for keep_arg in ["--keep=2", "--keep=1S"]: + output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg) + assert re.search(r"Keeping archive \(rule: keep #\d\):\s+test-1", output) + assert re.search(r"Keeping archive \(rule: keep #\d\):\s+test-2", output) + assert re.search(r"Would prune:\s+test-3", output) + + +@freeze_time(datetime(2023, 12, 31, 23, 59, 59, tzinfo=None)) +def test_prune_keep_int_or_flexibledelta_zero(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_archive_ts(archiver, "test", 2023, 12, 31, 23, 59, 59) + for keep_arg in ["--keep=0", "--keep=0S"]: + output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg) + assert re.search(r"Would prune:\s+test", output) + + +@freeze_time(datetime(2023, 12, 31, 23, 59, 59, tzinfo=None)) +def test_prune_keep_secondly_int_or_flexibledelta(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_archive_ts(archiver, "test-1", 2023, 12, 31, 23, 59, 58) + _create_archive_ts(archiver, "test-2", 2023, 12, 31, 23, 59, 57, 1) + _create_archive_ts(archiver, "test-3", 2023, 12, 31, 23, 59, 57) + _create_archive_ts(archiver, "test-4", 2023, 12, 31, 23, 59, 56, 999999) + for keep_arg in ["--keep-secondly=2", "--keep-secondly=2S"]: + output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg).splitlines() + assert re.search(r"Keeping archive \(rule: secondly #1\):\s+test-1", output.pop(0)) + assert re.search(r"Keeping archive \(rule: secondly #2\):\s+test-2", output.pop(0)) + assert re.search(r"Would prune:\s+test-3", output.pop(0)) + assert re.search(r"Would prune:\s+test-4", output.pop(0)) + + +@freeze_time(datetime(2023, 12, 31, 23, 59, 0, tzinfo=None)) +def test_prune_keep_minutely_int_or_flexibledelta(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_archive_ts(archiver, "test-1", 2023, 12, 31, 23, 58) + _create_archive_ts(archiver, "test-2", 2023, 12, 31, 23, 57, 1) + _create_archive_ts(archiver, "test-3", 2023, 12, 31, 23, 57) + _create_archive_ts(archiver, "test-4", 2023, 12, 31, 23, 56, 0, 1) # Last possible microsecond + _create_archive_ts(archiver, "test-5", 2023, 12, 31, 23, 56) + for keep_arg in ["--keep-minutely=3", "--keep-minutely=3M"]: + output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg).splitlines() + assert re.search(r"Keeping archive \(rule: minutely #1\):\s+test-1", output.pop(0)) + assert re.search(r"Keeping archive \(rule: minutely #2\):\s+test-2", output.pop(0)) + assert re.search(r"Would prune:\s+test-3", output.pop(0)) + assert re.search(r"Keeping archive \(rule: minutely #3\):\s+test-4", output.pop(0)) + assert re.search(r"Would prune:\s+test-5", output.pop(0)) + + +@freeze_time(datetime(2023, 12, 31, 23, 0, 0, tzinfo=None)) +def test_prune_keep_hourly_int_or_flexibledelta(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_archive_ts(archiver, "test-1", 2023, 12, 31, 22) + _create_archive_ts(archiver, "test-2", 2023, 12, 31, 21, us=1) + _create_archive_ts(archiver, "test-3", 2023, 12, 31, 21) + _create_archive_ts(archiver, "test-4", 2023, 12, 31, 20, us=1) # Last possible microsecond + _create_archive_ts(archiver, "test-5", 2023, 12, 31, 20) + for keep_arg in ["--keep-hourly=3", "--keep-hourly=3H"]: + output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg).splitlines() + assert re.search(r"Keeping archive \(rule: hourly #1\):\s+test-1", output.pop(0)) + assert re.search(r"Keeping archive \(rule: hourly #2\):\s+test-2", output.pop(0)) + assert re.search(r"Would prune:\s+test-3", output.pop(0)) + assert re.search(r"Keeping archive \(rule: hourly #3\):\s+test-4", output.pop(0)) + assert re.search(r"Would prune:\s+test-5", output.pop(0)) + + +@freeze_time(datetime(2023, 12, 31, 12, 0, 0, tzinfo=None)) +def test_prune_keep_daily_int_or_flexibledelta(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_archive_ts(archiver, "test-1", 2023, 12, 30) + _create_archive_ts(archiver, "test-2", 2023, 12, 29, S=1) + _create_archive_ts(archiver, "test-3", 2023, 12, 29) + _create_archive_ts(archiver, "test-4", 2023, 12, 28, 12, us=1) # Last possible microsecond + _create_archive_ts(archiver, "test-5", 2023, 12, 28, 12) + for keep_arg in ["--keep-daily=3", "--keep-daily=3d"]: + output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg).splitlines() + assert re.search(r"Keeping archive \(rule: daily #1\):\s+test-1", output.pop(0)) + assert re.search(r"Keeping archive \(rule: daily #2\):\s+test-2", output.pop(0)) + assert re.search(r"Would prune:\s+test-3", output.pop(0)) + assert re.search(r"Keeping archive \(rule: daily #3\):\s+test-4", output.pop(0)) + assert re.search(r"Would prune:\s+test-5", output.pop(0)) + + +@freeze_time(datetime(2023, 12, 31, 0, 0, 0, tzinfo=None)) +def test_prune_keep_weekly_int_or_flexibledelta(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_archive_ts(archiver, "test-1", 2023, 12, 24) + _create_archive_ts(archiver, "test-2", 2023, 12, 17, us=1) + _create_archive_ts(archiver, "test-3", 2023, 12, 17) + _create_archive_ts(archiver, "test-4", 2023, 12, 10, us=1) # Last possible microsecond + _create_archive_ts(archiver, "test-5", 2023, 12, 10) + for keep_arg in ["--keep-weekly=3", "--keep-weekly=3w"]: + output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg).splitlines() + assert re.search(r"Keeping archive \(rule: weekly #1\):\s+test-1", output.pop(0)) + assert re.search(r"Keeping archive \(rule: weekly #2\):\s+test-2", output.pop(0)) + assert re.search(r"Would prune:\s+test-3", output.pop(0)) + assert re.search(r"Keeping archive \(rule: weekly #3\):\s+test-4", output.pop(0)) + assert re.search(r"Would prune:\s+test-5", output.pop(0)) + + +@freeze_time(datetime(2023, 12, 31, 0, 0, 0, tzinfo=None)) +def test_prune_keep_monthly_int_or_flexibledelta(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_archive_ts(archiver, "test-1", 2023, 11, 30) + _create_archive_ts(archiver, "test-2", 2023, 10, 30, us=1) + _create_archive_ts(archiver, "test-3", 2023, 10, 30) + _create_archive_ts(archiver, "test-4", 2023, 9, 30, us=1) # Last possible microsecond + _create_archive_ts(archiver, "test-5", 2023, 9, 30) + for keep_arg in ["--keep-monthly=3", "--keep-monthly=3m"]: + output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg).splitlines() + assert re.search(r"Keeping archive \(rule: monthly #1\):\s+test-1", output.pop(0)) + assert re.search(r"Keeping archive \(rule: monthly #2\):\s+test-2", output.pop(0)) + assert re.search(r"Would prune:\s+test-3", output.pop(0)) + assert re.search(r"Keeping archive \(rule: monthly #3\):\s+test-4", output.pop(0)) + assert re.search(r"Would prune:\s+test-5", output.pop(0)) + + +# 2023-12-31 is Sunday, week 52. Makes these week calculations a little easier. +@freeze_time(datetime(2023, 12, 31, 0, 0, 0, tzinfo=None)) +def test_prune_keep_13weekly_int_or_flexibledelta(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_archive_ts(archiver, "test-1", 2023, 10, 1) + _create_archive_ts(archiver, "test-2", 2023, 7, 2, us=1) + _create_archive_ts(archiver, "test-3", 2023, 7, 2) + _create_archive_ts(archiver, "test-4", 2023, 4, 2, us=1) # Last possible microsecond + _create_archive_ts(archiver, "test-5", 2023, 4, 2) + for keep_arg in ["--keep-13weekly=3", "--keep-13weekly=39w"]: + output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg).splitlines() + assert re.search(r"Keeping archive \(rule: quarterly_13weekly #1\):\s+test-1", output.pop(0)) + assert re.search(r"Keeping archive \(rule: quarterly_13weekly #2\):\s+test-2", output.pop(0)) + assert re.search(r"Would prune:\s+test-3", output.pop(0)) + assert re.search(r"Keeping archive \(rule: quarterly_13weekly #3\):\s+test-4", output.pop(0)) + assert re.search(r"Would prune:\s+test-5", output.pop(0)) + + +@freeze_time(datetime(2023, 12, 31, 0, 0, 0, tzinfo=None)) +def test_prune_keep_3monthly_int_or_flexibledelta(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_archive_ts(archiver, "test-1", 2023, 9, 30) # 31st December - 3 calendar months + _create_archive_ts(archiver, "test-2", 2023, 6, 30, us=1) + _create_archive_ts(archiver, "test-3", 2023, 6, 30) + _create_archive_ts(archiver, "test-4", 2023, 3, 31, us=1) # Last possible microsecond + _create_archive_ts(archiver, "test-5", 2023, 3, 31) + for keep_arg in ["--keep-3monthly=3", f"--keep-3monthly={(datetime.now()-datetime(2023, 3, 31)).days}d"]: + output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg).splitlines() + assert re.search(r"Keeping archive \(rule: quarterly_3monthly #1\):\s+test-1", output.pop(0)) + assert re.search(r"Keeping archive \(rule: quarterly_3monthly #2\):\s+test-2", output.pop(0)) + assert re.search(r"Would prune:\s+test-3", output.pop(0)) + assert re.search(r"Keeping archive \(rule: quarterly_3monthly #3\):\s+test-4", output.pop(0)) + assert re.search(r"Would prune:\s+test-5", output.pop(0)) + + +@freeze_time(datetime(2023, 12, 31, 0, 0, 0, tzinfo=None)) +def test_prune_keep_yearly_int_or_flexibledelta(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_archive_ts(archiver, "test-1", 2022, 12, 31) + _create_archive_ts(archiver, "test-2", 2021, 12, 31, us=1) + _create_archive_ts(archiver, "test-3", 2021, 12, 31) + _create_archive_ts(archiver, "test-4", 2020, 12, 31, us=1) # Last possible microsecond + _create_archive_ts(archiver, "test-5", 2020, 12, 31) + for keep_arg in ["--keep-yearly=3", "--keep-yearly=3y"]: + output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg).splitlines() + assert re.search(r"Keeping archive \(rule: yearly #1\):\s+test-1", output.pop(0)) + assert re.search(r"Keeping archive \(rule: yearly #2\):\s+test-2", output.pop(0)) + assert re.search(r"Would prune:\s+test-3", output.pop(0)) + assert re.search(r"Keeping archive \(rule: yearly #3\):\s+test-4", output.pop(0)) + assert re.search(r"Would prune:\s+test-5", output.pop(0)) + + +@freeze_time(datetime(2025, 12, 24, 12, 0, 0, tzinfo=None)) +def test_prune_fuzzy_days(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_archive_ts(archiver, "test-1", 2025, 12, 23, 12, us=1) + _create_archive_ts(archiver, "test-2", 2025, 12, 23, 10) + output_nonfuzzy = cmd(archiver, "prune", "--list", "--dry-run", "--keep=1d").splitlines() + assert re.search(r"Keeping archive \(rule: keep #1\):\s+test-1", output_nonfuzzy.pop(0)) + assert re.search(r"Would prune:\s+test-2", output_nonfuzzy.pop(0)) + output_fuzzy = cmd(archiver, "prune", "--list", "--dry-run", "--keep=1dz").splitlines() + assert re.search(r"Keeping archive \(rule: keep #1\):\s+test-1", output_fuzzy.pop(0)) + + +@freeze_time(datetime(2025, 12, 24, 12, 0, 0, tzinfo=None)) # A Wednesday +def test_prune_fuzzy_weeks(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_archive_ts(archiver, "test-1", 2025, 12, 17, 12, us=1) # The previous Wednesday + _create_archive_ts(archiver, "test-2", 2025, 12, 15, 10) # Monday, start of previous week + output_nonfuzzy = cmd(archiver, "prune", "--list", "--dry-run", "--keep=1w").splitlines() + assert re.search(r"Keeping archive \(rule: keep #1\):\s+test-1", output_nonfuzzy.pop(0)) + assert re.search(r"Would prune:\s+test-2", output_nonfuzzy.pop(0)) + output_fuzzy = cmd(archiver, "prune", "--list", "--dry-run", "--keep=1wz").splitlines() + assert re.search(r"Keeping archive \(rule: keep #1\):\s+test-1", output_fuzzy.pop(0)) + assert re.search(r"Keeping archive \(rule: keep #2\):\s+test-2", output_fuzzy.pop(0)) + + +@freeze_time(datetime(2025, 12, 29, 12, 0, 0, tzinfo=None)) # Wednesday, end of year +def test_prune_repository_timedelta_everything_exact_vs_fuzzy(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + + # Archiving strategy: One archive is made at 12:00 every day and assumed to take 0 seconds. Pruning is done + # immediately afterwards. It is now 2025-12-29 12:00, the last Monday of the year, the 12:00 archive has been + # created and we are now running prune maintenance. + + # ---- daily: last 2 weeks ---- + # Assuming every archive is created at exactly 12:00:00 and we run prune also at exactly 12:00:00, + # "--keep-daily=2w" will keep THIRTEEN archives (assuming one every day) as the one from exactly 2 weeks ago falls + # one microsecond outside of range. + _create_archive_ts(archiver, "2025-12-29", 2025, 12, 29, 12) # daily 1 + _create_archive_ts(archiver, "2025-12-28", 2025, 12, 28, 12) # daily 2 + _create_archive_ts(archiver, "2025-12-27", 2025, 12, 27, 12) # daily 3 + _create_archive_ts(archiver, "2025-12-26", 2025, 12, 26, 12) # daily 4 + _create_archive_ts(archiver, "2025-12-25", 2025, 12, 25, 12) # daily 5 + _create_archive_ts(archiver, "2025-12-24", 2025, 12, 24, 12) # daily 6 + _create_archive_ts(archiver, "2025-12-23", 2025, 12, 23, 12) # daily 7 + _create_archive_ts(archiver, "2025-12-22", 2025, 12, 22, 12) # daily 8 + # Archiver not run on 2025-12-21 (daily 10) + _create_archive_ts(archiver, "2025-12-20", 2025, 12, 20, 12) # daily 10 + _create_archive_ts(archiver, "2025-12-19", 2025, 12, 19, 12) # daily 11 + _create_archive_ts(archiver, "2025-12-18", 2025, 12, 18, 12) # daily 12 + # Fuzzy by week includes the Tuesday and Wednesday 12:00 archives, as well as the one from the preceding Monday + _create_archive_ts(archiver, "2025-12-17", 2025, 12, 17, 12) # daily fuzzy 1 + _create_archive_ts(archiver, "2025-12-16", 2025, 12, 16, 12) # daily fuzzy 2 + _create_archive_ts(archiver, "2025-12-15", 2025, 12, 15, 12) # daily fuzzy 3 + # Fuzzy cutoff on crossing Sunday-Monday boundary. These would not exist in the exact case as stated in the example. + _create_archive_ts(archiver, "2025-12-14", 2025, 12, 14, 12) # daily fuzzy 4 + _create_archive_ts(archiver, "2025-12-13", 2025, 12, 13, 12) # daily fuzzy 5 + _create_archive_ts(archiver, "2025-12-12", 2025, 12, 12, 12) # daily fuzzy 6 + _create_archive_ts(archiver, "2025-12-11", 2025, 12, 11, 12) # daily fuzzy 7 + _create_archive_ts(archiver, "2025-12-10", 2025, 12, 10, 12) # daily fuzzy 8 + _create_archive_ts(archiver, "2025-12-09", 2025, 12, 9, 12) # daily fuzzy 9 + _create_archive_ts(archiver, "2025-12-08", 2025, 12, 8, 12) # daily fuzzy 10 + + # ---- weekly: last 6 months --- + # _create_archive_ts(archiver, "2025-12-14", 2025, 12, 14, 12) # weekly 1 (Duplicate with daily fuzzy 1) + _create_archive_ts(archiver, "2025-12-07", 2025, 12, 7, 12) # weekly 2 + _create_archive_ts(archiver, "2025-11-30", 2025, 11, 30, 12) # weekly 3 + _create_archive_ts(archiver, "2025-11-23", 2025, 11, 23, 12) # weekly 4 + _create_archive_ts(archiver, "2025-11-16", 2025, 11, 16, 12) # weekly 5 + _create_archive_ts(archiver, "2025-11-09", 2025, 11, 9, 12) # weekly 6 + _create_archive_ts(archiver, "2025-11-02", 2025, 11, 2, 12) # weekly 7 + _create_archive_ts(archiver, "2025-10-26", 2025, 10, 26, 12) # weekly 8 + _create_archive_ts(archiver, "2025-10-19", 2025, 10, 19, 12) # weekly 9 + _create_archive_ts(archiver, "2025-10-12", 2025, 10, 12, 12) # weekly 10 + _create_archive_ts(archiver, "2025-10-05", 2025, 10, 5, 12) # weekly 11 + _create_archive_ts( + archiver, "2025-09-27", 2025, 9, 27, 12 + ) # weekly 12 (archiver not run on Sunday 09-28, so 09-27 is kept) + _create_archive_ts(archiver, "2025-09-21", 2025, 9, 21, 12) # weekly 13 + _create_archive_ts(archiver, "2025-09-14", 2025, 9, 14, 12) # weekly 14 + _create_archive_ts(archiver, "2025-09-07", 2025, 9, 7, 12) # weekly 15 + _create_archive_ts(archiver, "2025-08-31", 2025, 8, 31, 12) # weekly 16 + _create_archive_ts(archiver, "2025-08-24", 2025, 8, 24, 12) # weekly 17 + _create_archive_ts(archiver, "2025-08-17", 2025, 8, 17, 12) # weekly 18 + _create_archive_ts(archiver, "2025-08-10", 2025, 8, 10, 12) # weekly 19 + _create_archive_ts(archiver, "2025-08-03", 2025, 8, 3, 12) # weekly 20 + _create_archive_ts(archiver, "2025-07-27", 2025, 7, 27, 12) # weekly 21 + _create_archive_ts(archiver, "2025-07-20", 2025, 7, 20, 12) # weekly 22 + _create_archive_ts(archiver, "2025-07-13", 2025, 7, 13, 12) # weekly 23 + _create_archive_ts(archiver, "2025-07-06", 2025, 7, 6, 12) # weekly 24 + # 12-31 minus 6 months is 06-30, meaning fuzzy month catches all backups in June + _create_archive_ts(archiver, "2025-06-29", 2025, 6, 29, 12) # weekly fuzzy 1 + _create_archive_ts(archiver, "2025-06-22", 2025, 6, 22, 12) # weekly fuzzy 2 + _create_archive_ts(archiver, "2025-06-15", 2025, 6, 15, 12) # weekly fuzzy 3 + _create_archive_ts(archiver, "2025-06-08", 2025, 6, 8, 12) # weekly fuzzy 4 + _create_archive_ts(archiver, "2025-06-01", 2025, 6, 1, 12) # weekly fuzzy 5 + + # ---- monthly: last year ---- + # Last Sunday each month (lowest granularity kept when monthly retention kick in) for the rest of the year + # _create_archive_ts(archiver, "2025-06-29", 2025, 6, 29, 12) # monthly 1 (Duplicate with weekly fuzzy 1) + _create_archive_ts(archiver, "2025-05-25", 2025, 5, 25, 12) # monhtly 2 + _create_archive_ts(archiver, "2025-04-27", 2025, 4, 27, 12) # monhtly 3 + _create_archive_ts(archiver, "2025-03-30", 2025, 3, 30, 12) # monhtly 4 + _create_archive_ts(archiver, "2025-02-23", 2025, 2, 23, 12) # monhtly 5 + _create_archive_ts(archiver, "2025-01-26", 2025, 1, 26, 12) # monhtly 6 + # Last Sunday each month through 2024 as per fuzzy year delta + _create_archive_ts(archiver, "2024-12-29", 2024, 12, 29, 12) # monthly fuzzy 1 + _create_archive_ts(archiver, "2024-11-24", 2024, 11, 24, 12) # monthly fuzzy 2 + _create_archive_ts(archiver, "2024-10-27", 2024, 10, 27, 12) # monthly fuzzy 3 + _create_archive_ts(archiver, "2024-09-29", 2024, 9, 29, 12) # monthly fuzzy 4 + _create_archive_ts(archiver, "2024-08-25", 2024, 8, 25, 12) # monthly fuzzy 5 + _create_archive_ts(archiver, "2024-07-28", 2024, 7, 28, 12) # monthly fuzzy 6 + _create_archive_ts(archiver, "2024-06-30", 2024, 6, 30, 12) # monthly fuzzy 7 + _create_archive_ts(archiver, "2024-05-26", 2024, 5, 26, 12) # monthly fuzzy 8 + _create_archive_ts(archiver, "2024-04-28", 2024, 4, 28, 12) # monthly fuzzy 9 + _create_archive_ts(archiver, "2024-03-31", 2024, 3, 31, 12) # monthly fuzzy 10 + _create_archive_ts(archiver, "2024-02-25", 2024, 2, 25, 12) # monthly fuzzy 11 + _create_archive_ts(archiver, "2024-01-28", 2024, 1, 28, 12) # monthly fuzzy 12 + + # ---- yearly: exactly 3 ----- + # _create_archive_ts(archiver, "2024-12-29", 2024, 12, 29, 12) # yearly 1 (Duplicate with monthly fuzzy 1) + _create_archive_ts(archiver, "2023-12-31", 2023, 12, 31, 12) # yearly 2 + _create_archive_ts(archiver, "2022-12-25", 2022, 12, 25, 12) # yearly 3 + _create_archive_ts(archiver, "2021-12-26", 2021, 12, 26, 12) # yearly 4 + + # ---- Exact deltas ------------------------------------------------ + output_exact = cmd( + archiver, + "prune", + "--list", + "--dry-run", + "--keep-daily=2w", + "--keep-weekly=6m", + "--keep-monthly=1y", + "--keep-yearly=3", + ) + print("Prune output (exact):") + print(output_exact) + output_exact = list(reversed(output_exact.splitlines())) + + # Daily within 2 weeks + assert re.search(r"Keeping archive \(rule: daily #1\):\s+2025-12-29", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: daily #2\):\s+2025-12-28", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: daily #3\):\s+2025-12-27", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: daily #4\):\s+2025-12-26", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: daily #5\):\s+2025-12-25", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: daily #6\):\s+2025-12-24", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: daily #7\):\s+2025-12-23", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: daily #8\):\s+2025-12-22", output_exact.pop()) + # Nothing on 2025-12-21 + assert re.search(r"Keeping archive \(rule: daily #9\):\s+2025-12-20", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: daily #10\):\s+2025-12-19", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: daily #11\):\s+2025-12-18", output_exact.pop()) + # Would match with daily by fuzzy week + assert re.search(r"Keeping archive \(rule: daily #12\):\s+2025-12-17", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: daily #13\):\s+2025-12-16", output_exact.pop()) + assert re.search(r"Would prune:\s+2025-12-15", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #1\):\s+2025-12-14", output_exact.pop()) + assert re.search(r"Would prune:\s+2025-12-13", output_exact.pop()) + assert re.search(r"Would prune:\s+2025-12-12", output_exact.pop()) + assert re.search(r"Would prune:\s+2025-12-11", output_exact.pop()) + assert re.search(r"Would prune:\s+2025-12-10", output_exact.pop()) + assert re.search(r"Would prune:\s+2025-12-09", output_exact.pop()) + assert re.search(r"Would prune:\s+2025-12-08", output_exact.pop()) + + # Weekly within 6 months + assert re.search(r"Keeping archive \(rule: weekly #2\):\s+2025-12-07", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #3\):\s+2025-11-30", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #4\):\s+2025-11-23", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #5\):\s+2025-11-16", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #6\):\s+2025-11-09", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #7\):\s+2025-11-02", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #8\):\s+2025-10-26", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #9\):\s+2025-10-19", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #10\):\s+2025-10-12", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #11\):\s+2025-10-05", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #12\):\s+2025-09-27", output_exact.pop()) # (Not 09-28) + assert re.search(r"Keeping archive \(rule: weekly #13\):\s+2025-09-21", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #14\):\s+2025-09-14", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #15\):\s+2025-09-07", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #16\):\s+2025-08-31", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #17\):\s+2025-08-24", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #18\):\s+2025-08-17", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #19\):\s+2025-08-10", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #20\):\s+2025-08-03", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #21\):\s+2025-07-27", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #22\):\s+2025-07-20", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #23\):\s+2025-07-13", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: weekly #24\):\s+2025-07-06", output_exact.pop()) + # Would match with weekly by fuzzy month + assert re.search(r"Keeping archive \(rule: monthly #1\):\s+2025-06-29", output_exact.pop()) + assert re.search(r"Would prune:\s+2025-06-22", output_exact.pop()) + assert re.search(r"Would prune:\s+2025-06-15", output_exact.pop()) + assert re.search(r"Would prune:\s+2025-06-08", output_exact.pop()) + assert re.search(r"Would prune:\s+2025-06-01", output_exact.pop()) + + # Monthly within 1 year + assert re.search(r"Keeping archive \(rule: monthly #2\):\s+2025-05-25", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: monthly #3\):\s+2025-04-27", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: monthly #4\):\s+2025-03-30", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: monthly #5\):\s+2025-02-23", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: monthly #6\):\s+2025-01-26", output_exact.pop()) + # Would match with monthly by fuzzy year + assert re.search(r"Keeping archive \(rule: yearly #1\):\s+2024-12-29", output_exact.pop()) + assert re.search(r"Would prune:\s+2024-11-24", output_exact.pop()) + assert re.search(r"Would prune:\s+2024-10-27", output_exact.pop()) + assert re.search(r"Would prune:\s+2024-09-29", output_exact.pop()) + assert re.search(r"Would prune:\s+2024-08-25", output_exact.pop()) + assert re.search(r"Would prune:\s+2024-07-28", output_exact.pop()) + assert re.search(r"Would prune:\s+2024-06-30", output_exact.pop()) + assert re.search(r"Would prune:\s+2024-05-26", output_exact.pop()) + assert re.search(r"Would prune:\s+2024-04-28", output_exact.pop()) + assert re.search(r"Would prune:\s+2024-03-31", output_exact.pop()) + assert re.search(r"Would prune:\s+2024-02-25", output_exact.pop()) + assert re.search(r"Would prune:\s+2024-01-28", output_exact.pop()) + + # Yearly x3 + assert re.search(r"Keeping archive \(rule: yearly #2\):\s+2023-12-31", output_exact.pop()) + assert re.search(r"Keeping archive \(rule: yearly #3\):\s+2022-12-25", output_exact.pop()) + # Would match if yearly #1 matched with fuzzy by month instead + assert re.search(r"Would prune:\s+2021-12-26", output_exact.pop()) + assert len(output_exact) == 0 + + # ---- Exact deltas ------------------------------------------------ + output_fuzzy = cmd( + archiver, + "prune", + "--list", + "--dry-run", + "--keep-daily=2wz", + "--keep-weekly=6mz", + "--keep-monthly=1yz", + "--keep-yearly=3", + ) + print("Prune output (fuzzy):") + print(output_fuzzy) + output_fuzzy = list(reversed(output_fuzzy.splitlines())) + + # Daily within 2 weeks + assert re.search(r"Keeping archive \(rule: daily #1\):\s+2025-12-29", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: daily #2\):\s+2025-12-28", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: daily #3\):\s+2025-12-27", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: daily #4\):\s+2025-12-26", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: daily #5\):\s+2025-12-25", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: daily #6\):\s+2025-12-24", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: daily #7\):\s+2025-12-23", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: daily #8\):\s+2025-12-22", output_fuzzy.pop()) + # Nothing on 2025-12-21 + assert re.search(r"Keeping archive \(rule: daily #9\):\s+2025-12-20", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: daily #10\):\s+2025-12-19", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: daily #11\):\s+2025-12-18", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: daily #12\):\s+2025-12-17", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: daily #13\):\s+2025-12-16", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: daily #14\):\s+2025-12-15", output_fuzzy.pop()) + + # Weekly within 6 months + assert re.search(r"Keeping archive \(rule: weekly #1\):\s+2025-12-14", output_fuzzy.pop()) + assert re.search(r"Would prune:\s+2025-12-13", output_fuzzy.pop()) + assert re.search(r"Would prune:\s+2025-12-12", output_fuzzy.pop()) + assert re.search(r"Would prune:\s+2025-12-11", output_fuzzy.pop()) + assert re.search(r"Would prune:\s+2025-12-10", output_fuzzy.pop()) + assert re.search(r"Would prune:\s+2025-12-09", output_fuzzy.pop()) + assert re.search(r"Would prune:\s+2025-12-08", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #2\):\s+2025-12-07", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #3\):\s+2025-11-30", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #4\):\s+2025-11-23", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #5\):\s+2025-11-16", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #6\):\s+2025-11-09", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #7\):\s+2025-11-02", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #8\):\s+2025-10-26", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #9\):\s+2025-10-19", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #10\):\s+2025-10-12", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #11\):\s+2025-10-05", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #12\):\s+2025-09-27", output_fuzzy.pop()) # (Not 09-28) + assert re.search(r"Keeping archive \(rule: weekly #13\):\s+2025-09-21", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #14\):\s+2025-09-14", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #15\):\s+2025-09-07", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #16\):\s+2025-08-31", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #17\):\s+2025-08-24", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #18\):\s+2025-08-17", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #19\):\s+2025-08-10", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #20\):\s+2025-08-03", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #21\):\s+2025-07-27", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #22\):\s+2025-07-20", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #23\):\s+2025-07-13", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #24\):\s+2025-07-06", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #25\):\s+2025-06-29", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #26\):\s+2025-06-22", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #27\):\s+2025-06-15", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #28\):\s+2025-06-08", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: weekly #29\):\s+2025-06-01", output_fuzzy.pop()) + + # Monthly within 1 year + assert re.search(r"Keeping archive \(rule: monthly #1\):\s+2025-05-25", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: monthly #2\):\s+2025-04-27", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: monthly #3\):\s+2025-03-30", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: monthly #4\):\s+2025-02-23", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: monthly #5\):\s+2025-01-26", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: monthly #6\):\s+2024-12-29", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: monthly #7\):\s+2024-11-24", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: monthly #8\):\s+2024-10-27", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: monthly #9\):\s+2024-09-29", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: monthly #10\):\s+2024-08-25", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: monthly #11\):\s+2024-07-28", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: monthly #12\):\s+2024-06-30", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: monthly #13\):\s+2024-05-26", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: monthly #14\):\s+2024-04-28", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: monthly #15\):\s+2024-03-31", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: monthly #16\):\s+2024-02-25", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: monthly #17\):\s+2024-01-28", output_fuzzy.pop()) + + # Yearly x3 + assert re.search(r"Keeping archive \(rule: yearly #1\):\s+2023-12-31", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: yearly #2\):\s+2022-12-25", output_fuzzy.pop()) + assert re.search(r"Keeping archive \(rule: yearly #3\):\s+2021-12-26", output_fuzzy.pop()) + assert len(output_fuzzy) == 0 + + +def test_prune_no_args(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + with pytest.raises(CommandError) as error: + cmd(archiver, "prune") + output = str(error.value) + assert re.search(r"At least one of the .* settings must be specified.", output) + assert re.search(r"keep(?!-)", output) + flags = [ + "last", + "within", + "secondly", + "minutely", + "hourly", + "daily", + "weekly", + "monthly", + "yearly", + "13weekly", + "3monthly", + ] + for flag in flags: + assert f"keep-{flag}" in output diff --git a/src/borg/testsuite/helpers/parseformat_test.py b/src/borg/testsuite/helpers/parseformat_test.py index 85600c819e..a0637fc474 100644 --- a/src/borg/testsuite/helpers/parseformat_test.py +++ b/src/borg/testsuite/helpers/parseformat_test.py @@ -1,5 +1,6 @@ import base64 import os +import re from argparse import ArgumentTypeError from datetime import datetime, timezone @@ -16,6 +17,7 @@ format_file_size, parse_file_size, interval, + int_or_flexibledelta, partial_format, clean_lines, format_line, @@ -25,7 +27,7 @@ eval_escapes, ChunkerParams, ) -from ...helpers.time import format_timedelta, parse_timestamp +from ...helpers.time import format_timedelta, parse_timestamp, FlexibleDelta def test_bin_to_hex(): @@ -376,6 +378,7 @@ def test_format_timedelta(): @pytest.mark.parametrize( "timeframe, num_secs", [ + ("0S", 0), ("5S", 5), ("2M", 2 * 60), ("1H", 60 * 60), @@ -392,9 +395,9 @@ def test_interval(timeframe, num_secs): @pytest.mark.parametrize( "invalid_interval, error_tuple", [ - ("H", ('Invalid number "": expected positive integer',)), - ("-1d", ('Invalid number "-1": expected positive integer',)), - ("food", ('Invalid number "foo": expected positive integer',)), + ("H", ('Invalid number "": expected nonnegative integer',)), + ("-1d", ('Invalid number "-1": expected nonnegative integer',)), + ("food", ('Invalid number "foo": expected nonnegative integer',)), ], ) def test_interval_time_unit(invalid_interval, error_tuple): @@ -403,10 +406,50 @@ def test_interval_time_unit(invalid_interval, error_tuple): assert exc.value.args == error_tuple -def test_interval_number(): +@pytest.mark.parametrize( + "invalid_input, error_regex", + [ + ("x", r'^Unexpected time unit "x": choose from'), + ("-1t", r'^Unexpected time unit "t": choose from'), + ("fool", r'^Unexpected time unit "l": choose from'), + ("abc", r'^Unexpected time unit "c": choose from'), + (" abc ", r'^Unexpected time unit " ": choose from'), + ], +) +def test_interval_invalid_time_format(invalid_input, error_regex): + with pytest.raises(ArgumentTypeError) as exc: + interval(invalid_input) + assert re.search(error_regex, exc.value.args[0]) + + +@pytest.mark.parametrize( + "input, result", + [ + ("0", 0), + ("5", 5), + (" 999 ", 999), + ("0S", FlexibleDelta(count=0, unit="S", fuzzy=False)), + ("5S", FlexibleDelta(count=5, unit="S", fuzzy=False)), + ("1m", FlexibleDelta(count=1, unit="m", fuzzy=False)), + ("1mz", FlexibleDelta(count=1, unit="m", fuzzy=True)), + ], +) +def test_int_or_flexibledelta(input, result): + assert int_or_flexibledelta(input) == result + + +@pytest.mark.parametrize( + "invalid_input, error_regex", + [ + ("H", r"Value is neither an integer nor an interval:"), + ("-1d", r"Value is neither an integer nor an interval:"), + ("food", r"Value is neither an integer nor an interval:"), + ], +) +def test_int_or_flexibledelta_time_unit(invalid_input, error_regex): with pytest.raises(ArgumentTypeError) as exc: - interval("5") - assert exc.value.args == ('Unexpected time unit "5": choose from y, m, w, d, H, M, S',) + int_or_flexibledelta(invalid_input) + assert re.search(error_regex, exc.value.args[0]) def test_parse_timestamp(): diff --git a/src/borg/testsuite/helpers/time_test.py b/src/borg/testsuite/helpers/time_test.py index 7dcffc8278..edd96d4563 100644 --- a/src/borg/testsuite/helpers/time_test.py +++ b/src/borg/testsuite/helpers/time_test.py @@ -1,7 +1,8 @@ import pytest from datetime import datetime, timezone +from zoneinfo import ZoneInfo -from ...helpers.time import safe_ns, safe_s, SUPPORT_32BIT_PLATFORMS +from ...helpers.time import safe_ns, safe_s, SUPPORT_32BIT_PLATFORMS, FlexibleDelta def utcfromtimestamp(timestamp): @@ -36,3 +37,75 @@ def test_safe_timestamps(): utcfromtimestamp(beyond_y10k) assert utcfromtimestamp(safe_s(beyond_y10k)) > datetime(2262, 1, 1) assert utcfromtimestamp(safe_ns(beyond_y10k) / 1000000000) > datetime(2262, 1, 1) + + +def test_flexible_delta(): + # Year delta across leap year + delta_1year = FlexibleDelta.parse("1y") + dt_2023 = datetime(year=2023, month=12, day=24) + assert delta_1year.add_to(dt_2023) == datetime(year=2024, month=12, day=23) + assert delta_1year.add_to(dt_2023, calendar=True) == datetime(year=2024, month=12, day=24) + assert delta_1year.subtract_from(dt_2023) == datetime(year=2022, month=12, day=24) + assert delta_1year.subtract_from(dt_2023, calendar=True) == datetime(year=2022, month=12, day=24) + + delta_1month = FlexibleDelta.parse("1m") + + # Month delta across leap day + dt_leap_february = datetime(year=2024, month=2, day=20) + assert delta_1month.add_to(dt_leap_february) == datetime(year=2024, month=3, day=22) + assert delta_1month.add_to(dt_leap_february, calendar=True) == datetime(year=2024, month=3, day=20) + assert delta_1month.subtract_from(dt_leap_february) == datetime(year=2024, month=1, day=20) + assert delta_1month.subtract_from(dt_leap_february, calendar=True) == datetime(year=2024, month=1, day=20) + + # Month delta across non-leap day February + dt_nonleap_february = datetime(year=2025, month=2, day=20) + assert delta_1month.add_to(dt_nonleap_february) == datetime(year=2025, month=3, day=23) + assert delta_1month.add_to(dt_nonleap_february, calendar=True) == datetime(year=2025, month=3, day=20) + + # Month delta across 31-day month boundary + dt_july = datetime(year=2025, month=7, day=20) + assert delta_1month.add_to(dt_july) == datetime(year=2025, month=8, day=20) + assert delta_1month.add_to(dt_july, calendar=True) == datetime(year=2025, month=8, day=20) + + # Month delta across 30-day month boundary + dt_july = datetime(year=2025, month=6, day=20) + assert delta_1month.add_to(dt_july) == datetime(year=2025, month=7, day=21) + assert delta_1month.add_to(dt_july, calendar=True) == datetime(year=2025, month=7, day=20) + + # Day delta across summe/winter time change + delta_1day = FlexibleDelta.parse("1d") + dt_oslo_wintertime = datetime(year=2026, month=3, day=28, hour=10, tzinfo=ZoneInfo("Europe/Oslo")) + assert delta_1day.add_to(dt_oslo_wintertime) == datetime( + year=2026, month=3, day=29, hour=10, tzinfo=ZoneInfo("Europe/Oslo") + ) + assert delta_1day.add_to(dt_oslo_wintertime, calendar=True) == datetime( + year=2026, month=3, day=29, hour=10, tzinfo=ZoneInfo("Europe/Oslo") + ) + + # Fuzzy day delta + delta_fuzzy_1day = FlexibleDelta.parse("1dz", fuzzyable=True) + dt = datetime(year=2026, month=1, day=1, hour=12) + assert delta_fuzzy_1day.add_to(dt) == datetime(year=2026, month=1, day=3, hour=0) # +1day -> end of day + assert delta_fuzzy_1day.subtract_from(dt) == datetime(year=2025, month=12, day=31, hour=0) # -1day -> start of day + + # Fuzzy month delta + delta_fuzzy_1month = FlexibleDelta.parse("1mz", fuzzyable=True) + dt = datetime(year=2026, month=1, day=30) + assert delta_fuzzy_1month.add_to(dt) == datetime( + year=2026, month=4, day=1 + ) # End of next month (31 days) (for inclusive check, ie. +1μs) + assert delta_fuzzy_1month.add_to(dt, calendar=True) == datetime( + year=2026, month=3, day=1 + ) # End of next month (for inclusive check, ie. +1μs) + assert delta_fuzzy_1month.subtract_from(dt) == datetime(year=2025, month=12, day=1) # Start of previous month + + # Fuzzy week delta + delta_fuzzy_1week = FlexibleDelta.parse("1wz", fuzzyable=True) + dt = datetime(year=2024, month=2, day=28) # A Wednesday, leap year + assert delta_fuzzy_1week.add_to(dt) == datetime( + year=2024, month=3, day=11 + ) # End of next week (for inclusive check, ie. +1μs) + assert delta_fuzzy_1week.add_to(dt, calendar=True) == datetime( + year=2024, month=3, day=11 + ) # End of next week (for inclusive check, ie. +1μs) + assert delta_fuzzy_1week.subtract_from(dt) == datetime(year=2024, month=2, day=19) # Start of previous week