feat: add clean_broken_retention CLI command

ChangeLog.md

@@ -1,3 +1,8 @@
+# unreleased
+
+NEW FEATURES
+- add `clean_broken_retention` CLI command — walks top-level of remote `path` and `object_disks_path` and batch-deletes (with retry) every entry that is not present in the live backup list and not matched by any `--keep=<glob>`. Dry-run by default; pass `--commit` to actually delete. Useful for cleaning up orphans left by failed retention runs
+
 # v2.6.43
 
 NEW FEATURES

Manual.md

@@ -310,6 +310,24 @@ OPTIONS:
    --config value, -c value                   Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG]
    --environment-override value, --env value  override any environment variable via CLI parameter
 
+```
+### CLI command - clean_broken_retention
+```
+NAME:
+   clickhouse-backup clean_broken_retention - Remove orphan entries under remote `path` and `object_disks_path` that are not in the live backup list
+
+USAGE:
+   clickhouse-backup clean_broken_retention [--commit] [--keep=glob ...]
+
+DESCRIPTION:
+   Walks top-level of remote `path` and `object_disks_path`, batch-deletes (with retry) every entry that is not a live backup and does not match any --keep glob. Runs in dry-run mode unless --commit is set.
+
+OPTIONS:
+   --config value, -c value                   Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG]
+   --environment-override value, --env value  override any environment variable via CLI parameter
+   --keep value                               Glob (path.Match syntax) of backup names to preserve in addition to live backups; can be passed multiple times
+   --commit                                   Actually delete orphans; without this flag the command only logs what would be deleted
+
 ```
 ### CLI command - watch
 ```
@@ -345,6 +363,14 @@ Look at the system.parts partition and partition_id fields for details https://c
    --delete, --delete-source, --delete-local                                       explicitly delete local backup during upload
 
 ```
+### CLI command - acvp
+```
+NAME:
+   clickhouse-backup acvp - Run ACVP wrapper protocol over stdin/stdout
+
+USAGE:
+   clickhouse-backup acvp
+```
 ### CLI command - server
 ```
 NAME:

ReadMe.md

@@ -972,6 +972,24 @@ OPTIONS:
    --config value, -c value                   Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG]
    --environment-override value, --env value  override any environment variable via CLI parameter
 
+```
+### CLI command - clean_broken_retention
+```
+NAME:
+   clickhouse-backup clean_broken_retention - Remove orphan entries under remote `path` and `object_disks_path` that are not in the live backup list
+
+USAGE:
+   clickhouse-backup clean_broken_retention [--commit] [--keep=glob ...]
+
+DESCRIPTION:
+   Walks top-level of remote `path` and `object_disks_path`, batch-deletes (with retry) every entry that is not a live backup and does not match any --keep glob. Runs in dry-run mode unless --commit is set.
+
+OPTIONS:
+   --config value, -c value                   Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG]
+   --environment-override value, --env value  override any environment variable via CLI parameter
+   --keep value                               Glob (path.Match syntax) of backup names to preserve in addition to live backups; can be passed multiple times
+   --commit                                   Actually delete orphans; without this flag the command only logs what would be deleted
+
 ```
 ### CLI command - watch
 ```
@@ -1007,6 +1025,14 @@ Look at the system.parts partition and partition_id fields for details https://c
    --delete, --delete-source, --delete-local                                       explicitly delete local backup during upload
 
 ```
+### CLI command - acvp
+```
+NAME:
+   clickhouse-backup acvp - Run ACVP wrapper protocol over stdin/stdout
+
+USAGE:
+   clickhouse-backup acvp
+```
 ### CLI command - server
 ```
 NAME:

cmd/clickhouse-backup/main.go

@@ -717,6 +717,30 @@ func main() {
 			},
 			Flags: cliapp.Flags,
 		},
+		{
+			Name:        "clean_broken_retention",
+			Usage:       "Remove orphan entries under remote `path` and `object_disks_path` that are not in the live backup list",
+			UsageText:   "clickhouse-backup clean_broken_retention [--commit] [--include=glob ...] [--exclude=glob ...]",
+			Description: "Walks top-level of remote `path` and `object_disks_path`, batch-deletes (with retry) every entry that is not a live backup and is not excluded by --exclude globs and is matched by --include globs (if provided). Object disk orphans are deleted in parallel with progress tracking. Pass --commit to actually delete; without it the command only logs what would be deleted.",
+			Action: func(c *cli.Context) error {
+				b := backup.NewBackuper(config.GetConfigFromCli(c))
+				return b.CleanBrokenRetention(status.NotFromAPI, c.StringSlice("include"), c.StringSlice("exclude"), c.Bool("commit"))
+			},
+			Flags: append(cliapp.Flags,
+				cli.StringSliceFlag{
+					Name:  "include",
+					Usage: "Glob (path.Match syntax) to scope cleanup only to backup names matching these patterns; can be passed multiple times; if omitted, all orphans are candidates",
+				},
+				cli.StringSliceFlag{
+					Name:  "exclude",
+					Usage: "Glob (path.Match syntax) of backup names to preserve even if they appear as orphans; can be passed multiple times",
+				},
+				cli.BoolFlag{
+					Name:  "commit",
+					Usage: "Actually delete orphans; without this flag the command only logs what would be deleted",
+				},
+		),
+	},
 
 		{
 			Name:        "watch",

generate_manual.sh

@@ -16,7 +16,9 @@ cmds=(
   clean
   clean_remote_broken
   clean_local_broken
+  clean_broken_retention
   watch
+  acvp
   server
 )
 for cmd in  ${cmds[@]}; do

pkg/backup/backuper.go

@@ -392,6 +392,24 @@ func (b *Backuper) buildEmbeddedLocationAZBLOB() string {
 	return fmt.Sprintf("DefaultEndpointsProtocol=%s;AccountName=%s;AccountKey=%s;BlobEndpoint=%s;", b.cfg.AzureBlob.EndpointSchema, b.cfg.AzureBlob.AccountName, b.cfg.AzureBlob.AccountKey, azblobBackupURL.String())
 }
 
+func (b *Backuper) getBackupPath() (string, error) {
+	switch b.cfg.General.RemoteStorage {
+	case "s3":
+		return b.cfg.S3.Path, nil
+	case "azblob":
+		return b.cfg.AzureBlob.Path, nil
+	case "gcs":
+		return b.cfg.GCS.Path, nil
+	case "cos":
+		return b.cfg.COS.Path, nil
+	case "ftp":
+		return b.cfg.FTP.Path, nil
+	case "sftp":
+		return b.cfg.SFTP.Path, nil
+	}
+	return "", errors.Errorf("getBackupPath: unsupported remote_storage: %s", b.cfg.General.RemoteStorage)
+}
+
 func (b *Backuper) getObjectDiskPath() (string, error) {
 	if b.cfg.General.RemoteStorage == "s3" {
 		return b.cfg.S3.ObjectDiskPath, nil