hotfix v2.6.44: retry batch delete in cleanBackupObjectDisks (#1356)

ChangeLog.md

@@ -1,8 +1,13 @@
-# unreleased
+# v2.7.0 unreleased
 
 NEW FEATURES
 - add `clean_broken_retention` CLI command — walks top-level of remote `path` and `object_disks_path` and batch-deletes (with retry) every entry that is not present in the live backup list and not matched by any `--keep=<glob>`. Dry-run by default; pass `--commit` to actually delete. Useful for cleaning up orphans left by failed retention runs
 
+# v2.6.44 (hotfix released only docker image)
+
+BUG FIXES
+- fix [1356](https://github.com/Altinity/clickhouse-backup/issues/1356), retry batch deletion in `cleanBackupObjectDisks` so transient errors (e.g. GCS 503) during retention no longer leave orphaned objects in `object_disks_path`
+
 # v2.6.43
 
 NEW FEATURES

cmd/clickhouse-backup/main.go

@@ -745,8 +745,8 @@ func main() {
 					Name:  "commit",
 					Usage: "Actually delete orphans; without this flag the command only logs what would be deleted",
 				},
-		),
-	},
+			),
+		},
 
 		{
 			Name:        "watch",

pkg/config/config.go

@@ -276,7 +276,7 @@ type ClickHouseConfig struct {
 	Debug                            bool              `yaml:"debug" envconfig:"CLICKHOUSE_DEBUG"`
 	// ForceRebalance triggers disk rebalancing during download even when the backup's disk
 	// name exists on the target, allowing distribution across JBOD disks under the same storage policy
-	ForceRebalance                   bool              `yaml:"force_rebalance" envconfig:"CLICKHOUSE_FORCE_REBALANCE"`
+	ForceRebalance bool `yaml:"force_rebalance" envconfig:"CLICKHOUSE_FORCE_REBALANCE"`
 }
 
 type APIConfig struct {

test/integration/fips_test.go

@@ -155,6 +155,12 @@ func TestFIPS(t *testing.T) {
 		r.Equal(0, len(inProgressActions), "inProgressActions=%+v", inProgressActions)
 		env.DockerExecNoError(r, "clickhouse", "pkill", "-n", "-f", "clickhouse-backup-fips")
 	}
+	// WTF =( why this works?
+	fipsOnlyBackupName := fmt.Sprintf("fips_only_backup_%d", rand.Int())
+	out, err := env.DockerExecOut("clickhouse", "bash", "-ce", "GODEBUG=fips140=only,x509debug=2,tls13=1 LOG_LEVEL=debug clickhouse-backup-fips -c /etc/clickhouse-backup/config-s3-fips.yml create_remote --tables="+t.Name()+".fips_table "+fipsOnlyBackupName+" 2>&1")
+	r.NoError(err, "FIPS-compatible clickhouse-backup -> clickhouse-server connection return error: %v, output: %s", err, out)
+
+	// WTF =( WHY THIS IS STOP WORKS, it was work in https://github.com/Altinity/clickhouse-backup/actions/runs/25434757510 and https://github.com/Altinity/clickhouse-backup/commit/92db680d1bdbc34855949634c140e3a11f8b96be =(
 	// P1: Test create_remote shall doesn't work with clickhouse-server which not compatible with FIPS
 	// Diagnostic: dump what clickhouse-server offers on 9440 so we can see why
 	// the fips140=only client succeeds or fails to negotiate a handshake.
@@ -166,11 +172,6 @@ func TestFIPS(t *testing.T) {
 	//log.Debug().Msgf("[fips-diag] supported TLS1.3 groups on clickhouse:9440:\n%s", curves)
 	//tssl, _ := env.DockerExecOut("clickhouse", "bash", "-ce", "rm -rf /tmp/testssl-ch* && /opt/testssl/testssl.sh -p -e --color 0 --disable-rating --quiet -n min --mode parallel clickhouse:9440 2>&1 | tail -120")
 	//log.Debug().Msgf("[fips-diag] testssl.sh against clickhouse:9440:\n%s", tssl)
-
-	fipsOnlyBackupName := fmt.Sprintf("fips_only_backup_%d", rand.Int())
-	out, err := env.DockerExecOut("clickhouse", "bash", "-ce", "GODEBUG=fips140=only,x509debug=2,tls13=1 LOG_LEVEL=debug clickhouse-backup-fips -c /etc/clickhouse-backup/config-s3-fips.yml create_remote --tables="+t.Name()+".fips_table "+fipsOnlyBackupName+" 2>&1")
-	r.NoError(err, "FIPS-compatible clickhouse-backup -> clickhouse-server connection return error: %v, output: %s", err, out)
-	// WTF =( WHY THIS IS STOP WORKS, it was work in https://github.com/Altinity/clickhouse-backup/actions/runs/25434757510 and https://github.com/Altinity/clickhouse-backup/commit/92db680d1bdbc34855949634c140e3a11f8b96be =(
 	// r.NoError(err, "FIPS-compatible clickhouse-backup -> FIPS-incompatible clickhouse-server connection shall return error: %s", out)
 	// r.Contains(out, "is not allowed in FIPS 140-only mode")
 	env.DockerExecNoError(r, "clickhouse", "bash", "-ce", "clickhouse-backup-fips -c /etc/clickhouse-backup/config-s3-fips.yml delete local "+fipsOnlyBackupName)