From c2f7b496e26a919e3219370dda9a32a835603b70 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=90=B4=E6=9D=A8=E5=B8=86?=
 <39647285+leno23@users.noreply.github.com>
Date: Sun, 17 May 2026 16:03:58 +0800
Subject: [PATCH] fix: skip missing remote table metadata

Treat object-store metadata 404s as permanent so download skips the
affected table immediately instead of exhausting the retry backoff.
---
 pkg/backup/download.go      | 28 ++++++++++++++++++++++++++++
 pkg/backup/download_test.go | 17 +++++++++++++++++
 2 files changed, 45 insertions(+)

diff --git a/pkg/backup/download.go b/pkg/backup/download.go
index c469e488a..69a808f77 100644
--- a/pkg/backup/download.go
+++ b/pkg/backup/download.go
@@ -40,6 +40,18 @@ var (
 	ErrBackupIsAlreadyExists = errors.New("backup is already exists")
 )
 
+func isRemoteMetadataNotFound(err error) bool {
+	if err == nil {
+		return false
+	}
+	message := strings.ToLower(err.Error())
+	return strings.Contains(message, "doesn't exist") ||
+		strings.Contains(message, "key not found") ||
+		strings.Contains(message, "nosuchkey") ||
+		strings.Contains(message, "statuscode 404") ||
+		strings.Contains(message, "statuscode: 404")
+}
+
 func (b *Backuper) Download(backupName string, tablePattern string, partitions []string, schemaOnly, rbacOnly, configsOnly, namedCollectionsOnly, resume bool, hardlinkExistsFiles bool, backupVersion string, commandId int) error {
 	if pidCheckErr := pidlock.CheckAndCreatePidFile(backupName, "download"); pidCheckErr != nil {
 		return errors.WithMessage(pidCheckErr, "CheckAndCreatePidFile")
@@ -487,10 +499,15 @@ func (b *Backuper) downloadTableMetadata(ctx context.Context, backupName string,
 			}
 		}
 		var tmBody []byte
+		metadataNotFound := false
 		retry := retrier.New(retrier.ExponentialBackoff(b.cfg.General.RetriesOnFailure, common.AddRandomJitter(b.cfg.General.RetriesDuration, b.cfg.General.RetriesJitter)), b)
 		err := retry.RunCtx(ctx, func(ctx context.Context) error {
 			tmReader, err := b.dst.GetFileReader(ctx, remoteMetadataFile)
 			if err != nil {
+				if isRemoteMetadataNotFound(err) {
+					metadataNotFound = true
+					return nil
+				}
 				return errors.Wrapf(err, "can't GetFileReader(%s) error", remoteMetadataFile)
 			}
 			tmBody, err = io.ReadAll(tmReader)
@@ -503,11 +520,22 @@ func (b *Backuper) downloadTableMetadata(ctx context.Context, backupName string,
 			}
 			return nil
 		})
+		// Missing metadata is permanent: do not burn retries, and skip the missing table.
+		if metadataNotFound {
+			logger.Warn().Str("remoteMetadataFile", remoteMetadataFile).Msg("metadata file not found on remote, skipping")
+			if strings.HasSuffix(localMetadataFile, ".json") {
+				return nil, size, nil
+			}
+			continue
+		}
 		// sql file could be not present in incremental backup
 		if err != nil && strings.HasSuffix(localMetadataFile, ".sql") {
 			log.Warn().Str("localMetadataFile", localMetadataFile).Err(err).Send()
 			continue
 		}
+		if err != nil {
+			return nil, 0, err
+		}
 
 		if err = os.MkdirAll(path.Dir(localMetadataFile), 0755); err != nil {
 			return nil, 0, errors.WithMessage(err, "MkdirAll metadata dir")
diff --git a/pkg/backup/download_test.go b/pkg/backup/download_test.go
index 27ea7434e..a1d34a171 100644
--- a/pkg/backup/download_test.go
+++ b/pkg/backup/download_test.go
@@ -1,6 +1,7 @@
 package backup
 
 import (
+	"errors"
 	"regexp"
 	"testing"
 	"time"
@@ -91,6 +92,22 @@ var remoteBackup = storage.Backup{
 	UploadDate: time.Now(),
 }
 
+func TestIsRemoteMetadataNotFound(t *testing.T) {
+	notFoundMessages := []string{
+		"object doesn't exist",
+		"key not found: metadata/default/test.json",
+		"NoSuchKey: The specified key does not exist",
+		"operation error S3: GetObject, https response error StatusCode: 404",
+		"StatusCode 404",
+	}
+	for _, msg := range notFoundMessages {
+		assert.True(t, isRemoteMetadataNotFound(errors.New(msg)), msg)
+	}
+
+	assert.False(t, isRemoteMetadataNotFound(nil))
+	assert.False(t, isRemoteMetadataNotFound(errors.New("temporary network timeout")))
+}
+
 func TestReBalanceTablesMetadataIfDiskNotExists_Files_NoErrors(t *testing.T) {
 	remoteBackup.DataFormat = "tar"
 	baseTable := metadata.TableMetadata{