From a3af5e5b162039a1ae2a40173c3ca770d72aca3c Mon Sep 17 00:00:00 2001 From: Remco Beckers Date: Tue, 3 Feb 2026 13:44:45 +0100 Subject: [PATCH 01/10] STAC-23457 Restore stackpacks as well --- cmd/settings/restore.go | 6 ++- cmd/stackgraph/restore.go | 6 ++- internal/foundation/config/config.go | 18 +++++---- .../config/testdata/validConfigMapConfig.yaml | 3 ++ .../scripts/restore-settings-backup.sh | 38 +++++++++++++++++++ .../scripts/restore-stackgraph-backup.sh | 23 +++++++++++ 6 files changed, 84 insertions(+), 10 deletions(-) diff --git a/cmd/settings/restore.go b/cmd/settings/restore.go index 905f488..41e24ad 100644 --- a/cmd/settings/restore.go +++ b/cmd/settings/restore.go @@ -28,13 +28,14 @@ var ( useLatest bool background bool skipConfirmation bool + skipStackpacks bool ) func restoreCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { cmd := &cobra.Command{ Use: "restore", Short: "Restore Settings from a backup archive", - Long: `Restore Settings data from a backup archive stored in S3. Can use --latest or --archive to specify which backup to restore.`, + Long: `Restore Settings data from a backup archive stored in S3. Automatically also restores Stackpacks backup that was made at the same time, it can be skipped with --skip-stackpacks. Can use --latest or --archive to specify which backup to restore.`, Run: func(_ *cobra.Command, _ []string) { cmdutils.Run(globalFlags, runRestore, cmdutils.StorageIsNotRequired) }, @@ -45,6 +46,7 @@ func restoreCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { cmd.Flags().BoolVar(&background, "background", false, "Run restore job in background without waiting for completion") cmd.Flags().BoolVarP(&skipConfirmation, "yes", "y", false, "Skip confirmation prompt") cmd.Flags().BoolVar(&fromPVC, "from-old-pvc", false, "Restore backup from legacy PVC instead of S3") + cmd.Flags().BoolVar(&skipStackpacks, "skip-stackpacks", false, "Skip restoring stackpacks backup") cmd.MarkFlagsMutuallyExclusive("archive", "latest") cmd.MarkFlagsOneRequired("archive", "latest") @@ -192,12 +194,14 @@ func buildEnvVar(extraEnvVar []corev1.EnvVar, config *config.Config) []corev1.En commonVar := []corev1.EnvVar{ {Name: "BACKUP_CONFIGURATION_BUCKET_NAME", Value: config.Settings.Bucket}, {Name: "BACKUP_CONFIGURATION_S3_PREFIX", Value: config.Settings.S3Prefix}, + {Name: "BACKUP_CONFIGURATION_STACKPACKS_S3_PREFIX", Value: config.Settings.StackpacksS3Prefix}, {Name: "MINIO_ENDPOINT", Value: fmt.Sprintf("%s:%d", storageService.Name, storageService.Port)}, {Name: "STACKSTATE_BASE_URL", Value: config.Settings.Restore.BaseURL}, {Name: "RECEIVER_BASE_URL", Value: config.Settings.Restore.ReceiverBaseURL}, {Name: "PLATFORM_VERSION", Value: config.Settings.Restore.PlatformVersion}, {Name: "ZOOKEEPER_QUORUM", Value: config.Settings.Restore.ZookeeperQuorum}, {Name: "BACKUP_CONFIGURATION_UPLOAD_REMOTE", Value: strconv.FormatBool(config.GlobalBackupEnabled())}, + {Name: "SKIP_STACKPACKS", Value: strconv.FormatBool(skipStackpacks)}, } if fromPVC { // Force PVC mode in the shell script, suppress local bucket diff --git a/cmd/stackgraph/restore.go b/cmd/stackgraph/restore.go index a56fc82..19cd50b 100644 --- a/cmd/stackgraph/restore.go +++ b/cmd/stackgraph/restore.go @@ -35,13 +35,14 @@ var ( useLatest bool background bool skipConfirmation bool + skipStackpacks bool ) func restoreCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { cmd := &cobra.Command{ Use: "restore", Short: "Restore Stackgraph from a backup archive", - Long: `Restore Stackgraph data from a backup archive stored in S3. Can use --latest or --archive to specify which backup to restore.`, + Long: `Restore Stackgraph data from a backup archive stored in S3. Automatically also restores Stackpacks backup that was made at the same time, it can be skipped with --skip-stackpacks. Can use --latest or --archive to specify which backup to restore.`, Run: func(_ *cobra.Command, _ []string) { cmdutils.Run(globalFlags, runRestore, cmdutils.StorageIsRequired) }, @@ -51,6 +52,7 @@ func restoreCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { cmd.Flags().BoolVar(&useLatest, "latest", false, "Restore from the most recent backup") cmd.Flags().BoolVar(&background, "background", false, "Run restore job in background without waiting for completion") cmd.Flags().BoolVarP(&skipConfirmation, "yes", "y", false, "Skip confirmation prompt") + cmd.Flags().BoolVar(&skipStackpacks, "skip-stackpacks", false, "Skip restoring stackpacks backup") cmd.MarkFlagsMutuallyExclusive("archive", "latest") cmd.MarkFlagsOneRequired("archive", "latest") @@ -268,9 +270,11 @@ func buildRestoreEnvVars(backupFile string, config *config.Config) []corev1.EnvV {Name: "FORCE_DELETE", Value: purgeStackgraphDataFlag}, {Name: "BACKUP_STACKGRAPH_BUCKET_NAME", Value: config.Stackgraph.Bucket}, {Name: "BACKUP_STACKGRAPH_S3_PREFIX", Value: config.Stackgraph.S3Prefix}, + {Name: "BACKUP_STACKGRAPH_STACKPACKS_S3_PREFIX", Value: config.Stackgraph.StackpacksS3Prefix}, {Name: "BACKUP_STACKGRAPH_MULTIPART_ARCHIVE", Value: strconv.FormatBool(config.Stackgraph.MultipartArchive)}, {Name: "MINIO_ENDPOINT", Value: fmt.Sprintf("%s:%d", storageService.Name, storageService.Port)}, {Name: "ZOOKEEPER_QUORUM", Value: config.Stackgraph.Restore.ZookeeperQuorum}, + {Name: "SKIP_STACKPACKS", Value: strconv.FormatBool(skipStackpacks)}, } } diff --git a/internal/foundation/config/config.go b/internal/foundation/config/config.go index e944301..2f7c054 100644 --- a/internal/foundation/config/config.go +++ b/internal/foundation/config/config.go @@ -135,10 +135,11 @@ type StorageConfig struct { // StackgraphConfig holds Stackgraph backup-specific configuration type StackgraphConfig struct { - Bucket string `yaml:"bucket" validate:"required"` - S3Prefix string `yaml:"s3Prefix"` - MultipartArchive bool `yaml:"multipartArchive" validate:"boolean"` - Restore StackgraphRestoreConfig `yaml:"restore" validate:"required"` + Bucket string `yaml:"bucket" validate:"required"` + S3Prefix string `yaml:"s3Prefix"` + StackpacksS3Prefix string `yaml:"stackpacksS3Prefix"` + MultipartArchive bool `yaml:"multipartArchive" validate:"boolean"` + Restore StackgraphRestoreConfig `yaml:"restore" validate:"required"` } type VictoriaMetricsConfig struct { @@ -169,10 +170,11 @@ type StackgraphRestoreConfig struct { } type SettingsConfig struct { - Bucket string `yaml:"bucket" validate:"required"` - S3Prefix string `yaml:"s3Prefix"` - LocalBucket string `yaml:"localBucket"` - Restore SettingsRestoreConfig `yaml:"restore" validate:"required"` + Bucket string `yaml:"bucket" validate:"required"` + S3Prefix string `yaml:"s3Prefix"` + StackpacksS3Prefix string `yaml:"stackpacksS3Prefix"` + LocalBucket string `yaml:"localBucket"` + Restore SettingsRestoreConfig `yaml:"restore" validate:"required"` } type SettingsRestoreConfig struct { diff --git a/internal/foundation/config/testdata/validConfigMapConfig.yaml b/internal/foundation/config/testdata/validConfigMapConfig.yaml index 76aca9a..e36900e 100644 --- a/internal/foundation/config/testdata/validConfigMapConfig.yaml +++ b/internal/foundation/config/testdata/validConfigMapConfig.yaml @@ -76,6 +76,8 @@ stackgraph: bucket: sts-stackgraph-backup # S3 prefix path for backups s3Prefix: "" + # S3 prefix path for stackpacks backups + stackpacksS3Prefix: "stackpacks/" # Archive split to multiple parts multipartArchive: true # Restore configuration @@ -139,6 +141,7 @@ victoriaMetrics: settings: bucket: sts-settings-backup s3Prefix: "" + stackpacksS3Prefix: "stackpacks/" restore: scaleDownLabelSelector: "observability.suse.com/scalable-during-settings-restore=true" loggingConfigConfigMap: suse-observability-logging diff --git a/internal/scripts/scripts/restore-settings-backup.sh b/internal/scripts/scripts/restore-settings-backup.sh index 68b95c4..2309054 100644 --- a/internal/scripts/scripts/restore-settings-backup.sh +++ b/internal/scripts/scripts/restore-settings-backup.sh @@ -55,4 +55,42 @@ fi echo "=== Restoring settings backup from \"${BACKUP_FILE}\"..." /opt/docker/bin/settings-backup -Dlogback.configurationFile=/opt/docker/etc_log/logback.xml -restore "${RESTORE_FILE}" +echo "=== Settings restore complete" + +# === StackPacks Restore === +if [ "${SKIP_STACKPACKS:-false}" == "true" ]; then + echo "=== Skipping StackPacks restore (--skip-stackpacks flag set)" +else + export STACKPACKS_BACKUP_DIR="${BACKUP_DIR}/stackpacks" + + # Construct stackpacks backup filename from the original backup file + STACKPACKS_FILE="${BACKUP_FILE}.stackpacks.zip" + STACKPACKS_RESTORE_FILE="${STACKPACKS_BACKUP_DIR}/${STACKPACKS_FILE}" + + echo "=== Checking for StackPacks backup \"${STACKPACKS_FILE}\"..." + + # Check local PVC first, then try S3 if not found and remote is enabled + if [ ! -f "${STACKPACKS_RESTORE_FILE}" ] && [ "$BACKUP_CONFIGURATION_UPLOAD_REMOTE" == "true" ]; then + # Ensure AWS credentials are set for S3 access + export AWS_ACCESS_KEY_ID + AWS_ACCESS_KEY_ID="$(cat /aws-keys/accesskey)" + export AWS_SECRET_ACCESS_KEY + AWS_SECRET_ACCESS_KEY="$(cat /aws-keys/secretkey)" + + # Check if file exists in S3 + if sts-toolbox aws s3 ls --endpoint "http://${MINIO_ENDPOINT}" --region minio --bucket "${BACKUP_CONFIGURATION_BUCKET_NAME}" --prefix "${BACKUP_CONFIGURATION_STACKPACKS_S3_PREFIX}${STACKPACKS_FILE}" 2>/dev/null | grep -q "${STACKPACKS_FILE}"; then + echo "=== Downloading StackPacks backup from S3..." + sts-toolbox aws s3 cp --endpoint "http://${MINIO_ENDPOINT}" --region minio "s3://${BACKUP_CONFIGURATION_BUCKET_NAME}/${BACKUP_CONFIGURATION_STACKPACKS_S3_PREFIX}${STACKPACKS_FILE}" "${TMP_DIR}/${STACKPACKS_FILE}" + STACKPACKS_RESTORE_FILE="${TMP_DIR}/${STACKPACKS_FILE}" + fi + fi + + if [ -f "${STACKPACKS_RESTORE_FILE}" ]; then + echo "=== Restoring StackPacks from \"${STACKPACKS_FILE}\"..." + /opt/docker/bin/stack-packs-backup -Dlogback.configurationFile=/opt/docker/etc_log/logback.xml -restore "${STACKPACKS_RESTORE_FILE}" + echo "=== StackPacks restore complete" + else + echo "=== WARNING: StackPacks backup \"${STACKPACKS_FILE}\" not found, skipping StackPacks restore" + fi +fi echo "===" diff --git a/internal/scripts/scripts/restore-stackgraph-backup.sh b/internal/scripts/scripts/restore-stackgraph-backup.sh index 2e89c30..0f3a9c4 100644 --- a/internal/scripts/scripts/restore-stackgraph-backup.sh +++ b/internal/scripts/scripts/restore-stackgraph-backup.sh @@ -30,4 +30,27 @@ fi echo "=== Importing StackGraph data from \"${BACKUP_FILE}\"..." /opt/docker/bin/stackstate-server -Dlogback.configurationFile=/opt/docker/etc_log/logback.xml -import "${TMP_DIR}/${BACKUP_FILE}" "${FORCE_DELETE}" +echo "=== StackGraph restore complete" + +# === StackPacks Restore === +if [ "${SKIP_STACKPACKS:-false}" == "true" ]; then + echo "=== Skipping StackPacks restore (--skip-stackpacks flag set)" +else + # Construct stackpacks backup filename from the original backup file + STACKPACKS_FILE="${BACKUP_FILE}.stackpacks.zip" + + echo "=== Checking for StackPacks backup \"${STACKPACKS_FILE}\" in bucket \"${BACKUP_STACKGRAPH_BUCKET_NAME}\"..." + + # Check if stackpacks backup exists in S3 + if sts-toolbox aws s3 ls --endpoint "http://${MINIO_ENDPOINT}" --region minio --bucket "${BACKUP_STACKGRAPH_BUCKET_NAME}" --prefix "${BACKUP_STACKGRAPH_STACKPACKS_S3_PREFIX}${STACKPACKS_FILE}" 2>/dev/null | grep -q "${STACKPACKS_FILE}"; then + echo "=== Downloading StackPacks backup..." + sts-toolbox aws s3 cp --endpoint "http://${MINIO_ENDPOINT}" --region minio "s3://${BACKUP_STACKGRAPH_BUCKET_NAME}/${BACKUP_STACKGRAPH_STACKPACKS_S3_PREFIX}${STACKPACKS_FILE}" "${TMP_DIR}/${STACKPACKS_FILE}" + + echo "=== Restoring StackPacks from \"${STACKPACKS_FILE}\"..." + /opt/docker/bin/stack-packs-backup -Dlogback.configurationFile=/opt/docker/etc_log/logback.xml -restore "${TMP_DIR}/${STACKPACKS_FILE}" + echo "=== StackPacks restore complete" + else + echo "=== WARNING: StackPacks backup \"${STACKPACKS_FILE}\" not found in S3, skipping StackPacks restore" + fi +fi echo "===" From 13531bd0f83fb20cf57c51ccafc984935e06f390 Mon Sep 17 00:00:00 2001 From: Remco Beckers Date: Fri, 6 Feb 2026 14:39:05 +0100 Subject: [PATCH 02/10] STAC-23457 Rename filterBackupObjects --- cmd/settings/list.go | 3 ++- cmd/stackgraph/list.go | 2 +- cmd/stackgraph/restore.go | 2 +- internal/clients/s3/filter.go | 4 ++-- internal/clients/s3/filter_test.go | 14 +++++++------- 5 files changed, 13 insertions(+), 12 deletions(-) diff --git a/cmd/settings/list.go b/cmd/settings/list.go index 6072e3a..388d91c 100644 --- a/cmd/settings/list.go +++ b/cmd/settings/list.go @@ -29,6 +29,7 @@ const ( isMultiPartArchive = false expectedListJobPodCount = 1 expectedListJobContainerCount = 1 + backupFileNameRegex = `^sts-backup-.*\.sty$` ) // Shared flag for --from-old-pvc, used by both list and restore commands @@ -182,7 +183,7 @@ func getBackupListFromS3(appCtx *app.Context) ([]BackupFileInfo, error) { } // Filter objects based on whether the archive is split or not - filteredObjects := s3client.FilterBackupObjects(result.Contents, isMultiPartArchive) + filteredObjects := s3client.FilterMultipartBackupObjects(result.Contents, isMultiPartArchive) var backups []BackupFileInfo for _, obj := range filteredObjects { diff --git a/cmd/stackgraph/list.go b/cmd/stackgraph/list.go index 1d169d9..5b242a5 100644 --- a/cmd/stackgraph/list.go +++ b/cmd/stackgraph/list.go @@ -63,7 +63,7 @@ func runList(appCtx *app.Context) error { } // Filter objects based on whether the archive is split or not - filteredObjects := s3client.FilterBackupObjects(result.Contents, multipartArchive) + filteredObjects := s3client.FilterMultipartBackupObjects(result.Contents, multipartArchive) // Sort by LastModified time (most recent first) sort.Slice(filteredObjects, func(i, j int) bool { diff --git a/cmd/stackgraph/restore.go b/cmd/stackgraph/restore.go index 19cd50b..1bb8688 100644 --- a/cmd/stackgraph/restore.go +++ b/cmd/stackgraph/restore.go @@ -180,7 +180,7 @@ func getLatestBackup(k8sClient *k8s.Client, namespace string, config *config.Con } // Filter objects based on whether the archive is split or not - filteredObjects := s3client.FilterBackupObjects(result.Contents, multipartArchive) + filteredObjects := s3client.FilterMultipartBackupObjects(result.Contents, multipartArchive) if len(filteredObjects) == 0 { return "", fmt.Errorf("no backups found in bucket %s", bucket) diff --git a/internal/clients/s3/filter.go b/internal/clients/s3/filter.go index 5787352..403aecb 100644 --- a/internal/clients/s3/filter.go +++ b/internal/clients/s3/filter.go @@ -19,10 +19,10 @@ type Object struct { Size int64 } -// FilterBackupObjects filters S3 objects based on whether the archive is split or not +// FilterMultipartBackupObjects filters S3 objects based on whether the archive is split or not // If it is not multipartArchive, it filters out multipart archives (files ending with .digits) // Otherwise, it groups multipart archives by base name and sums their sizes -func FilterBackupObjects(objects []s3types.Object, multipartArchive bool) []Object { +func FilterMultipartBackupObjects(objects []s3types.Object, multipartArchive bool) []Object { if !multipartArchive { return filterNonMultipart(objects) } diff --git a/internal/clients/s3/filter_test.go b/internal/clients/s3/filter_test.go index 2cf9b86..ef88d88 100644 --- a/internal/clients/s3/filter_test.go +++ b/internal/clients/s3/filter_test.go @@ -79,7 +79,7 @@ func TestFilterBackupObjects_SingleFileMode(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := FilterBackupObjects(tt.objects, false) + result := FilterMultipartBackupObjects(tt.objects, false) assert.Equal(t, tt.expectedCount, len(result)) @@ -158,7 +158,7 @@ func TestFilterBackupObjects_MultipartMode(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := FilterBackupObjects(tt.objects, tt.multipartArchive) + result := FilterMultipartBackupObjects(tt.objects, tt.multipartArchive) assert.Equal(t, tt.expectedCount, len(result)) @@ -196,14 +196,14 @@ func TestFilterBackupObjects_ObjectMetadata(t *testing.T) { } // Test single file mode - result := FilterBackupObjects(objects, false) + result := FilterMultipartBackupObjects(objects, false) assert.Equal(t, 1, len(result)) assert.Equal(t, "backup-2024-01-01.tar.gz", result[0].Key) assert.Equal(t, int64(1234567890), result[0].Size) assert.Equal(t, now.Unix(), result[0].LastModified.Unix()) // Test multipart mode - should group parts and sum sizes - result = FilterBackupObjects(objects, true) + result = FilterMultipartBackupObjects(objects, true) assert.Equal(t, 2, len(result)) // tar.gz file + grouped multipart // Find the multipart archive result @@ -275,7 +275,7 @@ func TestFilterBackupObjects_EdgeCases(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := FilterBackupObjects(tt.objects, tt.multipartArchive) + result := FilterMultipartBackupObjects(tt.objects, tt.multipartArchive) assert.Equal(t, tt.expectedCount, len(result)) }) } @@ -331,7 +331,7 @@ func TestFilterBackupObjects_RealWorldScenarios(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := FilterBackupObjects(tt.objects, tt.multipartArchive) + result := FilterMultipartBackupObjects(tt.objects, tt.multipartArchive) assert.Equal(t, tt.expectedCount, len(result), "Scenario: %s", tt.scenario) }) } @@ -348,7 +348,7 @@ func TestFilterBackupObjects_SizeSummation(t *testing.T) { {Key: aws.String("sts-backup-20251029-0924.graph.01"), Size: aws.Int64(6567239)}, } - result := FilterBackupObjects(objects, true) + result := FilterMultipartBackupObjects(objects, true) // Should have 3 grouped archives assert.Equal(t, 3, len(result)) From bcb426ab3941674ab2493d08beb1777347a48cd1 Mon Sep 17 00:00:00 2001 From: Remco Beckers Date: Fri, 6 Feb 2026 15:18:55 +0100 Subject: [PATCH 03/10] STAC-23457 Apply more filtering to settings/stackgraph backups To hide the settings backups and to make the backup name copy/pasteable --- cmd/settings/list.go | 23 ++++ cmd/stackgraph/list.go | 11 ++ internal/clients/s3/filter.go | 45 ++++++++ internal/clients/s3/filter_test.go | 170 +++++++++++++++++++++++++++++ 4 files changed, 249 insertions(+) diff --git a/cmd/settings/list.go b/cmd/settings/list.go index 388d91c..fc99fc1 100644 --- a/cmd/settings/list.go +++ b/cmd/settings/list.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "regexp" "slices" "sort" "strconv" @@ -185,6 +186,13 @@ func getBackupListFromS3(appCtx *app.Context) ([]BackupFileInfo, error) { // Filter objects based on whether the archive is split or not filteredObjects := s3client.FilterMultipartBackupObjects(result.Contents, isMultiPartArchive) + // Filter to only include direct children of the prefix that match the backup filename pattern, + // and strip the prefix from the key + filteredObjects, err = s3client.FilterByPrefixAndRegex(filteredObjects, prefix, backupFileNameRegex) + if err != nil { + return nil, fmt.Errorf("failed to filter objects: %w", err) + } + var backups []BackupFileInfo for _, obj := range filteredObjects { row := BackupFileInfo{ @@ -299,6 +307,9 @@ func getBackupListFromPVC(appCtx *app.Context) ([]BackupFileInfo, error) { return nil, fmt.Errorf("failed to parse list job output: %w", err) } + // Filter by backup filename pattern + files = filterBackupsByRegex(files, backupFileNameRegex) + return files, nil } @@ -377,3 +388,15 @@ func ParseListJobOutput(input string) ([]BackupFileInfo, error) { return files, nil } + +// filterBackupsByRegex filters BackupFileInfo by matching filename against a regex pattern +func filterBackupsByRegex(backups []BackupFileInfo, pattern string) []BackupFileInfo { + re := regexp.MustCompile(pattern) + var filtered []BackupFileInfo + for _, b := range backups { + if re.MatchString(b.Filename) { + filtered = append(filtered, b) + } + } + return filtered +} diff --git a/cmd/stackgraph/list.go b/cmd/stackgraph/list.go index 5b242a5..c9b86b7 100644 --- a/cmd/stackgraph/list.go +++ b/cmd/stackgraph/list.go @@ -17,6 +17,10 @@ import ( "github.com/stackvista/stackstate-backup-cli/internal/orchestration/portforward" ) +const ( + backupFileNameRegex = `^sts-backup-.*\.graph$` +) + func listCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { return &cobra.Command{ Use: "list", @@ -65,6 +69,13 @@ func runList(appCtx *app.Context) error { // Filter objects based on whether the archive is split or not filteredObjects := s3client.FilterMultipartBackupObjects(result.Contents, multipartArchive) + // Filter to only include direct children of the prefix that match the backup filename pattern, + // and strip the prefix from the key + filteredObjects, err = s3client.FilterByPrefixAndRegex(filteredObjects, prefix, backupFileNameRegex) + if err != nil { + return fmt.Errorf("failed to filter objects: %w", err) + } + // Sort by LastModified time (most recent first) sort.Slice(filteredObjects, func(i, j int) bool { return filteredObjects[i].LastModified.After(filteredObjects[j].LastModified) diff --git a/internal/clients/s3/filter.go b/internal/clients/s3/filter.go index 403aecb..f44ceb7 100644 --- a/internal/clients/s3/filter.go +++ b/internal/clients/s3/filter.go @@ -1,6 +1,8 @@ package s3 import ( + "fmt" + "regexp" "strings" "time" @@ -141,6 +143,49 @@ func getBaseName(key string) (string, bool) { return key, false } +// FilterByPrefixAndRegex filters objects to only include direct children of the given prefix +// that match the specified regex pattern. It excludes objects in nested subdirectories and +// strips the prefix from the key, returning just the filename portion. +// +// For example, with prefix "backups/" and pattern `^sts-backup-.*\.graph$`: +// - "backups/sts-backup-20240101.graph" -> included, Key becomes "sts-backup-20240101.graph" +// - "backups/other-file.txt" -> excluded (doesn't match pattern) +// - "backups/subdir/sts-backup-20240101.graph" -> excluded (nested) +func FilterByPrefixAndRegex(objects []Object, prefix string, pattern string) ([]Object, error) { + re, err := regexp.Compile(pattern) + if err != nil { + return nil, fmt.Errorf("invalid regex pattern: %w", err) + } + + var filtered []Object + for _, obj := range objects { + // Strip the prefix from the key + relativePath := strings.TrimPrefix(obj.Key, prefix) + + // Skip if the relative path contains a slash (indicating nested directory) + if strings.Contains(relativePath, "/") { + continue + } + + // Skip empty relative paths (the prefix itself) + if relativePath == "" { + continue + } + + // Check if the filename matches the regex pattern + if !re.MatchString(relativePath) { + continue + } + + filtered = append(filtered, Object{ + Key: relativePath, + LastModified: obj.LastModified, + Size: obj.Size, + }) + } + return filtered, nil +} + func FilterByCommonPrefix(objects []s3types.CommonPrefix) []Object { var filteredObjects []Object diff --git a/internal/clients/s3/filter_test.go b/internal/clients/s3/filter_test.go index ef88d88..1e64a86 100644 --- a/internal/clients/s3/filter_test.go +++ b/internal/clients/s3/filter_test.go @@ -364,3 +364,173 @@ func TestFilterBackupObjects_SizeSummation(t *testing.T) { assert.Equal(t, int64(109206155), sizeMap["sts-backup-20251029-0300.graph"]) // 104857600 + 4348555 assert.Equal(t, int64(111424839), sizeMap["sts-backup-20251029-0924.graph"]) // 104857600 + 6567239 } + +// TestFilterByPrefixAndRegex tests the combined filtering by prefix and regex pattern +func TestFilterByPrefixAndRegex(t *testing.T) { + now := time.Now() + + tests := []struct { + name string + objects []Object + prefix string + pattern string + expectedKeys []string + expectError bool + }{ + { + name: "filters stackgraph backups with prefix and .graph extension", + objects: []Object{ + {Key: "backups/sts-backup-20240101.graph", Size: 1000, LastModified: now}, + {Key: "backups/sts-backup-20240102.graph", Size: 2000, LastModified: now}, + {Key: "backups/other-file.txt", Size: 500, LastModified: now}, + {Key: "backups/sts-backup-20240103.tar.gz", Size: 3000, LastModified: now}, + }, + prefix: "backups/", + pattern: `^sts-backup-.*\.graph$`, + expectedKeys: []string{"sts-backup-20240101.graph", "sts-backup-20240102.graph"}, + expectError: false, + }, + { + name: "filters settings backups with .sty extension", + objects: []Object{ + {Key: "settings/sts-backup-20240101.sty", Size: 1000, LastModified: now}, + {Key: "settings/sts-backup-20240102.sty", Size: 2000, LastModified: now}, + {Key: "settings/other-file.txt", Size: 500, LastModified: now}, + {Key: "settings/sts-backup-20240103.graph", Size: 3000, LastModified: now}, + }, + prefix: "settings/", + pattern: `^sts-backup-.*\.sty$`, + expectedKeys: []string{"sts-backup-20240101.sty", "sts-backup-20240102.sty"}, + expectError: false, + }, + { + name: "excludes nested files even if they match pattern", + objects: []Object{ + {Key: "backups/sts-backup-20240101.graph", Size: 1000, LastModified: now}, + {Key: "backups/old/sts-backup-20240102.graph", Size: 2000, LastModified: now}, + {Key: "backups/archive/2023/sts-backup-20230101.graph", Size: 3000, LastModified: now}, + }, + prefix: "backups/", + pattern: `^sts-backup-.*\.graph$`, + expectedKeys: []string{"sts-backup-20240101.graph"}, + expectError: false, + }, + { + name: "works with empty prefix", + objects: []Object{ + {Key: "sts-backup-20240101.graph", Size: 1000, LastModified: now}, + {Key: "sts-backup-20240102.graph", Size: 2000, LastModified: now}, + {Key: "subdir/sts-backup-20240103.graph", Size: 3000, LastModified: now}, + {Key: "other-file.txt", Size: 500, LastModified: now}, + }, + prefix: "", + pattern: `^sts-backup-.*\.graph$`, + expectedKeys: []string{"sts-backup-20240101.graph", "sts-backup-20240102.graph"}, + expectError: false, + }, + { + name: "returns empty slice when no matches", + objects: []Object{ + {Key: "backups/other-file.txt", Size: 500, LastModified: now}, + {Key: "backups/another-file.log", Size: 100, LastModified: now}, + }, + prefix: "backups/", + pattern: `^sts-backup-.*\.graph$`, + expectedKeys: []string{}, + expectError: false, + }, + { + name: "handles empty object list", + objects: []Object{}, + prefix: "backups/", + pattern: `^sts-backup-.*\.graph$`, + expectedKeys: []string{}, + expectError: false, + }, + { + name: "returns error for invalid regex", + objects: []Object{ + {Key: "backups/sts-backup-20240101.graph", Size: 1000, LastModified: now}, + }, + prefix: "backups/", + pattern: `[invalid`, + expectedKeys: nil, + expectError: true, + }, + { + name: "excludes the prefix directory itself", + objects: []Object{ + {Key: "backups/", Size: 0, LastModified: now}, + {Key: "backups/sts-backup-20240101.graph", Size: 1000, LastModified: now}, + }, + prefix: "backups/", + pattern: `^sts-backup-.*\.graph$`, + expectedKeys: []string{"sts-backup-20240101.graph"}, + expectError: false, + }, + { + name: "returns empty when all files are nested", + objects: []Object{ + {Key: "backups/old/sts-backup-20240101.graph", Size: 1000, LastModified: now}, + {Key: "backups/archive/sts-backup-20240102.graph", Size: 2000, LastModified: now}, + }, + prefix: "backups/", + pattern: `^sts-backup-.*\.graph$`, + expectedKeys: []string{}, + expectError: false, + }, + { + name: "filters with complex regex pattern", + objects: []Object{ + {Key: "backups/sts-backup-20240101-1200.graph", Size: 1000, LastModified: now}, + {Key: "backups/sts-backup-20240102-1300.graph", Size: 2000, LastModified: now}, + {Key: "backups/sts-backup-invalid.graph", Size: 500, LastModified: now}, + {Key: "backups/sts-backup-20240103.graph", Size: 3000, LastModified: now}, + }, + prefix: "backups/", + pattern: `^sts-backup-\d{8}-\d{4}\.graph$`, + expectedKeys: []string{"sts-backup-20240101-1200.graph", "sts-backup-20240102-1300.graph"}, + expectError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := FilterByPrefixAndRegex(tt.objects, tt.prefix, tt.pattern) + + if tt.expectError { + assert.Error(t, err) + assert.Nil(t, result) + return + } + + assert.NoError(t, err) + + resultKeys := make([]string, len(result)) + for i, obj := range result { + resultKeys[i] = obj.Key + } + + assert.Equal(t, tt.expectedKeys, resultKeys) + }) + } +} + +// TestFilterByPrefixAndRegex_PreservesMetadata tests that object metadata is preserved after filtering +func TestFilterByPrefixAndRegex_PreservesMetadata(t *testing.T) { + now := time.Now() + + objects := []Object{ + {Key: "backups/sts-backup-20240101.graph", Size: 1234567890, LastModified: now}, + {Key: "backups/other-file.txt", Size: 500, LastModified: now.Add(-24 * time.Hour)}, + {Key: "backups/nested/sts-backup-20240102.graph", Size: 999, LastModified: now}, + } + + result, err := FilterByPrefixAndRegex(objects, "backups/", `^sts-backup-.*\.graph$`) + + assert.NoError(t, err) + assert.Equal(t, 1, len(result)) + assert.Equal(t, "sts-backup-20240101.graph", result[0].Key) + assert.Equal(t, int64(1234567890), result[0].Size) + assert.Equal(t, now.Unix(), result[0].LastModified.Unix()) +} From b1b53859fa3fa022cc87a4e3eceaea9ce704a292 Mon Sep 17 00:00:00 2001 From: Remco Beckers Date: Fri, 6 Feb 2026 17:04:59 +0100 Subject: [PATCH 04/10] STAC-23457 Configuration for stackpacks restore --- cmd/settings/restore.go | 36 ++++++++++++++++++++++--- cmd/stackgraph/restore.go | 40 +++++++++++++++++++++++++--- internal/foundation/config/config.go | 30 ++++++++++++--------- 3 files changed, 86 insertions(+), 20 deletions(-) diff --git a/cmd/settings/restore.go b/cmd/settings/restore.go index 41e24ad..a27a4cb 100644 --- a/cmd/settings/restore.go +++ b/cmd/settings/restore.go @@ -215,17 +215,26 @@ func buildEnvVar(extraEnvVar []corev1.EnvVar, config *config.Config) []corev1.En // buildVolumeMounts constructs volume mounts for the restore job container func buildVolumeMounts(config *config.Config) []corev1.VolumeMount { - mounts := []corev1.VolumeMount{ + volumeMounts := []corev1.VolumeMount{ {Name: "backup-log", MountPath: "/opt/docker/etc_log"}, + {Name: "config-volume", MountPath: "/opt/docker/etc/application_stackstate.conf", SubPath: "application_stackstate.conf"}, {Name: "backup-restore-scripts", MountPath: "/backup-restore-scripts"}, {Name: "minio-keys", MountPath: "/aws-keys"}, {Name: "tmp-data", MountPath: "/tmp-data"}, } // Mount PVC in legacy mode or when --from-old-pvc is set if config.IsLegacyMode() || fromPVC { - mounts = append(mounts, corev1.VolumeMount{Name: "settings-backup-data", MountPath: "/settings-backup-data"}) + volumeMounts = append(volumeMounts, corev1.VolumeMount{Name: "settings-backup-data", MountPath: "/settings-backup-data"}) } - return mounts + + if config.Settings.Restore.StackpacksPVCName != "" { + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: "stackpacks-local", + MountPath: "/var/stackpacks_local", + }) + } + + return volumeMounts } // buildVolumes constructs volumes for the restore job pod @@ -241,6 +250,16 @@ func buildVolumes(config *config.Config, defaultMode int32) []corev1.Volume { }, }, }, + { + Name: "config-volume", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: config.Settings.Restore.StsBackupConfigConfigMapName, + }, + }, + }, + }, { Name: "backup-restore-scripts", VolumeSource: corev1.VolumeSource{ @@ -278,6 +297,17 @@ func buildVolumes(config *config.Config, defaultMode int32) []corev1.Volume { }, }) } + if config.Settings.Restore.StackpacksPVCName != "" { + volumes = append(volumes, corev1.Volume{ + Name: "stackpacks-local", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: config.Settings.Restore.StackpacksPVCName, + }, + }, + }) + } + return volumes } diff --git a/cmd/stackgraph/restore.go b/cmd/stackgraph/restore.go index 1bb8688..cd0569f 100644 --- a/cmd/stackgraph/restore.go +++ b/cmd/stackgraph/restore.go @@ -279,13 +279,23 @@ func buildRestoreEnvVars(backupFile string, config *config.Config) []corev1.EnvV } // buildRestoreVolumeMounts constructs volume mounts for the restore job container -func buildRestoreVolumeMounts() []corev1.VolumeMount { - return []corev1.VolumeMount{ +func buildRestoreVolumeMounts(config *config.Config) []corev1.VolumeMount { + volumeMounts := []corev1.VolumeMount{ {Name: "backup-log", MountPath: "/opt/docker/etc_log"}, + {Name: "config-volume", MountPath: "/opt/docker/etc/application_stackstate.conf", SubPath: "application_stackstate.conf"}, {Name: "backup-restore-scripts", MountPath: "/backup-restore-scripts"}, {Name: "minio-keys", MountPath: "/aws-keys"}, {Name: "tmp-data", MountPath: "/tmp-data"}, } + + if config.Settings.Restore.StackpacksPVCName != "" { + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: "stackpacks-local", + MountPath: "/var/stackpacks_local", + }) + } + + return volumeMounts } // buildRestoreInitContainers constructs init containers for the restore job @@ -308,7 +318,7 @@ func buildRestoreInitContainers(config *config.Config) []corev1.Container { // buildRestoreVolumes constructs volumes for the restore job pod func buildRestoreVolumes(jobName string, config *config.Config, defaultMode int32) []corev1.Volume { - return []corev1.Volume{ + volumes := []corev1.Volume{ { Name: "backup-log", VolumeSource: corev1.VolumeSource{ @@ -319,6 +329,16 @@ func buildRestoreVolumes(jobName string, config *config.Config, defaultMode int3 }, }, }, + { + Name: "config-volume", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: config.Stackgraph.Restore.StsBackupConfigConfigMapName, + }, + }, + }, + }, { Name: "backup-restore-scripts", VolumeSource: corev1.VolumeSource{ @@ -347,6 +367,18 @@ func buildRestoreVolumes(jobName string, config *config.Config, defaultMode int3 }, }, } + if config.Settings.Restore.StackpacksPVCName != "" { + volumes = append(volumes, corev1.Volume{ + Name: "stackpacks-local", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: config.Settings.Restore.StackpacksPVCName, + }, + }, + }) + } + + return volumes } // buildRestoreContainers constructs containers for the restore job @@ -360,7 +392,7 @@ func buildRestoreContainers(backupFile string, config *config.Config) []corev1.C Command: []string{"/backup-restore-scripts/restore-stackgraph-backup.sh"}, Env: buildRestoreEnvVars(backupFile, config), Resources: k8s.ConvertResources(config.Stackgraph.Restore.Job.Resources), - VolumeMounts: buildRestoreVolumeMounts(), + VolumeMounts: buildRestoreVolumeMounts(config), }, } } diff --git a/internal/foundation/config/config.go b/internal/foundation/config/config.go index 2f7c054..1c0c0d9 100644 --- a/internal/foundation/config/config.go +++ b/internal/foundation/config/config.go @@ -162,11 +162,13 @@ type S3Location struct { // StackgraphRestoreConfig holds Stackgraph restore-specific configuration type StackgraphRestoreConfig struct { - ScaleDownLabelSelector string `yaml:"scaleDownLabelSelector" validate:"required"` - LoggingConfigConfigMapName string `yaml:"loggingConfigConfigMap" validate:"required"` - ZookeeperQuorum string `yaml:"zookeeperQuorum" validate:"required"` - Job JobConfig `yaml:"job" validate:"required"` - PVC PVCConfig `yaml:"pvc" validate:"required"` + ScaleDownLabelSelector string `yaml:"scaleDownLabelSelector" validate:"required"` + LoggingConfigConfigMapName string `yaml:"loggingConfigConfigMap" validate:"required"` + StsBackupConfigConfigMapName string `yaml:"stsBackupConfigConfigMap" validate:"required"` + ZookeeperQuorum string `yaml:"zookeeperQuorum" validate:"required"` + Job JobConfig `yaml:"job" validate:"required"` + PVC PVCConfig `yaml:"pvc" validate:"required"` + StackpacksPVCName string `yaml:"stackpacksPvc"` } type SettingsConfig struct { @@ -178,14 +180,16 @@ type SettingsConfig struct { } type SettingsRestoreConfig struct { - ScaleDownLabelSelector string `yaml:"scaleDownLabelSelector" validate:"required"` - LoggingConfigConfigMapName string `yaml:"loggingConfigConfigMap" validate:"required"` - BaseURL string `yaml:"baseUrl" validate:"required"` - ReceiverBaseURL string `yaml:"receiverBaseUrl" validate:"required"` - PlatformVersion string `yaml:"platformVersion" validate:"required"` - ZookeeperQuorum string `yaml:"zookeeperQuorum" validate:"required"` - Job JobConfig `yaml:"job" validate:"required"` - PVC string `yaml:"pvc"` // Required only in legacy mode + ScaleDownLabelSelector string `yaml:"scaleDownLabelSelector" validate:"required"` + LoggingConfigConfigMapName string `yaml:"loggingConfigConfigMap" validate:"required"` + StsBackupConfigConfigMapName string `yaml:"stsBackupConfigConfigMap" validate:"required"` + BaseURL string `yaml:"baseUrl" validate:"required"` + ReceiverBaseURL string `yaml:"receiverBaseUrl" validate:"required"` + PlatformVersion string `yaml:"platformVersion" validate:"required"` + ZookeeperQuorum string `yaml:"zookeeperQuorum" validate:"required"` + Job JobConfig `yaml:"job" validate:"required"` + PVC string `yaml:"pvc"` // Required only in legacy mode + StackpacksPVCName string `yaml:"stackpacksPvc"` } // ClickhouseConfig holds Clickhouse-specific configuration From 3f3c4b838b622abef980209a304acdf129ab374b Mon Sep 17 00:00:00 2001 From: Remco Beckers Date: Thu, 2 Apr 2026 15:13:17 +0200 Subject: [PATCH 05/10] STAC-23457 Filter out stackpacks from backup listing --- cmd/settings/list.go | 5 +++++ internal/clients/s3/filter_test.go | 13 +++++++++++++ 2 files changed, 18 insertions(+) diff --git a/cmd/settings/list.go b/cmd/settings/list.go index fc99fc1..f2e87c9 100644 --- a/cmd/settings/list.go +++ b/cmd/settings/list.go @@ -240,6 +240,11 @@ func getBackupListFromLocalBucket(appCtx *app.Context) ([]BackupFileInfo, error) filteredObjects := s3client.FilterBackupObjects(result.Contents, isMultiPartArchive) + filteredObjects, err = s3client.FilterByPrefixAndRegex(filteredObjects, "", backupFileNameRegex) + if err != nil { + return nil, fmt.Errorf("failed to filter objects: %w", err) + } + var backups []BackupFileInfo for _, obj := range filteredObjects { backups = append(backups, BackupFileInfo{ diff --git a/internal/clients/s3/filter_test.go b/internal/clients/s3/filter_test.go index 1e64a86..90365d2 100644 --- a/internal/clients/s3/filter_test.go +++ b/internal/clients/s3/filter_test.go @@ -479,6 +479,19 @@ func TestFilterByPrefixAndRegex(t *testing.T) { expectedKeys: []string{}, expectError: false, }, + { + name: "excludes stackpacks backups when listing settings local bucket (empty prefix)", + objects: []Object{ + {Key: "sts-backup-20240101.sty", Size: 1000, LastModified: now}, + {Key: "sts-backup-20240101.sty.stackpacks.zip", Size: 500, LastModified: now}, + {Key: "sts-backup-20240102.sty", Size: 2000, LastModified: now}, + {Key: "sts-backup-20240102.sty.stackpacks.zip", Size: 300, LastModified: now}, + }, + prefix: "", + pattern: `^sts-backup-.*\.sty$`, + expectedKeys: []string{"sts-backup-20240101.sty", "sts-backup-20240102.sty"}, + expectError: false, + }, { name: "filters with complex regex pattern", objects: []Object{ From 3d3edafab6a0f62a65f245c15fc72969bc599c20 Mon Sep 17 00:00:00 2001 From: Remco Beckers Date: Thu, 2 Apr 2026 17:35:51 +0200 Subject: [PATCH 06/10] STAC-23457 Load stackpack backups from S3proxy --- cmd/elasticsearch/list_test.go | 4 ++ cmd/settings/list.go | 2 +- cmd/settings/restore.go | 6 +- cmd/stackgraph/restore.go | 14 ++++- internal/clients/s3/filter_test.go | 2 +- internal/foundation/config/config.go | 47 +++++++++++++++- internal/foundation/config/config_test.go | 42 +++++++------- .../config/testdata/validConfigMapConfig.yaml | 11 +++- .../config/testdata/validConfigMapOnly.yaml | 10 +++- .../testdata/validStorageConfigMapConfig.yaml | 10 +++- .../testdata/validStorageConfigMapOnly.yaml | 10 +++- .../scripts/restore-settings-backup.sh | 55 +++++++++---------- 12 files changed, 141 insertions(+), 72 deletions(-) diff --git a/cmd/elasticsearch/list_test.go b/cmd/elasticsearch/list_test.go index e66cfb4..36e9a82 100644 --- a/cmd/elasticsearch/list_test.go +++ b/cmd/elasticsearch/list_test.go @@ -68,6 +68,7 @@ stackgraph: restore: scaleDownLabelSelector: "app=stackgraph" loggingConfigConfigMap: logging-config + stsBackupConfigConfigMap: backup-config zookeeperQuorum: "zookeeper:2181" job: image: backup:latest @@ -107,6 +108,7 @@ settings: restore: scaleDownLabelSelector: "app=settings" loggingConfigConfigMap: logging-config + stsBackupConfigConfigMap: backup-config baseUrl: "http://server:7070" receiverBaseUrl: "http://receiver:7077" platformVersion: "5.2.0" @@ -152,6 +154,7 @@ stackgraph: restore: scaleDownLabelSelector: "app=stackgraph" loggingConfigConfigMap: logging-config + stsBackupConfigConfigMap: backup-config zookeeperQuorum: "zookeeper:2181" job: image: backup:latest @@ -192,6 +195,7 @@ settings: restore: scaleDownLabelSelector: "app=settings" loggingConfigConfigMap: logging-config + stsBackupConfigConfigMap: backup-config baseUrl: "http://server:7070" receiverBaseUrl: "http://receiver:7077" platformVersion: "5.2.0" diff --git a/cmd/settings/list.go b/cmd/settings/list.go index f2e87c9..da8c341 100644 --- a/cmd/settings/list.go +++ b/cmd/settings/list.go @@ -238,7 +238,7 @@ func getBackupListFromLocalBucket(appCtx *app.Context) ([]BackupFileInfo, error) return nil, fmt.Errorf("failed to list objects in local bucket: %w", err) } - filteredObjects := s3client.FilterBackupObjects(result.Contents, isMultiPartArchive) + filteredObjects := s3client.FilterMultipartBackupObjects(result.Contents, isMultiPartArchive) filteredObjects, err = s3client.FilterByPrefixAndRegex(filteredObjects, "", backupFileNameRegex) if err != nil { diff --git a/cmd/settings/restore.go b/cmd/settings/restore.go index a27a4cb..5b7d978 100644 --- a/cmd/settings/restore.go +++ b/cmd/settings/restore.go @@ -196,9 +196,9 @@ func buildEnvVar(extraEnvVar []corev1.EnvVar, config *config.Config) []corev1.En {Name: "BACKUP_CONFIGURATION_S3_PREFIX", Value: config.Settings.S3Prefix}, {Name: "BACKUP_CONFIGURATION_STACKPACKS_S3_PREFIX", Value: config.Settings.StackpacksS3Prefix}, {Name: "MINIO_ENDPOINT", Value: fmt.Sprintf("%s:%d", storageService.Name, storageService.Port)}, - {Name: "STACKSTATE_BASE_URL", Value: config.Settings.Restore.BaseURL}, - {Name: "RECEIVER_BASE_URL", Value: config.Settings.Restore.ReceiverBaseURL}, - {Name: "PLATFORM_VERSION", Value: config.Settings.Restore.PlatformVersion}, + {Name: "STACKSTATE_BASE_URL", Value: config.GetBaseURL()}, + {Name: "RECEIVER_BASE_URL", Value: config.GetReceiverBaseURL()}, + {Name: "PLATFORM_VERSION", Value: config.GetPlatformVersion()}, {Name: "ZOOKEEPER_QUORUM", Value: config.Settings.Restore.ZookeeperQuorum}, {Name: "BACKUP_CONFIGURATION_UPLOAD_REMOTE", Value: strconv.FormatBool(config.GlobalBackupEnabled())}, {Name: "SKIP_STACKPACKS", Value: strconv.FormatBool(skipStackpacks)}, diff --git a/cmd/stackgraph/restore.go b/cmd/stackgraph/restore.go index cd0569f..07680f8 100644 --- a/cmd/stackgraph/restore.go +++ b/cmd/stackgraph/restore.go @@ -5,7 +5,6 @@ import ( "fmt" "sort" "strconv" - "strings" "time" "github.com/aws/aws-sdk-go-v2/aws" @@ -182,6 +181,13 @@ func getLatestBackup(k8sClient *k8s.Client, namespace string, config *config.Con // Filter objects based on whether the archive is split or not filteredObjects := s3client.FilterMultipartBackupObjects(result.Contents, multipartArchive) + // Filter to only include direct children of the prefix that match the backup filename pattern, + // and strip the prefix from the key + filteredObjects, err = s3client.FilterByPrefixAndRegex(filteredObjects, prefix, backupFileNameRegex) + if err != nil { + return "", fmt.Errorf("failed to filter objects: %w", err) + } + if len(filteredObjects) == 0 { return "", fmt.Errorf("no backups found in bucket %s", bucket) } @@ -190,8 +196,7 @@ func getLatestBackup(k8sClient *k8s.Client, namespace string, config *config.Con sort.Slice(filteredObjects, func(i, j int) bool { return filteredObjects[i].LastModified.After(filteredObjects[j].LastModified) }) - latestBackup := strings.TrimPrefix(filteredObjects[0].Key, prefix) - return latestBackup, nil + return filteredObjects[0].Key, nil } // buildPVCSpec builds a PVCSpec from configuration @@ -273,6 +278,9 @@ func buildRestoreEnvVars(backupFile string, config *config.Config) []corev1.EnvV {Name: "BACKUP_STACKGRAPH_STACKPACKS_S3_PREFIX", Value: config.Stackgraph.StackpacksS3Prefix}, {Name: "BACKUP_STACKGRAPH_MULTIPART_ARCHIVE", Value: strconv.FormatBool(config.Stackgraph.MultipartArchive)}, {Name: "MINIO_ENDPOINT", Value: fmt.Sprintf("%s:%d", storageService.Name, storageService.Port)}, + {Name: "STACKSTATE_BASE_URL", Value: config.GetBaseURL()}, + {Name: "RECEIVER_BASE_URL", Value: config.GetReceiverBaseURL()}, + {Name: "PLATFORM_VERSION", Value: config.GetPlatformVersion()}, {Name: "ZOOKEEPER_QUORUM", Value: config.Stackgraph.Restore.ZookeeperQuorum}, {Name: "SKIP_STACKPACKS", Value: strconv.FormatBool(skipStackpacks)}, } diff --git a/internal/clients/s3/filter_test.go b/internal/clients/s3/filter_test.go index 90365d2..7880c3a 100644 --- a/internal/clients/s3/filter_test.go +++ b/internal/clients/s3/filter_test.go @@ -366,7 +366,7 @@ func TestFilterBackupObjects_SizeSummation(t *testing.T) { } // TestFilterByPrefixAndRegex tests the combined filtering by prefix and regex pattern -func TestFilterByPrefixAndRegex(t *testing.T) { +func TestFilterByPrefixAndRegex(t *testing.T) { //nolint:funlen // Table-driven test now := time.Now() tests := []struct { diff --git a/internal/foundation/config/config.go b/internal/foundation/config/config.go index 1c0c0d9..905d623 100644 --- a/internal/foundation/config/config.go +++ b/internal/foundation/config/config.go @@ -20,12 +20,49 @@ type Config struct { Elasticsearch ElasticsearchConfig `yaml:"elasticsearch" validate:"required"` Minio MinioConfig `yaml:"minio"` Storage StorageConfig `yaml:"storage"` + Stackstate StackstateConfig `yaml:"stackstate"` Stackgraph StackgraphConfig `yaml:"stackgraph" validate:"required"` Settings SettingsConfig `yaml:"settings" validate:"required"` VictoriaMetrics VictoriaMetricsConfig `yaml:"victoriaMetrics" validate:"required"` Clickhouse ClickhouseConfig `yaml:"clickhouse" validate:"required"` } +// StackstateConfig holds platform-wide configuration shared across restore operations. +// These values are used by both Settings and Stackgraph restore jobs. +// When set, they take precedence over the per-restore-type fields in SettingsRestoreConfig. +type StackstateConfig struct { + BaseURL string `yaml:"baseUrl"` + ReceiverBaseURL string `yaml:"receiverBaseUrl"` + PlatformVersion string `yaml:"platformVersion"` +} + +// GetBaseURL returns the StackState base URL, preferring the top-level stackstate section +// over the legacy settings.restore.baseUrl for backward compatibility. +func (c *Config) GetBaseURL() string { + if c.Stackstate.BaseURL != "" { + return c.Stackstate.BaseURL + } + return c.Settings.Restore.BaseURL +} + +// GetReceiverBaseURL returns the receiver base URL, preferring the top-level stackstate section +// over the legacy settings.restore.receiverBaseUrl for backward compatibility. +func (c *Config) GetReceiverBaseURL() string { + if c.Stackstate.ReceiverBaseURL != "" { + return c.Stackstate.ReceiverBaseURL + } + return c.Settings.Restore.ReceiverBaseURL +} + +// GetPlatformVersion returns the platform version, preferring the top-level stackstate section +// over the legacy settings.restore.platformVersion for backward compatibility. +func (c *Config) GetPlatformVersion() string { + if c.Stackstate.PlatformVersion != "" { + return c.Stackstate.PlatformVersion + } + return c.Settings.Restore.PlatformVersion +} + // IsLegacyMode returns true when the configuration uses the legacy Minio config. // Legacy mode is detected by the presence of the Minio config with a non-empty service name. func (c *Config) IsLegacyMode() bool { @@ -183,9 +220,9 @@ type SettingsRestoreConfig struct { ScaleDownLabelSelector string `yaml:"scaleDownLabelSelector" validate:"required"` LoggingConfigConfigMapName string `yaml:"loggingConfigConfigMap" validate:"required"` StsBackupConfigConfigMapName string `yaml:"stsBackupConfigConfigMap" validate:"required"` - BaseURL string `yaml:"baseUrl" validate:"required"` - ReceiverBaseURL string `yaml:"receiverBaseUrl" validate:"required"` - PlatformVersion string `yaml:"platformVersion" validate:"required"` + BaseURL string `yaml:"baseUrl"` + ReceiverBaseURL string `yaml:"receiverBaseUrl"` + PlatformVersion string `yaml:"platformVersion"` ZookeeperQuorum string `yaml:"zookeeperQuorum" validate:"required"` Job JobConfig `yaml:"job" validate:"required"` PVC string `yaml:"pvc"` // Required only in legacy mode @@ -394,6 +431,10 @@ func LoadConfig(clientset kubernetes.Interface, namespace, configMapName, secret } // Custom validation: either minio or storage must be configured + if config.GetBaseURL() == "" || config.GetReceiverBaseURL() == "" || config.GetPlatformVersion() == "" { + return nil, fmt.Errorf("configuration validation failed: baseUrl, receiverBaseUrl, and platformVersion must be set in either 'stackstate' or 'settings.restore'") + } + if config.Minio.Service.Name == "" && config.Storage.Service.Name == "" { return nil, fmt.Errorf("configuration validation failed: either 'minio' or 'storage' must be configured") } diff --git a/internal/foundation/config/config_test.go b/internal/foundation/config/config_test.go index 2719e87..2b36732 100644 --- a/internal/foundation/config/config_test.go +++ b/internal/foundation/config/config_test.go @@ -625,9 +625,10 @@ func TestConfig_StructValidation(t *testing.T) { S3Prefix: "", MultipartArchive: true, Restore: StackgraphRestoreConfig{ - ScaleDownLabelSelector: "app=stackgraph", - LoggingConfigConfigMapName: "logging-config", - ZookeeperQuorum: "zookeeper:2181", + ScaleDownLabelSelector: "app=stackgraph", + LoggingConfigConfigMapName: "logging-config", + StsBackupConfigConfigMapName: "backup-config", + ZookeeperQuorum: "zookeeper:2181", Job: JobConfig{ Image: "backup:latest", WaitImage: "wait:latest", @@ -682,13 +683,14 @@ func TestConfig_StructValidation(t *testing.T) { Bucket: "settings-backup", S3Prefix: "", Restore: SettingsRestoreConfig{ - ScaleDownLabelSelector: "app=settings", - LoggingConfigConfigMapName: "logging-config", - BaseURL: "http://server:7070", - ReceiverBaseURL: "http://receiver:7077", - PlatformVersion: "5.2.0", - ZookeeperQuorum: "zookeeper:2181", - PVC: "suse-observability-settings-backup-data", + ScaleDownLabelSelector: "app=settings", + LoggingConfigConfigMapName: "logging-config", + StsBackupConfigConfigMapName: "backup-config", + BaseURL: "http://server:7070", + ReceiverBaseURL: "http://receiver:7077", + PlatformVersion: "5.2.0", + ZookeeperQuorum: "zookeeper:2181", + PVC: "suse-observability-settings-backup-data", Job: JobConfig{ Image: "settings-backup:latest", WaitImage: "wait:latest", @@ -772,9 +774,10 @@ func TestConfig_StructValidation(t *testing.T) { S3Prefix: "", MultipartArchive: true, Restore: StackgraphRestoreConfig{ - ScaleDownLabelSelector: "app=stackgraph", - LoggingConfigConfigMapName: "logging-config", - ZookeeperQuorum: "zookeeper:2181", + ScaleDownLabelSelector: "app=stackgraph", + LoggingConfigConfigMapName: "logging-config", + StsBackupConfigConfigMapName: "backup-config", + ZookeeperQuorum: "zookeeper:2181", Job: JobConfig{ Image: "backup:latest", WaitImage: "wait:latest", @@ -829,12 +832,13 @@ func TestConfig_StructValidation(t *testing.T) { Bucket: "settings-backup", S3Prefix: "", Restore: SettingsRestoreConfig{ - ScaleDownLabelSelector: "app=settings", - LoggingConfigConfigMapName: "logging-config", - BaseURL: "http://server:7070", - ReceiverBaseURL: "http://receiver:7077", - PlatformVersion: "5.2.0", - ZookeeperQuorum: "zookeeper:2181", + ScaleDownLabelSelector: "app=settings", + LoggingConfigConfigMapName: "logging-config", + StsBackupConfigConfigMapName: "backup-config", + BaseURL: "http://server:7070", + ReceiverBaseURL: "http://receiver:7077", + PlatformVersion: "5.2.0", + ZookeeperQuorum: "zookeeper:2181", Job: JobConfig{ Image: "settings-backup:latest", WaitImage: "wait:latest", diff --git a/internal/foundation/config/testdata/validConfigMapConfig.yaml b/internal/foundation/config/testdata/validConfigMapConfig.yaml index e36900e..9083aad 100644 --- a/internal/foundation/config/testdata/validConfigMapConfig.yaml +++ b/internal/foundation/config/testdata/validConfigMapConfig.yaml @@ -59,6 +59,12 @@ elasticsearch: # Pattern for indices to restore from snapshot (comma-separated glob patterns) indicesPattern: sts*,.ds-sts_k8s_logs* +# StackState platform configuration shared across restore operations +stackstate: + baseUrl: "http://suse-observability-server:7070" + receiverBaseUrl: "http://suse-observability-receiver:7077" + platformVersion: "5.2.0" + # Minio configuration for S3-compatible storage minio: enabled: true @@ -86,6 +92,7 @@ stackgraph: scaleDownLabelSelector: "observability.suse.com/scalable-during-stackgraph-restore=true" # ConfigMap containing logging configuration loggingConfigConfigMap: suse-observability-logging + stsBackupConfigConfigMap: suse-observability-backup-config # Zookeeper quorum connection string zookeeperQuorum: "suse-observability-zookeeper:2181" # Job configuration @@ -145,9 +152,7 @@ settings: restore: scaleDownLabelSelector: "observability.suse.com/scalable-during-settings-restore=true" loggingConfigConfigMap: suse-observability-logging - baseUrl: "http://suse-observability-server:7070" - receiverBaseUrl: "http://suse-observability-receiver:7077" - platformVersion: "5.2.0" + stsBackupConfigConfigMap: suse-observability-backup-config zookeeperQuorum: "suse-observability-zookeeper:2181" pvc: "suse-observability-settings-backup-data" job: diff --git a/internal/foundation/config/testdata/validConfigMapOnly.yaml b/internal/foundation/config/testdata/validConfigMapOnly.yaml index dbf8ea1..5d4dde6 100644 --- a/internal/foundation/config/testdata/validConfigMapOnly.yaml +++ b/internal/foundation/config/testdata/validConfigMapOnly.yaml @@ -66,6 +66,11 @@ elasticsearch: # Pattern for indices to restore from snapshot (comma-separated glob patterns) indicesPattern: sts*,.ds-sts_k8s_logs* +stackstate: + baseUrl: "http://suse-observability-server:7070" + receiverBaseUrl: "http://suse-observability-receiver:7077" + platformVersion: "5.2.0" + # Minio configuration for S3-compatible storage minio: enabled: true @@ -83,6 +88,7 @@ stackgraph: restore: scaleDownLabelSelector: "observability.suse.com/scalable-during-stackgraph-restore=true" loggingConfigConfigMap: suse-observability-logging + stsBackupConfigConfigMap: suse-observability-backup-config zookeeperQuorum: "suse-observability-zookeeper:2181" job: labels: @@ -132,9 +138,7 @@ settings: restore: scaleDownLabelSelector: "observability.suse.com/scalable-during-settings-restore=true" loggingConfigConfigMap: suse-observability-logging - baseUrl: "http://suse-observability-server:7070" - receiverBaseUrl: "http://suse-observability-receiver:7077" - platformVersion: "5.2.0" + stsBackupConfigConfigMap: suse-observability-backup-config zookeeperQuorum: "suse-observability-zookeeper:2181" pvc: "suse-observability-settings-backup-data" job: diff --git a/internal/foundation/config/testdata/validStorageConfigMapConfig.yaml b/internal/foundation/config/testdata/validStorageConfigMapConfig.yaml index 5b928f8..af09cdb 100644 --- a/internal/foundation/config/testdata/validStorageConfigMapConfig.yaml +++ b/internal/foundation/config/testdata/validStorageConfigMapConfig.yaml @@ -61,6 +61,11 @@ elasticsearch: # Pattern for indices to restore from snapshot (comma-separated glob patterns) indicesPattern: sts*,.ds-sts_k8s_logs* +stackstate: + baseUrl: "http://suse-observability-server:7070" + receiverBaseUrl: "http://suse-observability-receiver:7077" + platformVersion: "5.2.0" + # Storage configuration for S3-compatible storage (new mode, replaces Minio) storage: globalBackupEnabled: true @@ -87,6 +92,7 @@ stackgraph: scaleDownLabelSelector: "observability.suse.com/scalable-during-stackgraph-restore=true" # ConfigMap containing logging configuration loggingConfigConfigMap: suse-observability-logging + stsBackupConfigConfigMap: suse-observability-backup-config # Zookeeper quorum connection string zookeeperQuorum: "suse-observability-zookeeper:2181" # Job configuration @@ -147,9 +153,7 @@ settings: restore: scaleDownLabelSelector: "observability.suse.com/scalable-during-settings-restore=true" loggingConfigConfigMap: suse-observability-logging - baseUrl: "http://suse-observability-server:7070" - receiverBaseUrl: "http://suse-observability-receiver:7077" - platformVersion: "5.2.0" + stsBackupConfigConfigMap: suse-observability-backup-config zookeeperQuorum: "suse-observability-zookeeper:2181" job: labels: diff --git a/internal/foundation/config/testdata/validStorageConfigMapOnly.yaml b/internal/foundation/config/testdata/validStorageConfigMapOnly.yaml index 9498e59..7642b80 100644 --- a/internal/foundation/config/testdata/validStorageConfigMapOnly.yaml +++ b/internal/foundation/config/testdata/validStorageConfigMapOnly.yaml @@ -68,6 +68,11 @@ elasticsearch: # Pattern for indices to restore from snapshot (comma-separated glob patterns) indicesPattern: sts*,.ds-sts_k8s_logs* +stackstate: + baseUrl: "http://suse-observability-server:7070" + receiverBaseUrl: "http://suse-observability-receiver:7077" + platformVersion: "5.2.0" + # Storage configuration for S3-compatible storage (new mode, replaces Minio) storage: globalBackupEnabled: true @@ -86,6 +91,7 @@ stackgraph: restore: scaleDownLabelSelector: "observability.suse.com/scalable-during-stackgraph-restore=true" loggingConfigConfigMap: suse-observability-logging + stsBackupConfigConfigMap: suse-observability-backup-config zookeeperQuorum: "suse-observability-zookeeper:2181" job: labels: @@ -137,9 +143,7 @@ settings: restore: scaleDownLabelSelector: "observability.suse.com/scalable-during-settings-restore=true" loggingConfigConfigMap: suse-observability-logging - baseUrl: "http://suse-observability-server:7070" - receiverBaseUrl: "http://suse-observability-receiver:7077" - platformVersion: "5.2.0" + stsBackupConfigConfigMap: suse-observability-backup-config zookeeperQuorum: "suse-observability-zookeeper:2181" job: labels: diff --git a/internal/scripts/scripts/restore-settings-backup.sh b/internal/scripts/scripts/restore-settings-backup.sh index 2309054..0b42ffa 100644 --- a/internal/scripts/scripts/restore-settings-backup.sh +++ b/internal/scripts/scripts/restore-settings-backup.sh @@ -15,8 +15,9 @@ download_from_s3() { local bucket="$1" local prefix="$2" local dest="$3" - echo "=== Downloading Settings backup \"${BACKUP_FILE}\" from bucket \"${bucket}\"..." - sts-toolbox aws s3 --endpoint "http://${MINIO_ENDPOINT}" --region minio cp "s3://${bucket}/${prefix}${BACKUP_FILE}" "${dest}/${BACKUP_FILE}" + local backup_file="$4" + echo "=== Downloading Settings backup \"${backup_file}\" from bucket \"${bucket}\"..." + sts-toolbox aws s3 --endpoint "http://${MINIO_ENDPOINT}" --region minio cp "s3://${bucket}/${prefix}${backup_file}" "${dest}/${backup_file}" } RESTORE_FILE="" @@ -28,11 +29,11 @@ elif [ -n "${BACKUP_CONFIGURATION_LOCAL_BUCKET:-}" ]; then # New mode: no PVC, download from local bucket first, fall back to remote bucket setup_aws_credentials - if download_from_s3 "${BACKUP_CONFIGURATION_LOCAL_BUCKET}" "" "${TMP_DIR}"; then + if download_from_s3 "${BACKUP_CONFIGURATION_LOCAL_BUCKET}" "" "${TMP_DIR}" "${BACKUP_FILE}"; then RESTORE_FILE="${TMP_DIR}/${BACKUP_FILE}" elif [ "${BACKUP_CONFIGURATION_UPLOAD_REMOTE}" == "true" ]; then echo "=== Backup not found in local bucket, trying remote bucket..." - if download_from_s3 "${BACKUP_CONFIGURATION_BUCKET_NAME}" "${BACKUP_CONFIGURATION_S3_PREFIX}" "${TMP_DIR}"; then + if download_from_s3 "${BACKUP_CONFIGURATION_BUCKET_NAME}" "${BACKUP_CONFIGURATION_S3_PREFIX}" "${TMP_DIR}" "${BACKUP_FILE}"; then RESTORE_FILE="${TMP_DIR}/${BACKUP_FILE}" fi fi @@ -43,7 +44,7 @@ else if [ "$BACKUP_CONFIGURATION_UPLOAD_REMOTE" == "true" ] && [ ! -f "${RESTORE_FILE}" ]; then setup_aws_credentials - download_from_s3 "${BACKUP_CONFIGURATION_BUCKET_NAME}" "${BACKUP_CONFIGURATION_S3_PREFIX}" "${TMP_DIR}" + download_from_s3 "${BACKUP_CONFIGURATION_BUCKET_NAME}" "${BACKUP_CONFIGURATION_S3_PREFIX}" "${TMP_DIR}" "${BACKUP_FILE}" RESTORE_FILE="${TMP_DIR}/${BACKUP_FILE}" fi fi @@ -58,39 +59,33 @@ echo "=== Restoring settings backup from \"${BACKUP_FILE}\"..." echo "=== Settings restore complete" # === StackPacks Restore === +# StackPacks backups are always stored, next to the settings backup. if [ "${SKIP_STACKPACKS:-false}" == "true" ]; then echo "=== Skipping StackPacks restore (--skip-stackpacks flag set)" else - export STACKPACKS_BACKUP_DIR="${BACKUP_DIR}/stackpacks" - # Construct stackpacks backup filename from the original backup file STACKPACKS_FILE="${BACKUP_FILE}.stackpacks.zip" - STACKPACKS_RESTORE_FILE="${STACKPACKS_BACKUP_DIR}/${STACKPACKS_FILE}" - - echo "=== Checking for StackPacks backup \"${STACKPACKS_FILE}\"..." - - # Check local PVC first, then try S3 if not found and remote is enabled - if [ ! -f "${STACKPACKS_RESTORE_FILE}" ] && [ "$BACKUP_CONFIGURATION_UPLOAD_REMOTE" == "true" ]; then - # Ensure AWS credentials are set for S3 access - export AWS_ACCESS_KEY_ID - AWS_ACCESS_KEY_ID="$(cat /aws-keys/accesskey)" - export AWS_SECRET_ACCESS_KEY - AWS_SECRET_ACCESS_KEY="$(cat /aws-keys/secretkey)" + STACKPACKS_RESTORE_FILE="" + + echo "=== Checking for StackPacks backup \"${STACKPACKS_FILE}\" in bucket \"${BACKUP_CONFIGURATION_LOCAL_BUCKET}\"..." + setup_aws_credentials - # Check if file exists in S3 - if sts-toolbox aws s3 ls --endpoint "http://${MINIO_ENDPOINT}" --region minio --bucket "${BACKUP_CONFIGURATION_BUCKET_NAME}" --prefix "${BACKUP_CONFIGURATION_STACKPACKS_S3_PREFIX}${STACKPACKS_FILE}" 2>/dev/null | grep -q "${STACKPACKS_FILE}"; then - echo "=== Downloading StackPacks backup from S3..." - sts-toolbox aws s3 cp --endpoint "http://${MINIO_ENDPOINT}" --region minio "s3://${BACKUP_CONFIGURATION_BUCKET_NAME}/${BACKUP_CONFIGURATION_STACKPACKS_S3_PREFIX}${STACKPACKS_FILE}" "${TMP_DIR}/${STACKPACKS_FILE}" - STACKPACKS_RESTORE_FILE="${TMP_DIR}/${STACKPACKS_FILE}" - fi + if download_from_s3 "${BACKUP_CONFIGURATION_LOCAL_BUCKET}" "${BACKUP_CONFIGURATION_STACKPACKS_S3_PREFIX}" "${TMP_DIR}" "${STACKPACKS_FILE}"; then + STACKPACKS_RESTORE_FILE="${TMP_DIR}/${STACKPACKS_FILE}" + elif [ "${BACKUP_CONFIGURATION_UPLOAD_REMOTE}" == "true" ]; then + echo "=== StackPacks backup not found in kubernetes settings storage, trying main backups storage..." + if download_from_s3 "${BACKUP_CONFIGURATION_BUCKET_NAME}" "${BACKUP_CONFIGURATION_STACKPACKS_S3_PREFIX}" "${TMP_DIR}" "${STACKPACKS_FILE}"; then + STACKPACKS_RESTORE_FILE="${TMP_DIR}/${STACKPACKS_FILE}" + fi fi - if [ -f "${STACKPACKS_RESTORE_FILE}" ]; then - echo "=== Restoring StackPacks from \"${STACKPACKS_FILE}\"..." - /opt/docker/bin/stack-packs-backup -Dlogback.configurationFile=/opt/docker/etc_log/logback.xml -restore "${STACKPACKS_RESTORE_FILE}" - echo "=== StackPacks restore complete" - else - echo "=== WARNING: StackPacks backup \"${STACKPACKS_FILE}\" not found, skipping StackPacks restore" + if [ -z "${STACKPACKS_RESTORE_FILE}" ] || [ ! -f "${STACKPACKS_RESTORE_FILE}" ]; then + echo "=== WARNING: StackPacks backup \"${STACKPACKS_FILE}\" not found, skipping StackPacks restore" + exit 0 fi + + echo "=== Restoring StackPacks from \"${STACKPACKS_FILE}\"..." + /opt/docker/bin/stack-packs-backup -Dlogback.configurationFile=/opt/docker/etc_log/logback.xml -restore "${STACKPACKS_RESTORE_FILE}" + echo "=== StackPacks restore complete" fi echo "===" From 9b1a52a7673b6d1de9db31014081f8a28922c299 Mon Sep 17 00:00:00 2001 From: Remco Beckers Date: Fri, 3 Apr 2026 10:52:16 +0200 Subject: [PATCH 07/10] STAC-23457 Set stackpacks local path based on local PVC presence --- cmd/settings/restore.go | 3 +++ cmd/stackgraph/restore.go | 12 ++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/cmd/settings/restore.go b/cmd/settings/restore.go index 5b7d978..2aa6df9 100644 --- a/cmd/settings/restore.go +++ b/cmd/settings/restore.go @@ -209,6 +209,9 @@ func buildEnvVar(extraEnvVar []corev1.EnvVar, config *config.Config) []corev1.En } else if config.Settings.LocalBucket != "" { commonVar = append(commonVar, corev1.EnvVar{Name: "BACKUP_CONFIGURATION_LOCAL_BUCKET", Value: config.Settings.LocalBucket}) } + if config.Settings.Restore.StackpacksPVCName != "" { + commonVar = append(commonVar, corev1.EnvVar{Name: "CONFIG_FORCE_stackstate_stackPacks_localStackPacksUri", Value: "/var/stackpacks_local"}) + } commonVar = append(commonVar, extraEnvVar...) return commonVar } diff --git a/cmd/stackgraph/restore.go b/cmd/stackgraph/restore.go index 07680f8..666da6b 100644 --- a/cmd/stackgraph/restore.go +++ b/cmd/stackgraph/restore.go @@ -270,7 +270,7 @@ func createRestoreJob(k8sClient *k8s.Client, namespace, jobName, backupFile stri // buildRestoreEnvVars constructs environment variables for the restore job func buildRestoreEnvVars(backupFile string, config *config.Config) []corev1.EnvVar { storageService := config.GetStorageService() - return []corev1.EnvVar{ + env := []corev1.EnvVar{ {Name: "BACKUP_FILE", Value: backupFile}, {Name: "FORCE_DELETE", Value: purgeStackgraphDataFlag}, {Name: "BACKUP_STACKGRAPH_BUCKET_NAME", Value: config.Stackgraph.Bucket}, @@ -284,6 +284,10 @@ func buildRestoreEnvVars(backupFile string, config *config.Config) []corev1.EnvV {Name: "ZOOKEEPER_QUORUM", Value: config.Stackgraph.Restore.ZookeeperQuorum}, {Name: "SKIP_STACKPACKS", Value: strconv.FormatBool(skipStackpacks)}, } + if config.Stackgraph.Restore.StackpacksPVCName != "" { + env = append(env, corev1.EnvVar{Name: "CONFIG_FORCE_stackstate_stackPacks_localStackPacksUri", Value: "/var/stackpacks_local"}) + } + return env } // buildRestoreVolumeMounts constructs volume mounts for the restore job container @@ -296,7 +300,7 @@ func buildRestoreVolumeMounts(config *config.Config) []corev1.VolumeMount { {Name: "tmp-data", MountPath: "/tmp-data"}, } - if config.Settings.Restore.StackpacksPVCName != "" { + if config.Stackgraph.Restore.StackpacksPVCName != "" { volumeMounts = append(volumeMounts, corev1.VolumeMount{ Name: "stackpacks-local", MountPath: "/var/stackpacks_local", @@ -375,12 +379,12 @@ func buildRestoreVolumes(jobName string, config *config.Config, defaultMode int3 }, }, } - if config.Settings.Restore.StackpacksPVCName != "" { + if config.Stackgraph.Restore.StackpacksPVCName != "" { volumes = append(volumes, corev1.Volume{ Name: "stackpacks-local", VolumeSource: corev1.VolumeSource{ PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: config.Settings.Restore.StackpacksPVCName, + ClaimName: config.Stackgraph.Restore.StackpacksPVCName, }, }, }) From c93de1bdd245b3c17be0ea7b4e30e00b352753e3 Mon Sep 17 00:00:00 2001 From: Remco Beckers Date: Fri, 3 Apr 2026 11:17:00 +0200 Subject: [PATCH 08/10] STAC-23457 Refactor and cleanup configuration --- cmd/elasticsearch/list_test.go | 4 -- cmd/settings/restore.go | 23 ++------ cmd/stackgraph/restore.go | 23 ++------ internal/foundation/config/config.go | 59 +++++++++---------- internal/foundation/config/config_test.go | 12 ++-- .../config/testdata/validConfigMapConfig.yaml | 5 +- .../config/testdata/validConfigMapOnly.yaml | 5 +- .../testdata/validStorageConfigMapConfig.yaml | 5 +- .../testdata/validStorageConfigMapOnly.yaml | 5 +- 9 files changed, 53 insertions(+), 88 deletions(-) diff --git a/cmd/elasticsearch/list_test.go b/cmd/elasticsearch/list_test.go index 36e9a82..e66cfb4 100644 --- a/cmd/elasticsearch/list_test.go +++ b/cmd/elasticsearch/list_test.go @@ -68,7 +68,6 @@ stackgraph: restore: scaleDownLabelSelector: "app=stackgraph" loggingConfigConfigMap: logging-config - stsBackupConfigConfigMap: backup-config zookeeperQuorum: "zookeeper:2181" job: image: backup:latest @@ -108,7 +107,6 @@ settings: restore: scaleDownLabelSelector: "app=settings" loggingConfigConfigMap: logging-config - stsBackupConfigConfigMap: backup-config baseUrl: "http://server:7070" receiverBaseUrl: "http://receiver:7077" platformVersion: "5.2.0" @@ -154,7 +152,6 @@ stackgraph: restore: scaleDownLabelSelector: "app=stackgraph" loggingConfigConfigMap: logging-config - stsBackupConfigConfigMap: backup-config zookeeperQuorum: "zookeeper:2181" job: image: backup:latest @@ -195,7 +192,6 @@ settings: restore: scaleDownLabelSelector: "app=settings" loggingConfigConfigMap: logging-config - stsBackupConfigConfigMap: backup-config baseUrl: "http://server:7070" receiverBaseUrl: "http://receiver:7077" platformVersion: "5.2.0" diff --git a/cmd/settings/restore.go b/cmd/settings/restore.go index 2aa6df9..57321f9 100644 --- a/cmd/settings/restore.go +++ b/cmd/settings/restore.go @@ -209,8 +209,8 @@ func buildEnvVar(extraEnvVar []corev1.EnvVar, config *config.Config) []corev1.En } else if config.Settings.LocalBucket != "" { commonVar = append(commonVar, corev1.EnvVar{Name: "BACKUP_CONFIGURATION_LOCAL_BUCKET", Value: config.Settings.LocalBucket}) } - if config.Settings.Restore.StackpacksPVCName != "" { - commonVar = append(commonVar, corev1.EnvVar{Name: "CONFIG_FORCE_stackstate_stackPacks_localStackPacksUri", Value: "/var/stackpacks_local"}) + if config.Stackpacks != nil { + commonVar = append(commonVar, corev1.EnvVar{Name: "CONFIG_FORCE_stackstate_stackPacks_localStackPacksUri", Value: config.Stackpacks.LocalStackPacksUri}) } commonVar = append(commonVar, extraEnvVar...) return commonVar @@ -220,7 +220,6 @@ func buildEnvVar(extraEnvVar []corev1.EnvVar, config *config.Config) []corev1.En func buildVolumeMounts(config *config.Config) []corev1.VolumeMount { volumeMounts := []corev1.VolumeMount{ {Name: "backup-log", MountPath: "/opt/docker/etc_log"}, - {Name: "config-volume", MountPath: "/opt/docker/etc/application_stackstate.conf", SubPath: "application_stackstate.conf"}, {Name: "backup-restore-scripts", MountPath: "/backup-restore-scripts"}, {Name: "minio-keys", MountPath: "/aws-keys"}, {Name: "tmp-data", MountPath: "/tmp-data"}, @@ -230,10 +229,10 @@ func buildVolumeMounts(config *config.Config) []corev1.VolumeMount { volumeMounts = append(volumeMounts, corev1.VolumeMount{Name: "settings-backup-data", MountPath: "/settings-backup-data"}) } - if config.Settings.Restore.StackpacksPVCName != "" { + if config.Stackpacks != nil && config.Stackpacks.PVC != "" { volumeMounts = append(volumeMounts, corev1.VolumeMount{ Name: "stackpacks-local", - MountPath: "/var/stackpacks_local", + MountPath: config.Stackpacks.LocalStackPacksUri, }) } @@ -253,16 +252,6 @@ func buildVolumes(config *config.Config, defaultMode int32) []corev1.Volume { }, }, }, - { - Name: "config-volume", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: config.Settings.Restore.StsBackupConfigConfigMapName, - }, - }, - }, - }, { Name: "backup-restore-scripts", VolumeSource: corev1.VolumeSource{ @@ -300,12 +289,12 @@ func buildVolumes(config *config.Config, defaultMode int32) []corev1.Volume { }, }) } - if config.Settings.Restore.StackpacksPVCName != "" { + if config.Stackpacks != nil && config.Stackpacks.PVC != "" { volumes = append(volumes, corev1.Volume{ Name: "stackpacks-local", VolumeSource: corev1.VolumeSource{ PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: config.Settings.Restore.StackpacksPVCName, + ClaimName: config.Stackpacks.PVC, }, }, }) diff --git a/cmd/stackgraph/restore.go b/cmd/stackgraph/restore.go index 666da6b..f68eda3 100644 --- a/cmd/stackgraph/restore.go +++ b/cmd/stackgraph/restore.go @@ -284,8 +284,8 @@ func buildRestoreEnvVars(backupFile string, config *config.Config) []corev1.EnvV {Name: "ZOOKEEPER_QUORUM", Value: config.Stackgraph.Restore.ZookeeperQuorum}, {Name: "SKIP_STACKPACKS", Value: strconv.FormatBool(skipStackpacks)}, } - if config.Stackgraph.Restore.StackpacksPVCName != "" { - env = append(env, corev1.EnvVar{Name: "CONFIG_FORCE_stackstate_stackPacks_localStackPacksUri", Value: "/var/stackpacks_local"}) + if config.Stackpacks != nil { + env = append(env, corev1.EnvVar{Name: "CONFIG_FORCE_stackstate_stackPacks_localStackPacksUri", Value: config.Stackpacks.LocalStackPacksUri}) } return env } @@ -294,16 +294,15 @@ func buildRestoreEnvVars(backupFile string, config *config.Config) []corev1.EnvV func buildRestoreVolumeMounts(config *config.Config) []corev1.VolumeMount { volumeMounts := []corev1.VolumeMount{ {Name: "backup-log", MountPath: "/opt/docker/etc_log"}, - {Name: "config-volume", MountPath: "/opt/docker/etc/application_stackstate.conf", SubPath: "application_stackstate.conf"}, {Name: "backup-restore-scripts", MountPath: "/backup-restore-scripts"}, {Name: "minio-keys", MountPath: "/aws-keys"}, {Name: "tmp-data", MountPath: "/tmp-data"}, } - if config.Stackgraph.Restore.StackpacksPVCName != "" { + if config.Stackpacks != nil && config.Stackpacks.PVC != "" { volumeMounts = append(volumeMounts, corev1.VolumeMount{ Name: "stackpacks-local", - MountPath: "/var/stackpacks_local", + MountPath: config.Stackpacks.LocalStackPacksUri, }) } @@ -341,16 +340,6 @@ func buildRestoreVolumes(jobName string, config *config.Config, defaultMode int3 }, }, }, - { - Name: "config-volume", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: config.Stackgraph.Restore.StsBackupConfigConfigMapName, - }, - }, - }, - }, { Name: "backup-restore-scripts", VolumeSource: corev1.VolumeSource{ @@ -379,12 +368,12 @@ func buildRestoreVolumes(jobName string, config *config.Config, defaultMode int3 }, }, } - if config.Stackgraph.Restore.StackpacksPVCName != "" { + if config.Stackpacks != nil && config.Stackpacks.PVC != "" { volumes = append(volumes, corev1.Volume{ Name: "stackpacks-local", VolumeSource: corev1.VolumeSource{ PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: config.Stackgraph.Restore.StackpacksPVCName, + ClaimName: config.Stackpacks.PVC, }, }, }) diff --git a/internal/foundation/config/config.go b/internal/foundation/config/config.go index 905d623..300dc24 100644 --- a/internal/foundation/config/config.go +++ b/internal/foundation/config/config.go @@ -20,45 +20,48 @@ type Config struct { Elasticsearch ElasticsearchConfig `yaml:"elasticsearch" validate:"required"` Minio MinioConfig `yaml:"minio"` Storage StorageConfig `yaml:"storage"` - Stackstate StackstateConfig `yaml:"stackstate"` + Stackpacks *StackpacksConfig `yaml:"stackpacks"` Stackgraph StackgraphConfig `yaml:"stackgraph" validate:"required"` Settings SettingsConfig `yaml:"settings" validate:"required"` VictoriaMetrics VictoriaMetricsConfig `yaml:"victoriaMetrics" validate:"required"` Clickhouse ClickhouseConfig `yaml:"clickhouse" validate:"required"` } -// StackstateConfig holds platform-wide configuration shared across restore operations. -// These values are used by both Settings and Stackgraph restore jobs. -// When set, they take precedence over the per-restore-type fields in SettingsRestoreConfig. -type StackstateConfig struct { - BaseURL string `yaml:"baseUrl"` - ReceiverBaseURL string `yaml:"receiverBaseUrl"` - PlatformVersion string `yaml:"platformVersion"` +// StackpacksConfig holds stackpacks-specific configuration shared across restore operations. +// This section is optional. When present, its values are provided to restore pods/scripts. +// BaseURL, ReceiverBaseURL, and PlatformVersion take precedence over the per-restore-type +// fields in SettingsRestoreConfig when set. +type StackpacksConfig struct { + BaseURL string `yaml:"baseUrl"` + ReceiverBaseURL string `yaml:"receiverBaseUrl"` + PlatformVersion string `yaml:"platformVersion"` + LocalStackPacksUri string `yaml:"localStackPacksUri" validate:"required"` + PVC string `yaml:"pvc"` } -// GetBaseURL returns the StackState base URL, preferring the top-level stackstate section +// GetBaseURL returns the StackState base URL, preferring the top-level stackpacks section // over the legacy settings.restore.baseUrl for backward compatibility. func (c *Config) GetBaseURL() string { - if c.Stackstate.BaseURL != "" { - return c.Stackstate.BaseURL + if c.Stackpacks != nil && c.Stackpacks.BaseURL != "" { + return c.Stackpacks.BaseURL } return c.Settings.Restore.BaseURL } -// GetReceiverBaseURL returns the receiver base URL, preferring the top-level stackstate section +// GetReceiverBaseURL returns the receiver base URL, preferring the top-level stackpacks section // over the legacy settings.restore.receiverBaseUrl for backward compatibility. func (c *Config) GetReceiverBaseURL() string { - if c.Stackstate.ReceiverBaseURL != "" { - return c.Stackstate.ReceiverBaseURL + if c.Stackpacks != nil && c.Stackpacks.ReceiverBaseURL != "" { + return c.Stackpacks.ReceiverBaseURL } return c.Settings.Restore.ReceiverBaseURL } -// GetPlatformVersion returns the platform version, preferring the top-level stackstate section +// GetPlatformVersion returns the platform version, preferring the top-level stackpacks section // over the legacy settings.restore.platformVersion for backward compatibility. func (c *Config) GetPlatformVersion() string { - if c.Stackstate.PlatformVersion != "" { - return c.Stackstate.PlatformVersion + if c.Stackpacks != nil && c.Stackpacks.PlatformVersion != "" { + return c.Stackpacks.PlatformVersion } return c.Settings.Restore.PlatformVersion } @@ -201,11 +204,9 @@ type S3Location struct { type StackgraphRestoreConfig struct { ScaleDownLabelSelector string `yaml:"scaleDownLabelSelector" validate:"required"` LoggingConfigConfigMapName string `yaml:"loggingConfigConfigMap" validate:"required"` - StsBackupConfigConfigMapName string `yaml:"stsBackupConfigConfigMap" validate:"required"` - ZookeeperQuorum string `yaml:"zookeeperQuorum" validate:"required"` - Job JobConfig `yaml:"job" validate:"required"` - PVC PVCConfig `yaml:"pvc" validate:"required"` - StackpacksPVCName string `yaml:"stackpacksPvc"` + ZookeeperQuorum string `yaml:"zookeeperQuorum" validate:"required"` + Job JobConfig `yaml:"job" validate:"required"` + PVC PVCConfig `yaml:"pvc" validate:"required"` } type SettingsConfig struct { @@ -219,14 +220,12 @@ type SettingsConfig struct { type SettingsRestoreConfig struct { ScaleDownLabelSelector string `yaml:"scaleDownLabelSelector" validate:"required"` LoggingConfigConfigMapName string `yaml:"loggingConfigConfigMap" validate:"required"` - StsBackupConfigConfigMapName string `yaml:"stsBackupConfigConfigMap" validate:"required"` - BaseURL string `yaml:"baseUrl"` - ReceiverBaseURL string `yaml:"receiverBaseUrl"` - PlatformVersion string `yaml:"platformVersion"` - ZookeeperQuorum string `yaml:"zookeeperQuorum" validate:"required"` - Job JobConfig `yaml:"job" validate:"required"` - PVC string `yaml:"pvc"` // Required only in legacy mode - StackpacksPVCName string `yaml:"stackpacksPvc"` + BaseURL string `yaml:"baseUrl"` + ReceiverBaseURL string `yaml:"receiverBaseUrl"` + PlatformVersion string `yaml:"platformVersion"` + ZookeeperQuorum string `yaml:"zookeeperQuorum" validate:"required"` + Job JobConfig `yaml:"job" validate:"required"` + PVC string `yaml:"pvc"` // Required only in legacy mode } // ClickhouseConfig holds Clickhouse-specific configuration diff --git a/internal/foundation/config/config_test.go b/internal/foundation/config/config_test.go index 2b36732..1bd5468 100644 --- a/internal/foundation/config/config_test.go +++ b/internal/foundation/config/config_test.go @@ -627,8 +627,7 @@ func TestConfig_StructValidation(t *testing.T) { Restore: StackgraphRestoreConfig{ ScaleDownLabelSelector: "app=stackgraph", LoggingConfigConfigMapName: "logging-config", - StsBackupConfigConfigMapName: "backup-config", - ZookeeperQuorum: "zookeeper:2181", + ZookeeperQuorum: "zookeeper:2181", Job: JobConfig{ Image: "backup:latest", WaitImage: "wait:latest", @@ -685,8 +684,7 @@ func TestConfig_StructValidation(t *testing.T) { Restore: SettingsRestoreConfig{ ScaleDownLabelSelector: "app=settings", LoggingConfigConfigMapName: "logging-config", - StsBackupConfigConfigMapName: "backup-config", - BaseURL: "http://server:7070", + BaseURL: "http://server:7070", ReceiverBaseURL: "http://receiver:7077", PlatformVersion: "5.2.0", ZookeeperQuorum: "zookeeper:2181", @@ -776,8 +774,7 @@ func TestConfig_StructValidation(t *testing.T) { Restore: StackgraphRestoreConfig{ ScaleDownLabelSelector: "app=stackgraph", LoggingConfigConfigMapName: "logging-config", - StsBackupConfigConfigMapName: "backup-config", - ZookeeperQuorum: "zookeeper:2181", + ZookeeperQuorum: "zookeeper:2181", Job: JobConfig{ Image: "backup:latest", WaitImage: "wait:latest", @@ -834,8 +831,7 @@ func TestConfig_StructValidation(t *testing.T) { Restore: SettingsRestoreConfig{ ScaleDownLabelSelector: "app=settings", LoggingConfigConfigMapName: "logging-config", - StsBackupConfigConfigMapName: "backup-config", - BaseURL: "http://server:7070", + BaseURL: "http://server:7070", ReceiverBaseURL: "http://receiver:7077", PlatformVersion: "5.2.0", ZookeeperQuorum: "zookeeper:2181", diff --git a/internal/foundation/config/testdata/validConfigMapConfig.yaml b/internal/foundation/config/testdata/validConfigMapConfig.yaml index 9083aad..60efe2a 100644 --- a/internal/foundation/config/testdata/validConfigMapConfig.yaml +++ b/internal/foundation/config/testdata/validConfigMapConfig.yaml @@ -60,10 +60,11 @@ elasticsearch: indicesPattern: sts*,.ds-sts_k8s_logs* # StackState platform configuration shared across restore operations -stackstate: +stackpacks: baseUrl: "http://suse-observability-server:7070" receiverBaseUrl: "http://suse-observability-receiver:7077" platformVersion: "5.2.0" + localStackPacksUri: "/var/stackpacks_local" # Minio configuration for S3-compatible storage minio: @@ -92,7 +93,6 @@ stackgraph: scaleDownLabelSelector: "observability.suse.com/scalable-during-stackgraph-restore=true" # ConfigMap containing logging configuration loggingConfigConfigMap: suse-observability-logging - stsBackupConfigConfigMap: suse-observability-backup-config # Zookeeper quorum connection string zookeeperQuorum: "suse-observability-zookeeper:2181" # Job configuration @@ -152,7 +152,6 @@ settings: restore: scaleDownLabelSelector: "observability.suse.com/scalable-during-settings-restore=true" loggingConfigConfigMap: suse-observability-logging - stsBackupConfigConfigMap: suse-observability-backup-config zookeeperQuorum: "suse-observability-zookeeper:2181" pvc: "suse-observability-settings-backup-data" job: diff --git a/internal/foundation/config/testdata/validConfigMapOnly.yaml b/internal/foundation/config/testdata/validConfigMapOnly.yaml index 5d4dde6..323993e 100644 --- a/internal/foundation/config/testdata/validConfigMapOnly.yaml +++ b/internal/foundation/config/testdata/validConfigMapOnly.yaml @@ -66,10 +66,11 @@ elasticsearch: # Pattern for indices to restore from snapshot (comma-separated glob patterns) indicesPattern: sts*,.ds-sts_k8s_logs* -stackstate: +stackpacks: baseUrl: "http://suse-observability-server:7070" receiverBaseUrl: "http://suse-observability-receiver:7077" platformVersion: "5.2.0" + localStackPacksUri: "/var/stackpacks_local" # Minio configuration for S3-compatible storage minio: @@ -88,7 +89,6 @@ stackgraph: restore: scaleDownLabelSelector: "observability.suse.com/scalable-during-stackgraph-restore=true" loggingConfigConfigMap: suse-observability-logging - stsBackupConfigConfigMap: suse-observability-backup-config zookeeperQuorum: "suse-observability-zookeeper:2181" job: labels: @@ -138,7 +138,6 @@ settings: restore: scaleDownLabelSelector: "observability.suse.com/scalable-during-settings-restore=true" loggingConfigConfigMap: suse-observability-logging - stsBackupConfigConfigMap: suse-observability-backup-config zookeeperQuorum: "suse-observability-zookeeper:2181" pvc: "suse-observability-settings-backup-data" job: diff --git a/internal/foundation/config/testdata/validStorageConfigMapConfig.yaml b/internal/foundation/config/testdata/validStorageConfigMapConfig.yaml index af09cdb..227d8a3 100644 --- a/internal/foundation/config/testdata/validStorageConfigMapConfig.yaml +++ b/internal/foundation/config/testdata/validStorageConfigMapConfig.yaml @@ -61,10 +61,11 @@ elasticsearch: # Pattern for indices to restore from snapshot (comma-separated glob patterns) indicesPattern: sts*,.ds-sts_k8s_logs* -stackstate: +stackpacks: baseUrl: "http://suse-observability-server:7070" receiverBaseUrl: "http://suse-observability-receiver:7077" platformVersion: "5.2.0" + localStackPacksUri: "/var/stackpacks_local" # Storage configuration for S3-compatible storage (new mode, replaces Minio) storage: @@ -92,7 +93,6 @@ stackgraph: scaleDownLabelSelector: "observability.suse.com/scalable-during-stackgraph-restore=true" # ConfigMap containing logging configuration loggingConfigConfigMap: suse-observability-logging - stsBackupConfigConfigMap: suse-observability-backup-config # Zookeeper quorum connection string zookeeperQuorum: "suse-observability-zookeeper:2181" # Job configuration @@ -153,7 +153,6 @@ settings: restore: scaleDownLabelSelector: "observability.suse.com/scalable-during-settings-restore=true" loggingConfigConfigMap: suse-observability-logging - stsBackupConfigConfigMap: suse-observability-backup-config zookeeperQuorum: "suse-observability-zookeeper:2181" job: labels: diff --git a/internal/foundation/config/testdata/validStorageConfigMapOnly.yaml b/internal/foundation/config/testdata/validStorageConfigMapOnly.yaml index 7642b80..d08832d 100644 --- a/internal/foundation/config/testdata/validStorageConfigMapOnly.yaml +++ b/internal/foundation/config/testdata/validStorageConfigMapOnly.yaml @@ -68,10 +68,11 @@ elasticsearch: # Pattern for indices to restore from snapshot (comma-separated glob patterns) indicesPattern: sts*,.ds-sts_k8s_logs* -stackstate: +stackpacks: baseUrl: "http://suse-observability-server:7070" receiverBaseUrl: "http://suse-observability-receiver:7077" platformVersion: "5.2.0" + localStackPacksUri: "/var/stackpacks_local" # Storage configuration for S3-compatible storage (new mode, replaces Minio) storage: @@ -91,7 +92,6 @@ stackgraph: restore: scaleDownLabelSelector: "observability.suse.com/scalable-during-stackgraph-restore=true" loggingConfigConfigMap: suse-observability-logging - stsBackupConfigConfigMap: suse-observability-backup-config zookeeperQuorum: "suse-observability-zookeeper:2181" job: labels: @@ -143,7 +143,6 @@ settings: restore: scaleDownLabelSelector: "observability.suse.com/scalable-during-settings-restore=true" loggingConfigConfigMap: suse-observability-logging - stsBackupConfigConfigMap: suse-observability-backup-config zookeeperQuorum: "suse-observability-zookeeper:2181" job: labels: From 6bc1893ad29dcda9405597609650a95530719424 Mon Sep 17 00:00:00 2001 From: Remco Beckers Date: Fri, 3 Apr 2026 12:21:02 +0200 Subject: [PATCH 09/10] STAC-23457 Add tests for mounts --- cmd/settings/restore.go | 5 +-- cmd/settings/restore_test.go | 64 ++++++++++++++++++++++++++++++++++ cmd/stackgraph/restore.go | 5 +-- cmd/stackgraph/restore_test.go | 61 ++++++++++++++++++++++++++++++++ 4 files changed, 131 insertions(+), 4 deletions(-) create mode 100644 cmd/settings/restore_test.go create mode 100644 cmd/stackgraph/restore_test.go diff --git a/cmd/settings/restore.go b/cmd/settings/restore.go index 57321f9..518e261 100644 --- a/cmd/settings/restore.go +++ b/cmd/settings/restore.go @@ -3,6 +3,7 @@ package settings import ( "fmt" "strconv" + "strings" "time" "github.com/spf13/cobra" @@ -229,10 +230,10 @@ func buildVolumeMounts(config *config.Config) []corev1.VolumeMount { volumeMounts = append(volumeMounts, corev1.VolumeMount{Name: "settings-backup-data", MountPath: "/settings-backup-data"}) } - if config.Stackpacks != nil && config.Stackpacks.PVC != "" { + if config.Stackpacks != nil && config.Stackpacks.PVC != "" && strings.HasPrefix(config.Stackpacks.LocalStackPacksUri, "file://") { volumeMounts = append(volumeMounts, corev1.VolumeMount{ Name: "stackpacks-local", - MountPath: config.Stackpacks.LocalStackPacksUri, + MountPath: strings.TrimPrefix(config.Stackpacks.LocalStackPacksUri, "file://"), }) } diff --git a/cmd/settings/restore_test.go b/cmd/settings/restore_test.go new file mode 100644 index 0000000..4f43b49 --- /dev/null +++ b/cmd/settings/restore_test.go @@ -0,0 +1,64 @@ +package settings + +import ( + "testing" + + "github.com/stackvista/stackstate-backup-cli/internal/foundation/config" + "github.com/stretchr/testify/assert" +) + +func TestBuildVolumeMounts_StackpacksLocalFileURI(t *testing.T) { + tests := []struct { + name string + stackpacks *config.StackpacksConfig + expectStackpacks bool + expectedMountPath string + }{ + { + name: "no stackpacks config", + stackpacks: nil, + expectStackpacks: false, + }, + { + name: "stackpacks with no PVC", + stackpacks: &config.StackpacksConfig{LocalStackPacksUri: "file:///var/stackpacks_local"}, + expectStackpacks: false, + }, + { + name: "stackpacks with file:// URI and PVC", + stackpacks: &config.StackpacksConfig{LocalStackPacksUri: "file:///var/stackpacks_local", PVC: "stackpacks-pvc"}, + expectStackpacks: true, + expectedMountPath: "/var/stackpacks_local", + }, + { + name: "stackpacks with non-file URI and PVC", + stackpacks: &config.StackpacksConfig{LocalStackPacksUri: "s3://my-bucket/stackpacks", PVC: "stackpacks-pvc"}, + expectStackpacks: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Ensure the package-level fromPVC flag does not interfere + fromPVC = false + + cfg := &config.Config{Stackpacks: tt.stackpacks} + mounts := buildVolumeMounts(cfg) + + var stackpacksMount *struct{ Name, MountPath string } + for _, m := range mounts { + if m.Name == "stackpacks-local" { + stackpacksMount = &struct{ Name, MountPath string }{m.Name, m.MountPath} + break + } + } + + if tt.expectStackpacks { + assert.NotNil(t, stackpacksMount, "expected stackpacks-local volume mount to be present") + assert.Equal(t, tt.expectedMountPath, stackpacksMount.MountPath) + } else { + assert.Nil(t, stackpacksMount, "expected stackpacks-local volume mount to be absent") + } + }) + } +} diff --git a/cmd/stackgraph/restore.go b/cmd/stackgraph/restore.go index f68eda3..6d3225a 100644 --- a/cmd/stackgraph/restore.go +++ b/cmd/stackgraph/restore.go @@ -5,6 +5,7 @@ import ( "fmt" "sort" "strconv" + "strings" "time" "github.com/aws/aws-sdk-go-v2/aws" @@ -299,10 +300,10 @@ func buildRestoreVolumeMounts(config *config.Config) []corev1.VolumeMount { {Name: "tmp-data", MountPath: "/tmp-data"}, } - if config.Stackpacks != nil && config.Stackpacks.PVC != "" { + if config.Stackpacks != nil && config.Stackpacks.PVC != "" && strings.HasPrefix(config.Stackpacks.LocalStackPacksUri, "file://") { volumeMounts = append(volumeMounts, corev1.VolumeMount{ Name: "stackpacks-local", - MountPath: config.Stackpacks.LocalStackPacksUri, + MountPath: strings.TrimPrefix(config.Stackpacks.LocalStackPacksUri, "file://"), }) } diff --git a/cmd/stackgraph/restore_test.go b/cmd/stackgraph/restore_test.go new file mode 100644 index 0000000..447c5fd --- /dev/null +++ b/cmd/stackgraph/restore_test.go @@ -0,0 +1,61 @@ +package stackgraph + +import ( + "testing" + + "github.com/stackvista/stackstate-backup-cli/internal/foundation/config" + "github.com/stretchr/testify/assert" +) + +func TestBuildRestoreVolumeMounts_StackpacksLocalFileURI(t *testing.T) { + tests := []struct { + name string + stackpacks *config.StackpacksConfig + expectStackpacks bool + expectedMountPath string + }{ + { + name: "no stackpacks config", + stackpacks: nil, + expectStackpacks: false, + }, + { + name: "stackpacks with no PVC", + stackpacks: &config.StackpacksConfig{LocalStackPacksUri: "file:///var/stackpacks_local"}, + expectStackpacks: false, + }, + { + name: "stackpacks with file:// URI and PVC", + stackpacks: &config.StackpacksConfig{LocalStackPacksUri: "file:///var/stackpacks_local", PVC: "stackpacks-pvc"}, + expectStackpacks: true, + expectedMountPath: "/var/stackpacks_local", + }, + { + name: "stackpacks with non-file URI and PVC", + stackpacks: &config.StackpacksConfig{LocalStackPacksUri: "s3://my-bucket/stackpacks", PVC: "stackpacks-pvc"}, + expectStackpacks: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := &config.Config{Stackpacks: tt.stackpacks} + mounts := buildRestoreVolumeMounts(cfg) + + var stackpacksMount *struct{ Name, MountPath string } + for _, m := range mounts { + if m.Name == "stackpacks-local" { + stackpacksMount = &struct{ Name, MountPath string }{m.Name, m.MountPath} + break + } + } + + if tt.expectStackpacks { + assert.NotNil(t, stackpacksMount, "expected stackpacks-local volume mount to be present") + assert.Equal(t, tt.expectedMountPath, stackpacksMount.MountPath) + } else { + assert.Nil(t, stackpacksMount, "expected stackpacks-local volume mount to be absent") + } + }) + } +} From d71d87d6b6ffb3fb3030e2fa6620ae3a2b37e218 Mon Sep 17 00:00:00 2001 From: Remco Beckers Date: Fri, 3 Apr 2026 17:08:26 +0200 Subject: [PATCH 10/10] STAC-23457 Linter fixes --- cmd/settings/restore.go | 10 +++--- cmd/settings/restore_test.go | 6 ++-- cmd/stackgraph/restore.go | 10 +++--- cmd/stackgraph/restore_test.go | 6 ++-- internal/foundation/config/config.go | 34 ++++++++++---------- internal/foundation/config/config_test.go | 38 +++++++++++------------ 6 files changed, 54 insertions(+), 50 deletions(-) diff --git a/cmd/settings/restore.go b/cmd/settings/restore.go index 518e261..5a8ea32 100644 --- a/cmd/settings/restore.go +++ b/cmd/settings/restore.go @@ -36,7 +36,9 @@ func restoreCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { cmd := &cobra.Command{ Use: "restore", Short: "Restore Settings from a backup archive", - Long: `Restore Settings data from a backup archive stored in S3. Automatically also restores Stackpacks backup that was made at the same time, it can be skipped with --skip-stackpacks. Can use --latest or --archive to specify which backup to restore.`, + Long: "Restore Settings data from a backup archive stored in S3. Automatically also restores " + + "Stackpacks backup that was made at the same time, it can be skipped with --skip-stackpacks. " + + "Can use --latest or --archive to specify which backup to restore.", Run: func(_ *cobra.Command, _ []string) { cmdutils.Run(globalFlags, runRestore, cmdutils.StorageIsNotRequired) }, @@ -211,7 +213,7 @@ func buildEnvVar(extraEnvVar []corev1.EnvVar, config *config.Config) []corev1.En commonVar = append(commonVar, corev1.EnvVar{Name: "BACKUP_CONFIGURATION_LOCAL_BUCKET", Value: config.Settings.LocalBucket}) } if config.Stackpacks != nil { - commonVar = append(commonVar, corev1.EnvVar{Name: "CONFIG_FORCE_stackstate_stackPacks_localStackPacksUri", Value: config.Stackpacks.LocalStackPacksUri}) + commonVar = append(commonVar, corev1.EnvVar{Name: "CONFIG_FORCE_stackstate_stackPacks_localStackPacksUri", Value: config.Stackpacks.LocalStackPacksURI}) } commonVar = append(commonVar, extraEnvVar...) return commonVar @@ -230,10 +232,10 @@ func buildVolumeMounts(config *config.Config) []corev1.VolumeMount { volumeMounts = append(volumeMounts, corev1.VolumeMount{Name: "settings-backup-data", MountPath: "/settings-backup-data"}) } - if config.Stackpacks != nil && config.Stackpacks.PVC != "" && strings.HasPrefix(config.Stackpacks.LocalStackPacksUri, "file://") { + if config.Stackpacks != nil && config.Stackpacks.PVC != "" && strings.HasPrefix(config.Stackpacks.LocalStackPacksURI, "file://") { volumeMounts = append(volumeMounts, corev1.VolumeMount{ Name: "stackpacks-local", - MountPath: strings.TrimPrefix(config.Stackpacks.LocalStackPacksUri, "file://"), + MountPath: strings.TrimPrefix(config.Stackpacks.LocalStackPacksURI, "file://"), }) } diff --git a/cmd/settings/restore_test.go b/cmd/settings/restore_test.go index 4f43b49..30233d1 100644 --- a/cmd/settings/restore_test.go +++ b/cmd/settings/restore_test.go @@ -21,18 +21,18 @@ func TestBuildVolumeMounts_StackpacksLocalFileURI(t *testing.T) { }, { name: "stackpacks with no PVC", - stackpacks: &config.StackpacksConfig{LocalStackPacksUri: "file:///var/stackpacks_local"}, + stackpacks: &config.StackpacksConfig{LocalStackPacksURI: "file:///var/stackpacks_local"}, expectStackpacks: false, }, { name: "stackpacks with file:// URI and PVC", - stackpacks: &config.StackpacksConfig{LocalStackPacksUri: "file:///var/stackpacks_local", PVC: "stackpacks-pvc"}, + stackpacks: &config.StackpacksConfig{LocalStackPacksURI: "file:///var/stackpacks_local", PVC: "stackpacks-pvc"}, expectStackpacks: true, expectedMountPath: "/var/stackpacks_local", }, { name: "stackpacks with non-file URI and PVC", - stackpacks: &config.StackpacksConfig{LocalStackPacksUri: "s3://my-bucket/stackpacks", PVC: "stackpacks-pvc"}, + stackpacks: &config.StackpacksConfig{LocalStackPacksURI: "s3://my-bucket/stackpacks", PVC: "stackpacks-pvc"}, expectStackpacks: false, }, } diff --git a/cmd/stackgraph/restore.go b/cmd/stackgraph/restore.go index 6d3225a..ee620da 100644 --- a/cmd/stackgraph/restore.go +++ b/cmd/stackgraph/restore.go @@ -42,7 +42,9 @@ func restoreCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command { cmd := &cobra.Command{ Use: "restore", Short: "Restore Stackgraph from a backup archive", - Long: `Restore Stackgraph data from a backup archive stored in S3. Automatically also restores Stackpacks backup that was made at the same time, it can be skipped with --skip-stackpacks. Can use --latest or --archive to specify which backup to restore.`, + Long: "Restore Stackgraph data from a backup archive stored in S3. Automatically also restores " + + "Stackpacks backup that was made at the same time, it can be skipped with --skip-stackpacks. " + + "Can use --latest or --archive to specify which backup to restore.", Run: func(_ *cobra.Command, _ []string) { cmdutils.Run(globalFlags, runRestore, cmdutils.StorageIsRequired) }, @@ -286,7 +288,7 @@ func buildRestoreEnvVars(backupFile string, config *config.Config) []corev1.EnvV {Name: "SKIP_STACKPACKS", Value: strconv.FormatBool(skipStackpacks)}, } if config.Stackpacks != nil { - env = append(env, corev1.EnvVar{Name: "CONFIG_FORCE_stackstate_stackPacks_localStackPacksUri", Value: config.Stackpacks.LocalStackPacksUri}) + env = append(env, corev1.EnvVar{Name: "CONFIG_FORCE_stackstate_stackPacks_localStackPacksUri", Value: config.Stackpacks.LocalStackPacksURI}) } return env } @@ -300,10 +302,10 @@ func buildRestoreVolumeMounts(config *config.Config) []corev1.VolumeMount { {Name: "tmp-data", MountPath: "/tmp-data"}, } - if config.Stackpacks != nil && config.Stackpacks.PVC != "" && strings.HasPrefix(config.Stackpacks.LocalStackPacksUri, "file://") { + if config.Stackpacks != nil && config.Stackpacks.PVC != "" && strings.HasPrefix(config.Stackpacks.LocalStackPacksURI, "file://") { volumeMounts = append(volumeMounts, corev1.VolumeMount{ Name: "stackpacks-local", - MountPath: strings.TrimPrefix(config.Stackpacks.LocalStackPacksUri, "file://"), + MountPath: strings.TrimPrefix(config.Stackpacks.LocalStackPacksURI, "file://"), }) } diff --git a/cmd/stackgraph/restore_test.go b/cmd/stackgraph/restore_test.go index 447c5fd..77de1a1 100644 --- a/cmd/stackgraph/restore_test.go +++ b/cmd/stackgraph/restore_test.go @@ -21,18 +21,18 @@ func TestBuildRestoreVolumeMounts_StackpacksLocalFileURI(t *testing.T) { }, { name: "stackpacks with no PVC", - stackpacks: &config.StackpacksConfig{LocalStackPacksUri: "file:///var/stackpacks_local"}, + stackpacks: &config.StackpacksConfig{LocalStackPacksURI: "file:///var/stackpacks_local"}, expectStackpacks: false, }, { name: "stackpacks with file:// URI and PVC", - stackpacks: &config.StackpacksConfig{LocalStackPacksUri: "file:///var/stackpacks_local", PVC: "stackpacks-pvc"}, + stackpacks: &config.StackpacksConfig{LocalStackPacksURI: "file:///var/stackpacks_local", PVC: "stackpacks-pvc"}, expectStackpacks: true, expectedMountPath: "/var/stackpacks_local", }, { name: "stackpacks with non-file URI and PVC", - stackpacks: &config.StackpacksConfig{LocalStackPacksUri: "s3://my-bucket/stackpacks", PVC: "stackpacks-pvc"}, + stackpacks: &config.StackpacksConfig{LocalStackPacksURI: "s3://my-bucket/stackpacks", PVC: "stackpacks-pvc"}, expectStackpacks: false, }, } diff --git a/internal/foundation/config/config.go b/internal/foundation/config/config.go index 300dc24..61d652b 100644 --- a/internal/foundation/config/config.go +++ b/internal/foundation/config/config.go @@ -32,10 +32,10 @@ type Config struct { // BaseURL, ReceiverBaseURL, and PlatformVersion take precedence over the per-restore-type // fields in SettingsRestoreConfig when set. type StackpacksConfig struct { - BaseURL string `yaml:"baseUrl"` - ReceiverBaseURL string `yaml:"receiverBaseUrl"` - PlatformVersion string `yaml:"platformVersion"` - LocalStackPacksUri string `yaml:"localStackPacksUri" validate:"required"` + BaseURL string `yaml:"baseUrl"` + ReceiverBaseURL string `yaml:"receiverBaseUrl"` + PlatformVersion string `yaml:"platformVersion"` + LocalStackPacksURI string `yaml:"localStackPacksUri" validate:"required"` PVC string `yaml:"pvc"` } @@ -202,11 +202,11 @@ type S3Location struct { // StackgraphRestoreConfig holds Stackgraph restore-specific configuration type StackgraphRestoreConfig struct { - ScaleDownLabelSelector string `yaml:"scaleDownLabelSelector" validate:"required"` - LoggingConfigConfigMapName string `yaml:"loggingConfigConfigMap" validate:"required"` - ZookeeperQuorum string `yaml:"zookeeperQuorum" validate:"required"` - Job JobConfig `yaml:"job" validate:"required"` - PVC PVCConfig `yaml:"pvc" validate:"required"` + ScaleDownLabelSelector string `yaml:"scaleDownLabelSelector" validate:"required"` + LoggingConfigConfigMapName string `yaml:"loggingConfigConfigMap" validate:"required"` + ZookeeperQuorum string `yaml:"zookeeperQuorum" validate:"required"` + Job JobConfig `yaml:"job" validate:"required"` + PVC PVCConfig `yaml:"pvc" validate:"required"` } type SettingsConfig struct { @@ -218,14 +218,14 @@ type SettingsConfig struct { } type SettingsRestoreConfig struct { - ScaleDownLabelSelector string `yaml:"scaleDownLabelSelector" validate:"required"` - LoggingConfigConfigMapName string `yaml:"loggingConfigConfigMap" validate:"required"` - BaseURL string `yaml:"baseUrl"` - ReceiverBaseURL string `yaml:"receiverBaseUrl"` - PlatformVersion string `yaml:"platformVersion"` - ZookeeperQuorum string `yaml:"zookeeperQuorum" validate:"required"` - Job JobConfig `yaml:"job" validate:"required"` - PVC string `yaml:"pvc"` // Required only in legacy mode + ScaleDownLabelSelector string `yaml:"scaleDownLabelSelector" validate:"required"` + LoggingConfigConfigMapName string `yaml:"loggingConfigConfigMap" validate:"required"` + BaseURL string `yaml:"baseUrl"` + ReceiverBaseURL string `yaml:"receiverBaseUrl"` + PlatformVersion string `yaml:"platformVersion"` + ZookeeperQuorum string `yaml:"zookeeperQuorum" validate:"required"` + Job JobConfig `yaml:"job" validate:"required"` + PVC string `yaml:"pvc"` // Required only in legacy mode } // ClickhouseConfig holds Clickhouse-specific configuration diff --git a/internal/foundation/config/config_test.go b/internal/foundation/config/config_test.go index 1bd5468..2719e87 100644 --- a/internal/foundation/config/config_test.go +++ b/internal/foundation/config/config_test.go @@ -625,9 +625,9 @@ func TestConfig_StructValidation(t *testing.T) { S3Prefix: "", MultipartArchive: true, Restore: StackgraphRestoreConfig{ - ScaleDownLabelSelector: "app=stackgraph", - LoggingConfigConfigMapName: "logging-config", - ZookeeperQuorum: "zookeeper:2181", + ScaleDownLabelSelector: "app=stackgraph", + LoggingConfigConfigMapName: "logging-config", + ZookeeperQuorum: "zookeeper:2181", Job: JobConfig{ Image: "backup:latest", WaitImage: "wait:latest", @@ -682,13 +682,13 @@ func TestConfig_StructValidation(t *testing.T) { Bucket: "settings-backup", S3Prefix: "", Restore: SettingsRestoreConfig{ - ScaleDownLabelSelector: "app=settings", - LoggingConfigConfigMapName: "logging-config", - BaseURL: "http://server:7070", - ReceiverBaseURL: "http://receiver:7077", - PlatformVersion: "5.2.0", - ZookeeperQuorum: "zookeeper:2181", - PVC: "suse-observability-settings-backup-data", + ScaleDownLabelSelector: "app=settings", + LoggingConfigConfigMapName: "logging-config", + BaseURL: "http://server:7070", + ReceiverBaseURL: "http://receiver:7077", + PlatformVersion: "5.2.0", + ZookeeperQuorum: "zookeeper:2181", + PVC: "suse-observability-settings-backup-data", Job: JobConfig{ Image: "settings-backup:latest", WaitImage: "wait:latest", @@ -772,9 +772,9 @@ func TestConfig_StructValidation(t *testing.T) { S3Prefix: "", MultipartArchive: true, Restore: StackgraphRestoreConfig{ - ScaleDownLabelSelector: "app=stackgraph", - LoggingConfigConfigMapName: "logging-config", - ZookeeperQuorum: "zookeeper:2181", + ScaleDownLabelSelector: "app=stackgraph", + LoggingConfigConfigMapName: "logging-config", + ZookeeperQuorum: "zookeeper:2181", Job: JobConfig{ Image: "backup:latest", WaitImage: "wait:latest", @@ -829,12 +829,12 @@ func TestConfig_StructValidation(t *testing.T) { Bucket: "settings-backup", S3Prefix: "", Restore: SettingsRestoreConfig{ - ScaleDownLabelSelector: "app=settings", - LoggingConfigConfigMapName: "logging-config", - BaseURL: "http://server:7070", - ReceiverBaseURL: "http://receiver:7077", - PlatformVersion: "5.2.0", - ZookeeperQuorum: "zookeeper:2181", + ScaleDownLabelSelector: "app=settings", + LoggingConfigConfigMapName: "logging-config", + BaseURL: "http://server:7070", + ReceiverBaseURL: "http://receiver:7077", + PlatformVersion: "5.2.0", + ZookeeperQuorum: "zookeeper:2181", Job: JobConfig{ Image: "settings-backup:latest", WaitImage: "wait:latest",