diff --git a/Manual.md b/Manual.md index b861a58b9..4946a1718 100644 --- a/Manual.md +++ b/Manual.md @@ -4,14 +4,16 @@ NAME: clickhouse-backup tables - List of tables, exclude skip_tables USAGE: - clickhouse-backup tables [--tables=.] [--remote-backup=] [--all] + clickhouse-backup tables [--tables=.
] [--remote-backup=] [--local-backup=] [-f, --format=] [--all] OPTIONS: --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] --environment-override value, --env value override any environment variable via CLI parameter --all, -a Print table even when match with skip_tables pattern --table value, --tables value, -t value List tables only match with table name patterns, separated by comma, allow ? and * as wildcard - --remote-backup value List tables from remote backup + --remote-backup value List tables from a remote backup, including per-table size and parts count + --local-backup value List tables from a local backup (read from disk, no live ClickHouse query), including per-table size and parts count + --format value, -f value Output format (text|json|yaml|csv|tsv) ``` ### CLI command - create @@ -193,7 +195,7 @@ Look at the system.parts partition and partition_id fields for details https://c --restore-schema-as-attach Use DETACH/ATTACH instead of DROP/CREATE for schema restoration --replicated-copy-to-detached Copy data to detached folder for Replicated*MergeTree tables but skip ATTACH PART step --skip-empty-tables Skip restoring tables that have no data (empty tables with only schema) - + ``` ### CLI command - restore_remote ``` @@ -231,7 +233,7 @@ Look at the system.parts partition and partition_id fields for details https://c --restore-schema-as-attach Use DETACH/ATTACH instead of DROP/CREATE for schema restoration --hardlink-exists-files Create hardlinks for existing files instead of downloading --skip-empty-tables Skip restoring tables that have no data (empty tables with only schema) - + ``` ### CLI command - delete ``` diff --git a/ReadMe.md b/ReadMe.md index bbd64e2ef..eae574143 100644 --- a/ReadMe.md +++ b/ReadMe.md @@ -429,17 +429,19 @@ Kill selected command from `GET /backup/actions` command list, kill process shou ### GET /backup/tables -Print list of tables: `curl -s localhost:7171/backup/tables | jq .`, exclude pattern matched tables from `skip_tables` configuration parameters +Print list of tables: `curl -s localhost:7171/backup/tables | jq .`, exclude pattern matched tables from `skip_tables` configuration parameters. - Optional query argument `table` works the same as the `--table=pattern` CLI argument. -- Optional query argument `remote_backup` or `remote-backup` works the same as `--remote-backup=name` CLI argument. +- Optional query argument `remote_backup` (or `remote-backup`) works the same as the `--remote-backup=name` CLI argument. The response then includes per-table `size`, `total_bytes`, `parts`, and `disks` (JSON array of disk names) fields read from the remote backup metadata. +- Optional query argument `local_backup` (or `local-backup`) works the same as the `--local-backup=name` CLI argument: it lists tables from a local backup directly from disk (no live ClickHouse query required), with `size`, `total_bytes`, `parts`, and `disks` (JSON array) fields. ### GET /backup/tables/all Print list of tables: `curl -s localhost:7171/backup/tables/all | jq .`, ignore `skip_tables` configuration parameters. - Optional query argument `table` works the same as the `--table=pattern` CLI argument. -- Optional query argument `remote_backup`or `remote-backup` works the same as `--remote-backup=name` CLI argument. +- Optional query argument `remote_backup` (or `remote-backup`) works the same as the `--remote-backup=name` CLI argument; response shape matches `GET /backup/tables` with the remote-backup parameter. +- Optional query argument `local_backup` (or `local-backup`) works the same as the `--local-backup=name` CLI argument; response shape matches `GET /backup/tables` with the local-backup parameter. ### POST /backup/create @@ -666,14 +668,16 @@ NAME: clickhouse-backup tables - List of tables, exclude skip_tables USAGE: - clickhouse-backup tables [--tables=.
] [--remote-backup=] [--all] + clickhouse-backup tables [--tables=.
] [--remote-backup=] [--local-backup=] [-f, --format=] [--all] OPTIONS: --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] --environment-override value, --env value override any environment variable via CLI parameter --all, -a Print table even when match with skip_tables pattern --table value, --tables value, -t value List tables only match with table name patterns, separated by comma, allow ? and * as wildcard - --remote-backup value List tables from remote backup + --remote-backup value List tables from a remote backup, including per-table size and parts count + --local-backup value List tables from a local backup (read from disk, no live ClickHouse query), including per-table size and parts count + --format value, -f value Output format (text|json|yaml|csv|tsv) ``` ### CLI command - create @@ -855,7 +859,7 @@ Look at the system.parts partition and partition_id fields for details https://c --restore-schema-as-attach Use DETACH/ATTACH instead of DROP/CREATE for schema restoration --replicated-copy-to-detached Copy data to detached folder for Replicated*MergeTree tables but skip ATTACH PART step --skip-empty-tables Skip restoring tables that have no data (empty tables with only schema) - + ``` ### CLI command - restore_remote ``` @@ -893,7 +897,7 @@ Look at the system.parts partition and partition_id fields for details https://c --restore-schema-as-attach Use DETACH/ATTACH instead of DROP/CREATE for schema restoration --hardlink-exists-files Create hardlinks for existing files instead of downloading --skip-empty-tables Skip restoring tables that have no data (empty tables with only schema) - + ``` ### CLI command - delete ``` diff --git a/cmd/clickhouse-backup/main.go b/cmd/clickhouse-backup/main.go index 1302abd18..41c3dfe75 100644 --- a/cmd/clickhouse-backup/main.go +++ b/cmd/clickhouse-backup/main.go @@ -87,10 +87,10 @@ func main() { { Name: "tables", Usage: "List of tables, exclude skip_tables", - UsageText: "clickhouse-backup tables [--tables=.
] [--remote-backup=] [--all]", + UsageText: "clickhouse-backup tables [--tables=.
] [--remote-backup=] [--local-backup=] [-f, --format=] [--all]", Action: func(c *cli.Context) error { b := backup.NewBackuper(config.GetConfigFromCli(c)) - return b.PrintTables(c.Bool("all"), c.String("table"), c.String("remote-backup")) + return b.PrintTables(c.Bool("all"), c.String("table"), c.String("remote-backup"), c.String("local-backup"), c.String("format")) }, Flags: append(cliapp.Flags, cli.BoolFlag{ @@ -106,7 +106,17 @@ func main() { cli.StringFlag{ Name: "remote-backup", Hidden: false, - Usage: "List tables from remote backup", + Usage: "List tables from a remote backup, including per-table size and parts count", + }, + cli.StringFlag{ + Name: "local-backup", + Hidden: false, + Usage: "List tables from a local backup (read from disk, no live ClickHouse query), including per-table size and parts count", + }, + cli.StringFlag{ + Name: "format, f", + Hidden: false, + Usage: "Output format (text|json|yaml|csv|tsv)", }, ), }, diff --git a/generate_manual.sh b/generate_manual.sh old mode 100644 new mode 100755 index 4ca7f62f9..9d16efede --- a/generate_manual.sh +++ b/generate_manual.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -CLICKHOUSE_BACKUP_BIN=${CLICKHOUSE_BACKUP_BIN:-build/linux/$(dpkg --print-architecture)/clickhouse-backup} -make clean ${CLICKHOUSE_BACKUP_BIN} +CLICKHOUSE_BACKUP_BIN=${CLICKHOUSE_BACKUP_BIN:-build/$(go env GOOS)/$(go env GOARCH)/clickhouse-backup} +make clean ${CLICKHOUSE_BACKUP_BIN} >&2 cmds=( tables create diff --git a/pkg/backup/list.go b/pkg/backup/list.go index 9bf9c202d..e9f9ca25b 100644 --- a/pkg/backup/list.go +++ b/pkg/backup/list.go @@ -15,6 +15,7 @@ import ( "time" "github.com/Altinity/clickhouse-backup/v2/pkg/clickhouse" + "github.com/Altinity/clickhouse-backup/v2/pkg/common" "github.com/Altinity/clickhouse-backup/v2/pkg/custom" "github.com/Altinity/clickhouse-backup/v2/pkg/metadata" "github.com/Altinity/clickhouse-backup/v2/pkg/status" @@ -525,61 +526,544 @@ func (b *Backuper) GetTables(ctx context.Context, tablePattern string) ([]clickh return allTables, nil } -// PrintTables - print all tables suitable for backup -func (b *Backuper) PrintTables(printAll bool, tablePattern, remoteBackup string) error { - var err error +// TableRow is the output projection used by the `tables` command for non-text formats. +// Disks is exposed as a structured list in JSON/YAML and as a comma-joined string in CSV/TSV. +type TableRow struct { + Database string `json:"database" yaml:"database" csv:"database"` + Table string `json:"table" yaml:"table" csv:"table"` + TotalBytes uint64 `json:"total_bytes" yaml:"total_bytes" csv:"total_bytes"` + Size string `json:"size" yaml:"size" csv:"size"` + Parts int `json:"parts" yaml:"parts" csv:"parts"` + Disks []string `json:"disks" yaml:"disks" csv:"-"` + DisksStr string `json:"-" yaml:"-" csv:"disks"` + Skip bool `json:"skip" yaml:"skip" csv:"skip"` + BackupType string `json:"backup_type,omitempty" yaml:"backup_type,omitempty" csv:"backup_type"` +} + +// InfoResult wraps a per-backup result with aggregate fields for JSON/YAML output of +// `tables --local-backup` / `tables --remote-backup`. +type InfoResult struct { + BackupName string `json:"backup_name" yaml:"backup_name"` + BackupType string `json:"backup_type" yaml:"backup_type"` + TablePattern string `json:"table_pattern,omitempty" yaml:"table_pattern,omitempty"` + TableCount int `json:"table_count" yaml:"table_count"` + TotalBytes uint64 `json:"total_bytes" yaml:"total_bytes"` + TotalSize string `json:"total_size" yaml:"total_size"` + TotalParts int `json:"total_parts" yaml:"total_parts"` + Tables []TableRow `json:"tables" yaml:"tables"` +} + +// tableSection groups rows of a single backup location for layered text/JSON output. +type tableSection struct { + BackupName string + BackupType string + TablePattern string + Rows []TableRow +} + +// PrintTables - print all tables suitable for backup. +// When localBackup or remoteBackup is set, list tables from the corresponding backup +// (per-table size and parts count are read from `metadata.TableMetadata`); both flags may +// be set simultaneously to render `local` and `remote` sections in one go. +// Otherwise tables are read from the live ClickHouse server. +// `format` controls output: text (default), json, yaml, csv, tsv. +func (b *Backuper) PrintTables(printAll bool, tablePattern, remoteBackup, localBackup, format string) error { ctx, cancel, _ := status.Current.GetContextWithCancel(status.NotFromAPI) defer cancel() - if err = b.ch.Connect(); err != nil { + if err := b.ch.Connect(); err != nil { return errors.Wrap(err, "can't connect to clickhouse") } defer b.ch.Close() - w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', tabwriter.DiscardEmptyColumns) - if remoteBackup == "" { - if err = b.printTablesLocal(ctx, tablePattern, printAll, w); err != nil { - return errors.WithMessage(err, "PrintTables printTablesLocal") + + if localBackup == "" && remoteBackup == "" { + rows, err := b.collectTablesFromLive(ctx, tablePattern) + if err != nil { + return err + } + if !printAll { + rows = filterSkippedRows(rows) + } + return printLiveTableRows(rows, format) + } + + var sections []tableSection + if localBackup != "" { + rows, err := b.collectTablesFromLocalBackup(ctx, localBackup, tablePattern) + if err != nil { + return err } - } else { - if err = b.printTablesRemote(ctx, remoteBackup, tablePattern, printAll, w); err != nil { - return errors.WithMessage(err, "PrintTables printTablesRemote") + if !printAll { + rows = filterSkippedRows(rows) } + sections = append(sections, tableSection{ + BackupName: localBackup, + BackupType: "local", + TablePattern: tablePattern, + Rows: rows, + }) + } + if remoteBackup != "" { + rows, err := b.collectTablesFromRemoteBackup(ctx, remoteBackup, tablePattern) + if err != nil { + return err + } + if !printAll { + rows = filterSkippedRows(rows) + } + sections = append(sections, tableSection{ + BackupName: remoteBackup, + BackupType: "remote", + TablePattern: tablePattern, + Rows: rows, + }) + } + return printBackupSections(sections, format) +} +// GetTableRowsForLocalBackup returns per-table rows (db, table, size, parts, disks, skip) +// for a local backup, reading metadata from disk; intended for callers like the REST API. +// When printAll is false, tables matching skip_tables are filtered out. +func (b *Backuper) GetTableRowsForLocalBackup(ctx context.Context, backupName, tablePattern string, printAll bool) ([]TableRow, error) { + if err := b.ch.Connect(); err != nil { + return nil, errors.Wrap(err, "can't connect to clickhouse") + } + defer b.ch.Close() + rows, err := b.collectTablesFromLocalBackup(ctx, backupName, tablePattern) + if err != nil { + return nil, err } - if err := w.Flush(); err != nil { - log.Error().Msgf("can't flush tabular writer error: %v", err) + if printAll { + return rows, nil } - return nil + return filterSkippedRows(rows), nil } -func (b *Backuper) printTablesLocal(ctx context.Context, tablePattern string, printAll bool, w *tabwriter.Writer) error { - logger := log.With().Str("logger", "PrintTablesLocal").Logger() +// GetTableRowsForRemoteBackup returns per-table rows (db, table, size, parts, disks, skip) +// for a remote backup, downloading per-table metadata; intended for callers like the REST API. +// When printAll is false, tables matching skip_tables are filtered out. +func (b *Backuper) GetTableRowsForRemoteBackup(ctx context.Context, backupName, tablePattern string, printAll bool) ([]TableRow, error) { + if err := b.ch.Connect(); err != nil { + return nil, errors.Wrap(err, "can't connect to clickhouse") + } + defer b.ch.Close() + rows, err := b.collectTablesFromRemoteBackup(ctx, backupName, tablePattern) + if err != nil { + return nil, err + } + if printAll { + return rows, nil + } + return filterSkippedRows(rows), nil +} + +func filterSkippedRows(rows []TableRow) []TableRow { + out := rows[:0] + for _, r := range rows { + if !r.Skip { + out = append(out, r) + } + } + return out +} + +func (b *Backuper) collectTablesFromLive(ctx context.Context, tablePattern string) ([]TableRow, error) { allTables, err := b.GetTables(ctx, tablePattern) if err != nil { - return errors.WithMessage(err, "printTablesLocal GetTables") + return nil, errors.WithMessage(err, "collectTablesFromLive GetTables") } disks, err := b.ch.GetDisks(ctx, false) if err != nil { - return errors.WithMessage(err, "printTablesLocal GetDisks") + return nil, errors.WithMessage(err, "collectTablesFromLive GetDisks") } + rows := make([]TableRow, 0, len(allTables)) for _, table := range allTables { - if table.Skip && !printAll { - continue - } var tableDisks []string for disk := range clickhouse.GetDisksByPaths(disks, table.DataPaths) { tableDisks = append(tableDisks, disk) } - if table.Skip { - if bytes, err := fmt.Fprintf(w, "%s.%s\t%s\t%v\tskip\n", table.Database, table.Name, utils.FormatBytes(table.TotalBytes), strings.Join(tableDisks, ",")); err != nil { - logger.Error().Msgf("fmt.Fprintf write %d bytes return error: %v", bytes, err) + sort.Strings(tableDisks) + rows = append(rows, TableRow{ + Database: table.Database, + Table: table.Name, + TotalBytes: table.TotalBytes, + Size: utils.FormatBytes(table.TotalBytes), + Disks: tableDisks, + DisksStr: strings.Join(tableDisks, ","), + Skip: table.Skip, + BackupType: string(table.BackupType), + }) + } + return rows, nil +} + +func (b *Backuper) collectTablesFromLocalBackup(ctx context.Context, backupName, tablePattern string) ([]TableRow, error) { + localBackup, _, err := b.getLocalBackup(ctx, backupName, nil) + if err != nil { + return nil, errors.WithMessage(err, "collectTablesFromLocalBackup getLocalBackup") + } + filtered := filterBackupTablesByPattern(localBackup.Tables, tablePattern) + if len(filtered) == 0 && tablePattern != "" { + log.Warn().Msgf("no tables matching pattern '%s' found in local backup '%s'", tablePattern, backupName) + } + rows := make([]TableRow, 0, len(filtered)) + metadataPath := path.Join(b.DefaultDataPath, "backup", backupName, "metadata") + for _, t := range filtered { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + tableName := fmt.Sprintf("%s.%s", t.Database, t.Table) + tmFile := path.Join(metadataPath, common.TablePathEncode(t.Database), fmt.Sprintf("%s.json", common.TablePathEncode(t.Table))) + var tm metadata.TableMetadata + if _, err := tm.Load(tmFile); err != nil { + log.Warn().Str("table", tableName).Err(err).Msg("can't load table metadata, skipping size/parts") + rows = append(rows, TableRow{Database: t.Database, Table: t.Table, Disks: []string{}, Skip: b.shouldSkipByTableName(tableName) || IsInformationSchema(t.Database)}) + continue + } + rows = append(rows, b.tableRowFromMetadata(t, &tm)) + } + return rows, nil +} + +func (b *Backuper) collectTablesFromRemoteBackup(ctx context.Context, backupName, tablePattern string) ([]TableRow, error) { + if b.cfg.General.RemoteStorage == "none" || b.cfg.General.RemoteStorage == "custom" { + return nil, errors.New("`tables --remote-backup` does not support `none` and `custom` remote storage") + } + ownDst := false + if b.dst == nil { + bd, err := storage.NewBackupDestination(ctx, b.cfg, b.ch, "") + if err != nil { + return nil, errors.WithMessage(err, "collectTablesFromRemoteBackup NewBackupDestination") + } + if err := bd.Connect(ctx); err != nil { + return nil, errors.Wrap(err, "can't connect to remote storage") + } + b.dst = bd + ownDst = true + } + defer func() { + if ownDst { + if err := b.dst.Close(ctx); err != nil { + log.Warn().Msgf("can't close BackupDestination error: %v", err) } + b.dst = nil + } + }() + + backupList, err := b.dst.BackupList(ctx, true, backupName) + if err != nil { + return nil, errors.WithMessage(err, "collectTablesFromRemoteBackup BackupList") + } + var remoteBackupMeta *storage.Backup + for i := range backupList { + if backupList[i].BackupName == backupName { + remoteBackupMeta = &backupList[i] + break + } + } + if remoteBackupMeta == nil { + return nil, errors.Errorf("backup '%s' not found on remote storage", backupName) + } + + filtered := filterBackupTablesByPattern(remoteBackupMeta.Tables, tablePattern) + if len(filtered) == 0 && tablePattern != "" { + log.Warn().Msgf("no tables matching pattern '%s' found in remote backup '%s'", tablePattern, backupName) + } + rows := make([]TableRow, 0, len(filtered)) + for _, t := range filtered { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + tableName := fmt.Sprintf("%s.%s", t.Database, t.Table) + tmPath := path.Join(backupName, "metadata", common.TablePathEncode(t.Database), fmt.Sprintf("%s.json", common.TablePathEncode(t.Table))) + tmReader, err := b.dst.GetFileReader(ctx, tmPath) + if err != nil { + log.Warn().Str("table", tableName).Err(err).Msg("can't read remote table metadata, skipping size/parts") + rows = append(rows, TableRow{Database: t.Database, Table: t.Table, Disks: []string{}, Skip: b.shouldSkipByTableName(tableName) || IsInformationSchema(t.Database)}) continue } - if bytes, err := fmt.Fprintf(w, "%s.%s\t%s\t%v\t%v\n", table.Database, table.Name, utils.FormatBytes(table.TotalBytes), strings.Join(tableDisks, ","), table.BackupType); err != nil { - logger.Error().Msgf("fmt.Fprintf write %d bytes return error: %v", bytes, err) + data, readErr := io.ReadAll(tmReader) + if closeErr := tmReader.Close(); closeErr != nil { + log.Warn().Err(closeErr).Str("path", tmPath).Msg("can't close reader") + } + if readErr != nil { + return nil, errors.Wrapf(readErr, "io.ReadAll(%s)", tmPath) } + var tm metadata.TableMetadata + if jsonErr := json.Unmarshal(data, &tm); jsonErr != nil { + log.Warn().Str("table", tableName).Err(jsonErr).Msg("can't unmarshal remote table metadata, skipping size/parts") + rows = append(rows, TableRow{Database: t.Database, Table: t.Table, Disks: []string{}, Skip: b.shouldSkipByTableName(tableName) || IsInformationSchema(t.Database)}) + continue + } + rows = append(rows, b.tableRowFromMetadata(t, &tm)) } - return nil + return rows, nil +} + +func (b *Backuper) tableRowFromMetadata(t metadata.TableTitle, tm *metadata.TableMetadata) TableRow { + tableName := fmt.Sprintf("%s.%s", t.Database, t.Table) + disks := []string{} + for disk := range tm.Size { + disks = append(disks, disk) + } + sort.Strings(disks) + partCount := 0 + for _, parts := range tm.Parts { + partCount += len(parts) + } + return TableRow{ + Database: t.Database, + Table: t.Table, + TotalBytes: tm.TotalBytes, + Size: utils.FormatBytes(tm.TotalBytes), + Parts: partCount, + Disks: disks, + DisksStr: strings.Join(disks, ","), + Skip: IsInformationSchema(t.Database) || b.shouldSkipByTableName(tableName), + } +} + +// filterBackupTablesByPattern keeps only tables matching any comma-separated glob pattern. +// Empty pattern returns the input unchanged. +func filterBackupTablesByPattern(tables []metadata.TableTitle, tablePattern string) []metadata.TableTitle { + if tablePattern == "" { + return tables + } + patterns := strings.Split(tablePattern, ",") + // https://github.com/Altinity/clickhouse-backup/issues/1091 + replacer := strings.NewReplacer("/", "_", `\`, "_") + result := make([]metadata.TableTitle, 0, len(tables)) + for _, t := range tables { + tableName := fmt.Sprintf("%s.%s", t.Database, t.Table) + for _, p := range patterns { + p = strings.Trim(p, " \t\r\n") + if p == "*" { + result = append(result, t) + break + } + if matched, _ := filepath.Match(replacer.Replace(p), replacer.Replace(tableName)); matched { + result = append(result, t) + break + } + } + } + return result +} + +// sortTableRows sorts rows by database.table for deterministic, human-friendly output. +func sortTableRows(rows []TableRow) { + sort.Slice(rows, func(i, j int) bool { + if rows[i].Database != rows[j].Database { + return rows[i].Database < rows[j].Database + } + return rows[i].Table < rows[j].Table + }) +} + +// printLiveTableRows renders rows from the live ClickHouse server (no backup header / TOTAL). +func printLiveTableRows(rows []TableRow, format string) error { + switch format { + case "json": + data, err := json.Marshal(rows) + if err != nil { + return errors.WithMessage(err, "printLiveTableRows json.Marshal") + } + fmt.Println(string(data)) + return nil + case "yaml": + data, err := yaml.Marshal(rows) + if err != nil { + return errors.WithMessage(err, "printLiveTableRows yaml.Marshal") + } + fmt.Print(string(data)) + return nil + case "csv": + s, err := gocsv.MarshalString(rows) + if err != nil { + return errors.WithMessage(err, "printLiveTableRows csv MarshalString") + } + fmt.Print(s) + return nil + case "tsv": + gocsv.SetCSVWriter(func(out io.Writer) *gocsv.SafeCSVWriter { + writer := gocsv.NewSafeCSVWriter(csv.NewWriter(out)) + writer.Comma = '\t' + return writer + }) + s, err := gocsv.MarshalString(rows) + if err != nil { + return errors.WithMessage(err, "printLiveTableRows tsv MarshalString") + } + fmt.Print(s) + return nil + case "text", "": + w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', tabwriter.DiscardEmptyColumns) + for _, r := range rows { + marker := r.BackupType + if r.Skip { + marker = "skip" + } + if _, err := fmt.Fprintf(w, "%s.%s\t%s\t%d\t%s\t%s\n", r.Database, r.Table, r.Size, r.Parts, r.DisksStr, marker); err != nil { + log.Error().Msgf("printLiveTableRows Fprintf error: %v", err) + } + } + return w.Flush() + } + return errors.Errorf("unknown format '%s', use one of: text, json, yaml, csv, tsv", format) +} + +// buildInfoResults transforms tableSections into the InfoResult projection (with totals +// computed from rendered rows). +func buildInfoResults(sections []tableSection) []InfoResult { + results := make([]InfoResult, 0, len(sections)) + for _, s := range sections { + var totalBytes uint64 + var totalParts int + for _, r := range s.Rows { + totalBytes += r.TotalBytes + totalParts += r.Parts + } + results = append(results, InfoResult{ + BackupName: s.BackupName, + BackupType: s.BackupType, + TablePattern: s.TablePattern, + TableCount: len(s.Rows), + TotalBytes: totalBytes, + TotalSize: utils.FormatBytes(totalBytes), + TotalParts: totalParts, + Tables: s.Rows, + }) + } + return results +} + +// printBackupSections renders one or more tableSection in the requested format. +// JSON/YAML get an InfoResult wrapper (single object for one section, array for several). +// CSV/TSV emit per-section blocks separated by a blank line. +// Text gets `Backup: (type)` header, optional `Filter:` line, sorted rows, and a TOTAL line. +func printBackupSections(sections []tableSection, format string) error { + for i := range sections { + sortTableRows(sections[i].Rows) + } + switch format { + case "json": + results := buildInfoResults(sections) + var data []byte + var err error + if len(results) == 1 { + data, err = json.MarshalIndent(results[0], "", " ") + } else { + data, err = json.MarshalIndent(results, "", " ") + } + if err != nil { + return errors.WithMessage(err, "printBackupSections json.Marshal") + } + fmt.Println(string(data)) + return nil + case "yaml": + results := buildInfoResults(sections) + var data []byte + var err error + if len(results) == 1 { + data, err = yaml.Marshal(results[0]) + } else { + data, err = yaml.Marshal(results) + } + if err != nil { + return errors.WithMessage(err, "printBackupSections yaml.Marshal") + } + fmt.Print(string(data)) + return nil + case "csv": + for i, s := range sections { + if i > 0 { + fmt.Println() + } + csvString, err := gocsv.MarshalString(s.Rows) + if err != nil { + return errors.WithMessage(err, "printBackupSections csv MarshalString") + } + fmt.Print(csvString) + } + return nil + case "tsv": + gocsv.SetCSVWriter(func(out io.Writer) *gocsv.SafeCSVWriter { + writer := gocsv.NewSafeCSVWriter(csv.NewWriter(out)) + writer.Comma = '\t' + return writer + }) + for i, s := range sections { + if i > 0 { + fmt.Println() + } + csvString, err := gocsv.MarshalString(s.Rows) + if err != nil { + return errors.WithMessage(err, "printBackupSections tsv MarshalString") + } + fmt.Print(csvString) + } + return nil + case "text", "": + for i, s := range sections { + if i > 0 { + fmt.Println() + } + if err := renderTextSection(s); err != nil { + return err + } + } + return nil + } + return errors.Errorf("unknown format '%s', use one of: text, json, yaml, csv, tsv", format) +} + +func renderTextSection(s tableSection) error { + w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', tabwriter.DiscardEmptyColumns) + if _, err := fmt.Fprintf(w, "Backup:\t%s (%s)\n", s.BackupName, s.BackupType); err != nil { + return err + } + if s.TablePattern != "" { + if _, err := fmt.Fprintf(w, "Filter:\t%s\n", s.TablePattern); err != nil { + return err + } + } + if len(s.Rows) == 0 { + fmt.Fprintln(w) + if _, err := fmt.Fprintln(w, "(no tables)"); err != nil { + return err + } + return w.Flush() + } + fmt.Fprintln(w) + if _, err := fmt.Fprintln(w, "TABLE\tSIZE\tPARTS\tDISKS\tFLAGS"); err != nil { + return err + } + if _, err := fmt.Fprintln(w, "-----\t----\t-----\t-----\t-----"); err != nil { + return err + } + var totalBytes uint64 + var totalParts int + for _, r := range s.Rows { + marker := r.BackupType + if r.Skip { + marker = "skip" + } + if _, err := fmt.Fprintf(w, "%s.%s\t%s\t%d\t%s\t%s\n", r.Database, r.Table, r.Size, r.Parts, r.DisksStr, marker); err != nil { + return err + } + totalBytes += r.TotalBytes + totalParts += r.Parts + } + if _, err := fmt.Fprintln(w, "-----\t----\t-----\t-----\t-----"); err != nil { + return err + } + if _, err := fmt.Fprintf(w, "TOTAL (%d tables)\t%s\t%d\t\t\n", len(s.Rows), utils.FormatBytes(totalBytes), totalParts); err != nil { + return err + } + return w.Flush() } func (b *Backuper) GetTablesRemote(ctx context.Context, backupName string, tablePattern string) ([]clickhouse.Table, error) { @@ -654,21 +1138,3 @@ func (b *Backuper) GetTablesRemote(ctx context.Context, backupName string, table return tables, nil } -// printTablesRemote https://github.com/Altinity/clickhouse-backup/issues/778 -func (b *Backuper) printTablesRemote(ctx context.Context, backupName string, tablePattern string, printAll bool, w *tabwriter.Writer) error { - tables, err := b.GetTablesRemote(ctx, backupName, tablePattern) - if err != nil { - return errors.WithMessage(err, "printTablesRemote GetTablesRemote") - } - - for _, t := range tables { - if t.Skip && !printAll { - continue - } - if bytes, err := fmt.Fprintf(w, "%s.%s\tskip=%v\n", t.Database, t.Name, t.Skip); err != nil { - log.Error().Msgf("fmt.Fprintf write %d bytes return error: %v", bytes, err) - } - } - - return nil -} diff --git a/pkg/server/server.go b/pkg/server/server.go index acb3e6f91..2c48529d5 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -775,7 +775,13 @@ func (api *APIServer) httpKillHandler(w http.ResponseWriter, r *http.Request) { } } -// httpTablesHandler - display list of tables +// httpTablesHandler - display list of tables. +// Query parameters: +// - table - filter by db.table glob pattern (comma-separated) +// - remote_backup - list tables from a remote backup (per-table size and parts) +// - local_backup - list tables from a local backup (per-table size and parts), no live ClickHouse query needed +// +// /backup/tables/all also returns tables that match skip_tables. func (api *APIServer) httpTablesHandler(w http.ResponseWriter, r *http.Request) { cfg, err := api.ReloadConfig(w, "tables") if err != nil { @@ -783,18 +789,34 @@ func (api *APIServer) httpTablesHandler(w http.ResponseWriter, r *http.Request) } b := backup.NewBackuper(cfg) q := r.URL.Query() - var tables []clickhouse.Table - // https://github.com/Altinity/clickhouse-backup/issues/778 + tablePattern := q.Get("table") + printAll := r.URL.Path == "/backup/tables/all" + + // https://github.com/Altinity/clickhouse-backup/issues/1388 + if localBackup, exists := api.getQueryParameter(q, "local_backup"); exists { + rows, err := b.GetTableRowsForLocalBackup(context.Background(), localBackup, tablePattern, printAll) + if err != nil { + api.writeError(w, http.StatusInternalServerError, "tables", err) + return + } + api.sendJSONEachRow(w, http.StatusOK, rows) + return + } if remoteBackup, exists := api.getQueryParameter(q, "remote_backup"); exists { - tables, err = b.GetTablesRemote(context.Background(), remoteBackup, q.Get("table")) - } else { - tables, err = b.GetTables(context.Background(), q.Get("table")) + rows, err := b.GetTableRowsForRemoteBackup(context.Background(), remoteBackup, tablePattern, printAll) + if err != nil { + api.writeError(w, http.StatusInternalServerError, "tables", err) + return + } + api.sendJSONEachRow(w, http.StatusOK, rows) + return } + tables, err := b.GetTables(context.Background(), tablePattern) if err != nil { api.writeError(w, http.StatusInternalServerError, "tables", err) return } - if r.URL.Path == "/backup/tables/all" { + if printAll { api.sendJSONEachRow(w, http.StatusOK, tables) return } diff --git a/test/integration/serverAPI_test.go b/test/integration/serverAPI_test.go index 6503a5d6f..c4cb134df 100644 --- a/test/integration/serverAPI_test.go +++ b/test/integration/serverAPI_test.go @@ -49,6 +49,8 @@ func TestServerAPI(t *testing.T) { testAPIBackupTablesRemote(r, env) + testAPIBackupTablesLocal(r, env) + testAPIBackupRestoreRemote(r, env) testAPIBackupStatus(r, env) @@ -525,6 +527,43 @@ func testAPIBackupTablesRemote(r *require.Assertions, env *TestEnvironment) { r.NotContains(out, "INFORMATION_SCHEMA") r.NotContains(out, "information_schema") r.NotContains(out, "command is already running") + // /backup/tables?remote_backup=... must include per-table size and parts (https://github.com/Altinity/clickhouse-backup/issues/1388). + r.Contains(out, `"size":`) + r.Contains(out, `"parts":`) + r.Contains(out, `"total_bytes":`) + r.Contains(out, `"disks":[`) +} + +// testAPIBackupTablesLocal exercises /backup/tables?local_backup= +// (https://github.com/Altinity/clickhouse-backup/issues/1388) — listing tables +// from a local backup without a live ClickHouse query, with size and parts. +func testAPIBackupTablesLocal(r *require.Assertions, env *TestEnvironment) { + log.Debug().Msg("Check /backup/tables?local_backup=z_backup_1") + out, err := env.DockerExecOut( + "clickhouse-backup", + "bash", "-xe", "-c", "curl -sfL \"http://localhost:7171/backup/tables?local_backup=z_backup_1\"", + ) + r.NoError(err, "%s\nunexpected GET /backup/tables?local_backup=z_backup_1 error: %v", out, err) + r.Contains(out, "long_schema") + r.NotContains(out, "Connection refused") + r.NotContains(out, "another operation is currently running") + r.NotContains(out, "\"status\":\"error\"") + r.NotContains(out, "system") + r.NotContains(out, "INFORMATION_SCHEMA") + r.NotContains(out, "information_schema") + r.Contains(out, `"size":`) + r.Contains(out, `"parts":`) + r.Contains(out, `"total_bytes":`) + r.Contains(out, `"disks":[`) + + // Filtered by table pattern. + out, err = env.DockerExecOut( + "clickhouse-backup", + "bash", "-xe", "-c", "curl -sfL \"http://localhost:7171/backup/tables?local_backup=z_backup_1&table=long_schema.t0\"", + ) + r.NoError(err, "%s\nunexpected GET /backup/tables?local_backup=z_backup_1&table=long_schema.t0 error: %v", out, err) + r.Contains(out, `"table":"t0"`) + r.NotContains(out, `"table":"t1"`) } func testAPIBackupVersion(r *require.Assertions, env *TestEnvironment) { diff --git a/test/integration/tables_test.go b/test/integration/tables_test.go new file mode 100644 index 000000000..e30fb53de --- /dev/null +++ b/test/integration/tables_test.go @@ -0,0 +1,146 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + "time" +) + +// TestTablesCommand exercises `clickhouse-backup tables` flags introduced together with +// https://github.com/Altinity/clickhouse-backup/issues/1388 — `--local-backup`, `--remote-backup`, +// and `--format` — verifying per-table size/parts breakdown for both local and remote backups. +func TestTablesCommand(t *testing.T) { + env, r := NewTestEnvironment(t) + env.connectWithWait(t, r, 500*time.Millisecond, 1*time.Second, 1*time.Minute) + + testBackupName := "test_backup_tables_cmd" + databaseList := []string{dbNameOrdinary, dbNameAtomic, dbNameReplicated, dbNameMySQL, dbNamePostgreSQL, Issue331Issue1091Atomic, Issue331Issue1091Ordinary} + dbNameAtomicTest := dbNameAtomic + "_" + t.Name() + + fullCleanup(t, r, env, []string{testBackupName}, []string{"remote", "local"}, databaseList, true, false, false, "config-s3.yml") + generateTestData(t, r, env, "S3", false, defaultTestData) + + // Live tables -- the no-backup case still works. + out, err := env.DockerExecOut("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "tables", "--tables", " "+dbNameAtomicTest+".*") + r.NoError(err, "%s\nunexpected tables error: %v", out, err) + r.Contains(out, dbNameAtomicTest) + + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "create_remote", testBackupName) + + // Local backup, text format -- should include the new size/parts/disks columns. + out, err = env.DockerExecOut("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "tables", "--local-backup", testBackupName) + r.NoError(err, "%s\nunexpected tables --local-backup error: %v", out, err) + r.Contains(out, dbNameAtomicTest) + + // Local backup, JSON format -- wrapped InfoResult with aggregate totals + tables[] array. + // Logs go to stderr; isolate stdout so the JSON parse is not polluted by INFO lines. + out, err = env.DockerExecOut("clickhouse-backup", "bash", "-c", "clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml tables --local-backup "+testBackupName+" --format json 2>/dev/null") + r.NoError(err, "%s\nunexpected tables --local-backup --format=json error: %v", out, err) + var localResult map[string]interface{} + r.NoError(json.Unmarshal([]byte(out), &localResult), "json output is not parseable: %s", out) + r.Equal(testBackupName, localResult["backup_name"], "backup_name mismatch: %v", localResult) + r.Equal("local", localResult["backup_type"], "backup_type mismatch: %v", localResult) + r.Contains(localResult, "total_bytes") + r.Contains(localResult, "total_size") + r.Contains(localResult, "total_parts") + r.Contains(localResult, "table_count") + localRowsRaw, ok := localResult["tables"].([]interface{}) + r.True(ok, "expected 'tables' array in InfoResult: %v", localResult) + r.NotEmpty(localRowsRaw, "expected at least one table row for local backup") + hasAtomicTest := false + for _, raw := range localRowsRaw { + row, _ := raw.(map[string]interface{}) + db, _ := row["database"].(string) + tbl, _ := row["table"].(string) + if db == dbNameAtomicTest { + hasAtomicTest = true + _, sizeOK := row["size"] + _, partsOK := row["parts"] + _, disksOK := row["disks"].([]interface{}) + r.True(sizeOK, "missing size for %s.%s in JSON output: %v", db, tbl, row) + r.True(partsOK, "missing parts for %s.%s in JSON output: %v", db, tbl, row) + r.True(disksOK, "disks should be array, got %T: %v", row["disks"], row) + } + } + r.True(hasAtomicTest, "expected database %s in local-backup JSON output: %s", dbNameAtomicTest, out) + + // Local backup with --tables pattern -- pattern must be applied. + out, err = env.DockerExecOut("clickhouse-backup", "bash", "-c", "clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml tables --local-backup "+testBackupName+" --tables '"+dbNameAtomicTest+".*' --format json 2>/dev/null") + r.NoError(err, "%s\nunexpected tables --local-backup --tables error: %v", out, err) + var localFiltered map[string]interface{} + r.NoError(json.Unmarshal([]byte(out), &localFiltered)) + r.Equal(dbNameAtomicTest+".*", localFiltered["table_pattern"], "table_pattern should be echoed in result: %v", localFiltered) + filteredRowsRaw, _ := localFiltered["tables"].([]interface{}) + for _, raw := range filteredRowsRaw { + row, _ := raw.(map[string]interface{}) + db, _ := row["database"].(string) + r.Equal(dbNameAtomicTest, db, "filtered output should only contain %s, got row=%v", dbNameAtomicTest, row) + } + + // Local backup, CSV format -- header line must include size and parts columns. + out, err = env.DockerExecOut("clickhouse-backup", "bash", "-c", "clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml tables --local-backup "+testBackupName+" --format csv 2>/dev/null") + r.NoError(err, "%s\nunexpected tables --local-backup --format=csv error: %v", out, err) + csvHead := strings.SplitN(out, "\n", 2)[0] + r.Contains(csvHead, "size") + r.Contains(csvHead, "parts") + r.Contains(out, dbNameAtomicTest) + + // Remote backup, JSON format -- size/parts must come back too. + out, err = env.DockerExecOut("clickhouse-backup", "bash", "-c", "clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml tables --remote-backup "+testBackupName+" --format json 2>/dev/null") + r.NoError(err, "%s\nunexpected tables --remote-backup --format=json error: %v", out, err) + var remoteResult map[string]interface{} + r.NoError(json.Unmarshal([]byte(out), &remoteResult), "json output is not parseable: %s", out) + r.Equal("remote", remoteResult["backup_type"], "backup_type mismatch: %v", remoteResult) + remoteRowsRaw, ok := remoteResult["tables"].([]interface{}) + r.True(ok, "expected 'tables' array in InfoResult: %v", remoteResult) + r.NotEmpty(remoteRowsRaw, "expected at least one table row for remote backup") + hasAtomicTest = false + for _, raw := range remoteRowsRaw { + row, _ := raw.(map[string]interface{}) + db, _ := row["database"].(string) + if db == dbNameAtomicTest { + hasAtomicTest = true + _, sizeOK := row["size"] + _, partsOK := row["parts"] + r.True(sizeOK, "missing size for remote row: %v", row) + r.True(partsOK, "missing parts for remote row: %v", row) + } + } + r.True(hasAtomicTest, "expected database %s in remote-backup JSON output: %s", dbNameAtomicTest, out) + + // Both --local-backup and --remote-backup -- JSON output is now an array of two InfoResults. + out, err = env.DockerExecOut("clickhouse-backup", "bash", "-c", "clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml tables --local-backup "+testBackupName+" --remote-backup "+testBackupName+" --format json 2>/dev/null") + r.NoError(err, "%s\nunexpected tables --local-backup --remote-backup --format=json error: %v", out, err) + var allResults []map[string]interface{} + r.NoError(json.Unmarshal([]byte(out), &allResults), "json output is not parseable: %s", out) + r.Len(allResults, 2, "expected two InfoResult entries (local + remote), got: %s", out) + types := []string{} + for _, sec := range allResults { + bt, _ := sec["backup_type"].(string) + types = append(types, bt) + } + r.Contains(types, "local", "missing local section, types=%v", types) + r.Contains(types, "remote", "missing remote section, types=%v", types) + + // Local backup, text format -- must include `Backup: name (local)` header and TOTAL row. + out, err = env.DockerExecOut("clickhouse-backup", "bash", "-c", "clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml tables --local-backup "+testBackupName+" 2>/dev/null") + r.NoError(err, "%s\nunexpected tables --local-backup text error: %v", out, err) + r.Contains(out, "Backup:") + r.Contains(out, "(local)") + r.Contains(out, "TABLE") + r.Contains(out, "TOTAL (") + r.Contains(out, dbNameAtomicTest) + + // Unknown format must fail with a clear error. + out, err = env.DockerExecOut("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "tables", "--local-backup", testBackupName, "--format", "bogus") + r.Error(err, "expected error for unknown format, got: %s", out) + r.Contains(strings.ToLower(out+fmt.Sprint(err)), "unknown format") + + fullCleanup(t, r, env, []string{testBackupName}, []string{"remote", "local"}, databaseList, true, true, true, "config-s3.yml") + env.checkObjectStorageIsEmpty(t, r, "S3") + env.Cleanup(t, r) +}