diff --git a/acceptance/experimental/air/get-ai-runtime/output.txt b/acceptance/experimental/air/get-ai-runtime/output.txt index 21719cb8d7..76bec7b5f5 100644 --- a/acceptance/experimental/air/get-ai-runtime/output.txt +++ b/acceptance/experimental/air/get-ai-runtime/output.txt @@ -47,6 +47,7 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 "attempt_number": 0, "experiment_name": "my-exp", "dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[NUMID]", - "mlflow_url": "[DATABRICKS_URL]/ml/experiments/exp1/runs/run1/artifacts/logs/node_0" + "mlflow_url": "[DATABRICKS_URL]/ml/experiments/exp1/runs/run1/artifacts/logs/node_0", + "est_remaining_seconds": null } } diff --git a/acceptance/experimental/air/get/output.txt b/acceptance/experimental/air/get/output.txt index ff5cbd2ab0..c593690bbc 100644 --- a/acceptance/experimental/air/get/output.txt +++ b/acceptance/experimental/air/get/output.txt @@ -47,7 +47,8 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 "attempt_number": 0, "experiment_name": "my-exp", "dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[NUMID]", - "mlflow_url": "[DATABRICKS_URL]/ml/experiments/exp1/runs/run1/artifacts/logs/node_0" + "mlflow_url": "[DATABRICKS_URL]/ml/experiments/exp1/runs/run1/artifacts/logs/node_0", + "est_remaining_seconds": null } } diff --git a/acceptance/experimental/air/list/output.txt b/acceptance/experimental/air/list/output.txt index 6c1eca7d7f..e2fd90f8b5 100644 --- a/acceptance/experimental/air/list/output.txt +++ b/acceptance/experimental/air/list/output.txt @@ -1,8 +1,8 @@ === list (text) >>> [CLI] experimental air list - Run ID Experiment Status Started Duration MLflow User Accelerators - [NUMID] qwen-train ● SUCCESS [TIMESTAMP] 12s qwen-train-001 [USERNAME] 8x H100 + Run ID Experiment Status Started Duration ETA MLflow User Accelerators + [NUMID] qwen-train ● SUCCESS [TIMESTAMP] 12s - qwen-train-001 [USERNAME] 8x H100 === list (json) >>> [CLI] experimental air list -o json @@ -25,8 +25,8 @@ === list --all-status (text, via AiTrainingService index) >>> [CLI] experimental air list --all-status - Run ID Experiment Status Started Duration MLflow User Accelerators - [NUMID] qwen-train ● SUCCESS [TIMESTAMP] 12s qwen-train-001 [USERNAME] 8x H100 + Run ID Experiment Status Started Duration ETA MLflow User Accelerators + [NUMID] qwen-train ● SUCCESS [TIMESTAMP] 12s - qwen-train-001 [USERNAME] 8x H100 === list --all-status (json) >>> [CLI] experimental air list --all-status -o json diff --git a/experimental/air/cmd/eta.go b/experimental/air/cmd/eta.go new file mode 100644 index 0000000000..b766a8e984 --- /dev/null +++ b/experimental/air/cmd/eta.go @@ -0,0 +1,197 @@ +package aircmd + +import ( + "cmp" + "context" + "fmt" + "math" + "slices" + "strconv" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/ml" +) + +// trainingETA is a best-effort estimate of the wall-clock time remaining for a +// running training job, derived from MLflow progress signals. +// +// It is only produced when MLflow carries both a progress metric and a known +// total. That combination is logged by HuggingFace Trainer's MLflow integration +// (a metric per training log carrying the global step and fractional epoch, plus +// `max_steps` / `num_train_epochs` as params); AIR itself logs only system +// metrics, which are a wall-clock heartbeat, not training progress. For a run +// without a known total there is no reliable denominator, so no estimate is +// shown rather than a misleading one. +type trainingETA struct { + // RemainingSeconds is the projected time to completion. + RemainingSeconds int64 + // Progress is a short human breadcrumb, e.g. "step 4120/10000" or "epoch 1.8/3". + Progress string +} + +// etaProgressMetric is the MLflow metric HuggingFace Trainer logs on every +// training log call. Each point carries the fractional epoch as its value and +// the global step as its MLflow step, so a single metric history drives both the +// step-based and epoch-based estimates. +const etaProgressMetric = "epoch" + +// etaWindowPoints bounds how many trailing history points feed the rate, so the +// estimate reflects recent throughput rather than a warmup-skewed whole-run +// average. +const etaWindowPoints = 10 + +// detailed renders the ETA for the single-run view: "~48m 20s · step 4120/10000". +func (e *trainingETA) detailed() string { + return fmt.Sprintf("~%s · %s", formatDuration(e.RemainingSeconds), e.Progress) +} + +// compact renders the ETA for the list table's ETA column: "~48m 20s". +func (e *trainingETA) compact() string { + return "~" + formatDuration(e.RemainingSeconds) +} + +// estimateTrainingETA fetches a running run's MLflow params and progress-metric +// history and projects the remaining time. It returns nil whenever an estimate +// can't be made (no known total, too little history, an API error): the ETA is a +// convenience, so any failure is logged and treated as "no estimate" rather than +// failing the command. +func estimateTrainingETA(ctx context.Context, w *databricks.WorkspaceClient, mlflowRunID string) *trainingETA { + if mlflowRunID == "" { + return nil + } + + resp, err := w.Experiments.GetRun(ctx, ml.GetRunRequest{RunId: mlflowRunID}) + if err != nil { + log.Debugf(ctx, "air: could not fetch MLflow run for ETA: %v", err) + return nil + } + if resp.Run == nil || resp.Run.Data == nil { + return nil + } + + // The total is only present for Trainer runs; skip the metric-history fetch + // entirely when it is absent, so a run we can't estimate costs one call, not two. + params := paramMap(resp.Run.Data.Params) + if _, hasSteps := positiveInt(params["max_steps"]); !hasSteps { + if _, hasEpochs := positiveFloat(params["num_train_epochs"]); !hasEpochs { + return nil + } + } + + history, err := w.Experiments.GetHistoryAll(ctx, ml.GetHistoryRequest{ + RunId: mlflowRunID, + MetricKey: etaProgressMetric, + }) + if err != nil { + log.Debugf(ctx, "air: could not fetch %q metric history for ETA: %v", etaProgressMetric, err) + return nil + } + return computeETA(params, history) +} + +// computeETA is the pure projection: given a run's params and the ascending +// history of the progress metric, it returns the remaining-time estimate, or nil +// when it can't be computed. Split from estimateTrainingETA so it can be tested +// without an API client. +// +// When max_steps is set (> 0) HuggingFace Trainer trains by step and overrides +// num_train_epochs, so the step total wins when both are present. +func computeETA(params map[string]string, history []ml.Metric) *trainingETA { + if len(history) < 2 { + return nil + } + + // History should already be ordered by step; sort by timestamp defensively so + // the rate is measured over increasing wall-clock time. + points := append([]ml.Metric(nil), history...) + slices.SortStableFunc(points, func(a, b ml.Metric) int { return cmp.Compare(a.Timestamp, b.Timestamp) }) + if len(points) > etaWindowPoints { + points = points[len(points)-etaWindowPoints:] + } + first, last := points[0], points[len(points)-1] + + elapsedSec := float64(last.Timestamp-first.Timestamp) / 1000.0 + if elapsedSec <= 0 { + return nil + } + + if maxSteps, ok := positiveInt(params["max_steps"]); ok { + return stepETA(first, last, elapsedSec, maxSteps) + } + if totalEpochs, ok := positiveFloat(params["num_train_epochs"]); ok { + return epochETA(first, last, elapsedSec, totalEpochs) + } + return nil +} + +// stepETA projects remaining time from the global step (the metric's MLflow +// step) against a known max_steps. +func stepETA(first, last ml.Metric, elapsedSec float64, maxSteps int64) *trainingETA { + current := last.Step + if current <= 0 || current >= maxSteps { + return nil + } + done := last.Step - first.Step + if done <= 0 { + return nil + } + perSec := float64(done) / elapsedSec + remaining := float64(maxSteps-current) / perSec + return &trainingETA{ + RemainingSeconds: int64(math.Round(remaining)), + Progress: fmt.Sprintf("step %d/%d", current, maxSteps), + } +} + +// epochETA projects remaining time from the fractional epoch (the metric's +// value) against a known num_train_epochs. +func epochETA(first, last ml.Metric, elapsedSec, totalEpochs float64) *trainingETA { + current := last.Value + if current <= 0 || current >= totalEpochs { + return nil + } + done := last.Value - first.Value + if done <= 0 { + return nil + } + perSec := done / elapsedSec + remaining := (totalEpochs - current) / perSec + return &trainingETA{ + RemainingSeconds: int64(math.Round(remaining)), + Progress: fmt.Sprintf("epoch %.1f/%s", current, trimFloat(totalEpochs)), + } +} + +// paramMap indexes MLflow params by key. +func paramMap(params []ml.Param) map[string]string { + m := make(map[string]string, len(params)) + for _, p := range params { + m[p.Key] = p.Value + } + return m +} + +// positiveInt parses s as an integer and reports it only when it is > 0. Trainer +// logs max_steps=-1 when training by epochs, which this rejects. +func positiveInt(s string) (int64, bool) { + n, err := strconv.ParseInt(s, 10, 64) + if err != nil || n <= 0 { + return 0, false + } + return n, true +} + +// positiveFloat parses s as a float and reports it only when it is > 0. +func positiveFloat(s string) (float64, bool) { + f, err := strconv.ParseFloat(s, 64) + if err != nil || f <= 0 { + return 0, false + } + return f, true +} + +// trimFloat renders a float without trailing zeros: 3.0 -> "3", 2.5 -> "2.5". +func trimFloat(f float64) string { + return strconv.FormatFloat(f, 'f', -1, 64) +} diff --git a/experimental/air/cmd/eta_test.go b/experimental/air/cmd/eta_test.go new file mode 100644 index 0000000000..7f2c57721a --- /dev/null +++ b/experimental/air/cmd/eta_test.go @@ -0,0 +1,94 @@ +package aircmd + +import ( + "testing" + + "github.com/databricks/databricks-sdk-go/service/ml" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// metric builds one epoch-metric history point: fractional epoch as the value, +// global step as the MLflow step, wall-clock as the timestamp (ms). +func metric(tsMillis, step int64, epoch float64) ml.Metric { + return ml.Metric{Key: "epoch", Timestamp: tsMillis, Step: step, Value: epoch} +} + +func TestComputeETA_StepMode(t *testing.T) { + // max_steps wins over num_train_epochs. 4000 steps in 100s => 40 steps/s; + // 5000 steps remain => 125s. + eta := computeETA( + map[string]string{"max_steps": "10000", "num_train_epochs": "3"}, + []ml.Metric{metric(0, 1000, 0.3), metric(100_000, 5000, 1.5)}, + ) + require.NotNil(t, eta) + assert.Equal(t, int64(125), eta.RemainingSeconds) + assert.Equal(t, "step 5000/10000", eta.Progress) +} + +func TestComputeETA_EpochMode(t *testing.T) { + // No usable max_steps (Trainer logs -1 when training by epochs), so the + // fractional epoch drives it: 1.0 epoch in 100s => 0.01 epoch/s; 1.5 remain + // of 3 => 150s. + eta := computeETA( + map[string]string{"max_steps": "-1", "num_train_epochs": "3"}, + []ml.Metric{metric(0, 100, 0.5), metric(100_000, 900, 1.5)}, + ) + require.NotNil(t, eta) + assert.Equal(t, int64(150), eta.RemainingSeconds) + assert.Equal(t, "epoch 1.5/3", eta.Progress) +} + +func TestComputeETA_UsesTrailingWindow(t *testing.T) { + // Early points are slow (10 steps over the first 100s), recent points fast + // (100 steps/10s). The window should reflect the recent rate, not the whole + // run's average. Build 12 points; only the last etaWindowPoints (10) count. + var pts []ml.Metric + pts = append(pts, metric(0, 0, 0), metric(100_000, 10, 0.1)) + // Ten fast points: +100 steps every 10s starting at step 10, t=100s. + for i := int64(1); i <= 10; i++ { + pts = append(pts, metric(100_000+i*10_000, 10+i*100, 0.1+float64(i)*0.1)) + } + eta := computeETA(map[string]string{"max_steps": "2000"}, pts) + require.NotNil(t, eta) + // Windowed rate is 10 steps/s; current step 1010, so 990 remain => 99s. + // (The whole-run average would be far slower and give a larger estimate.) + assert.Equal(t, int64(99), eta.RemainingSeconds) + assert.Equal(t, "step 1010/2000", eta.Progress) +} + +func TestComputeETA_NoEstimate(t *testing.T) { + base := []ml.Metric{metric(0, 100, 0.3), metric(100_000, 5000, 1.5)} + cases := []struct { + name string + params map[string]string + history []ml.Metric + }{ + {"no total param", map[string]string{}, base}, + {"max_steps not positive", map[string]string{"max_steps": "-1"}, base}, + {"num_train_epochs zero", map[string]string{"num_train_epochs": "0"}, base}, + {"total param unparseable", map[string]string{"max_steps": "lots"}, base}, + {"too few points", map[string]string{"max_steps": "10000"}, base[:1]}, + {"no elapsed time", map[string]string{"max_steps": "10000"}, []ml.Metric{metric(500, 100, 0.3), metric(500, 5000, 1.5)}}, + {"already at max steps", map[string]string{"max_steps": "5000"}, base}, + {"epoch already complete", map[string]string{"num_train_epochs": "1.5"}, base}, + {"step not advancing", map[string]string{"max_steps": "10000"}, []ml.Metric{metric(0, 5000, 1.5), metric(100_000, 5000, 1.5)}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Nil(t, computeETA(tc.params, tc.history)) + }) + } +} + +func TestTrainingETADisplay(t *testing.T) { + eta := &trainingETA{RemainingSeconds: 2900, Progress: "step 5000/10000"} + assert.Equal(t, "~48m 20s · step 5000/10000", eta.detailed()) + assert.Equal(t, "~48m 20s", eta.compact()) +} + +func TestTrimFloat(t *testing.T) { + assert.Equal(t, "3", trimFloat(3.0)) + assert.Equal(t, "2.5", trimFloat(2.5)) + assert.Equal(t, "0.1", trimFloat(0.1)) +} diff --git a/experimental/air/cmd/format.go b/experimental/air/cmd/format.go index de7046de3f..1dfb7e1184 100644 --- a/experimental/air/cmd/format.go +++ b/experimental/air/cmd/format.go @@ -84,6 +84,13 @@ var gpuDisplayNames = map[string]string{ "GPU_1xH100": "H100", } +// isRunning reports whether a run is currently executing (lifecycle RUNNING), as +// opposed to pending, terminal, or state-unknown. Only a running run has a +// meaningful remaining-time estimate. +func isRunning(run *jobs.Run) bool { + return run.State != nil && run.State.LifeCycleState == jobs.RunLifeCycleStateRunning +} + // runStatus returns the single status word to show for a run. The backend // reports two values: a lifecycle state (e.g. PENDING, RUNNING) and, once the // run has finished, a result state (e.g. SUCCESS, FAILED). The result state is diff --git a/experimental/air/cmd/get.go b/experimental/air/cmd/get.go index dcde5473d1..06975b44e5 100644 --- a/experimental/air/cmd/get.go +++ b/experimental/air/cmd/get.go @@ -29,6 +29,9 @@ type getData struct { ExperimentName *string `json:"experiment_name"` DashboardURL string `json:"dashboard_url"` MLflowURL *string `json:"mlflow_url"` + // ESTRemainingSeconds is a best-effort estimate of the time left for a + // running run, or null when one can't be made (see estimateTrainingETA). + ESTRemainingSeconds *int64 `json:"est_remaining_seconds"` // The fields below are pre-rendered text-view cells, excluded from JSON // (matching `air get --json`). Each shows "N/A" when its value is @@ -42,6 +45,9 @@ type getData struct { AcceleratorsDisplay string `json:"-"` EnvironmentDisplay string `json:"-"` MaxRetriesDisplay string `json:"-"` + // ETADisplay is the pre-rendered "ETA" cell ("~48m 20s · step 4120/10000"), + // set only for a running run with an estimable remaining time. + ETADisplay string `json:"-"` // TrainingConfigPath is the run's config file, downloaded for the config box. TrainingConfigPath string `json:"-"` // Sweep replaces the single-run view for foreach runs. @@ -155,6 +161,15 @@ func newGetCommand() *cobra.Command { if ids != nil { url := mlflowLogsURL(w.Config.Host, ids) data.MLflowURL = &url + // A remaining-time estimate only makes sense while the run is still + // training; a terminal run is either done or stopped mid-progress. + if isRunning(run) { + if eta := estimateTrainingETA(ctx, w, ids.RunID); eta != nil { + secs := eta.RemainingSeconds + data.ESTRemainingSeconds = &secs + data.ETADisplay = eta.detailed() + } + } } if task := findForEachTask(run); task != nil { data.Sweep = buildSweepInfo(ctx, w, task) diff --git a/experimental/air/cmd/list.go b/experimental/air/cmd/list.go index c5cc64aafd..4356edb90b 100644 --- a/experimental/air/cmd/list.go +++ b/experimental/air/cmd/list.go @@ -44,10 +44,13 @@ type listRow struct { StartedAt *string `json:"started_at"` IsSweep bool `json:"is_sweep"` - // Experiment, Duration, MLflowURL and Accelerators are table-only columns, - // omitted from JSON to match `air list --json`. - Experiment string `json:"-"` - Duration string `json:"-"` + // Experiment, Duration, ETA, MLflowURL and Accelerators are table-only + // columns, omitted from JSON to match `air list --json`. + Experiment string `json:"-"` + Duration string `json:"-"` + // ETA is a best-effort remaining-time estimate ("~48m 20s"), set only for a + // running run we could estimate; empty renders as "-". + ETA string `json:"-"` MLflowURL string `json:"-"` MLflowLabel string `json:"-"` RunURL string `json:"-"` @@ -308,18 +311,29 @@ func warnIfTruncated(ctx context.Context, f *runFetcher) { } } -// setMLflowLinks fills in each row's MLflow link, label, and experiment URL in -// parallel, best-effort: a row whose IDs can't be resolved keeps its "-" placeholder. +// setMLflowLinks fills in each row's MLflow link, label, and experiment URL - +// and, for a running run, its ETA - in parallel, best-effort: a row whose IDs +// can't be resolved keeps its "-" placeholder, and a run we can't estimate has +// no ETA. func setMLflowLinks(ctx context.Context, w *databricks.WorkspaceClient, host string, entries []listedRun) { var g errgroup.Group g.SetLimit(enrichConcurrency) for i := range entries { g.Go(func() error { - if ids := mlflowIDsForTask(ctx, w, entries[i].taskRunID); ids != nil { - entries[i].row.MLflowURL = mlflowLogsURL(host, ids) - name := fetchMLflowRunName(ctx, w, ids.RunID) - entries[i].row.MLflowLabel = mlflowRunLabel(name, ids.RunID) - entries[i].row.ExperimentURL = mlflowExperimentURL(host, ids) + ids := mlflowIDsForTask(ctx, w, entries[i].taskRunID) + if ids == nil { + return nil + } + entries[i].row.MLflowURL = mlflowLogsURL(host, ids) + name := fetchMLflowRunName(ctx, w, ids.RunID) + entries[i].row.MLflowLabel = mlflowRunLabel(name, ids.RunID) + entries[i].row.ExperimentURL = mlflowExperimentURL(host, ids) + // The ETA needs a progress-metric history fetch, so it's computed only + // for running rows (the only ones that can have one). + if entries[i].row.Status == string(jobs.RunLifeCycleStateRunning) { + if eta := estimateTrainingETA(ctx, w, ids.RunID); eta != nil { + entries[i].row.ETA = eta.compact() + } } return nil }) diff --git a/experimental/air/cmd/list_tui_render.go b/experimental/air/cmd/list_tui_render.go index ad70369230..f59f479623 100644 --- a/experimental/air/cmd/list_tui_render.go +++ b/experimental/air/cmd/list_tui_render.go @@ -37,7 +37,7 @@ func newListStyles(r *lipgloss.Renderer) listStyles { // listCols holds the computed width of each variable-width column. MLflow is // fixed (a short link) and the gutter is one cell. type listCols struct { - runID, experiment, status, started, duration, user, accel int + runID, experiment, status, started, duration, eta, user, accel int } // columnCap bounds the widest free-text columns so one long value can't dominate @@ -51,6 +51,7 @@ func computeListCols(rows []listRow) listCols { status: len("Status"), started: len("Started"), duration: len("Duration"), + eta: len("ETA"), user: len("User"), accel: len("Accelerators"), } @@ -60,6 +61,7 @@ func computeListCols(rows []listRow) listCols { c.status = max(c.status, lipgloss.Width("● "+r.Status)) c.started = max(c.started, lipgloss.Width(startedDisplay(r))) c.duration = max(c.duration, lipgloss.Width(r.Duration)) + c.eta = max(c.eta, lipgloss.Width(etaDisplay(r))) c.user = min(columnCap, max(c.user, lipgloss.Width(r.User))) c.accel = max(c.accel, lipgloss.Width(r.Accelerators)) } @@ -78,6 +80,7 @@ func (s listStyles) renderHeader(cols listCols) string { h("Status", cols.status, false), h("Started", cols.started, false), h("Duration", cols.duration, true), + h("ETA", cols.eta, true), h("MLflow", mlflowColWidth, false), h("User", cols.user, false), h("Accelerators", cols.accel, false), @@ -123,6 +126,7 @@ func (s listStyles) renderRow(cols listCols, r listRow, selected, links bool) st s.cell(base, "● "+r.Status, cols.status, fg(statusColor(r.Status)), false, false, ""), s.cell(base, startedDisplay(r), cols.started, fg(colN9), false, false, ""), s.cell(base, r.Duration, cols.duration, fg(colN9), true, false, ""), + s.cell(base, etaDisplay(r), cols.eta, fg(colAmber), true, false, ""), s.mlflowCell(base, r, selected, links), s.cell(base, r.User, cols.user, fg(colN9), false, false, ""), s.cell(base, r.Accelerators, cols.accel, fg(colN9), false, false, ""), @@ -191,6 +195,15 @@ func statusColor(status string) lipgloss.Color { } } +// etaDisplay is the row's remaining-time estimate, or "-" when there is none +// (a finished run, or a run we couldn't estimate). +func etaDisplay(r listRow) string { + if r.ETA == "" { + return "-" + } + return r.ETA +} + // startedDisplay trims the row's ISO start timestamp to second precision // ("2006-01-02T15:04:05"), or "-" when the run hasn't started. func startedDisplay(r listRow) string { diff --git a/experimental/air/cmd/list_tui_test.go b/experimental/air/cmd/list_tui_test.go index 8b52417ee2..e0b31250ba 100644 --- a/experimental/air/cmd/list_tui_test.go +++ b/experimental/air/cmd/list_tui_test.go @@ -16,7 +16,7 @@ import ( func testListRows() []listRow { return []listRow{ {RunID: "1", Experiment: "qwen-train", User: "me@example.com", Status: "SUCCESS", StartedAt: new("2026-06-05T17:32:39.000000+00:00"), Duration: "1m 14s", MLflowURL: "https://h/ml/experiments/E/runs/04c41514fbb0/artifacts/logs/node_0", MLflowLabel: "qwen-run-001", Accelerators: "8x H100"}, - {RunID: "2", Experiment: "llama-train", User: "me@example.com", Status: "RUNNING", StartedAt: new("2026-06-05T18:43:24.000000+00:00"), Duration: "3m 32s", MLflowURL: "-", MLflowLabel: "-", Accelerators: "1x A10"}, + {RunID: "2", Experiment: "llama-train", User: "me@example.com", Status: "RUNNING", StartedAt: new("2026-06-05T18:43:24.000000+00:00"), Duration: "3m 32s", ETA: "~48m 20s", MLflowURL: "-", MLflowLabel: "-", Accelerators: "1x A10"}, {RunID: "3", Experiment: "mixtral", User: "me@example.com", Status: "FAILED", StartedAt: nil, Duration: "-", MLflowURL: "-", MLflowLabel: "-", Accelerators: "-"}, } } @@ -158,8 +158,9 @@ func TestListModelView(t *testing.T) { assert.NotContains(t, out, "\x1b", "Ascii profile + no links should produce no escapes") for _, want := range []string{ - "Run ID", "Experiment", "Status", "Started", "Duration", "MLflow", "User", "Accelerators", + "Run ID", "Experiment", "Status", "Started", "Duration", "ETA", "MLflow", "User", "Accelerators", "qwen-train", "● SUCCESS", "● RUNNING", "● FAILED", + "~48m 20s", // ETA on the running row "qwen-run-001", // MLflow run label "2026-06-05T17:32:39", // started trimmed to seconds "▸", // selection gutter on the first row diff --git a/experimental/air/cmd/render.go b/experimental/air/cmd/render.go index 5d4cf9bb6d..641a7b8174 100644 --- a/experimental/air/cmd/render.go +++ b/experimental/air/cmd/render.go @@ -78,6 +78,7 @@ type runView struct { retries int maxRetries string duration string + eta string experiment string mlflowLabel string mlflowURL string @@ -110,6 +111,7 @@ func renderRunText(ctx context.Context, out io.Writer, w *databricks.WorkspaceCl retries: data.AttemptNumber, maxRetries: data.MaxRetriesDisplay, duration: data.DurationDisplay, + eta: data.ETADisplay, experiment: data.ExperimentDisplay, mlflowLabel: na, user: data.UserDisplay, @@ -376,12 +378,19 @@ func renderFields(p palette, colorOn bool, v runView) string { field(p, "Retries", p.n12.Render(strconv.Itoa(v.retries))), field(p, "Max Retries", p.n12.Render(v.maxRetries)), field(p, "Duration", p.n12.Render(v.duration)), + } + // The ETA is shown only for a running run we could estimate; amber marks it as + // live, in-progress information. + if v.eta != "" { + rows = append(rows, field(p, "ETA", p.amber.Render(v.eta))) + } + rows = append(rows, field(p, "Experiment", p.n12.Render(v.experiment)), field(p, "MLflow Run", link(colorOn, p.blue, v.mlflowLabel, v.mlflowURL)), field(p, "User", p.n12.Render(v.user)), field(p, "Accelerators", p.n12.Render(v.accelerators)), field(p, "Environment", p.n12.Render(v.environment)), - } + ) return strings.Join(rows, "\n") }