Improve client ergonomics for mult-sled jobs

README.md

@@ -48,23 +48,18 @@ test try running the command `echo $SUSH_JOB_ID`:
 ```
 sush# job start --wait "echo $SUSH_JOB_ID"
 ✅ Signed request for job `install-there-mutual-warfare-sound-live-order-man`
-✅ Job ID:      install-there-mutual-warfare-sound-live-order-man
-   Target:	a part:0001
-   Job status:	Stopped
-   Started at:	2026-08-05 16:27:58.824606210 UTC
-   Stopped at:	2026-08-05 16:27:58.892796324 UTC (68ms 190us 114ns)
-   Exit status:	0
-   Stdout len:	50 B
-   Stderr len:	0 B
-   Stdout hash:	ae5a3df9c5c629590af52baceb12021525c0be16994738977b2ed049f4f374bb
-   Stderr hash:	af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262
+✅ Job ID:    install-there-mutual-warfare-sound-live-order-man
+   a part:0001  Stopped, exit 0 (68ms 190us 114ns), 50 B out, 0 B err
 ✅ Job stdout:
 install-there-mutual-warfare-sound-live-order-man
 ```
 
 The `--wait` (`-w`) flag tells `sush` to wait for the job to stop before
-returning; the default behavior is to start the job and immediately return.
-You can check the status of a (running) job with `job status`.
+returning, showing a live status line per sled while it runs; the default
+behavior is to start the job and immediately return. You can check the
+status of a (running) job with `job status`, watch it the same way with
+`job status --wait`, or see the complete job status with
+`job status --full`.
 
 To run an interactive job with a pseudoterminal, you can use `job start
 --interactive` (`-i`):

client/src/cli.rs

@@ -4,6 +4,8 @@
 
 //! Possibly-interactive command-line interface.
 
+use std::collections::BTreeMap;
+use std::fmt;
 use std::io::{self, BufRead as _, Read as _, Write as _, stderr, stdin, stdout};
 use std::path::Path;
 use std::sync::{Arc, Mutex};
@@ -12,9 +14,10 @@ use std::time::{Duration, SystemTime};
 use bytesize::ByteSize;
 use chrono::TimeDelta;
 use humantime::format_duration;
-use indicatif::{ProgressBar, ProgressStyle};
+use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
 use rustix::io::ioctl_fionread;
 use serde_json::{json, to_string as to_json_string, to_string_pretty as to_json_string_pretty};
+use sled_hardware_types::BaseboardId;
 use x509_cert::Certificate;
 use x509_cert::der::Encode as _;
 
@@ -27,13 +30,14 @@ use sush_common::keys::{KeyId, Signature, SshPublicKey};
 
 use crate::AuthzSigner;
 use crate::commands::{CommandError, GlobalArgs};
-use crate::context::{CommandContext, OutputFormat};
+use crate::context::{CommandContext, OutputFormat, StatusDisplayStyle};
 
 #[derive(Clone, Debug, Default)]
 pub struct Cli {
     globals: Arc<Mutex<GlobalArgs>>,
     output: Arc<Mutex<OutputFormat>>,
     progress: Arc<Mutex<Option<ProgressBar>>>,
+    watch: Arc<Mutex<Option<Watch>>>,
     session: Arc<Mutex<Option<Session>>>,
     credentials: AuthzSigner,
 }
@@ -42,6 +46,57 @@ fn byte_size(len: u64) -> bytesize::Display {
     ByteSize::b(len).display().si()
 }
 
+fn format_elapsed_duration(duration: TimeDelta) -> String {
+    if let Ok(duration) = duration.to_std() {
+        format_duration(duration).to_string()
+    } else {
+        String::from("negative duration, times may be unreliable")
+    }
+}
+
+/// One sled's worth of job status, without the sled's name.
+fn short_status_row(status: &JobStatus) -> String {
+    match status {
+        JobStatus::Cancelled {
+            time_cancelled,
+            actor,
+            ..
+        } => format!("Cancelled at {time_cancelled} by {actor}"),
+        JobStatus::Queued {
+            time_queued, actor, ..
+        } => format!("Queued at {time_queued} by {actor}"),
+        JobStatus::Started { time_started, .. } => format!("Started at {time_started}"),
+        JobStatus::Stopped { result, output, .. } => {
+            let duration = format_elapsed_duration(status.time_elapsed());
+            let stdout_len = byte_size(output.stdout_len);
+            let stderr_len = byte_size(output.stderr_len);
+            let result = match result {
+                Ok(exit_status) => format!("exit {exit_status}"),
+                Err(err) => err.to_string(),
+            };
+            format!("Stopped, {result} ({duration}), {stdout_len} out, {stderr_len} err")
+        }
+        JobStatus::Error {
+            time_error, error, ..
+        } => format!("Error at {time_error}: {error}"),
+    }
+}
+
+/// Live per-sled status lines for a watched job.
+struct Watch {
+    multi: MultiProgress,
+    bars: BTreeMap<BaseboardId, ProgressBar>,
+    width: usize,
+}
+
+impl fmt::Debug for Watch {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("Watch")
+            .field("bars", &self.bars.keys())
+            .finish_non_exhaustive()
+    }
+}
+
 impl CommandContext for Cli {
     // Context management
 
@@ -209,7 +264,13 @@ impl CommandContext for Cli {
     fn job_stopped(&mut self, job_id: &JobId) {
         match self.get_output_format() {
             OutputFormat::Json => println!("{}", json!(job_id)),
-            OutputFormat::Text => println!("\r✅ Stopped job `{job_id}`"),
+            OutputFormat::Text => {
+                if let Some(watch) = self.watch.lock().unwrap().as_ref() {
+                    let _ = watch.multi.println(format!("✅ Stopped job `{job_id}`"));
+                } else {
+                    println!("\r✅ Stopped job `{job_id}`");
+                }
+            }
         }
     }
 
@@ -218,6 +279,13 @@ impl CommandContext for Cli {
         error
     }
 
+    fn job_output_target(&mut self, target: &BaseboardId) {
+        match self.get_output_format() {
+            OutputFormat::Json => println!("{}", json!({ "target": target.to_string() })),
+            OutputFormat::Text => println!("⟹  {target}  ⟸"),
+        }
+    }
+
     fn job_output(
         &mut self,
         _job_id: &JobId,
@@ -301,34 +369,61 @@ impl CommandContext for Cli {
         }
     }
 
-    fn job_polling_started(&mut self, job_id: &JobId, elapsed: Duration) {
-        let mut progress = self.progress.lock().unwrap();
-        if matches!(self.get_output_format(), OutputFormat::Text) && progress.is_none() {
-            let bar = ProgressBar::new_spinner();
-            bar.set_elapsed(elapsed);
-            bar.set_prefix(format!("Waiting for job `{job_id}`"));
-            bar.set_style(
-                ProgressStyle::with_template(
-                    "{spinner}  \
-                     {prefix} \
-                     [{elapsed_precise}] \
-                     {msg}",
-                )
-                .unwrap(),
-            );
-            *progress = Some(bar);
+    fn job_watch_started(&mut self, _job_id: &JobId) {
+        if matches!(self.get_output_format(), OutputFormat::Text) {
+            *self.watch.lock().unwrap() = Some(Watch {
+                multi: MultiProgress::new(),
+                bars: BTreeMap::new(),
+                width: 0,
+            });
+        }
+    }
+
+    fn job_watch_update(&mut self, status: &JobStatusMap) {
+        let mut guard = self.watch.lock().unwrap();
+        let Some(watch) = guard.as_mut() else { return };
+
+        // Widen every name column if a longer baseboard ID appears.
+        let width = status
+            .keys()
+            .map(|b| b.to_string().len())
+            .max()
+            .unwrap_or(0);
+        if width > watch.width {
+            watch.width = width;
+            for (baseboard_id, bar) in &watch.bars {
+                bar.set_prefix(format!("{:<width$}", baseboard_id.to_string()));
+            }
         }
-    }
+        let width = watch.width;
 
-    fn job_polling_update(&mut self, _job_id: &JobId) {
-        if let Some(progress) = self.progress.lock().unwrap().as_mut() {
-            progress.tick();
+        for (baseboard_id, status) in status {
+            if !watch.bars.contains_key(baseboard_id) {
+                let index = watch.bars.range(..baseboard_id).count();
+                let bar = watch.multi.insert(index, ProgressBar::new_spinner());
+                bar.set_style(ProgressStyle::with_template("{spinner}  {prefix}  {msg}").unwrap());
+                bar.set_prefix(format!("{:<width$}", baseboard_id.to_string()));
+                bar.enable_steady_tick(Duration::from_millis(100));
+                watch.bars.insert(baseboard_id.to_owned(), bar);
+            }
+            let bar = &watch.bars[baseboard_id];
+            if bar.is_finished() {
+                continue;
+            }
+            if status.is_terminal() {
+                bar.finish_with_message(short_status_row(status));
+            } else {
+                bar.set_message(short_status_row(status));
+            }
         }
     }
 
-    fn job_polling_finished(&mut self, _job_id: &JobId) {
-        if let Some(progress) = self.progress.lock().unwrap().take() {
-            progress.finish_and_clear();
+    fn job_watch_finished(&mut self, _job_id: &JobId) {
+        if let Some(watch) = self.watch.lock().unwrap().take() {
+            for bar in watch.bars.values() {
+                bar.finish_and_clear();
+            }
+            let _ = watch.multi.clear();
         }
     }
 
@@ -397,18 +492,33 @@ impl CommandContext for Cli {
         }
     }
 
-    fn job_status(&mut self, _job_id: &JobId, status: &JobStatusMap) {
-        fn format_elapsed_duration(duration: TimeDelta) -> String {
-            if let Ok(duration) = duration.to_std() {
-                format_duration(duration).to_string()
-            } else {
-                String::from("negative duration, times may be unreliable")
-            }
-        }
-
-        // TODO: parallel status display
+    fn job_status(&mut self, job_id: &JobId, status: &JobStatusMap, style: StatusDisplayStyle) {
         match self.get_output_format() {
             OutputFormat::Json => println!("{}", json!(job_status_to_json_map(status.clone()))),
+            OutputFormat::Text if matches!(style, StatusDisplayStyle::Short) => {
+                let icon = if status.values().any(|s| {
+                    matches!(
+                        s,
+                        JobStatus::Error { .. }
+                            | JobStatus::Cancelled { .. }
+                            | JobStatus::Stopped { result: Err(_), .. }
+                    )
+                }) {
+                    "❌"
+                } else {
+                    "✅"
+                };
+                let width = status
+                    .keys()
+                    .map(|b| b.to_string().len())
+                    .max()
+                    .unwrap_or(0);
+                println!("{icon} Job ID:\t{job_id}");
+                for (baseboard_id, status) in status {
+                    let id = baseboard_id.to_string();
+                    println!("   {id:<width$}  {}", short_status_row(status));
+                }
+            }
             OutputFormat::Text => {
                 for (baseboard_id, status) in status {
                     match status {