From 4b210c68bb84376616d7cf9119cb12fc3dc436c8 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 8 Jul 2026 15:14:27 +0200 Subject: [PATCH] feat(cli): add openshell conformance subcommand skeleton MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a hidden `openshell conformance` subcommand behind a `conformance` Cargo feature flag. The subcommand connects to a registered gateway and runs a suite of driver-agnostic conformance scenarios, reporting pass/fail per scenario and exiting non-zero on any failure. Implements the `lifecycle` scenario (create → ready → delete) as the first concrete scenario. The remaining five scenarios are stubbed as "not yet implemented" placeholders. Sandbox names follow the `conformance--` convention (Kubernetes RFC 1123 safe) to make orphaned sandboxes identifiable. Signed-off-by: Evan Lezar --- crates/openshell-cli/Cargo.toml | 1 + crates/openshell-cli/src/main.rs | 91 ++++++++++ crates/openshell-cli/src/run.rs | 293 +++++++++++++++++++++++++++++++ 3 files changed, 385 insertions(+) diff --git a/crates/openshell-cli/Cargo.toml b/crates/openshell-cli/Cargo.toml index 8e4cb3fb25..3ce1e5a556 100644 --- a/crates/openshell-cli/Cargo.toml +++ b/crates/openshell-cli/Cargo.toml @@ -86,6 +86,7 @@ workspace = true [features] bundled-z3 = ["openshell-prover/bundled-z3"] +conformance = [] [dev-dependencies] futures = { workspace = true } diff --git a/crates/openshell-cli/src/main.rs b/crates/openshell-cli/src/main.rs index e02c7c9cd5..f84cedd410 100644 --- a/crates/openshell-cli/src/main.rs +++ b/crates/openshell-cli/src/main.rs @@ -554,6 +554,27 @@ enum Commands { command: Option, }, + /// Validate that a gateway and its driver are functioning correctly. + /// + /// Runs a suite of conformance scenarios against the configured gateway + /// and reports pass/fail for each. Exits non-zero if any scenario fails. + /// + /// All sandboxes created by this command use the naming convention + /// `conformance--` and are cleaned up on completion. + /// If the command is interrupted, any remaining sandboxes can be + /// identified by the `conformance-` prefix. + /// + /// Examples: + /// openshell conformance run + /// openshell conformance run --filter lifecycle + /// openshell conformance run --timeout 120 + #[cfg(feature = "conformance")] + #[command(hide = true, help_template = SUBCOMMAND_HELP_TEMPLATE)] + Conformance { + #[command(subcommand)] + command: ConformanceCommands, + }, + // =================================================================== // ADDITIONAL COMMANDS // =================================================================== @@ -1186,6 +1207,47 @@ enum InferenceCommands { }, } +// ----------------------------------------------------------------------- +// Conformance commands +// ----------------------------------------------------------------------- + +#[cfg(feature = "conformance")] +#[derive(Subcommand, Debug)] +enum ConformanceCommands { + /// Run the conformance suite against the configured gateway. + /// + /// Connects to the gateway using the registered credentials and runs + /// each conformance scenario in sequence. Reports pass/fail per + /// scenario and exits non-zero if any scenario fails. + /// + /// Examples: + /// openshell conformance run + /// openshell conformance run --filter lifecycle + /// openshell conformance run --timeout 120 + #[command(help_template = LEAF_HELP_TEMPLATE, next_help_heading = "FLAGS")] + Run { + /// Only run scenarios whose name contains this substring. + #[arg(long, short = 'f')] + filter: Option, + + /// Seconds to wait for each scenario to complete. + #[arg(long, default_value = "300")] + timeout: u64, + + /// Output format. + #[arg(short = 'o', long = "output", value_enum, default_value_t = OutputFormat::Table)] + output: OutputFormat, + }, + + /// List available conformance scenarios without running them. + #[command(help_template = LEAF_HELP_TEMPLATE, next_help_heading = "FLAGS")] + List { + /// Output format. + #[arg(short = 'o', long = "output", value_enum, default_value_t = OutputFormat::Table)] + output: OutputFormat, + }, +} + // ----------------------------------------------------------------------- // Doctor (diagnostic) commands // ----------------------------------------------------------------------- @@ -2062,6 +2124,35 @@ async fn main() -> Result<()> { } }, + // ----------------------------------------------------------- + // Conformance commands + // ----------------------------------------------------------- + #[cfg(feature = "conformance")] + Some(Commands::Conformance { command }) => { + let ctx = resolve_gateway(&cli.gateway, &cli.gateway_endpoint)?; + let mut tls = tls.with_gateway_name(&ctx.name); + apply_auth(&mut tls, &ctx.name); + match command { + ConformanceCommands::Run { + filter, + timeout, + output, + } => { + run::conformance_run( + &ctx.endpoint, + &tls, + filter.as_deref(), + timeout, + output.as_str(), + ) + .await?; + } + ConformanceCommands::List { output } => { + run::conformance_list(output.as_str())?; + } + } + } + // ----------------------------------------------------------- // Doctor (diagnostic) commands // ----------------------------------------------------------- diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index f56bb71512..b6b40cb33f 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -1558,6 +1558,299 @@ pub fn gateway_admin_info(name: &str) -> Result<()> { /// /// Checks Docker connectivity and reports the result. Returns exit code 0 /// if all checks pass, 1 otherwise. +// ----------------------------------------------------------------------- +// Conformance suite +// ----------------------------------------------------------------------- + +/// A single conformance scenario definition. +#[cfg(feature = "conformance")] +struct Scenario { + /// Short lowercase-hyphenated name, used in sandbox naming and filtering. + name: &'static str, + /// Human-readable description shown in `conformance list`. + description: &'static str, +} + +#[cfg(feature = "conformance")] +fn all_scenarios() -> &'static [Scenario] { + &[ + Scenario { + name: "capabilities", + description: "GetCapabilities returns non-empty driver name, version, and default image", + }, + Scenario { + name: "lifecycle", + description: "Create → running → stop → delete completes without error", + }, + Scenario { + name: "not-found", + description: "Get/stop/delete for an unknown sandbox ID returns an appropriate error", + }, + Scenario { + name: "idempotent-delete", + description: "Deleting an already-deleted sandbox does not error", + }, + Scenario { + name: "validate", + description: "Invalid sandbox specs are rejected before creation", + }, + Scenario { + name: "concurrent", + description: "Two sandboxes created simultaneously do not interfere", + }, + ] +} + +/// Outcome of a single conformance scenario. +#[cfg(feature = "conformance")] +#[derive(serde::Serialize)] +struct ScenarioResult { + name: String, + passed: bool, + message: String, + duration_ms: u64, +} + +#[cfg(feature = "conformance")] +pub async fn conformance_run( + server: &str, + tls: &TlsOptions, + filter: Option<&str>, + timeout_secs: u64, + output: &str, +) -> Result<()> { + use std::time::Instant; + + let client = grpc_client(server, tls) + .await + .wrap_err("failed to connect to gateway")?; + + // Stable run-id for sandbox naming: seconds since Unix epoch, truncated + // to 8 hex digits. Keeps names Kubernetes RFC 1123 safe and short enough + // to read in `sandbox list` output. + let run_id = format!( + "{:08x}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() as u32 + ); + + let scenarios = all_scenarios() + .iter() + .filter(|s| filter.map_or(true, |f| s.name.contains(f))); + + let mut results: Vec = Vec::new(); + let mut any_failed = false; + + for scenario in scenarios { + let start = Instant::now(); + let timeout = Duration::from_secs(timeout_secs); + let mut client = client.clone(); + + let outcome = + tokio::time::timeout(timeout, run_scenario(scenario.name, &mut client, &run_id)) + .await + .unwrap_or_else(|_| { + Err(miette::miette!("scenario timed out after {timeout_secs}s")) + }); + + let passed = outcome.is_ok(); + if !passed { + any_failed = true; + } + + results.push(ScenarioResult { + name: scenario.name.to_string(), + passed, + message: match &outcome { + Ok(()) => "ok".to_string(), + Err(e) => format!("{e}"), + }, + duration_ms: start.elapsed().as_millis() as u64, + }); + } + + if crate::output::print_output_collection(output, &results, |r| { + serde_json::json!({ + "name": r.name, + "passed": r.passed, + "message": r.message, + "duration_ms": r.duration_ms, + }) + })? { + // structured output already printed + } else { + for r in &results { + let status = if r.passed { "PASS" } else { "FAIL" }; + println!( + " [{status}] {} ({}ms) — {}", + r.name, r.duration_ms, r.message + ); + } + } + + if any_failed { + return Err(miette::miette!("one or more conformance scenarios failed")); + } + Ok(()) +} + +/// Dispatch a scenario by name. +#[cfg(feature = "conformance")] +async fn run_scenario(name: &str, client: &mut crate::tls::GrpcClient, run_id: &str) -> Result<()> { + match name { + "lifecycle" => scenario_lifecycle(client, run_id).await, + _ => Err(miette::miette!("scenario '{name}' is not yet implemented")), + } +} + +/// Scenario: create → ready → delete. +/// +/// Creates a minimal sandbox, waits for it to reach the Ready phase, then +/// deletes it. Verifies that the sandbox appears in the list between create +/// and delete, and that delete reports it as deleted. +#[cfg(feature = "conformance")] +async fn scenario_lifecycle(client: &mut crate::tls::GrpcClient, run_id: &str) -> Result<()> { + let sandbox_name = format!("conformance-lifecycle-{run_id}"); + + // ── 1. Create ──────────────────────────────────────────────────────── + let response = client + .create_sandbox(CreateSandboxRequest { + name: sandbox_name.clone(), + spec: Some(SandboxSpec::default()), + labels: Default::default(), + }) + .await + .into_diagnostic() + .wrap_err("create_sandbox failed")?; + + let sandbox = response + .into_inner() + .sandbox + .ok_or_else(|| miette::miette!("create_sandbox response missing sandbox"))?; + let sandbox_id = sandbox.object_id().to_string(); + + // ── 2. Wait for Ready ──────────────────────────────────────────────── + let mut stream = client + .watch_sandbox(WatchSandboxRequest { + id: sandbox_id.clone(), + follow_status: true, + follow_logs: false, + follow_events: false, + log_tail_lines: 0, + event_tail: 0, + stop_on_terminal: false, + log_since_ms: 0, + log_sources: vec![], + log_min_level: String::new(), + }) + .await + .into_diagnostic() + .wrap_err("watch_sandbox failed")? + .into_inner(); + + let mut ready = false; + while let Some(item) = stream.next().await { + let evt = item + .into_diagnostic() + .wrap_err("watch_sandbox stream error")?; + if let Some(openshell_core::proto::sandbox_stream_event::Payload::Sandbox(s)) = evt.payload + { + match SandboxPhase::try_from(s.phase()).unwrap_or(SandboxPhase::Unknown) { + SandboxPhase::Ready => { + ready = true; + break; + } + SandboxPhase::Error => { + // Best-effort cleanup before returning the error. + let _ = client + .delete_sandbox(DeleteSandboxRequest { + name: sandbox_name.clone(), + }) + .await; + return Err(miette::miette!( + "sandbox entered Error phase before becoming Ready" + )); + } + _ => {} + } + } + } + + if !ready { + let _ = client + .delete_sandbox(DeleteSandboxRequest { + name: sandbox_name.clone(), + }) + .await; + return Err(miette::miette!( + "watch stream ended before sandbox reached Ready" + )); + } + + // ── 3. Verify it appears in the list ──────────────────────────────── + let list_response = client + .list_sandboxes(ListSandboxesRequest::default()) + .await + .into_diagnostic() + .wrap_err("list_sandboxes failed")?; + + let found = list_response + .into_inner() + .sandboxes + .iter() + .any(|s| s.object_name() == sandbox_name); + + if !found { + let _ = client + .delete_sandbox(DeleteSandboxRequest { + name: sandbox_name.clone(), + }) + .await; + return Err(miette::miette!( + "sandbox '{sandbox_name}' not found in list_sandboxes response after creation" + )); + } + + // ── 4. Delete ──────────────────────────────────────────────────────── + let del_response = client + .delete_sandbox(DeleteSandboxRequest { + name: sandbox_name.clone(), + }) + .await + .into_diagnostic() + .wrap_err("delete_sandbox failed")?; + + if !del_response.into_inner().deleted { + return Err(miette::miette!( + "delete_sandbox reported sandbox '{sandbox_name}' was not deleted" + )); + } + + Ok(()) +} + +#[cfg(feature = "conformance")] +pub fn conformance_list(output: &str) -> Result<()> { + let scenarios = all_scenarios(); + + if crate::output::print_output_collection(output, scenarios, |s| { + serde_json::json!({ + "name": s.name, + "description": s.description, + }) + })? { + return Ok(()); + } + + println!("{} conformance scenarios:", scenarios.len()); + for s in scenarios { + println!(" {:<20} {}", s.name, s.description); + } + Ok(()) +} + pub fn doctor_check() -> Result<()> { use std::io::Write; let mut stdout = std::io::stdout().lock();