diff --git a/Cargo.lock b/Cargo.lock index 7d475254..f3014d37 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2391,6 +2391,40 @@ dependencies = [ "tracing", ] +[[package]] +name = "dstack-auth" +version = "0.5.11" +dependencies = [ + "anyhow", + "clap", + "rocket", + "serde", + "serde_json", +] + +[[package]] +name = "dstack-cli" +version = "0.5.11" +dependencies = [ + "anyhow", + "clap", + "dstack-cli-core", + "serde_json", + "tokio", +] + +[[package]] +name = "dstack-cli-core" +version = "0.5.11" +dependencies = [ + "anyhow", + "dstack-vmm-rpc", + "http-client", + "rustix 0.38.44", + "serde_json", + "toml", +] + [[package]] name = "dstack-gateway" version = "0.5.11" @@ -2861,6 +2895,21 @@ dependencies = [ "serde_json", ] +[[package]] +name = "dstackup" +version = "0.5.11" +dependencies = [ + "anyhow", + "clap", + "dstack-cli-core", + "hex", + "reqwest", + "serde", + "serde_json", + "sha2 0.10.9", + "tokio", +] + [[package]] name = "dunce" version = "1.0.5" diff --git a/Cargo.toml b/Cargo.toml index 90bf4e9e..b873a680 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -63,6 +63,10 @@ members = [ "sdk/rust", "sdk/rust/types", "no_std_check", + "crates/dstack-cli-core", + "crates/dstack-cli", + "crates/dstackup", + "crates/dstack-auth", ] resolver = "2" @@ -75,6 +79,7 @@ dstack-gateway-rpc = { path = "gateway/rpc" } dstack-kms-rpc = { path = "kms/rpc" } dstack-guest-agent-rpc = { path = "guest-agent/rpc" } dstack-vmm-rpc = { path = "vmm/rpc" } +dstack-cli-core = { path = "crates/dstack-cli-core" } dstack-port-forward = { path = "port-forward" } cc-eventlog = { path = "cc-eventlog" } supervisor = { path = "supervisor" } diff --git a/README.md b/README.md index b0bb0dfe..09c2b806 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ AI providers ask users to trust them with sensitive data. But trust doesn't scal | Platform | Status | Attestation | |----------|--------|-------------| | **Bare metal TDX** | Available | TDX | +| **Bare metal AMD SEV-SNP** | Host support; requires an SNP-capable guest image | SEV-SNP | | **[Phala Cloud](https://cloud.phala.network)** | Available | TDX | | **GCP Confidential VMs** | Available | TDX + TPM | | **AWS Nitro Enclaves** | Available | NSM | @@ -66,15 +67,15 @@ services: - "8000:8000" ``` -Deploy to any Intel TDX host using a guest OS image from [meta-dstack releases](https://github.com/Dstack-TEE/meta-dstack/releases), or use [Phala Cloud](https://cloud.phala.network) for managed infrastructure. +Deploy to a self-hosted TDX machine with the `dstackup install` -> `dstack deploy` workflow, or use [Phala Cloud](https://cloud.phala.network) for managed infrastructure. AMD SEV-SNP hosts use the same workflow when the selected guest image includes `digest.sev.txt`. -Setting up dstack on your own hardware? See the [full deployment guide →](./docs/deployment.md) +Setting up dstack on your own hardware? Start with the [self-hosted quick onboarding guide](./docs/onboarding.md) ## Architecture ![Architecture](./docs/assets/arch.png) -Your container runs inside a Confidential VM (Intel TDX) with optional GPU isolation via NVIDIA Confidential Computing. The CPU TEE protects application logic; the GPU TEE protects model weights and inference data. +Your container runs inside a Confidential VM, such as Intel TDX or AMD SEV-SNP, with optional GPU isolation via NVIDIA Confidential Computing. The CPU TEE protects application logic; the GPU TEE protects model weights and inference data. **Core components:** @@ -107,6 +108,8 @@ Apps communicate with the guest agent via HTTP over `/var/run/dstack.sock`. Use - [Verification](./docs/verification.md) - How to verify TEE attestation **For Operators** +- [Hardware Enablement](./docs/hardware-enablement.md) - Prepare a TDX or AMD SEV-SNP host +- [Self-hosted Quick Onboarding](./docs/onboarding.md) - First app on one host - [Deployment](./docs/deployment.md) - Self-hosting on TDX hardware - [On-Chain Governance](./docs/onchain-governance.md) - Smart contract authorization - [Gateway](./docs/dstack-gateway.md) - Gateway configuration @@ -174,7 +177,7 @@ Yes. dstack runs on any Intel TDX-capable server. See the [deployment guide](./d
What TEE hardware is supported? -Currently: Intel TDX (4th/5th Gen Xeon) and NVIDIA Confidential Computing (H100, Blackwell). AMD SEV-SNP support is planned. +Currently: Intel TDX, AMD SEV-SNP, AWS Nitro Enclaves, GCP Confidential VMs, and NVIDIA Confidential Computing GPUs (H100, Blackwell).
diff --git a/crates/dstack-auth/Cargo.toml b/crates/dstack-auth/Cargo.toml new file mode 100644 index 00000000..d9fb1879 --- /dev/null +++ b/crates/dstack-auth/Cargo.toml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "dstack-auth" +version.workspace = true +edition.workspace = true +license.workspace = true + +[[bin]] +name = "dstack-auth" +path = "src/main.rs" + +[dependencies] +anyhow.workspace = true +clap.workspace = true +rocket = { workspace = true, features = ["json"] } +serde.workspace = true +serde_json.workspace = true diff --git a/crates/dstack-auth/src/main.rs b/crates/dstack-auth/src/main.rs new file mode 100644 index 00000000..217d9294 --- /dev/null +++ b/crates/dstack-auth/src/main.rs @@ -0,0 +1,320 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! `dstack-auth` — the single-operator KMS auth webhook (Rust reimplementation +//! of `auth-simple`). +//! +//! Runs on the host as `dstack-auth.service`; the KMS-in-CVM reaches it at +//! `http://10.0.2.2:` under user-mode networking and POSTs `BootInfo` to +//! `/bootAuth/app` (compose-hash allowlist) and `/bootAuth/kms` (mrAggregated +//! allowlist). The allowlist JSON is re-read on every request, so `dstack run` +//! can add an app without a restart. Fails closed: a missing/invalid allowlist +//! denies everything. +//! +//! Deliberate single-node deviation from `auth-simple`: it does NOT enforce +//! `tcbStatus == UpToDate`. Real TDX hosts routinely report a non-`UpToDate` +//! TCB (microcode / TDX-module behind), and in the single-node model the +//! operator already controls and trusts their own host, so a hard TCB gate +//! would be friction without a corresponding trust gain here. Re-add the check +//! (capture `tcbStatus`, deny unless `UpToDate`) if this grows into a +//! multi-tenant / hosted deployment. + +use anyhow::Result; +use clap::Parser; +use rocket::serde::json::Json; +use rocket::{get, post, routes, State}; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use std::collections::HashMap; +use std::path::PathBuf; + +#[derive(Parser, Clone)] +#[command( + name = "dstack-auth", + version, + about = "single-operator KMS auth webhook" +)] +struct Cli { + /// path to the allowlist JSON (re-read on every request). + #[arg(long, default_value = "/var/lib/dstack/auth-allowlist.json")] + config: PathBuf, + /// bind address. Defaults to loopback (reachable from CVMs at 10.0.2.2 via + /// user-mode networking, and not exposed externally). + #[arg(long, default_value = "127.0.0.1")] + address: String, + /// bind port. + #[arg(long, default_value_t = 8001)] + port: u16, +} + +/// boot info the KMS sends (camelCase; byte fields are hex strings). Only the +/// fields the allowlist checks are captured; the rest are ignored. +#[derive(Deserialize, Default)] +#[serde(rename_all = "camelCase", default)] +struct BootInfo { + mr_aggregated: String, + os_image_hash: String, + app_id: String, + compose_hash: String, + device_id: String, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct BootResponse { + is_allowed: bool, + gateway_app_id: String, + reason: String, +} + +#[derive(Deserialize, Default)] +#[serde(rename_all = "camelCase", default)] +struct Allowlist { + os_images: Vec, + gateway_app_id: String, + kms: KmsRules, + apps: HashMap, +} + +#[derive(Deserialize, Default)] +#[serde(rename_all = "camelCase", default)] +struct KmsRules { + mr_aggregated: Vec, + devices: Vec, + allow_any_device: bool, +} + +#[derive(Deserialize, Default)] +#[serde(rename_all = "camelCase", default)] +struct AppRules { + compose_hashes: Vec, + devices: Vec, + allow_any_device: bool, +} + +/// normalize a hex string for comparison: trim, drop a `0x`/`0X` prefix, +/// lowercase. MUST stay in sync with `dstack-cli-core::config::norm_hex` — both +/// `dstack run` (writing the allowlist) and this webhook (reading it) must +/// agree on the canonical form, or apps are silently denied. +fn norm(s: &str) -> String { + let s = s.trim(); + let s = s + .strip_prefix("0x") + .or_else(|| s.strip_prefix("0X")) + .unwrap_or(s); + s.to_lowercase() +} + +fn contains(list: &[String], value: &str) -> bool { + let v = norm(value); + list.iter().any(|x| norm(x) == v) +} + +/// matches auth-simple: an empty `devices` list means "any device" even when +/// `allowAnyDevice` is false (it only enforces a non-empty list). +fn device_ok(allow_any: bool, devices: &[String], device_id: &str) -> bool { + allow_any || devices.is_empty() || contains(devices, device_id) +} + +fn deny(al: &Allowlist, reason: &str) -> BootResponse { + BootResponse { + is_allowed: false, + gateway_app_id: al.gateway_app_id.clone(), + reason: reason.to_string(), + } +} + +fn allow(al: &Allowlist) -> BootResponse { + BootResponse { + is_allowed: true, + gateway_app_id: al.gateway_app_id.clone(), + reason: "ok".to_string(), + } +} + +fn check_app(info: &BootInfo, al: &Allowlist) -> BootResponse { + if !al.os_images.is_empty() && !contains(&al.os_images, &info.os_image_hash) { + return deny(al, "os image not allowed"); + } + let app_id = norm(&info.app_id); + let Some(app) = al + .apps + .iter() + .find(|(k, _)| norm(k) == app_id) + .map(|(_, v)| v) + else { + return deny(al, "app not registered"); + }; + if !contains(&app.compose_hashes, &info.compose_hash) { + return deny(al, "compose hash not allowed"); + } + if !device_ok(app.allow_any_device, &app.devices, &info.device_id) { + return deny(al, "device not allowed"); + } + allow(al) +} + +fn check_kms(info: &BootInfo, al: &Allowlist) -> BootResponse { + if !contains(&al.kms.mr_aggregated, &info.mr_aggregated) { + return deny(al, "kms mrAggregated not allowed"); + } + if !device_ok(al.kms.allow_any_device, &al.kms.devices, &info.device_id) { + return deny(al, "device not allowed"); + } + allow(al) +} + +/// load the allowlist, failing closed (deny-all) if it's missing or invalid. +fn load(path: &PathBuf) -> Allowlist { + match std::fs::read_to_string(path) { + Ok(body) => serde_json::from_str(&body).unwrap_or_else(|e| { + rocket::warn!("allowlist {} is invalid: {e}; denying all", path.display()); + Allowlist::default() + }), + Err(e) => { + rocket::warn!("allowlist {} unreadable: {e}; denying all", path.display()); + Allowlist::default() + } + } +} + +#[post("/bootAuth/app", data = "")] +fn boot_app(info: Json, cli: &State) -> Json { + let r = check_app(&info, &load(&cli.config)); + rocket::info!( + "bootAuth/app app={} compose={} -> allowed={} ({})", + norm(&info.app_id), + norm(&info.compose_hash), + r.is_allowed, + r.reason + ); + Json(r) +} + +#[post("/bootAuth/kms", data = "")] +fn boot_kms(info: Json, cli: &State) -> Json { + let r = check_kms(&info, &load(&cli.config)); + rocket::info!( + "bootAuth/kms mr={} -> allowed={} ({})", + norm(&info.mr_aggregated), + r.is_allowed, + r.reason + ); + Json(r) +} + +/// info endpoint the KMS GETs to populate its metadata. Single-node: no chain. +#[get("/")] +fn info() -> Json { + Json(json!({ + "status": "ok", + "kmsContractAddr": "", + "ethRpcUrl": "", + "gatewayAppId": "", + "chainId": 0, + "appImplementation": "" + })) +} + +#[rocket::main] +async fn main() -> Result<()> { + let cli = Cli::parse(); + let figment = rocket::Config::figment() + .merge(("address", cli.address.clone())) + .merge(("port", cli.port)); + rocket::custom(figment) + .manage(cli) + .mount("/", routes![info, boot_app, boot_kms]) + .launch() + .await + .map_err(|e| anyhow::anyhow!("auth webhook failed: {e}"))?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn allowlist() -> Allowlist { + serde_json::from_str( + r#"{ + "osImages": ["0xIMG"], + "kms": { "mrAggregated": ["0xMR"], "allowAnyDevice": true }, + "apps": { "0xApp1": { "composeHashes": ["0xHASH"], "allowAnyDevice": true } } + }"#, + ) + .unwrap() + } + + fn boot(app: &str, hash: &str, img: &str) -> BootInfo { + BootInfo { + app_id: app.into(), + compose_hash: hash.into(), + os_image_hash: img.into(), + ..Default::default() + } + } + + #[test] + fn app_allowed_with_normalized_hex() { + // differing 0x/case must still match. + let r = check_app(&boot("APP1", "hash", "img"), &allowlist()); + assert!(r.is_allowed, "{}", r.reason); + } + + #[test] + fn app_denied_unknown_app_hash_or_image() { + let al = allowlist(); + assert!(!check_app(&boot("0xnope", "0xHASH", "0xIMG"), &al).is_allowed); + assert!(!check_app(&boot("0xApp1", "0xnope", "0xIMG"), &al).is_allowed); + assert!(!check_app(&boot("0xApp1", "0xHASH", "0xnope"), &al).is_allowed); + } + + #[test] + fn kms_allowlist_and_empty_default() { + let al = allowlist(); + let info = BootInfo { + mr_aggregated: "0xMR".into(), + ..Default::default() + }; + assert!(check_kms(&info, &al).is_allowed); + // fail closed: empty allowlist denies (the single-node case never calls this). + assert!(!check_kms(&info, &Allowlist::default()).is_allowed); + } + + // wire-contract snapshot: BootInfo as the KMS serializes it (camelCase). + // Keep these field names in sync with the kms BootInfo. `#[serde(default)]` + // means extra fields are ignored AND a renamed field deserializes to "" — + // which fails closed, but silently — so this test pins the names we depend + // on: if the KMS renames one, the matching assertion here breaks first. + #[test] + fn deserializes_the_kms_bootinfo_wire_contract() { + let wire = r#"{ + "attestationMode": "dstack", + "mrAggregated": "0xAABB", + "osImageHash": "0xC2AA", + "mrSystem": "0xdead", + "appId": "0xApp1", + "composeHash": "0xHASH", + "instanceId": "0x01", + "deviceId": "0xDEV", + "keyProviderInfo": "kp", + "tcbStatus": "UpToDate", + "advisoryIds": [] + }"#; + let info: BootInfo = serde_json::from_str(wire).expect("kms BootInfo must deserialize"); + assert_eq!(norm(&info.mr_aggregated), "aabb"); + assert_eq!(norm(&info.os_image_hash), "c2aa"); + assert_eq!(norm(&info.app_id), "app1"); + assert_eq!(norm(&info.compose_hash), "hash"); + assert_eq!(norm(&info.device_id), "dev"); + // a check using this payload should pass against a matching allowlist. + let info2: BootInfo = serde_json::from_str(wire).unwrap(); + let al: Allowlist = serde_json::from_str( + r#"{"osImages":["0xC2AA"],"apps":{"0xApp1":{"composeHashes":["0xHASH"],"allowAnyDevice":true}}}"#, + ) + .unwrap(); + assert!(check_app(&info2, &al).is_allowed); + } +} diff --git a/crates/dstack-cli-core/Cargo.toml b/crates/dstack-cli-core/Cargo.toml new file mode 100644 index 00000000..ea74358b --- /dev/null +++ b/crates/dstack-cli-core/Cargo.toml @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "dstack-cli-core" +version.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +anyhow.workspace = true +http-client = { workspace = true, features = ["prpc"] } +dstack-vmm-rpc.workspace = true +serde_json.workspace = true +# advisory file locking (flock) for the allowlist/state read-modify-write; +# already in the dependency tree transitively, so no extra compile cost. +rustix = { version = "0.38", features = ["fs"] } + +[dev-dependencies] +toml.workspace = true diff --git a/crates/dstack-cli-core/src/compose.rs b/crates/dstack-cli-core/src/compose.rs new file mode 100644 index 00000000..5c395213 --- /dev/null +++ b/crates/dstack-cli-core/src/compose.rs @@ -0,0 +1,37 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! build the app-compose manifest — the JSON document the VMM hashes (to derive +//! the app id) and deploys. The raw docker-compose YAML is embedded as a string. + +use serde_json::json; + +/// build a minimal app-compose manifest from a docker-compose YAML body +/// (single-node, no gateway). +/// +/// `kms_enabled` selects KMS mode (deterministic, upgradeable per-app keys); +/// gateway and local-key-provider are off for the direct-port single-node flow. +pub fn build_app_compose(name: &str, docker_compose_yaml: &str, kms_enabled: bool) -> String { + let manifest = json!({ + "manifest_version": 2, + "name": name, + "runner": "docker-compose", + "docker_compose_file": docker_compose_yaml, + "kms_enabled": kms_enabled, + "gateway_enabled": false, + "local_key_provider_enabled": false, + "public_logs": true, + "public_sysinfo": true, + "no_instance_id": false, + // don't block boot on `chronyc waitsync` — the manifest default is true, + // but the single-node direct-port flow has no gateway/RA-TLS that needs a + // pre-synced clock, and the strict wait hard-fails (→ reboot loop) whenever + // chrony has no usable source. chronyd still syncs in the background. + // (NTS is also currently broken in guest images — see dstack#745.) + "secure_time": false, + }); + // pretty-print via Value's Display (`{:#}`) — infallible, and byte-identical + // to serde_json::to_string_pretty (avoids an expect on an unfailable Result). + format!("{manifest:#}") +} diff --git a/crates/dstack-cli-core/src/config.rs b/crates/dstack-cli-core/src/config.rs new file mode 100644 index 00000000..a0ca8241 --- /dev/null +++ b/crates/dstack-cli-core/src/config.rs @@ -0,0 +1,509 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! render the config files `dstackup install` writes: +//! +//! * `kms.toml` — embedded into the KMS-in-CVM app-compose; this is the +//! single-node config (webhook auth + `enforce_self_authorization = +//! false` + a set `auto_bootstrap_domain`, the combination validated to make +//! bootstrap hands-off). +//! * `auth-allowlist.json` — read by the host-side Rust auth webhook. +//! * `vmm.toml` — the host VMM config (gateway + auth-token gating off). + +use crate::host::Platform; +use anyhow::{Context, Result}; +use serde_json::json; +use std::path::Path; + +/// normalize a hex string for comparison: trim, drop a single `0x`/`0X` +/// prefix, lowercase. MUST stay in sync with `dstack-auth`'s `norm()` — the +/// webhook compares allowlist entries against KMS-supplied hashes with the same +/// rule, so a divergence here silently denies (or wrongly allows) apps. +pub fn norm_hex(s: &str) -> String { + let s = s.trim(); + let s = s + .strip_prefix("0x") + .or_else(|| s.strip_prefix("0X")) + .unwrap_or(s); + s.to_lowercase() +} + +/// register an app (id + compose hash) in the auth webhook's allowlist file, +/// so the KMS will issue keys to it. Read-modify-write; idempotent. +/// +/// Holds an exclusive lock for the whole read-modify-write (so two concurrent +/// `dstack run`s can't clobber each other) and writes atomically (so a crash or +/// partial write can't leave torn JSON — which the webhook would read as +/// deny-all). The stored hash is normalized so the on-disk file can't +/// accumulate visually-distinct-but-equal entries. +pub fn register_app_in_allowlist(path: &Path, app_id: &str, compose_hash: &str) -> Result<()> { + let _lock = crate::fsutil::lock_exclusive(path)?; + let body = match std::fs::read_to_string(path) { + Ok(b) => b, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => anyhow::bail!( + "allowlist {} does not exist — run `dstackup install` first, or check the --allowlist path", + path.display() + ), + Err(e) if e.kind() == std::io::ErrorKind::PermissionDenied => { + return Err(e).with_context(|| { + format!( + "reading allowlist {} (it is usually root-owned — run with sudo)", + path.display() + ) + }) + } + Err(e) => return Err(e).with_context(|| format!("reading allowlist {}", path.display())), + }; + let mut v: serde_json::Value = serde_json::from_str(&body).context("parsing allowlist json")?; + let apps = v + .get_mut("apps") + .and_then(|a| a.as_object_mut()) + .context("allowlist has no `apps` object")?; + let entry = apps + .entry(norm_hex(app_id)) + .or_insert_with(|| json!({ "composeHashes": [], "devices": [], "allowAnyDevice": true })); + let hashes = entry + .get_mut("composeHashes") + .and_then(|h| h.as_array_mut()) + .context("app entry missing `composeHashes`")?; + let norm = norm_hex(compose_hash); + let present = hashes + .iter() + .any(|h| h.as_str().map(|s| norm_hex(s) == norm).unwrap_or(false)); + if !present { + hashes.push(serde_json::Value::String(norm)); + } + crate::fsutil::write_atomic(path, &serde_json::to_string_pretty(&v)?) + .with_context(|| format!("writing allowlist {}", path.display()))?; + Ok(()) +} + +/// public OS-image download URL template used by the KMS image-hash verifier. +pub const DEFAULT_IMAGE_DOWNLOAD_URL: &str = + "https://download.dstack.org/os-images/mr_{OS_IMAGE_HASH}.tar.gz"; + +/// inputs that parameterize the rendered configs. +#[derive(Debug, Clone)] +pub struct HostConfig { + /// URL the KMS-in-CVM uses to reach the host auth webhook + /// (the host as seen from the CVM under user-mode networking, e.g. + /// `http://10.0.2.2:8001`). + pub auth_webhook_url: String, + /// KMS bootstrap domain — the host address as seen from the CVM + /// (e.g. `10.0.2.2`); the bootstrapped RPC cert is issued for this. + pub kms_bootstrap_domain: String, + /// OS image hash to allow apps to boot from (the measured guest image). + pub os_image_hash: String, + /// OS image download URL template (must contain `{OS_IMAGE_HASH}`). + pub image_download_url: String, + /// whether the KMS verifies the OS image hash on app key requests. + pub verify_os_image: bool, + /// confidential-computing platform (selects SNP-specific KMS settings). + pub platform: Platform, +} + +impl Default for HostConfig { + fn default() -> Self { + Self { + auth_webhook_url: "http://10.0.2.2:8001".to_string(), + kms_bootstrap_domain: "10.0.2.2".to_string(), + os_image_hash: String::new(), + image_download_url: DEFAULT_IMAGE_DOWNLOAD_URL.to_string(), + verify_os_image: true, + platform: Platform::Tdx, + } + } +} + +/// render the single-node KMS config (lives at `/kms/kms.toml` inside the CVM). +pub fn kms_toml(cfg: &HostConfig) -> String { + format!( + r#"# generated by `dstackup install` — single-node KMS + +[rpc] +address = "0.0.0.0" +port = 8000 + +[rpc.tls] +key = "/kms/certs/rpc.key" +certs = "/kms/certs/rpc.crt" + +[rpc.tls.mutual] +ca_certs = "/kms/certs/tmp-ca.crt" +mandatory = false + +[core] +cert_dir = "/kms/certs" +admin_token_hash = "" +# single-node: the KMS does not self-attest to its own auth API before +# bootstrap (it still attests the genesis keys via the guest agent, and app +# auth + per-app quote checks are unaffected). +enforce_self_authorization = false +{sev_snp} +[core.image] +verify = {verify} +cache_dir = "/kms/images" +download_url = "{download_url}" +download_timeout = "2m" + +[core.metrics] +enabled = false + +[core.auth_api] +type = "webhook" + +[core.auth_api.webhook] +url = "{webhook_url}" + +[core.onboard] +enabled = true +auto_bootstrap_domain = "{bootstrap_domain}" +address = "0.0.0.0" +port = 8000 +"#, + // AMD SEV-SNP gates EVERY key release (incl. the KMS's own bootstrap) on + // `sev_snp_key_release`, which defaults to false — so it must be set on + // SNP or the KMS refuses to release keys. Harmless/ignored on TDX. + sev_snp = match cfg.platform { + Platform::AmdSevSnp => "sev_snp_key_release = true\namd_kds_base_url = \"\"\n", + Platform::Tdx => "", + }, + verify = cfg.verify_os_image, + download_url = cfg.image_download_url, + webhook_url = cfg.auth_webhook_url, + bootstrap_domain = cfg.kms_bootstrap_domain, + ) +} + +/// render the host-side auth webhook allowlist. +/// +/// single-node (no gateway): the OS image is allowed, the KMS `mrAggregated` +/// allowlist is empty (no replication; self-bootstrap is hands-off), and per-app +/// compose hashes are added by `dstack run`. +pub fn auth_allowlist_json(cfg: &HostConfig) -> String { + let allowlist = json!({ + "osImages": if cfg.os_image_hash.is_empty() { + Vec::::new() + } else { + vec![cfg.os_image_hash.clone()] + }, + "kms": { + "mrAggregated": [], + "devices": [], + "allowAnyDevice": true + }, + "apps": {} + }); + // infallible pretty-print via Value's Display; see compose::build_app_compose. + format!("{allowlist:#}") +} + +/// default pinned, reproducibly-built KMS image (Docker Hub). +pub const DEFAULT_KMS_IMAGE: &str = "dstacktee/dstack-kms:0.5.11"; + +/// build the KMS-in-CVM app-compose manifest. An init script writes the +/// rendered `kms.toml` into the guest and the KMS container mounts it. On TDX +/// the CVM uses the SGX local key provider to seal the KMS root key; AMD +/// SEV-SNP has no such provider, so it's disabled there. +pub fn kms_app_compose(kms_toml: &str, kms_image: &str, platform: Platform) -> String { + let docker_compose = format!( + r#"services: + kms: + image: {kms_image} + volumes: + - kms-volume:/kms + - /var/run/dstack.sock:/var/run/dstack.sock + - /dstack/kms-config/kms.toml:/kms/kms.toml:ro + ports: + - "8000:8000" + restart: unless-stopped + command: sh -c 'mkdir -p /kms/certs /kms/images && exec dstack-kms -c /kms/kms.toml' +volumes: + kms-volume: +"# + ); + let init_script = format!( + "mkdir -p /dstack/kms-config\ncat > /dstack/kms-config/kms.toml <<'KMSTOML'\n{kms_toml}\nKMSTOML\ntrue\n" + ); + let manifest = json!({ + "manifest_version": 2, + "name": "dstack-kms", + "runner": "docker-compose", + "docker_compose_file": docker_compose, + "init_script": init_script, + "kms_enabled": false, + "gateway_enabled": false, + "local_key_provider_enabled": platform == Platform::Tdx, + "public_logs": true, + "public_sysinfo": true, + "public_tcbinfo": true, + "no_instance_id": false, + "secure_time": false, + "allowed_envs": [] + }); + // infallible pretty-print via Value's Display; see compose::build_app_compose. + format!("{manifest:#}") +} + +/// inputs for rendering `vmm.toml`. Defaults target a localhost dashboard and +/// reuse of an existing local key provider; the isolation knobs (ports, cid +/// range, prefix) let a fresh instance coexist with an existing VMM. +#[derive(Debug, Clone)] +pub struct VmmRender { + /// Rocket endpoint for the dashboard + management API + /// (e.g. `tcp:127.0.0.1:9080`, or `unix:`). + pub dashboard_addr: String, + /// guest image directory. + pub image_path: String, + /// qemu binary path. + pub qemu_path: String, + /// run directory for the supervisor socket/pid/log. + pub run_dir: String, + /// VM storage directory (isolated per install; default `~/.dstack-vmm/vm`). + pub vm_path: String, + /// supervisor binary path. + pub supervisor_exe: String, + /// CID pool start (raise to coexist with an existing VMM). + pub cid_start: u32, + /// CID pool size. + pub cid_pool_size: u32, + /// host-api vsock port (raise to coexist with an existing VMM on 10000). + pub host_api_port: u32, + /// local key-provider address (reuse the running one). + pub key_provider_addr: String, + /// local key-provider port. + pub key_provider_port: u32, + /// KMS URLs injected into app CVMs (the guest-visible KMS address). + pub kms_urls: Vec, + /// confidential-computing platform (selects qemu/share-mode for the CVMs). + pub platform: Platform, +} + +impl Default for VmmRender { + fn default() -> Self { + Self { + dashboard_addr: "tcp:127.0.0.1:9080".to_string(), + image_path: "/var/lib/dstack/images".to_string(), + qemu_path: "/usr/bin/qemu-system-x86_64".to_string(), + run_dir: "/var/lib/dstack/run".to_string(), + vm_path: "/var/lib/dstack/vm".to_string(), + supervisor_exe: "/usr/bin/dstack-supervisor".to_string(), + cid_start: 1000, + cid_pool_size: 1000, + host_api_port: 10000, + key_provider_addr: "127.0.0.1".to_string(), + key_provider_port: 3443, + kms_urls: Vec::new(), + platform: Platform::Tdx, + } + } +} + +/// render the host `vmm.toml`. Gateway and auth-token gating are off +/// (single-node direct-port access); CVMs use user-mode networking with host +/// port mapping. +pub fn vmm_toml(r: &VmmRender) -> String { + format!( + r#"# generated by `dstackup install` + +workers = 8 +max_blocking = 64 +ident = "dstack VMM" +temp_dir = "/tmp" +keep_alive = 10 +log_level = "info" +address = "{dashboard_addr}" +reuse = true +kms_url = "" +event_buffer_size = 20 +node_name = "" +run_path = "{vm_path}" + +[image] +path = "{image_path}" +registry = "" + +[cvm] +platform = "{platform}" +qemu_path = "{qemu_path}" +kms_urls = [{kms_urls}] +gateway_urls = [] +pccs_url = "" +docker_registry = "" +cid_start = {cid_start} +cid_pool_size = {cid_pool_size} +max_allocable_vcpu = 20 +max_allocable_memory_in_mb = 100_000 +qmp_socket = false +user = "" +use_mrconfigid = {use_mrconfigid} +qemu_pci_hole64_size = 0 +qemu_hotplug_off = false +host_share_mode = "{host_share_mode}" +qgs_port = 4050 + +[cvm.product] +sys_vendor = "dstack" +product_name = "dstack" + +[cvm.networking] +mode = "user" +net = "10.0.2.0/24" +dhcp_start = "10.0.2.10" +restrict = false +forward_service_enabled = false + +[cvm.port_mapping] +enabled = true +address = "127.0.0.1" +range = [ + {{ protocol = "tcp", from = 1, to = 20000 }}, +] + +[cvm.auto_restart] +enabled = true +interval = 20 + +[cvm.gpu] +enabled = false +listing = [] +exclude = [] +include = [] +allow_attach_all = false + +[gateway] +base_domain = "localhost" +port = 8082 +agent_port = 8090 + +[auth] +enabled = false +tokens = [] + +[supervisor] +exe = "{supervisor_exe}" +sock = "{run_dir}/supervisor.sock" +pid_file = "{run_dir}/supervisor.pid" +log_file = "{run_dir}/supervisor.log" +detached = true +auto_start = true + +[host_api] +ident = "dstack VMM" +address = "vsock:2" +port = {host_api_port} + +[key_provider] +enabled = true +address = "{kp_addr}" +port = {kp_port} +"#, + dashboard_addr = r.dashboard_addr, + image_path = r.image_path, + vm_path = r.vm_path, + qemu_path = r.qemu_path, + platform = r.platform.vmm_str(), + // SNP CVMs share the host dir via a virtual disk (9p doesn't play with + // SNP memory encryption) and bind measurements via mrconfigid. + use_mrconfigid = r.platform == Platform::AmdSevSnp, + host_share_mode = match r.platform { + Platform::AmdSevSnp => "vhd", + Platform::Tdx => "9p", + }, + kms_urls = r + .kms_urls + .iter() + .map(|u| format!("\"{u}\"")) + .collect::>() + .join(", "), + cid_start = r.cid_start, + cid_pool_size = r.cid_pool_size, + supervisor_exe = r.supervisor_exe, + run_dir = r.run_dir, + host_api_port = r.host_api_port, + kp_addr = r.key_provider_addr, + kp_port = r.key_provider_port, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn vmm_toml_is_valid_and_parameterized() { + let r = VmmRender { + dashboard_addr: "tcp:127.0.0.1:19080".into(), + cid_start: 2000, + host_api_port: 10001, + ..Default::default() + }; + let rendered = vmm_toml(&r); + assert!(rendered.contains(r#"address = "tcp:127.0.0.1:19080""#)); + assert!(rendered.contains("cid_start = 2000")); + assert!(rendered.contains("port = 10001")); + toml::from_str::(&rendered).expect("vmm.toml must be valid TOML"); + } + + #[test] + fn kms_toml_has_single_node_invariants() { + let cfg = HostConfig { + auth_webhook_url: "http://10.0.2.2:8001".into(), + kms_bootstrap_domain: "10.0.2.2".into(), + ..Default::default() + }; + let toml = kms_toml(&cfg); + assert!(toml.contains("enforce_self_authorization = false")); + assert!(toml.contains(r#"auto_bootstrap_domain = "10.0.2.2""#)); + assert!(toml.contains(r#"type = "webhook""#)); + assert!(toml.contains(r#"url = "http://10.0.2.2:8001""#)); + // sanity: it parses as TOML. + toml::from_str::(&toml).expect("kms.toml must be valid TOML"); + } + + #[test] + fn platform_specific_rendering() { + // TDX defaults: no SNP key-release, 9p share, mrconfigid off, SGX provider. + let tdx = kms_toml(&HostConfig::default()); + assert!(!tdx.contains("sev_snp_key_release")); + toml::from_str::(&tdx).expect("tdx kms.toml valid"); + let tdx_vmm = vmm_toml(&VmmRender::default()); + assert!(tdx_vmm.contains(r#"platform = "tdx""#)); + assert!(tdx_vmm.contains(r#"host_share_mode = "9p""#)); + assert!(tdx_vmm.contains("use_mrconfigid = false")); + toml::from_str::(&tdx_vmm).expect("tdx vmm.toml valid"); + assert!(kms_app_compose("x", "img", Platform::Tdx) + .contains(r#""local_key_provider_enabled": true"#)); + + // SNP: key-release gate set, vhd share, mrconfigid on, no local provider. + let snp = kms_toml(&HostConfig { + platform: Platform::AmdSevSnp, + ..Default::default() + }); + assert!(snp.contains("sev_snp_key_release = true")); + toml::from_str::(&snp).expect("snp kms.toml valid"); + let snp_vmm = vmm_toml(&VmmRender { + platform: Platform::AmdSevSnp, + ..Default::default() + }); + assert!(snp_vmm.contains(r#"platform = "amd-sev-snp""#)); + assert!(snp_vmm.contains(r#"host_share_mode = "vhd""#)); + assert!(snp_vmm.contains("use_mrconfigid = true")); + toml::from_str::(&snp_vmm).expect("snp vmm.toml valid"); + assert!(kms_app_compose("x", "img", Platform::AmdSevSnp) + .contains(r#""local_key_provider_enabled": false"#)); + } + + #[test] + fn allowlist_shape() { + let cfg = HostConfig { + os_image_hash: "0xabc".into(), + ..Default::default() + }; + let v: serde_json::Value = serde_json::from_str(&auth_allowlist_json(&cfg)).unwrap(); + assert_eq!(v["osImages"][0], "0xabc"); + assert_eq!(v["kms"]["mrAggregated"].as_array().unwrap().len(), 0); + assert!(v["apps"].as_object().unwrap().is_empty()); + } +} diff --git a/crates/dstack-cli-core/src/fsutil.rs b/crates/dstack-cli-core/src/fsutil.rs new file mode 100644 index 00000000..e03567e0 --- /dev/null +++ b/crates/dstack-cli-core/src/fsutil.rs @@ -0,0 +1,102 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! small filesystem helpers: atomic file replace + advisory locking. +//! +//! The allowlist and the install state file are read-modify-written from more +//! than one process (`dstack run` adds an app while the webhook reads; a second +//! `dstack run` can race the first). A torn write there is not cosmetic: the +//! auth webhook fails *closed* on invalid JSON, so a half-written allowlist +//! denies keys to every app on the host. These helpers make the write atomic +//! and serialize concurrent writers. + +use anyhow::{Context, Result}; +use std::ffi::OsString; +use std::fs::{File, OpenOptions}; +use std::io::Write; +use std::path::{Path, PathBuf}; + +/// `path` with `suffix` appended to its full name (not replacing the extension, +/// so `a/b.json` + `.tmp` → `a/b.json.tmp`, a sibling in the same directory). +fn sibling(path: &Path, suffix: &str) -> PathBuf { + let mut s: OsString = path.as_os_str().to_os_string(); + s.push(suffix); + PathBuf::from(s) +} + +/// atomically replace `path`'s contents: write a sibling temp file, fsync it, +/// rename it over the target, then fsync the directory. A reader (or a crash) +/// sees either the old file or the new one, never a fragment, and the rename is +/// durable across a power loss. `tmp` and `path` are in the same directory so +/// the rename is atomic. +pub fn write_atomic(path: &Path, contents: &str) -> Result<()> { + let tmp = sibling(path, ".tmp"); + let mut f = + File::create(&tmp).with_context(|| format!("creating temp file {}", tmp.display()))?; + f.write_all(contents.as_bytes()) + .with_context(|| format!("writing {}", tmp.display()))?; + f.sync_all() + .with_context(|| format!("syncing {}", tmp.display()))?; + drop(f); + std::fs::rename(&tmp, path) + .with_context(|| format!("renaming {} -> {}", tmp.display(), path.display()))?; + // fsync the containing directory so the rename itself survives a crash. + if let Some(dir) = path.parent().filter(|d| !d.as_os_str().is_empty()) { + if let Ok(d) = File::open(dir) { + let _ = d.sync_all(); + } + } + Ok(()) +} + +/// acquire an exclusive advisory lock tied to `path` (held on a sibling +/// `.lock` file). The lock releases when the returned guard is dropped — +/// including on process exit, so a crash never leaves a stale lock. Hold it +/// around a read-modify-write of `path` to serialize concurrent processes. +#[must_use = "the lock is released when the returned guard is dropped"] +pub fn lock_exclusive(path: &Path) -> Result { + let lock_path = sibling(path, ".lock"); + let f = OpenOptions::new() + .create(true) + .truncate(false) + .write(true) + .open(&lock_path) + .with_context(|| format!("opening lock {}", lock_path.display()))?; + rustix::fs::flock(&f, rustix::fs::FlockOperation::LockExclusive) + .with_context(|| format!("locking {}", lock_path.display()))?; + Ok(f) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn atomic_write_replaces_contents() { + let dir = std::env::temp_dir().join(format!("dstack-fsutil-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let p = dir.join("x.json"); + write_atomic(&p, "one").unwrap(); + assert_eq!(std::fs::read_to_string(&p).unwrap(), "one"); + write_atomic(&p, "two").unwrap(); + assert_eq!(std::fs::read_to_string(&p).unwrap(), "two"); + // no temp file left behind. + assert!(!sibling(&p, ".tmp").exists()); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn lock_is_reentrant_within_process_after_drop() { + let dir = std::env::temp_dir().join(format!("dstack-fslock-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let p = dir.join("y.json"); + std::fs::write(&p, "{}").unwrap(); + { + let _g = lock_exclusive(&p).unwrap(); + } + // re-acquire after the first guard dropped. + let _g2 = lock_exclusive(&p).unwrap(); + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/crates/dstack-cli-core/src/host.rs b/crates/dstack-cli-core/src/host.rs new file mode 100644 index 00000000..3a8688fe --- /dev/null +++ b/crates/dstack-cli-core/src/host.rs @@ -0,0 +1,277 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! host environment checks used by `dstackup` — SGX presence and the primary IP. + +use anyhow::{bail, Result}; +use std::net::{IpAddr, UdpSocket}; +use std::path::Path; + +/// presence of the SGX device nodes the local key provider needs. +#[derive(Debug, Clone, Copy)] +pub struct Sgx { + pub enclave: bool, + pub provision: bool, +} + +impl Sgx { + pub fn ok(&self) -> bool { + self.enclave && self.provision + } +} + +/// check for `/dev/sgx_enclave` and `/dev/sgx_provision`. +pub fn check_sgx() -> Sgx { + Sgx { + enclave: Path::new("/dev/sgx_enclave").exists(), + provision: Path::new("/dev/sgx_provision").exists(), + } +} + +/// check for the AMD secure processor device the host VMM needs for SEV-SNP. +pub fn check_sev() -> bool { + Path::new("/dev/sev").exists() +} + +/// require SGX, with a clear message if it is missing (design decision: fail fast +/// rather than silently degrade to a host-mode KMS with no real attestation). +pub fn require_sgx() -> Result<()> { + let sgx = check_sgx(); + if !sgx.ok() { + let mut missing = Vec::new(); + if !sgx.enclave { + missing.push("/dev/sgx_enclave"); + } + if !sgx.provision { + missing.push("/dev/sgx_provision"); + } + bail!( + "sgx not available (missing {}); dstack requires Intel SGX for the local key provider — enable SGX in BIOS, or run on a TDX+SGX host", + missing.join(", ") + ); + } + Ok(()) +} + +/// the confidential-computing platform a host launches CVMs on. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Platform { + /// Intel TDX (with an SGX-backed local key provider). + #[default] + Tdx, + /// AMD SEV-SNP. + AmdSevSnp, +} + +impl Platform { + /// the `[cvm] platform` value the VMM expects in `vmm.toml`. + pub fn vmm_str(self) -> &'static str { + match self { + Platform::Tdx => "tdx", + Platform::AmdSevSnp => "amd-sev-snp", + } + } + + /// parse a `--platform` value: `tdx` | `amd-sev-snp` | `auto` (None). + pub fn parse_opt(s: &str) -> Result> { + match s { + "auto" => Ok(None), + "tdx" => Ok(Some(Platform::Tdx)), + "amd-sev-snp" | "sev-snp" | "snp" => Ok(Some(Platform::AmdSevSnp)), + other => bail!("unknown --platform '{other}' (expected: auto | tdx | amd-sev-snp)"), + } + } + + /// auto-detect from `/proc/cpuinfo` (AMD SNP advertises the `sev_snp` flag; + /// Intel TDX hosts advertise `tdx_host_platform`). None if neither is found. + pub fn detect() -> Option { + let info = std::fs::read_to_string("/proc/cpuinfo").ok()?; + let has = |flag: &str| { + info.lines() + .any(|l| l.starts_with("flags") && l.split_whitespace().any(|f| f == flag)) + }; + if has("sev_snp") { + Some(Platform::AmdSevSnp) + } else if has("tdx_host_platform") { + Some(Platform::Tdx) + } else { + None + } + } +} + +/// require the host to actually support `platform`, with a clear message. +/// TDX needs the SGX device nodes (for the local key provider); AMD SEV-SNP +/// needs `/dev/sev` (the AMD secure processor). +pub fn require_platform(platform: Platform) -> Result<()> { + match platform { + Platform::Tdx => require_sgx(), + Platform::AmdSevSnp => { + if check_sev() { + Ok(()) + } else { + bail!( + "amd sev-snp not available (missing /dev/sev); this host can't launch SNP CVMs — enable SEV-SNP in BIOS and load kvm_amd, or pass --platform tdx" + ) + } + } + } +} + +/// best-effort primary routable IPv4 of this host. +/// +/// uses the standard UDP-connect trick: connecting a datagram socket sends no +/// packets but makes the kernel pick the source address it would route from. +pub fn detect_host_ip() -> Result { + let socket = UdpSocket::bind("0.0.0.0:0")?; + socket.connect("8.8.8.8:80")?; + Ok(socket.local_addr()?.ip()) +} + +/// whether `ip` is a link-local address (169.254/16) — usable, but a poor +/// default for a dashboard SAN or KMS bootstrap domain. +pub fn is_link_local(ip: &IpAddr) -> bool { + matches!(ip, IpAddr::V4(v4) if v4.is_link_local()) +} + +/// CID windows already spoken for on this host. vsock CIDs are a global +/// resource, so a second VMM must avoid these. Two sources, unioned: +/// +/// * the `[cid_start, cid_start+cid_pool_size)` pool of every other running +/// `dstack-vmm` (read from the `-c ` it was launched with) — this +/// catches the reserved pool even when that VMM has no live CVM right now, +/// and +/// * any live `guest-cid=` from a running QEMU, as a 1-wide range (covers a +/// VMM whose config we couldn't read). +/// +/// Best-effort: unreadable cmdlines/configs are skipped. Ranges are half-open +/// `[start, end)`. +pub fn occupied_cid_ranges() -> Vec<(u32, u32)> { + let mut ranges = Vec::new(); + let Ok(entries) = std::fs::read_dir("/proc") else { + return ranges; + }; + for entry in entries.flatten() { + let Ok(data) = std::fs::read(entry.path().join("cmdline")) else { + continue; + }; + // cmdline is NUL-separated argv. + let args: Vec = data + .split(|&b| b == 0) + .filter(|s| !s.is_empty()) + .map(|s| String::from_utf8_lossy(s).into_owned()) + .collect(); + if args.is_empty() { + continue; + } + // (a) another dstack-vmm's reserved pool, from its config. + let is_vmm = Path::new(&args[0]).file_name().and_then(|f| f.to_str()) == Some("dstack-vmm"); + if is_vmm { + if let Some(cfg) = arg_value(&args, "-c").or_else(|| arg_value(&args, "--config")) { + if let Some((start, size)) = read_cid_pool(&cfg) { + ranges.push((start, start.saturating_add(size))); + } + } + } + // (b) any live guest-cid token. + for arg in &args { + for tok in arg.split([',', ' ']) { + if let Some(rest) = tok.strip_prefix("guest-cid=") { + if let Ok(n) = rest.trim().parse::() { + ranges.push((n, n.saturating_add(1))); + } + } + } + } + } + ranges +} + +/// value following `flag` in an argv (`-c foo` → `foo`). +fn arg_value(args: &[String], flag: &str) -> Option { + args.windows(2).find(|w| w[0] == flag).map(|w| w[1].clone()) +} + +/// read `[cvm]` `cid_start` / `cid_pool_size` from a vmm.toml by line scan +/// (avoids a toml dependency; tolerates partial configs — size defaults 1000). +fn read_cid_pool(config_path: &str) -> Option<(u32, u32)> { + let text = std::fs::read_to_string(config_path).ok()?; + let mut start = None; + let mut size = None; + for line in text.lines() { + let l = line.trim(); + if let Some(v) = l.strip_prefix("cid_start") { + start = parse_toml_u32(v); + } else if let Some(v) = l.strip_prefix("cid_pool_size") { + size = parse_toml_u32(v); + } + } + Some((start?, size.unwrap_or(1000))) +} + +/// parse the `= ` that follows a key (tolerating a trailing `# comment`). +fn parse_toml_u32(after_key: &str) -> Option { + after_key + .trim_start() + .strip_prefix('=')? + .split('#') + .next()? + .trim() + .parse() + .ok() +} + +/// host-api vsock ports reserved by other running `dstack-vmm` processes (read +/// from each one's `-c `), so a fresh install can avoid colliding on the +/// host's vsock port space. Best-effort; sorted, deduped. +pub fn other_vmm_host_api_ports() -> Vec { + let mut ports = Vec::new(); + let Ok(entries) = std::fs::read_dir("/proc") else { + return ports; + }; + for entry in entries.flatten() { + let Ok(data) = std::fs::read(entry.path().join("cmdline")) else { + continue; + }; + let args: Vec = data + .split(|&b| b == 0) + .filter(|s| !s.is_empty()) + .map(|s| String::from_utf8_lossy(s).into_owned()) + .collect(); + if args.is_empty() { + continue; + } + if Path::new(&args[0]).file_name().and_then(|f| f.to_str()) != Some("dstack-vmm") { + continue; + } + if let Some(cfg) = arg_value(&args, "-c").or_else(|| arg_value(&args, "--config")) { + if let Some(p) = read_host_api_port(&cfg) { + ports.push(p); + } + } + } + ports.sort_unstable(); + ports.dedup(); + ports +} + +/// read the `[host_api]` `port` from a vmm.toml (section-aware: `port` appears +/// under several tables, so we only read the one inside `[host_api]`). +fn read_host_api_port(config_path: &str) -> Option { + let text = std::fs::read_to_string(config_path).ok()?; + let mut in_host_api = false; + for line in text.lines() { + let l = line.trim(); + if l.starts_with('[') { + in_host_api = l == "[host_api]"; + } else if in_host_api { + if let Some(v) = l.strip_prefix("port") { + if let Some(p) = parse_toml_u32(v) { + return Some(p); + } + } + } + } + None +} diff --git a/crates/dstack-cli-core/src/layout.rs b/crates/dstack-cli-core/src/layout.rs new file mode 100644 index 00000000..08d44a5a --- /dev/null +++ b/crates/dstack-cli-core/src/layout.rs @@ -0,0 +1,237 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! Filesystem layout shared by `dstackup` and the local `dstack` client. + +use anyhow::{bail, Result}; +use std::os::unix::ffi::OsStrExt; +use std::path::{Path, PathBuf}; + +pub const DEFAULT_BIN_DIR: &str = "/usr/local/bin"; +pub const DEFAULT_LIBEXEC_DIR: &str = "/usr/local/libexec/dstack"; +pub const DEFAULT_SHARE_DIR: &str = "/usr/local/share/dstack"; +pub const DEFAULT_CONFIG_DIR: &str = "/etc/dstack"; +pub const DEFAULT_STATE_DIR: &str = "/var/lib/dstack"; +pub const DEFAULT_CACHE_DIR: &str = "/var/cache/dstack"; +pub const DEFAULT_RUN_DIR: &str = "/run/dstack"; +pub const STATE_FILE: &str = "dstackup-state.json"; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct InstallLayout { + /// Explicit installation root supplied through `--prefix`. + /// + /// `None` means the default system-wide FHS layout is in use. + pub root: Option, + pub bin_dir: PathBuf, + pub libexec_dir: PathBuf, + pub share_dir: PathBuf, + pub config_dir: PathBuf, + pub state_dir: PathBuf, + pub cache_dir: PathBuf, + pub run_dir: PathBuf, +} + +impl InstallLayout { + pub fn new(prefix: Option<&str>) -> Self { + match prefix { + Some(prefix) => { + let root = PathBuf::from(prefix); + Self { + root: Some(root.clone()), + bin_dir: root.join("bin"), + libexec_dir: root.join("libexec/dstack"), + share_dir: root.join("share/dstack"), + config_dir: root.join("etc/dstack"), + state_dir: root.join("var/lib/dstack"), + cache_dir: root.join("var/cache/dstack"), + run_dir: root.join("run/dstack"), + } + } + None => Self { + root: None, + bin_dir: PathBuf::from(DEFAULT_BIN_DIR), + libexec_dir: PathBuf::from(DEFAULT_LIBEXEC_DIR), + share_dir: PathBuf::from(DEFAULT_SHARE_DIR), + config_dir: PathBuf::from(DEFAULT_CONFIG_DIR), + state_dir: PathBuf::from(DEFAULT_STATE_DIR), + cache_dir: PathBuf::from(DEFAULT_CACHE_DIR), + run_dir: PathBuf::from(DEFAULT_RUN_DIR), + }, + } + } + + pub fn state_path(&self) -> PathBuf { + self.state_dir.join(STATE_FILE) + } + + pub fn image_dir(&self) -> PathBuf { + self.state_dir.join("images") + } + + pub fn source_dir(&self) -> PathBuf { + self.cache_dir.join("source") + } + + pub fn cargo_target_dir(&self) -> PathBuf { + self.cache_dir.join("target") + } + + pub fn key_provider_dir(&self) -> PathBuf { + self.share_dir.join("key-provider-build") + } + + pub fn hello_nginx_compose(&self) -> PathBuf { + self.share_dir + .join("examples/hello-nginx/docker-compose.yaml") + } + + pub fn state_path_for_prefix(prefix: Option<&str>) -> PathBuf { + Self::new(prefix).state_path() + } + + pub fn image_dir_for_prefix(prefix: Option<&str>) -> PathBuf { + Self::new(prefix).image_dir() + } + + pub fn is_default(&self) -> bool { + self.root.is_none() + } + + pub fn all_dirs_absolute(&self) -> bool { + [ + &self.bin_dir, + &self.libexec_dir, + &self.share_dir, + &self.config_dir, + &self.state_dir, + &self.cache_dir, + &self.run_dir, + ] + .into_iter() + .all(|path| path.is_absolute()) + } + + pub fn validate(&self) -> Result<()> { + if let Some(root) = &self.root { + validate_install_prefix(root)?; + } + for (name, path) in [ + ("bin dir", &self.bin_dir), + ("libexec dir", &self.libexec_dir), + ("share dir", &self.share_dir), + ("config dir", &self.config_dir), + ("state dir", &self.state_dir), + ("cache dir", &self.cache_dir), + ("run dir", &self.run_dir), + ] { + validate_owned_dir(name, path)?; + } + Ok(()) + } +} + +pub fn path_string(path: &Path) -> String { + path.display().to_string() +} + +pub fn validate_install_prefix(prefix: &Path) -> Result<()> { + validate_absolute_path("--prefix", prefix)?; + validate_no_dot_segments("--prefix", prefix)?; + Ok(()) +} + +pub fn validate_owned_path(name: &str, path: &Path) -> Result<()> { + validate_owned_dir(name, path) +} + +fn validate_owned_dir(name: &str, path: &Path) -> Result<()> { + validate_absolute_path(name, path)?; + validate_no_dot_segments(name, path)?; + Ok(()) +} + +fn validate_no_dot_segments(name: &str, path: &Path) -> Result<()> { + for segment in path.as_os_str().as_bytes().split(|byte| *byte == b'/') { + if segment == b"." || segment == b".." { + bail!("{name} must not contain . or .. path components"); + } + } + Ok(()) +} + +fn validate_absolute_path(name: &str, path: &Path) -> Result<()> { + if !path.is_absolute() { + bail!("{name} must be an absolute path"); + } + if path == Path::new("/") { + bail!("{name} must not be /"); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn default_layout_uses_system_paths() { + let layout = InstallLayout::new(None); + assert_eq!(layout.bin_dir, PathBuf::from("/usr/local/bin")); + assert_eq!( + layout.libexec_dir, + PathBuf::from("/usr/local/libexec/dstack") + ); + assert_eq!(layout.share_dir, PathBuf::from("/usr/local/share/dstack")); + assert_eq!(layout.config_dir, PathBuf::from("/etc/dstack")); + assert_eq!(layout.state_dir, PathBuf::from("/var/lib/dstack")); + assert_eq!(layout.cache_dir, PathBuf::from("/var/cache/dstack")); + assert_eq!(layout.run_dir, PathBuf::from("/run/dstack")); + assert_eq!( + layout.state_path(), + PathBuf::from("/var/lib/dstack/dstackup-state.json") + ); + } + + #[test] + fn prefix_layout_is_self_contained() { + let layout = InstallLayout::new(Some("/opt/dstack-a")); + assert_eq!(layout.bin_dir, PathBuf::from("/opt/dstack-a/bin")); + assert_eq!( + layout.libexec_dir, + PathBuf::from("/opt/dstack-a/libexec/dstack") + ); + assert_eq!( + layout.share_dir, + PathBuf::from("/opt/dstack-a/share/dstack") + ); + assert_eq!(layout.config_dir, PathBuf::from("/opt/dstack-a/etc/dstack")); + assert_eq!( + layout.state_dir, + PathBuf::from("/opt/dstack-a/var/lib/dstack") + ); + assert_eq!( + layout.cache_dir, + PathBuf::from("/opt/dstack-a/var/cache/dstack") + ); + assert_eq!(layout.run_dir, PathBuf::from("/opt/dstack-a/run/dstack")); + } + + #[test] + fn prefix_validation_rejects_root_and_parent_components() { + for bad in ["relative", "/", "/opt/../dstack", "/opt/./dstack"] { + assert!( + validate_install_prefix(Path::new(bad)).is_err(), + "{bad:?} should be rejected" + ); + } + validate_install_prefix(Path::new("/opt/dstack-a")).unwrap(); + } + + #[test] + fn layout_validation_rejects_root_owned_dirs() { + let mut layout = InstallLayout::new(Some("/opt/dstack-a")); + layout.share_dir = PathBuf::from("/"); + assert!(layout.validate().is_err()); + } +} diff --git a/crates/dstack-cli-core/src/lib.rs b/crates/dstack-cli-core/src/lib.rs new file mode 100644 index 00000000..f86b9eaa --- /dev/null +++ b/crates/dstack-cli-core/src/lib.rs @@ -0,0 +1,26 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! shared internals for the `dstack` (client) and `dstackup` (host setup) binaries. +//! +//! `vmm` is a thin typed client over the VMM `Vmm` prpc service; `compose` builds +//! the app-compose manifest; `ports` does host-port allocation; `config` renders +//! the config files `dstackup install` writes; `fsutil` provides the atomic +//! write + advisory lock the allowlist/state files need. + +/// re-export the generated VMM rpc types (VmConfiguration, PortMapping, …). +pub use dstack_vmm_rpc as rpc; + +/// identifier string attached to outbound RPC calls. +pub fn user_agent() -> String { + format!("dstack-cli/{}", env!("CARGO_PKG_VERSION")) +} + +pub mod compose; +pub mod config; +pub mod fsutil; +pub mod host; +pub mod layout; +pub mod ports; +pub mod vmm; diff --git a/crates/dstack-cli-core/src/ports.rs b/crates/dstack-cli-core/src/ports.rs new file mode 100644 index 00000000..93b730d7 --- /dev/null +++ b/crates/dstack-cli-core/src/ports.rs @@ -0,0 +1,62 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! host-port helpers. The VMM does not auto-allocate host ports, so the client +//! picks a free one and passes it explicitly in the VM configuration. + +use anyhow::{bail, Context, Result}; +use dstack_vmm_rpc::PortMapping; +use std::net::TcpListener; + +/// pick a currently-free TCP port on loopback by binding to port 0. +/// +/// inherently racy (the port could be taken before the VMM binds it), but fine +/// for a single interactive deploy; the VMM will surface a bind conflict. +pub fn free_local_port() -> Result { + let listener = TcpListener::bind("127.0.0.1:0").context("failed to find a free host port")?; + let port = listener.local_addr()?.port(); + Ok(port) +} + +/// whether `addr:port` can be bound right now. Best-effort and racy, but it +/// catches the common "another service already owns this port" case so an +/// install can refuse before it starts changing the host. +pub fn tcp_port_free(addr: &str, port: u16) -> bool { + TcpListener::bind((addr, port)).is_ok() +} + +/// parse a `--port` spec into a [`PortMapping`], auto-allocating the host port +/// when it is omitted, `0`, or `auto`. Accepted forms: +/// +/// * `` — auto host port, tcp, 127.0.0.1 +/// * `:` — tcp, 127.0.0.1 +/// * `::` +/// * `:::` +pub fn parse_port(spec: &str) -> Result { + let parts: Vec<&str> = spec.split(':').collect(); + let (proto, addr, host, vm) = match parts.as_slice() { + [vm] => ("tcp", "127.0.0.1", "auto", *vm), + [host, vm] => ("tcp", "127.0.0.1", *host, *vm), + [proto, host, vm] => (*proto, "127.0.0.1", *host, *vm), + [proto, addr, host, vm] => (*proto, *addr, *host, *vm), + _ => bail!( + "invalid --port '{spec}': expected vm | host:vm | proto:host:vm | proto:addr:host:vm" + ), + }; + let vm_port: u32 = vm + .parse() + .with_context(|| format!("invalid vm port in '{spec}'"))?; + let host_port: u32 = if host.is_empty() || host == "auto" || host == "0" { + free_local_port()? as u32 + } else { + host.parse() + .with_context(|| format!("invalid host port in '{spec}'"))? + }; + Ok(PortMapping { + protocol: proto.to_string(), + host_address: addr.to_string(), + host_port, + vm_port, + }) +} diff --git a/crates/dstack-cli-core/src/vmm.rs b/crates/dstack-cli-core/src/vmm.rs new file mode 100644 index 00000000..6105a236 --- /dev/null +++ b/crates/dstack-cli-core/src/vmm.rs @@ -0,0 +1,130 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! thin typed client over the VMM `Vmm` prpc service. +//! +//! talks to a local VMM over its unix control socket, or a remote VMM over an +//! http(s) endpoint. prpc calls go to `/prpc/?json`; a few endpoints +//! (e.g. `/logs`) are plain HTTP and are reached with [`http_client::http_request`]. + +use anyhow::{anyhow, bail, Result}; +use dstack_vmm_rpc::vmm_client::VmmClient; +use dstack_vmm_rpc::{Id, StatusRequest, StatusResponse, VmConfiguration}; +use http_client::http_request; +use http_client::prpc::PrpcClient; + +/// default local VMM control socket (created by `dstackup install`). +pub const DEFAULT_HOST: &str = "unix:/var/run/dstack/vmm.sock"; + +/// a connection to a VMM — local unix socket or remote http endpoint. +pub struct Vmm { + rpc: VmmClient, + /// base string usable with [`http_request`] for non-prpc endpoints. + base: String, +} + +impl Vmm { + /// connect to a VMM addressed by `host`: + /// `unix:/path/to/vmm.sock` (local) or `http(s)://host:port` (remote). + pub fn connect(host: &str) -> Result { + let host = host.trim(); + if let Some(sock) = host.strip_prefix("unix:") { + let rpc = VmmClient::new(PrpcClient::new_unix(sock.to_string(), "/prpc".to_string())); + Ok(Self { + rpc, + base: format!("unix:{sock}"), + }) + } else if host.starts_with("http://") || host.starts_with("https://") { + let base = host.trim_end_matches('/').to_string(); + let rpc = VmmClient::new(PrpcClient::new(format!("{base}/prpc"))); + Ok(Self { rpc, base }) + } else { + bail!( + "unsupported host '{host}': expected unix:/path/to/vmm.sock or http(s)://host:port" + ); + } + } + + /// whether this connection targets a local unix socket. + pub fn is_local(&self) -> bool { + self.base.starts_with("unix:") + } + + /// list deployed VMs (brief: no full configuration). + pub async fn status(&self) -> Result { + self.rpc + .status(StatusRequest { + brief: true, + ..Default::default() + }) + .await + .map_err(|e| anyhow!("vmm Status rpc failed: {e}")) + } + + /// compute the compose hash for a VM configuration (no side effects). + /// the app id is the first 40 hex chars of this hash. takes `&cfg` and + /// clones once because the generated prpc client consumes its argument. + pub async fn get_compose_hash(&self, cfg: &VmConfiguration) -> Result { + self.rpc + .get_compose_hash(cfg.clone()) + .await + .map(|c| c.hash) + .map_err(|e| anyhow!("vmm GetComposeHash rpc failed: {e}")) + } + + /// create (and, unless `cfg.stopped`, start) a VM; returns the new VM id. + pub async fn create_vm(&self, cfg: VmConfiguration) -> Result { + self.rpc + .create_vm(cfg) + .await + .map(|id| id.id) + .map_err(|e| anyhow!("vmm CreateVm rpc failed: {e}")) + } + + /// stop a VM by id, keeping its disk (so its keys survive a re-install). + pub async fn stop_vm(&self, id: &str) -> Result<()> { + self.rpc + .stop_vm(Id { id: id.to_string() }) + .await + .map_err(|e| anyhow!("vmm StopVm rpc failed: {e}")) + } + + /// remove (and stop) a VM by id. + pub async fn remove_vm(&self, id: &str) -> Result<()> { + self.rpc + .remove_vm(Id { id: id.to_string() }) + .await + .map_err(|e| anyhow!("vmm RemoveVm rpc failed: {e}")) + } + + /// whether a VM with the given id currently exists. + pub async fn has_vm(&self, id: &str) -> bool { + match self.status().await { + Ok(s) => s.vms.iter().any(|v| v.id == id), + Err(_) => false, + } + } + + /// fetch the last `lines` log lines for a VM (non-following). + /// + /// `/logs` is a plain-HTTP `GET` endpoint. Only the local unix-socket + /// transport is wired today; remote `dstack logs` lands with the TLS+token + /// transport (the shared `http_request` honors the method on the unix/vsock + /// paths but hardcodes `POST` on the remote http path, and an unauthenticated + /// remote log endpoint shouldn't be reachable before that exists anyway). + pub async fn logs(&self, id: &str, lines: u32) -> Result { + if !self.is_local() { + bail!( + "`dstack logs` over a remote endpoint isn't wired yet (lands with the \ + TLS+token transport); use the local VMM socket for now" + ); + } + let path = format!("/logs?id={id}&follow=false&ansi=false&lines={lines}"); + let (status, body) = http_request("GET", &self.base, &path, b"").await?; + if status != 200 { + bail!("vmm /logs returned status {status}"); + } + Ok(String::from_utf8_lossy(&body).into_owned()) + } +} diff --git a/crates/dstack-cli/Cargo.toml b/crates/dstack-cli/Cargo.toml new file mode 100644 index 00000000..1ff3c41b --- /dev/null +++ b/crates/dstack-cli/Cargo.toml @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "dstack-cli" +version.workspace = true +edition.workspace = true +license.workspace = true + +# the package is dstack-cli (clear it's a CLI, not the dstack project), but the +# binary stays `dstack`. +[[bin]] +name = "dstack" +path = "src/main.rs" + +[dependencies] +anyhow.workspace = true +clap.workspace = true +dstack-cli-core.workspace = true +serde_json.workspace = true +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/crates/dstack-cli/src/main.rs b/crates/dstack-cli/src/main.rs new file mode 100644 index 00000000..4fae8201 --- /dev/null +++ b/crates/dstack-cli/src/main.rs @@ -0,0 +1,507 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! `dstack` — client for deploying and managing apps on a dstack host. +//! +//! Works against a local VMM (unix socket) or a remote one (`--host` + `--token`). +//! Setup/host tasks live in the separate `dstackup` binary. +//! +//! Command names follow the `phala` CLI where it makes sense (`deploy`, `apps`, +//! `logs`, a global `-j/--json`). + +use anyhow::{bail, Context, Result}; +use clap::{Parser, Subcommand}; +use dstack_cli_core::layout::InstallLayout; +use dstack_cli_core::vmm::{Vmm, DEFAULT_HOST}; +use dstack_cli_core::{compose, ports, rpc}; + +#[derive(Parser)] +#[command( + name = "dstack", + version, + about = "client for deploying and managing dstack apps" +)] +struct Cli { + /// VMM endpoint: `unix:/path/to/vmm.sock` (local) or `http(s)://host:port` (remote). + /// Defaults to the local `dstackup install` endpoint, then the local control socket. + #[arg(long, global = true)] + host: Option, + + /// local `dstackup install` prefix to read defaults from. Omit for the default system install. + #[arg(long, global = true, value_name = "DIR")] + prefix: Option, + + /// auth token for a remote VMM. + #[arg(long, global = true)] + token: Option, + + /// machine-readable JSON output (honored by `deploy` and `apps`). + #[arg(long, short = 'j', global = true)] + json: bool, + + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + /// Deploy an app from a docker-compose file. + Deploy { + /// path to the docker-compose file. + compose: Option, + /// path to the docker-compose file. + #[arg(long = "compose", short = 'c', value_name = "PATH")] + compose_file: Option, + /// app name. + #[arg(long, short = 'n', default_value = "app")] + name: String, + /// guest OS image name. Defaults to the image selected by `dstackup install`. + #[arg(long)] + image: Option, + /// vCPUs. + #[arg(long, default_value_t = 2)] + vcpu: u32, + /// memory in MB (matches vmm-cli's default; images with a large + /// initramfs-rootfs may need more — raise it if the guest fails early). + #[arg(long, default_value_t = 1024)] + memory: u32, + /// disk size in GB. + #[arg(long, default_value_t = 20)] + disk: u32, + /// expose a port: `vm` | `host:vm` | `proto:host:vm` | `proto:addr:host:vm` + /// (host omitted/`auto`/`0` ⇒ a free host port is picked). Repeatable. + #[arg(long = "port", value_name = "SPEC")] + ports: Vec, + /// deploy in non-KMS mode (ephemeral keys; no KMS required). + #[arg(long)] + no_kms: bool, + /// register the app's compose hash in this auth-allowlist.json. Defaults + /// to the local allowlist from `dstackup install`. + #[arg(long, value_name = "PATH")] + allowlist: Option, + /// build + hash the compose and print it, without deploying. + #[arg(long)] + dry_run: bool, + }, + /// List deployed apps. + Apps, + /// Show recent logs for an app. + Logs { + /// app, instance, or VM id. + id: String, + /// number of trailing log lines to fetch. + #[arg(long, default_value_t = 200)] + lines: u32, + }, + /// Show details for an app. + Info { + /// app or instance id. + id: String, + }, + /// Scaffold a new app project in the current directory. + Init, +} + +#[tokio::main] +async fn main() -> Result<()> { + let cli = Cli::parse(); + // remote-auth wiring lands with the TLS+token transport. + let _ = &cli.token; + let defaults = LocalDefaults::read(cli.prefix.as_deref()); + let use_local_defaults = cli.host.is_none(); + let host = cli + .host + .clone() + .or_else(|| defaults.as_ref().and_then(|d| d.client_url.clone())) + .unwrap_or_else(|| DEFAULT_HOST.to_string()); + let json = cli.json; + + match cli.command { + Command::Apps => cmd_apps(&host, json).await, + Command::Logs { id, lines } => cmd_logs(&host, &id, lines).await, + Command::Deploy { + compose, + compose_file, + name, + image, + vcpu, + memory, + disk, + ports, + no_kms, + allowlist, + dry_run, + } => { + let compose = resolve_compose_arg(compose, compose_file)?; + let image = if use_local_defaults { + image.or_else(|| defaults.as_ref().and_then(|d| d.image.clone())) + } else { + image + }; + let allowlist = if use_local_defaults { + allowlist.or_else(|| { + (!no_kms) + .then(|| defaults.as_ref().and_then(LocalDefaults::allowlist_path)) + .flatten() + }) + } else { + allowlist + }; + cmd_deploy( + &host, + &compose, + &name, + image.as_deref(), + vcpu, + memory, + disk, + &ports, + no_kms, + allowlist.as_deref(), + dry_run, + json, + ) + .await + } + Command::Info { .. } => stub("info"), + Command::Init => stub("init"), + } +} + +fn resolve_compose_arg(positional: Option, flagged: Option) -> Result { + match (positional, flagged) { + (Some(path), None) | (None, Some(path)) => Ok(path), + (Some(_), Some(_)) => bail!("pass the compose file once: either as or with -c"), + (None, None) => bail!("missing compose file: pass -c "), + } +} + +struct LocalDefaults { + client_url: Option, + image: Option, + allowlist_path: Option, +} + +impl LocalDefaults { + fn read(prefix: Option<&str>) -> Option { + let path = InstallLayout::state_path_for_prefix(prefix); + let body = std::fs::read_to_string(path).ok()?; + let v: serde_json::Value = serde_json::from_str(&body).ok()?; + Some(Self::from_value(&v)) + } + + fn from_value(v: &serde_json::Value) -> Self { + Self { + client_url: v + .get("client_url") + .and_then(|x| x.as_str()) + .filter(|s| !s.is_empty()) + .map(str::to_string), + image: v + .get("image") + .and_then(|x| x.as_str()) + .filter(|s| !s.is_empty()) + .map(str::to_string), + allowlist_path: v + .get("allowlist_path") + .and_then(|x| x.as_str()) + .filter(|s| !s.is_empty()) + .map(str::to_string), + } + } + + fn allowlist_path(&self) -> Option { + self.allowlist_path.clone() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_local_install_defaults() { + let value = serde_json::json!({ + "client_url": "http://127.0.0.1:19080", + "image": "dstack-0.5.11", + "allowlist_path": "/tmp/dstack/etc/dstack/auth-allowlist.json" + }); + let defaults = LocalDefaults::from_value(&value); + assert_eq!( + defaults.client_url.as_deref(), + Some("http://127.0.0.1:19080") + ); + assert_eq!(defaults.image.as_deref(), Some("dstack-0.5.11")); + assert_eq!( + defaults.allowlist_path().as_deref(), + Some("/tmp/dstack/etc/dstack/auth-allowlist.json") + ); + } + + #[test] + fn reads_local_install_defaults_from_prefix() { + let install_root = + std::env::temp_dir().join(format!("dstack-cli-state-test-{}", std::process::id())); + let state_dir = install_root.join("var/lib/dstack"); + std::fs::create_dir_all(&state_dir).unwrap(); + std::fs::write( + state_dir.join(dstack_cli_core::layout::STATE_FILE), + r#"{ + "client_url": "http://127.0.0.1:29080", + "image": "dstack-0.5.12", + "allowlist_path": "/tmp/custom-dstack/etc/dstack/auth-allowlist.json" + }"#, + ) + .unwrap(); + + let prefix = dstack_cli_core::layout::path_string(&install_root); + let defaults = LocalDefaults::read(Some(&prefix)).unwrap(); + assert_eq!( + defaults.client_url.as_deref(), + Some("http://127.0.0.1:29080") + ); + assert_eq!(defaults.image.as_deref(), Some("dstack-0.5.12")); + assert_eq!( + defaults.allowlist_path().as_deref(), + Some("/tmp/custom-dstack/etc/dstack/auth-allowlist.json") + ); + + let _ = std::fs::remove_dir_all(install_root); + } + + #[test] + fn parses_phala_style_deploy_flags() { + let cli = Cli::parse_from([ + "dstack", + "deploy", + "-n", + "hello", + "-c", + "examples/hello-nginx/docker-compose.yaml", + "--port", + "8080:80", + ]); + match cli.command { + Command::Deploy { + compose, + compose_file, + name, + ports, + .. + } => { + assert_eq!(compose, None); + assert_eq!( + compose_file.as_deref(), + Some("examples/hello-nginx/docker-compose.yaml") + ); + assert_eq!(name, "hello"); + assert_eq!(ports, vec!["8080:80"]); + } + _ => panic!("expected deploy command"), + } + } +} + +#[allow(clippy::too_many_arguments)] +async fn cmd_deploy( + host: &str, + compose_path: &str, + name: &str, + image: Option<&str>, + vcpu: u32, + memory: u32, + disk: u32, + port_specs: &[String], + no_kms: bool, + allowlist: Option<&str>, + dry_run: bool, + json: bool, +) -> Result<()> { + let yaml = std::fs::read_to_string(compose_path) + .with_context(|| format!("reading compose file '{compose_path}'"))?; + let app_compose = compose::build_app_compose(name, &yaml, !no_kms); + + let mut port_maps = Vec::new(); + for spec in port_specs { + port_maps.push(ports::parse_port(spec)?); + } + + let mut cfg = rpc::VmConfiguration { + name: name.to_string(), + image: image.unwrap_or_default().to_string(), + compose_file: app_compose.clone(), + vcpu, + memory, + disk_size: disk, + ports: port_maps.clone(), + ..Default::default() + }; + + let vmm = Vmm::connect(host)?; + let hash = vmm.get_compose_hash(&cfg).await?; + let app_id = short(&hash, 40); + cfg.app_id = Some(app_id.clone()); + if !json { + println!("compose hash: {hash}"); + println!("app id: {app_id}"); + } + + if dry_run { + if json { + print_json(&serde_json::json!({ + "composeHash": hash, + "appId": app_id, + "appCompose": app_compose, + "dryRun": true, + })); + } else { + println!("--- app-compose ---\n{app_compose}"); + println!("(dry run — not deploying)"); + } + return Ok(()); + } + if cfg.image.is_empty() { + bail!( + "an image is required to deploy: run `dstackup install` first, or pass --image " + ); + } + + // register the compose hash so the KMS will issue keys (KMS mode, local). + if let Some(path) = allowlist { + dstack_cli_core::config::register_app_in_allowlist( + std::path::Path::new(path), + &app_id, + &hash, + ) + .with_context(|| format!("registering app in {path}"))?; + if !json { + println!("registered compose hash in {path}"); + println!( + " (the KMS issues keys only if this is the allowlist its auth webhook serves)" + ); + } + } else if !no_kms && !json { + println!("note: no --allowlist given; a KMS-mode app needs its compose hash registered to get keys"); + } + + let id = vmm.create_vm(cfg).await?; + if json { + let ports: Vec<_> = port_maps + .iter() + .map(|p| { + serde_json::json!({ + "vmPort": p.vm_port, + "hostPort": p.host_port, + "hostAddress": host_addr(p), + }) + }) + .collect(); + print_json(&serde_json::json!({ + "vmId": id, + "appId": app_id, + "composeHash": hash, + "ports": ports, + })); + } else { + println!("deployed: vm {id}"); + if port_maps.is_empty() { + println!("(no ports mapped — add --port to expose the app)"); + } + for p in &port_maps { + println!( + " app :{} -> http://{}:{}/", + p.vm_port, + host_addr(p), + p.host_port + ); + } + } + Ok(()) +} + +fn stub(name: &str) -> Result<()> { + // exit non-zero so `dstack && next` doesn't proceed as if it worked. + bail!( + "dstack {name}: not yet implemented ({})", + dstack_cli_core::user_agent() + ) +} + +async fn cmd_apps(host: &str, json: bool) -> Result<()> { + let vmm = Vmm::connect(host)?; + let resp = vmm.status().await?; + if json { + let arr: Vec<_> = resp + .vms + .iter() + .map(|vm| { + serde_json::json!({ + "id": vm.id, + "name": vm.name, + "status": vm.status, + "uptime": vm.uptime, + "appId": vm.app_id, + }) + }) + .collect(); + print_json(&serde_json::Value::Array(arr)); + return Ok(()); + } + if resp.vms.is_empty() { + println!("no apps deployed"); + return Ok(()); + } + println!( + "{:<14} {:<22} {:<10} {:<14} APP ID", + "ID", "NAME", "STATUS", "UPTIME" + ); + for vm in resp.vms { + println!( + "{:<14} {:<22} {:<10} {:<14} {}", + short(&vm.id, 12), + trunc(&vm.name, 22), + trunc(&vm.status, 10), + trunc(&vm.uptime, 14), + short(&vm.app_id, 40), + ); + } + Ok(()) +} + +async fn cmd_logs(host: &str, id: &str, lines: u32) -> Result<()> { + let vmm = Vmm::connect(host)?; + let logs = vmm.logs(id, lines).await?; + print!("{logs}"); + Ok(()) +} + +/// the host address a port maps to (loopback when unset). +fn host_addr(p: &rpc::PortMapping) -> &str { + if p.host_address.is_empty() { + "127.0.0.1" + } else { + &p.host_address + } +} + +/// print a value as pretty JSON (infallible via Value's Display). +fn print_json(v: &serde_json::Value) { + println!("{v:#}"); +} + +/// first `n` chars of an id-like string. +fn short(s: &str, n: usize) -> String { + s.chars().take(n).collect() +} + +/// truncate to `n` chars with an ellipsis if longer. +fn trunc(s: &str, n: usize) -> String { + if s.chars().count() <= n { + s.to_string() + } else { + let mut out: String = s.chars().take(n.saturating_sub(1)).collect(); + out.push('…'); + out + } +} diff --git a/crates/dstackup/Cargo.toml b/crates/dstackup/Cargo.toml new file mode 100644 index 00000000..cfc3a812 --- /dev/null +++ b/crates/dstackup/Cargo.toml @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "dstackup" +version.workspace = true +edition.workspace = true +license.workspace = true + +[[bin]] +name = "dstackup" +path = "src/main.rs" + +[dependencies] +anyhow.workspace = true +clap.workspace = true +dstack-cli-core.workspace = true +hex = { workspace = true, features = ["alloc"] } +# reqwest (+ rustls/hyper) is already linked via dstack-cli-core's prpc client, +# so using it for http here adds ~no binary cost and drops the curl dependency. +reqwest = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +sha2.workspace = true +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "time"] } diff --git a/crates/dstackup/src/cid.rs b/crates/dstackup/src/cid.rs new file mode 100644 index 00000000..f0cc44ae --- /dev/null +++ b/crates/dstackup/src/cid.rs @@ -0,0 +1,84 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! pick a vsock CID window that doesn't collide with a VMM already on the host. + +use anyhow::{bail, Result}; + +/// size of a VMM's CID pool (matches `config::VmmRender` default). +const CID_POOL_SIZE: u32 = 1000; +/// default CID pool start when nothing else is using that range. +const DEFAULT_CID_START: u32 = 1000; + +/// whether `[start, start+CID_POOL_SIZE)` intersects any occupied range. +fn cid_window_overlaps(start: u32, occupied: &[(u32, u32)]) -> bool { + let end = start.saturating_add(CID_POOL_SIZE); + occupied.iter().any(|&(s, e)| start < e && s < end) +} + +/// the lowest pool-aligned CID block at or above every occupied range. We jump +/// above the highest reservation rather than packing into a free gap below it — +/// simpler, and the result is always collision-free. +fn next_free_cid_block(occupied: &[(u32, u32)]) -> u32 { + let max_end = occupied + .iter() + .map(|&(_, e)| e) + .max() + .unwrap_or(DEFAULT_CID_START); + (max_end.div_ceil(CID_POOL_SIZE) * CID_POOL_SIZE).max(DEFAULT_CID_START) +} + +/// choose a CID window `[start, start+CID_POOL_SIZE)` that won't collide with a +/// VMM already running on this host. With an explicit `--cid-start`, honor it +/// but refuse on overlap; without one, use the default unless it's taken, then +/// move to the next free block. +pub(crate) fn pick_cid_start(explicit: Option, occupied: &[(u32, u32)]) -> Result { + match explicit { + Some(n) => { + if cid_window_overlaps(n, occupied) { + bail!( + "--cid-start {n} overlaps a CID range already reserved on this host; \ + pick a free start, e.g. --cid-start {}", + next_free_cid_block(occupied) + ); + } + Ok(n) + } + None if !cid_window_overlaps(DEFAULT_CID_START, occupied) => Ok(DEFAULT_CID_START), + None => { + let start = next_free_cid_block(occupied); + println!(" [ok] cid-start {start} (avoids CIDs already reserved by another VMM)"); + Ok(start) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cid_default_when_range_free() { + assert_eq!(pick_cid_start(None, &[]).unwrap(), 1000); + // a pool entirely above the default window leaves it free. + assert_eq!(pick_cid_start(None, &[(2000, 3000)]).unwrap(), 1000); + } + + #[test] + fn cid_auto_offsets_past_an_existing_vmm() { + // another VMM reserving [1000,2000) -> jump to 2000. + assert_eq!(pick_cid_start(None, &[(1000, 2000)]).unwrap(), 2000); + // its reserved pool plus a stray live CVM at 2500 -> jump past it. + assert_eq!( + pick_cid_start(None, &[(1000, 2000), (2500, 2501)]).unwrap(), + 3000 + ); + } + + #[test] + fn cid_explicit_honored_or_refused() { + assert_eq!(pick_cid_start(Some(2000), &[(1000, 2000)]).unwrap(), 2000); + assert!(pick_cid_start(Some(1000), &[(1000, 2000)]).is_err()); + } +} diff --git a/crates/dstackup/src/cli.rs b/crates/dstackup/src/cli.rs new file mode 100644 index 00000000..65dc8156 --- /dev/null +++ b/crates/dstackup/src/cli.rs @@ -0,0 +1,235 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! command-line interface (clap definitions). + +use clap::{Args, Parser, Subcommand}; +use dstack_cli_core::config; + +pub(crate) const DEFAULT_VMM_BIN: &str = "dstack-vmm"; +pub(crate) const DEFAULT_AUTH_BIN: &str = "dstack-auth"; +pub(crate) const DEFAULT_SUPERVISOR_BIN: &str = "supervisor"; +pub(crate) const DEFAULT_SOURCE_REPO: &str = "https://github.com/Dstack-TEE/dstack"; +pub(crate) const DEFAULT_SOURCE_REF: &str = "master"; + +#[derive(Parser)] +#[command(name = "dstackup", version, about = "set up and manage a dstack host")] +pub(crate) struct Cli { + /// VMM control socket / endpoint to talk to. Defaults to the local install state, + /// then the local control socket. + #[arg(long, global = true)] + pub(crate) host: Option, + + #[command(subcommand)] + pub(crate) command: Command, +} + +#[derive(Subcommand)] +// `Install` carries all the host-setup flags; the size gap to `Status`/`Destroy` +// is irrelevant for a CLI enum constructed once at startup. +#[allow(clippy::large_enum_variant)] +pub(crate) enum Command { + /// Bring up the host stack: SGX preflight, render configs, and start the + /// VMM + auth webhook. (Gramine bring-up and KMS-in-CVM bootstrap follow.) + Install(InstallOpts), + /// Show the health of the host stack. + Status { + /// installation root to inspect. Omit for the default system install. + #[arg(long, value_name = "DIR")] + prefix: Option, + }, + /// Download or list guest OS images. + #[command(subcommand)] + Image(ImageCmd), + /// Tear down the deployment (keeps configs + KMS keys unless --purge). + Destroy { + /// installation root to tear down. Omit for the default system install. + #[arg(long, value_name = "DIR")] + prefix: Option, + /// also wipe generated config, state, cache, runtime files, and KMS keys. + #[arg(long)] + purge: bool, + }, +} + +/// where guest images live, shared by every `image` subcommand and resolved the +/// same way `install` does: `--image-path` if given, else the layout image dir. +#[derive(Args)] +pub(crate) struct ImageLoc { + /// image directory (overrides the layout image dir, e.g. an external store). + #[arg(long)] + pub(crate) image_path: Option, + /// installation root. Omit for the default system install. + #[arg(long, value_name = "DIR")] + pub(crate) prefix: Option, +} + +impl ImageLoc { + /// the resolved image directory. + pub(crate) fn dir(&self) -> String { + crate::image::resolve_image_dir(self.image_path.as_deref(), self.prefix.as_deref()) + } +} + +/// `dstackup image` subcommands. +#[derive(Subcommand)] +pub(crate) enum ImageCmd { + /// Download a guest OS image from meta-dstack releases. + Pull { + /// image version to fetch (default: the latest release). + #[arg(long, value_name = "VERSION")] + version: Option, + /// fetch the gpu (nvidia) image instead of the cpu one. + #[arg(long)] + gpu: bool, + #[command(flatten)] + loc: ImageLoc, + /// re-download even if the image is already present. + #[arg(long)] + force: bool, + /// proceed even if the release publishes no sha256 to verify against. + #[arg(long)] + insecure: bool, + }, + /// List guest OS images already present locally. + List { + #[command(flatten)] + loc: ImageLoc, + }, + /// Remove one or more local guest OS images. + #[command(visible_alias = "remove")] + Rm { + /// image name(s) to delete (as shown by `dstackup image list`). + #[arg(value_name = "NAME", required = true)] + names: Vec, + #[command(flatten)] + loc: ImageLoc, + }, +} + +/// flags for `dstackup install`. +#[derive(Args)] +pub(crate) struct InstallOpts { + /// expose the dashboard on this IP (default: bind localhost only — + /// reach it via an SSH tunnel). + #[arg(long, value_name = "IP")] + pub(crate) expose: Option, + + /// guest OS image name or release version to deploy. + #[arg(long, value_name = "VERSION")] + pub(crate) image: Option, + + /// confidential-computing platform: `auto` (detect) | `tdx` | `amd-sev-snp`. + #[arg(long, default_value = "auto")] + pub(crate) platform: String, + + /// installation root. Omit for the default system install. + #[arg(long, value_name = "DIR")] + pub(crate) prefix: Option, + + /// systemd instance suffix: units become `dstack-vmm-` etc., + /// so a fresh install coexists with an existing `dstack-vmm.service`. + #[arg(long)] + pub(crate) instance: Option, + + /// guest image directory (default: the layout image directory). + #[arg(long)] + pub(crate) image_path: Option, + + /// dstack source checkout used to build managed binaries. + /// Defaults to the current checkout, or a source cache under the install layout. + #[arg(long, value_name = "DIR")] + pub(crate) source: Option, + + /// Git repository used when dstackup needs to populate the source cache. + #[arg(long, default_value = DEFAULT_SOURCE_REPO)] + pub(crate) source_repo: String, + + /// Git ref used when dstackup needs to populate the source cache. + #[arg(long, default_value = DEFAULT_SOURCE_REF)] + pub(crate) source_ref: String, + + /// directory where dstackup installs user-facing dstack binaries. + #[arg(long, value_name = "DIR")] + pub(crate) bin_dir: Option, + + /// directory where dstackup installs private host daemon binaries. + #[arg(long, value_name = "DIR")] + pub(crate) libexec_dir: Option, + + /// directory where dstackup installs static assets and examples. + #[arg(long, value_name = "DIR")] + pub(crate) share_dir: Option, + + /// use the configured binaries as-is; do not build or install managed binaries. + #[arg(long)] + pub(crate) skip_managed_binaries: bool, + + /// dstack-vmm binary. + #[arg(long, default_value = DEFAULT_VMM_BIN)] + pub(crate) vmm_bin: String, + + /// dstack-auth binary. + #[arg(long, default_value = DEFAULT_AUTH_BIN)] + pub(crate) auth_bin: String, + + /// supervisor binary. + #[arg(long, default_value = DEFAULT_SUPERVISOR_BIN)] + pub(crate) supervisor_bin: String, + + /// qemu binary. + #[arg(long, default_value = "/usr/bin/qemu-system-x86_64")] + pub(crate) qemu: String, + + /// dashboard TCP port. + #[arg(long, default_value_t = 9080)] + pub(crate) dashboard_port: u16, + + /// auth webhook port. + #[arg(long, default_value_t = 8001)] + pub(crate) auth_port: u16, + + /// host-api vsock port (raise to coexist with an existing VMM on 10000). + #[arg(long, default_value_t = 10000)] + pub(crate) host_api_port: u32, + + /// CID pool start (default: auto — the first free block, so it coexists + /// with any VMM already running on this host). + #[arg(long)] + pub(crate) cid_start: Option, + + /// use an existing key provider at ADDR:PORT instead of running our own. + #[arg(long, value_name = "ADDR:PORT")] + pub(crate) use_existing_key_provider: Option, + + /// port for our own key provider (when not using an existing one). + #[arg(long, default_value_t = 3443)] + pub(crate) key_provider_port: u16, + + /// key-provider build/compose directory (to start our own). + #[arg(long)] + pub(crate) key_provider_src: Option, + + /// KMS container image. + #[arg(long, default_value = config::DEFAULT_KMS_IMAGE)] + pub(crate) kms_image: String, + + /// host port for the KMS RPC (default: an auto-picked free port). + #[arg(long)] + pub(crate) kms_port: Option, + + /// skip the KMS-in-CVM deploy (bring up VMM + auth only). + #[arg(long)] + pub(crate) no_kms: bool, + + /// proceed even if the app OS image can't be pinned (missing platform + /// digest) — apps will boot any unmeasured image and still get keys. not + /// recommended. + #[arg(long)] + pub(crate) allow_unpinned_image: bool, + + /// render + write configs only; do not start any process. + #[arg(long)] + pub(crate) no_start: bool, +} diff --git a/crates/dstackup/src/destroy.rs b/crates/dstackup/src/destroy.rs new file mode 100644 index 00000000..2ff6ab4f --- /dev/null +++ b/crates/dstackup/src/destroy.rs @@ -0,0 +1,178 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! `dstackup destroy` — tear down what `install` started. + +use crate::state::{read_state, state_path}; +use crate::systemd::{remove_unit, systemctl, tool}; +use anyhow::{Context, Result}; +use dstack_cli_core::layout::InstallLayout; +use dstack_cli_core::vmm::Vmm; +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; + +/// tear down what `install` started; idempotent. Keeps generated config, state, +/// and KMS keys unless `--purge`. +pub(crate) async fn cmd_destroy(prefix: Option<&str>, purge: bool) -> Result<()> { + let layout = InstallLayout::new(prefix); + layout.validate()?; + println!("dstackup destroy ({})", layout.state_dir.display()); + match read_state(&layout.state_dir) { + Some(st) => { + // gracefully stop the KMS CVM first so its keys flush to disk + // (unless we purge). Stopping the VMM unit below reaps the supervisor + // and the CVM qemu via the unit's cgroup. Look it up by recorded id + // AND by name, so an install that died before persisting kms_vm_id + // (or a torn state file) doesn't leave the CVM orphaned. + if let Ok(vmm) = Vmm::connect(&st.client_url) { + let mut target = st.kms_vm_id.clone(); + if target.is_none() { + if let Ok(s) = vmm.status().await { + target = s + .vms + .iter() + .find(|v| v.name == "dstack-kms") + .map(|v| v.id.clone()); + } + } + if let Some(id) = target { + if vmm.has_vm(&id).await { + let _ = vmm.stop_vm(&id).await; + println!(" stopping KMS CVM (vm {id})"); + } + } + } + // stop + remove the units. `systemctl stop` is synchronous and tears + // down the whole unit cgroup (VMM + supervisor + CVM qemu), so the + // host is back to baseline when this returns. + if !st.vmm_unit.is_empty() { + remove_unit(&st.vmm_unit); + println!(" stopped {}.service", st.vmm_unit); + } + if !st.auth_unit.is_empty() { + remove_unit(&st.auth_unit); + println!(" stopped {}.service", st.auth_unit); + } + systemctl(&["daemon-reload"]); + // stop our own key provider, if we started one. + if let Some(project) = &st.kp_own_project { + let _ = tool("docker") + .args(["compose", "-p", project, "down"]) + .status(); + println!(" stopped key provider (project {project})"); + } + // remove the runtime-state marker so a later install starts fresh. + let _ = fs::remove_file(state_path(&layout.state_dir)); + } + None => println!( + " no install state at {} (nothing running to stop)", + layout.state_dir.display() + ), + } + + if purge { + purge_layout(&layout)?; + } else { + println!( + " configs kept at {}; state + KMS keys kept at {} (use --purge to wipe)", + layout.config_dir.display(), + layout.state_dir.display() + ); + } + Ok(()) +} + +fn purge_layout(layout: &InstallLayout) -> Result<()> { + for dir in [ + &layout.config_dir, + &layout.state_dir, + &layout.cache_dir, + &layout.run_dir, + ] { + remove_dir_all_if_exists(dir)?; + } + + if let Some(root) = &layout.root { + remove_dir_all_if_exists(&layout.share_dir)?; + remove_dir_all_if_exists(&layout.libexec_dir)?; + + for file in [ + layout.bin_dir.join("dstack"), + layout.bin_dir.join("dstackup"), + ] { + remove_file_if_exists(&file)?; + } + + for dir in [ + &layout.bin_dir, + &layout.libexec_dir, + &layout.share_dir, + &layout.config_dir, + &layout.state_dir, + &layout.cache_dir, + &layout.run_dir, + ] { + remove_empty_parents(dir, root)?; + } + remove_empty_dir(&layout.bin_dir)?; + remove_empty_dir(root)?; + } + Ok(()) +} + +fn remove_dir_all_if_exists(dir: &Path) -> Result<()> { + match fs::remove_dir_all(dir) { + Ok(()) => { + println!(" purged {}", dir.display()); + Ok(()) + } + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e).with_context(|| format!("purging {}", dir.display())), + } +} + +fn remove_file_if_exists(path: &Path) -> Result<()> { + match fs::remove_file(path) { + Ok(()) => { + println!(" removed {}", path.display()); + Ok(()) + } + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e).with_context(|| format!("removing {}", path.display())), + } +} + +fn remove_empty_parents(path: &Path, stop_at: &Path) -> Result<()> { + let mut current = path.parent().map(PathBuf::from); + while let Some(dir) = current { + if !dir.starts_with(stop_at) { + break; + } + if dir == stop_at { + break; + } + remove_empty_dir(&dir)?; + current = dir.parent().map(PathBuf::from); + } + Ok(()) +} + +fn remove_empty_dir(dir: &Path) -> Result<()> { + match fs::remove_dir(dir) { + Ok(()) => { + println!(" removed empty dir {}", dir.display()); + Ok(()) + } + Err(e) + if matches!( + e.kind(), + io::ErrorKind::NotFound | io::ErrorKind::DirectoryNotEmpty + ) => + { + Ok(()) + } + Err(e) => Err(e).with_context(|| format!("removing empty dir {}", dir.display())), + } +} diff --git a/crates/dstackup/src/image.rs b/crates/dstackup/src/image.rs new file mode 100644 index 00000000..2b0c9376 --- /dev/null +++ b/crates/dstackup/src/image.rs @@ -0,0 +1,718 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! `dstackup image` — fetch, list, and remove guest OS images. +//! +//! Images are published as release tarballs at `Dstack-TEE/meta-dstack`. There +//! are two variants — cpu (`dstack-`) and gpu (`dstack-nvidia-`). +//! `install` validates the selected image against the platform-specific digest +//! it needs before starting the host stack: TDX uses `digest.txt`, and +//! SEV-SNP uses `digest.sev.txt`. HTTP + checksum are native (reqwest is +//! already linked via the prpc client; sha2 verifies inline); only `tar` is +//! shelled out, since GNU tar is ubiquitous and battle-tested on archive edges. + +use crate::cli::ImageCmd; +use crate::systemd::tool; +use anyhow::{bail, Context, Result}; +use dstack_cli_core::layout::{path_string, validate_owned_path, InstallLayout}; +use serde::Deserialize; +use sha2::{Digest, Sha256}; +use std::fs; +use std::io::Write; +use std::path::Path; +use std::time::SystemTime; + +const REPO: &str = "Dstack-TEE/meta-dstack"; +pub(crate) const RELEASES_URL: &str = "https://github.com/Dstack-TEE/meta-dstack/releases"; + +/// the single rule for where images live: `--image-path` if given, else the +/// image directory from the install layout. `install` and every image subcommand resolve through +/// here, so they can't drift. +pub(crate) fn resolve_image_dir(image_path: Option<&str>, prefix: Option<&str>) -> String { + image_path + .map(str::to_string) + .unwrap_or_else(|| path_string(&InstallLayout::image_dir_for_prefix(prefix))) +} + +pub(crate) fn validate_image_dir(image_dir: &str) -> Result<()> { + validate_owned_path("image directory", Path::new(image_dir)) +} + +#[derive(Deserialize)] +struct Release { + tag_name: String, + assets: Vec, +} + +#[derive(Deserialize)] +struct Asset { + name: String, + browser_download_url: String, + /// `"sha256:"` when the release publishes one (newer releases do); we + /// verify the download against it. absent on older releases. + #[serde(default)] + digest: Option, +} + +struct PullSpec { + version: String, + gpu: bool, +} + +pub(crate) async fn cmd_image(cmd: ImageCmd) -> Result<()> { + match cmd { + ImageCmd::Pull { + version, + gpu, + loc, + force, + insecure, + } => { + let image_dir = loc.dir(); + validate_image_dir(&image_dir)?; + pull(version.as_deref(), gpu, &image_dir, force, insecure).await?; + Ok(()) + } + ImageCmd::List { loc } => { + let image_dir = loc.dir(); + validate_image_dir(&image_dir)?; + list(&image_dir) + } + ImageCmd::Rm { names, loc } => { + let image_dir = loc.dir(); + validate_image_dir(&image_dir)?; + remove(&names, &image_dir) + } + } +} + +/// download a guest image from the latest (or a specific) meta-dstack release. +pub(crate) async fn pull( + version: Option<&str>, + gpu: bool, + image_dir: &str, + force: bool, + insecure: bool, +) -> Result { + println!( + "dstackup image pull — {} image", + if gpu { "gpu (nvidia)" } else { "cpu" } + ); + let release = fetch_release(version).await?; + let ver = release.tag_name.trim_start_matches('v'); + + // the unpacked dir is usually `dstack[-nvidia]-`; check that first so a + // repeat pull is a cheap no-op instead of re-fetching a few hundred MB. + let expected = format!("dstack-{}{ver}", if gpu { "nvidia-" } else { "" }); + if !force + && Path::new(image_dir) + .join(&expected) + .join("metadata.json") + .exists() + { + println!(" [ok] {expected} already present (use --force to re-download)"); + return Ok(expected); + } + + let asset = pick_asset(&release.assets, gpu).with_context(|| { + format!( + "no {} image tarball in meta-dstack release {} (assets: {})", + if gpu { "gpu" } else { "cpu" }, + release.tag_name, + release + .assets + .iter() + .map(|a| a.name.as_str()) + .collect::>() + .join(", ") + ) + })?; + // never trust the asset name into a filesystem path (github forbids `/` in + // asset names, but don't rely on that structurally). + if !valid_image_name(&asset.name) { + bail!( + "refusing release asset with an unsafe name {:?}", + asset.name + ); + } + println!(" [..] release {} -> {}", release.tag_name, asset.name); + + fs::create_dir_all(image_dir).with_context(|| format!("creating {image_dir}"))?; + + // download → verify checksum → unpack into a dot-prefixed staging dir → + // adopt (atomic rename) only once metadata.json is present. so a truncated + // download or a tar that dies mid-unpack can never masquerade as a valid + // image. temp artifacts are dot-prefixed (skipped by listings) and cleaned + // up regardless of outcome. (the `valid_image_name` check above is + // load-bearing for these two joins — keep it before any path use.) + let tmp = Path::new(image_dir).join(format!(".{}.partial", asset.name)); + let staging = Path::new(image_dir).join(format!(".{}.staging", asset.name)); + let _ = fs::remove_file(&tmp); + let _ = fs::remove_dir_all(&staging); + let adopted = stage_image(asset, image_dir, &tmp, &staging, insecure).await; + let _ = fs::remove_file(&tmp); + let _ = fs::remove_dir_all(&staging); + let name = adopted?; + + println!(" [ok] image ready: {name}"); + println!(" deploy with: dstackup install --image {name} (or: dstack deploy -c --image {name})"); + Ok(name) +} + +/// download, verify, unpack into `staging`, and atomically move the unpacked +/// image dir into `image_dir`. returns the image's (unpacked) directory name. +async fn stage_image( + asset: &Asset, + image_dir: &str, + tmp: &Path, + staging: &Path, + insecure: bool, +) -> Result { + download_verified( + &asset.browser_download_url, + tmp, + asset.digest.as_deref(), + insecure, + ) + .await?; + fs::create_dir_all(staging).with_context(|| format!("creating {}", staging.display()))?; + extract(&tmp.to_string_lossy(), &staging.to_string_lossy())?; + // the unpacked dir name needn't match the asset name (e.g. a `-uki` asset), + // so find the dir that actually holds a metadata.json. + let inner = image_subdirs(&staging.to_string_lossy()) + .into_iter() + .find(|d| staging.join(d).join("metadata.json").exists()) + .context("unpacked tarball has no image dir with a metadata.json")?; + let dest = Path::new(image_dir).join(&inner); + let _ = fs::remove_dir_all(&dest); + fs::rename(staging.join(&inner), &dest) + .with_context(|| format!("moving image into {}", dest.display()))?; + Ok(inner) +} + +/// stream the download to `dest`, hashing as it goes, and verify against the +/// release's `"sha256:"` digest in the same pass — fail closed on mismatch, +/// and fail closed when no digest is published unless `insecure`. github +/// 302-redirects to its object store; reqwest follows that by default. +/// +/// the `std::fs` writes here are synchronous inside an async fn; that's fine for +/// this single-task CLI (nothing else runs on the executor), and not worth a +/// `spawn_blocking` dance. +async fn download_verified( + url: &str, + dest: &Path, + expected: Option<&str>, + insecure: bool, +) -> Result<()> { + // fail closed BEFORE downloading hundreds of MB if we can't verify it. + if expected.is_none() && !insecure { + bail!("this release publishes no sha256 digest to verify the download against — pass --insecure to proceed unverified (not recommended)"); + } + let mut resp = reqwest::get(url) + .await + .with_context(|| format!("requesting {url}"))? + .error_for_status() + .with_context(|| format!("download failed from {url}"))?; + let total = resp.content_length(); + println!( + " [..] downloading{}...", + total + .map(|n| format!(" {} MB", n / 1_048_576)) + .unwrap_or_default() + ); + let mut file = + fs::File::create(dest).with_context(|| format!("creating {}", dest.display()))?; + let mut hasher = Sha256::new(); + let mut done: u64 = 0; + let mut next_pct = 25u64; + let mut next_bytes = 50 * 1_048_576u64; + while let Some(chunk) = resp.chunk().await.context("reading download stream")? { + hasher.update(&chunk); + file.write_all(&chunk) + .with_context(|| format!("writing {}", dest.display()))?; + done += chunk.len() as u64; + match total.filter(|t| *t > 0) { + // known size: percentage milestones. + Some(total) => { + let pct = done * 100 / total; + if pct >= next_pct { + println!(" [..] {pct}%"); + next_pct = (pct / 25 + 1) * 25; + } + } + // chunked / unknown size: byte milestones, so it's never silent. + None => { + if done >= next_bytes { + println!(" [..] {} MB", done / 1_048_576); + next_bytes += 50 * 1_048_576; + } + } + } + } + let _ = file.sync_all(); + + let Some(expected) = expected else { + println!(" [!] no sha256 digest published - integrity not verified (--insecure)"); + return Ok(()); + }; + let want = expected + .strip_prefix("sha256:") + .unwrap_or(expected) + .to_lowercase(); + let got = hex::encode(hasher.finalize()); + if got != want { + bail!("image checksum mismatch (expected {want}, got {got}) — refusing a tampered or corrupt download"); + } + println!(" [ok] sha256 verified"); + Ok(()) +} + +fn list(image_dir: &str) -> Result<()> { + let imgs = installed_images(image_dir); + if imgs.is_empty() { + println!("{}", no_image_message(image_dir)); + return Ok(()); + } + println!("images in {image_dir} (newest last):"); + for name in &imgs { + println!(" {name}"); + } + Ok(()) +} + +/// delete one or more local images by name (the `/` dir). +fn remove(names: &[String], image_dir: &str) -> Result<()> { + let mut removed = 0; + for name in names { + // a name must be a plain dir component — never a path that could escape + // image_dir (`..`, `/foo`) and delete something we don't own. + if !valid_image_name(name) { + bail!("invalid image name {name:?} (expected a plain image name, see `dstackup image list`)"); + } + let dir = Path::new(image_dir).join(name); + if !dir.is_dir() { + println!(" [!] {name}: not found in {image_dir}"); + continue; + } + fs::remove_dir_all(&dir).with_context(|| format!("removing {}", dir.display()))?; + println!(" [ok] removed {name}"); + removed += 1; + } + if removed == 0 { + bail!("removed nothing (see `dstackup image list`)"); + } + Ok(()) +} + +/// a removable image name is a single path component, never `.`/`..` or a path +/// (so `rm` can't be tricked into deleting outside the image dir). +fn valid_image_name(name: &str) -> bool { + !name.is_empty() + && name != "." + && name != ".." + && !name.starts_with('.') + && !name.contains('/') + && !name.contains('\\') +} + +/// resolve which guest image `install` should use: an explicit `--image` if +/// given, else the newest image present locally. `require` (KMS mode, which +/// boots a CVM at install time) makes "none" a hard error with download +/// guidance; otherwise it returns `None` and prints a gentle note. +pub(crate) fn resolve_image( + image_dir: &str, + requested: Option<&str>, + require: bool, +) -> Result> { + if let Some(name) = requested { + if !valid_image_name(name) { + bail!("invalid image name {name:?} (expected a plain image name, see `dstackup image list`)"); + } + if Path::new(image_dir) + .join(name) + .join("metadata.json") + .exists() + { + return Ok(Some(name.to_string())); + } + bail!("{}", missing_named_image_message(image_dir, name)); + } + let mut imgs = installed_images(image_dir); + if let Some(newest) = imgs.pop() { + if imgs.is_empty() { + println!(" [ok] using image {newest}"); + } else { + println!( + " [ok] using image {newest} (newest by fetch time; also present: {} — pass --image to choose)", + imgs.join(", ") + ); + } + return Ok(Some(newest)); + } + if require { + bail!("{}", no_image_message(image_dir)); + } + println!(" [!] no guest image in {image_dir} - `dstack deploy -c ` will need one (`dstackup image pull`)"); + Ok(None) +} + +/// resolve the image for install. If KMS mode needs an image and there is no +/// local image yet, fetch the latest CPU image through the same verified pull +/// path as `dstackup image pull`, then resolve from disk again. +pub(crate) async fn resolve_or_pull_image( + image_dir: &str, + requested: Option<&str>, + require: bool, + required_digest: Option<&str>, +) -> Result> { + if let Some(name) = requested { + if !valid_image_name(name) { + bail!("invalid image name {name:?} (expected a plain image name, see `dstackup image list`)"); + } + if Path::new(image_dir) + .join(name) + .join("metadata.json") + .exists() + { + return Ok(Some(name.to_string())); + } + if let Some(spec) = pull_spec(name) { + println!(" [..] image {name} not found locally; downloading it"); + let pulled = pull(Some(&spec.version), spec.gpu, image_dir, false, false).await?; + return Ok(Some(pulled)); + } + return resolve_image(image_dir, Some(name), require); + } + + let mut imgs = installed_images(image_dir); + let skipped = retain_images_with_digest(&mut imgs, image_dir, required_digest); + if let Some(newest) = imgs.pop() { + if !skipped.is_empty() { + println!( + " [!] ignoring image(s) without {}: {}", + required_digest.unwrap_or("required digest"), + skipped.join(", ") + ); + } + if imgs.is_empty() { + println!(" [ok] using image {newest}"); + } else { + println!( + " [ok] using image {newest} (newest by fetch time; also present: {} - pass --image to choose)", + imgs.join(", ") + ); + } + return Ok(Some(newest)); + } + + if !require { + if let Some(digest) = required_digest { + if skipped.is_empty() { + println!(" [!] no guest image in {image_dir} with {digest} - `dstack deploy -c ` will need one (`dstackup image pull`)"); + } else { + println!( + " [!] no guest image in {image_dir} with {digest}; ignored {} - `dstack deploy -c ` will need one (`dstackup image pull`)", + skipped.join(", ") + ); + } + } else { + println!(" [!] no guest image in {image_dir} - `dstack deploy -c ` will need one (`dstackup image pull`)"); + } + return Ok(None); + } + + if let Some(digest) = required_digest { + if skipped.is_empty() { + println!( + " [..] no local guest image with {digest} found; downloading the latest cpu image" + ); + } else { + println!( + " [..] no local guest image with {digest} found (ignored {}); downloading the latest cpu image", + skipped.join(", ") + ); + } + } else { + println!(" [..] no local guest image found; downloading the latest cpu image"); + } + let pulled = pull(None, false, image_dir, false, false).await?; + + if Path::new(image_dir) + .join(&pulled) + .join("metadata.json") + .exists() + { + Ok(Some(pulled)) + } else { + bail!("downloaded image {pulled}, but it is not available in {image_dir}") + } +} + +fn retain_images_with_digest( + imgs: &mut Vec, + image_dir: &str, + required_digest: Option<&str>, +) -> Vec { + let Some(required_digest) = required_digest else { + return Vec::new(); + }; + let mut skipped = Vec::new(); + imgs.retain(|name| { + let has_digest = Path::new(image_dir) + .join(name) + .join(required_digest) + .is_file(); + if !has_digest { + skipped.push(name.clone()); + } + has_digest + }); + skipped +} + +fn pull_spec(name: &str) -> Option { + if !valid_image_name(name) { + return None; + } + if let Some(version) = name.strip_prefix("dstack-nvidia-") { + return release_version(version).map(|version| PullSpec { version, gpu: true }); + } + if let Some(version) = name.strip_prefix("dstack-") { + return release_version(version).map(|version| PullSpec { + version, + gpu: false, + }); + } + release_version(name).map(|version| PullSpec { + version, + gpu: false, + }) +} + +fn release_version(version: &str) -> Option { + let version = version.trim_start_matches('v'); + let mut chars = version.chars(); + if !chars.next().is_some_and(|c| c.is_ascii_digit()) { + return None; + } + if !chars.all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '_')) { + return None; + } + Some(version.to_string()) +} + +/// the `dstackup image pull` invocation that targets `image_dir` — bare for the +/// default dir, else with the explicit `--image-path` so it's copy-paste correct. +fn pull_cmd(image_dir: &str) -> String { + if image_dir == resolve_image_dir(None, None) { + "dstackup image pull".to_string() + } else { + format!("dstackup image pull --image-path {image_dir}") + } +} + +/// the friendly "no image — here's how to get one" message. +pub(crate) fn no_image_message(image_dir: &str) -> String { + let pull = pull_cmd(image_dir); + format!( + "no guest image found in {image_dir}\n\n\ + download the latest with:\n \ + {pull} # cpu image\n \ + {pull} --gpu # gpu (nvidia) image\n\n\ + images are published at {RELEASES_URL}" + ) +} + +fn missing_named_image_message(image_dir: &str, name: &str) -> String { + let pull = pull_cmd(image_dir); + format!( + "image '{name}' not found in {image_dir}\n\n\ + download it with:\n \ + {pull} --version \n\n\ + or see what's available locally:\n \ + dstackup image list" + ) +} + +/// GET the latest (or a tagged) release JSON from the github api. +async fn fetch_release(version: Option<&str>) -> Result { + let url = match version { + Some(v) => format!( + "https://api.github.com/repos/{REPO}/releases/tags/v{}", + v.trim_start_matches('v') + ), + None => format!("https://api.github.com/repos/{REPO}/releases/latest"), + }; + reqwest::Client::new() + .get(&url) + .header("user-agent", "dstackup") + .header("accept", "application/vnd.github+json") + .send() + .await + .context("requesting the github release")? + .error_for_status() + .with_context(|| { + format!("github release lookup failed; check the version exists at {RELEASES_URL}") + })? + .json() + .await + .context("parsing github release json") +} + +/// pick the cpu or gpu image tarball from a release's assets, skipping `-dev` +/// builds. cpu = `dstack-...`, gpu = `dstack-nvidia-...`. +fn pick_asset(assets: &[Asset], gpu: bool) -> Option<&Asset> { + assets.iter().find(|a| { + let n = a.name.as_str(); + if !n.ends_with(".tar.gz") || n.contains("-dev") { + return false; + } + let is_gpu = n.starts_with("dstack-nvidia-"); + if gpu { + is_gpu + } else { + n.starts_with("dstack-") && !is_gpu + } + }) +} + +fn extract(tarball: &str, into: &str) -> Result<()> { + println!(" [..] unpacking..."); + // `tar` already refuses absolute/`..` members; drop owner/perms from the + // (root-run) extraction so a hostile member set can't carry setuid/ownership. + let ok = tool("tar") + .args([ + "-xzf", + tarball, + "-C", + into, + "--no-same-owner", + "--no-same-permissions", + ]) + .status() + .context("running tar")? + .success(); + if !ok { + bail!("failed to unpack {tarball}"); + } + Ok(()) +} + +/// subdirectory names directly under `dir`, excluding dot-prefixed entries (our +/// `.partial`/`.staging` scratch, and never a real image name). +fn image_subdirs(dir: &str) -> Vec { + let Ok(rd) = fs::read_dir(dir) else { + return Vec::new(); + }; + rd.flatten() + .filter(|e| e.path().is_dir()) + .filter_map(|e| e.file_name().into_string().ok()) + .filter(|n| !n.starts_with('.')) + .collect() +} + +/// valid local images (a subdir with a `metadata.json`), oldest first so the +/// caller can `.pop()` the newest. "newest" = most recently fetched (mtime), +/// which is the right default after a `pull`. +fn installed_images(image_dir: &str) -> Vec { + let mut v: Vec<(SystemTime, String)> = image_subdirs(image_dir) + .into_iter() + .filter(|d| Path::new(image_dir).join(d).join("metadata.json").exists()) + .map(|d| { + let mtime = fs::metadata(Path::new(image_dir).join(&d)) + .and_then(|m| m.modified()) + .unwrap_or(SystemTime::UNIX_EPOCH); + (mtime, d) + }) + .collect(); + v.sort_by_key(|(t, _)| *t); + v.into_iter().map(|(_, n)| n).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn asset(name: &str) -> Asset { + Asset { + name: name.to_string(), + browser_download_url: format!("https://x/{name}"), + digest: None, + } + } + + #[test] + fn picks_cpu_and_gpu_skipping_dev() { + let assets = vec![ + asset("dstack-dev-0.5.11.tar.gz"), + asset("dstack-0.5.11.tar.gz"), + asset("dstack-nvidia-dev-0.5.11.tar.gz"), + asset("dstack-nvidia-0.5.11.tar.gz"), + asset("checksums.txt"), + ]; + assert_eq!( + pick_asset(&assets, false).unwrap().name, + "dstack-0.5.11.tar.gz" + ); + assert_eq!( + pick_asset(&assets, true).unwrap().name, + "dstack-nvidia-0.5.11.tar.gz" + ); + } + + #[test] + fn gpu_only_release_has_no_cpu_asset() { + let assets = vec![asset("dstack-nvidia-0.6.0.a2-uki.tar.gz")]; + assert!(pick_asset(&assets, false).is_none()); + assert_eq!( + pick_asset(&assets, true).unwrap().name, + "dstack-nvidia-0.6.0.a2-uki.tar.gz" + ); + } + + #[test] + fn messages_mention_the_pull_command() { + assert!(no_image_message("/d").contains("dstackup image pull")); + assert!(missing_named_image_message("/d", "x").contains("dstackup image pull")); + } + + #[test] + fn rm_rejects_path_escapes() { + assert!(valid_image_name("dstack-0.5.11")); + for bad in ["", ".", "..", ".partial", "/etc", "a/b", "..\\x"] { + assert!(!valid_image_name(bad), "{bad:?} should be rejected"); + } + } + + #[test] + fn image_dir_rejects_root_and_relative_paths() { + for bad in ["/", "images", "/var/lib/../dstack/images"] { + assert!( + validate_image_dir(bad).is_err(), + "{bad:?} should be rejected" + ); + } + validate_image_dir("/var/lib/dstack/images").unwrap(); + } + + #[test] + fn parses_requested_image_for_pull() { + let cpu = pull_spec("dstack-0.5.11").unwrap(); + assert_eq!(cpu.version, "0.5.11"); + assert!(!cpu.gpu); + + let gpu = pull_spec("dstack-nvidia-0.5.11").unwrap(); + assert_eq!(gpu.version, "0.5.11"); + assert!(gpu.gpu); + + let bare = pull_spec("v0.5.11").unwrap(); + assert_eq!(bare.version, "0.5.11"); + assert!(!bare.gpu); + + assert!(pull_spec("").is_none()); + assert!(pull_spec("custom-local-image").is_none()); + assert!(pull_spec("dstack-dev-0.5.11").is_none()); + assert!(pull_spec("a/b").is_none()); + } +} diff --git a/crates/dstackup/src/install.rs b/crates/dstackup/src/install.rs new file mode 100644 index 00000000..7a0dbc49 --- /dev/null +++ b/crates/dstackup/src/install.rs @@ -0,0 +1,1192 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! `dstackup install` — bring up the host stack and bootstrap the KMS-in-CVM. + +use crate::cid::pick_cid_start; +use crate::cli::{InstallOpts, DEFAULT_AUTH_BIN, DEFAULT_SUPERVISOR_BIN, DEFAULT_VMM_BIN}; +use crate::state::{read_state, write, write_state, State}; +use crate::systemd::{auth_unit_file, install_unit, tool, unit_active, unit_name, vmm_unit_file}; +use anyhow::{bail, Context, Result}; +use dstack_cli_core::config::{self, HostConfig, VmmRender}; +use dstack_cli_core::host::Platform; +use dstack_cli_core::layout::{path_string, InstallLayout}; +use dstack_cli_core::vmm::Vmm; +use dstack_cli_core::{host, ports, rpc}; +use std::env; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command as PCommand; +use std::time::Duration; + +const USER_BINARIES: &[(&str, &str)] = &[("dstack", "dstack")]; + +const DAEMON_BINARIES: &[(&str, &str)] = &[ + ("dstack-auth", "dstack-auth"), + ("dstack-vmm", "dstack-vmm"), + ("supervisor", "supervisor"), +]; + +pub(crate) async fn cmd_install(mut o: InstallOpts) -> Result<()> { + // --expose is not safe yet: the rendered vmm.toml binds the VM-control + // plane with neither TLS nor an auth token (the management RPCs are not + // behind an auth guard), so exposing it would hand deploy/destroy to anyone + // who can reach the IP. Refuse until the TLS+token transport lands; the + // supported path is localhost + an SSH tunnel. + if let Some(ip) = &o.expose { + bail!( + "--expose {ip} is not yet safe: it would bind the VM-control plane on \ + {ip}:{port} with no TLS and no auth. reach the dashboard over an SSH \ + tunnel instead: ssh -L {port}:127.0.0.1:{port} ", + port = o.dashboard_port + ); + } + + println!("dstackup install — preflight"); + + // 1. resolve the platform (auto-detects the host) and gate on the host + // actually supporting it: TDX needs SGX (local key provider); SNP needs + // /dev/sev. + let platform = match host::Platform::parse_opt(&o.platform)? { + Some(p) => p, + None => host::Platform::detect().unwrap_or(host::Platform::Tdx), + }; + host::require_platform(platform)?; + println!(" [ok] platform: {}", platform.vmm_str()); + + // 2. host IP (informational; used as the bind/SAN when --expose is set). + match host::detect_host_ip() { + Ok(ip) if host::is_link_local(&ip) => { + println!(" [!] host ip {ip} is link-local") + } + Ok(ip) => println!(" [ok] host ip: {ip}"), + Err(e) => println!(" [!] could not detect host ip: {e}"), + } + + // 3. resolve paths (no side effects yet). The image dir resolves through the + // same helper the `image` subcommands use, so `install --prefix X` and + // `image pull --prefix X` always agree on where images live. + let mut layout = InstallLayout::new(o.prefix.as_deref()); + apply_layout_overrides(&mut layout, &o); + validate_layout(&layout)?; + validate_install_opts(&o)?; + let explicit_prefix = o.prefix.is_some(); + let images = crate::image::resolve_image_dir(o.image_path.as_deref(), o.prefix.as_deref()); + crate::image::validate_image_dir(&images)?; + let mut st = read_state(&layout.state_dir).unwrap_or_default(); + + let bind = o.expose.clone().unwrap_or_else(|| "127.0.0.1".to_string()); + let dashboard_addr = format!("tcp:{bind}:{}", o.dashboard_port); + let client_url = format!("http://{bind}:{}", o.dashboard_port); + let kms_port = resolve_kms_port(&o, &st)?; + + // 4. preflight - fail BEFORE any side effect (image download, key provider, + // dirs, units), so a CID/port clash can't half-install the host. + let cid_start = pick_cid_start(o.cid_start, &host::occupied_cid_ranges())?; + let kms_owned = kms_port_owned(&st, &client_url, kms_port, o.no_kms).await; + let port_plan = tcp_port_plan(&o, &st, platform, &bind, &client_url, kms_port, kms_owned); + preflight_ports(&port_plan)?; + + // 5. resolve the guest image: explicit --image, else the newest present + // locally. In KMS mode, bootstrap needs an image now; if the image store is + // empty, download the latest CPU image through the verified image path. + // Pinning is validated before installing managed binaries, so an + // incompatible image fails without leaving a half-built host install. + let required_digest = + (!o.no_kms && !o.allow_unpinned_image).then_some(os_image_digest_file(platform)); + o.image = crate::image::resolve_or_pull_image( + &images, + o.image.as_deref(), + !o.no_kms, + required_digest, + ) + .await?; + let os_image_hash = resolve_image_pin(&o, &images, platform)?; + + // 6. install the binaries managed by dstackup. The bootstrap installer only + // installs dstackup; this step owns the local dstack CLI and host daemons. + prepare_managed_binaries(&mut o, &layout)?; + + // 7. lay out the installation directories. + for dir in [ + layout.config_dir.clone(), + layout.state_dir.clone(), + layout.state_dir.join("certs"), + layout.run_dir.clone(), + ] { + fs::create_dir_all(&dir).with_context(|| format!("creating {}", dir.display()))?; + } + + // 8. resolve the key provider - run our own unless told to use an existing + // one (TDX only; SNP has no SGX gramine provider). + let (kp_addr, kp_port, kp_own_project) = + resolve_key_provider(&o, platform, !o.no_start, &layout)?; + + // KMS host port + URL, resolved during preflight so conflicts are caught + // before image download, builds, or systemd writes. + let kms_urls = if o.no_kms { + vec![] + } else { + vec![format!("https://10.0.2.2:{kms_port}")] + }; + + // 9. render configs. + let vmm = config::vmm_toml(&VmmRender { + dashboard_addr: dashboard_addr.clone(), + image_path: images.clone(), + qemu_path: o.qemu.clone(), + run_dir: layout.run_dir.display().to_string(), + vm_path: layout.state_dir.join("vm").display().to_string(), + supervisor_exe: o.supervisor_bin.clone(), + cid_start, + host_api_port: o.host_api_port, + key_provider_addr: kp_addr, + key_provider_port: kp_port as u32, + kms_urls: kms_urls.clone(), + platform, + ..Default::default() + }); + // the KMS-in-CVM reaches the host auth webhook at 10.0.2.2:. + // The KMS's own image download-verify stays off for the single-node flow + // (it would need a published image source), but we PIN the app OS image in + // the webhook allowlist (resolved in preflight, fail-closed): digest.txt + // holds the measured image hash the KMS reports for an app, so an app cannot + // boot under a different, unmeasured image and still receive keys. + // bootAuth/kms ignores osImages, so the KMS bootstrap itself is unaffected. + let host_cfg = HostConfig { + auth_webhook_url: format!("http://10.0.2.2:{}", o.auth_port), + os_image_hash: os_image_hash.unwrap_or_default(), + verify_os_image: false, + platform, + ..Default::default() + }; + let kms = config::kms_toml(&host_cfg); + let allowlist = config::auth_allowlist_json(&host_cfg); + + let vmm_path = layout.config_dir.join("vmm.toml"); + let kms_path = layout.config_dir.join("kms.toml"); + let allow_path = layout.config_dir.join("auth-allowlist.json"); + write(&vmm_path, &vmm)?; + write(&kms_path, &kms)?; + write(&allow_path, &allowlist)?; + println!( + " [ok] wrote {}, {}, {}", + vmm_path.display(), + kms_path.display(), + allow_path.display() + ); + + if o.no_start { + println!(" (--no-start: configs written; not starting any process)"); + return Ok(()); + } + + st.prefix = layout.state_dir.display().to_string(); + st.install_prefix = layout.root.as_ref().map(|p| p.display().to_string()); + st.config_dir = layout.config_dir.display().to_string(); + st.state_dir = layout.state_dir.display().to_string(); + st.cache_dir = layout.cache_dir.display().to_string(); + st.run_dir = layout.run_dir.display().to_string(); + st.allowlist_path = allow_path.display().to_string(); + st.client_url = client_url.clone(); + st.auth_port = o.auth_port; + st.platform = platform.vmm_str().to_string(); + st.image = o.image.clone(); + let instance = effective_instance(&o, &layout, explicit_prefix); + let auth_unit = unit_name("auth", &instance); + let vmm_unit = unit_name("vmm", &instance); + + // 10. auth webhook systemd unit (idempotent). + if unit_active(&auth_unit) { + println!(" [ok] {auth_unit}.service already active"); + } else { + install_unit( + &auth_unit, + &auth_unit_file(&o.auth_bin, &allow_path, o.auth_port, &layout.state_dir), + ) + .context("installing the auth webhook unit")?; + println!( + " [ok] started {auth_unit}.service on 127.0.0.1:{}", + o.auth_port + ); + } + st.auth_unit = auth_unit.clone(); + + // 11. VMM systemd unit (idempotent). + if vmm_reachable(&client_url).await { + println!(" [ok] VMM already serving at {client_url}"); + } else { + install_unit( + &vmm_unit, + &vmm_unit_file(&o.vmm_bin, &vmm_path, &layout.state_dir, &auth_unit), + ) + .context("installing the VMM unit")?; + println!(" [ok] started {vmm_unit}.service"); + print!(" [..] waiting for VMM at {client_url} "); + if wait_ready(&client_url, Duration::from_secs(25)).await { + println!("=> ready"); + } else { + println!("=> not ready within timeout (journalctl -u {vmm_unit})"); + } + } + st.vmm_unit = vmm_unit.clone(); + + // persist what we have so far (so a later step / destroy can see it). + st.kp_own_project = kp_own_project; + write_state(&layout.state_dir, &st)?; + + // 12. deploy + bootstrap the KMS-in-CVM (idempotent). + if o.no_kms { + println!(" (--no-kms: skipping KMS deploy)"); + } else { + let vmm = Vmm::connect(&client_url)?; + let existing = match &st.kms_vm_id { + Some(id) if vmm.has_vm(id).await => Some(id.clone()), + _ => None, + }; + if let Some(id) = existing { + println!(" [ok] KMS CVM already deployed (vm {id})"); + } else { + let img = o + .image + .clone() + .context("kms deploy needs --image (or pass --no-kms)")?; + let compose = config::kms_app_compose(&kms, &o.kms_image, platform); + let cfg = rpc::VmConfiguration { + name: "dstack-kms".into(), + image: img.clone(), + compose_file: compose, + vcpu: 4, + memory: 8192, + disk_size: 20, + ports: vec![rpc::PortMapping { + protocol: "tcp".into(), + host_address: "127.0.0.1".into(), + host_port: kms_port as u32, + vm_port: 8000, + }], + ..Default::default() + }; + println!(" [..] deploying KMS CVM (os {img}, kms {})", o.kms_image); + let vm_id = vmm + .create_vm(cfg) + .await + .context("createVm for the kms cvm failed")?; + print!(" [..] waiting for KMS CVM boot (vm {vm_id}) "); + match wait_kms_vm_booted(&vmm, &vm_id, Duration::from_secs(240)).await { + KmsVmBootState::Ready => println!("=> booted"), + KmsVmBootState::Failed(reason) => { + println!("=> failed ({reason}; check `dstack logs {vm_id}` / VMM log)") + } + KmsVmBootState::Pending => { + println!("=> not ready in time (check `dstack logs {vm_id}` / VMM log)") + } + } + st.kms_vm_id = Some(vm_id); + st.kms_url = format!("https://10.0.2.2:{kms_port}"); + write_state(&layout.state_dir, &st)?; + } + } + + println!(); + println!("dashboard: {client_url} (localhost — reach it via an SSH tunnel)"); + if !st.kms_url.is_empty() { + println!( + "kms: {} (apps reach it via this address)", + st.kms_url + ); + } + let dstack_cmd = if layout.is_default() { + "dstack".to_string() + } else { + path_string(&layout.bin_dir.join("dstack")) + }; + println!( + "deploy an app with: sudo {dstack_cmd} deploy -c {} --port :", + layout.hello_nginx_compose().display() + ); + Ok(()) +} + +fn apply_layout_overrides(layout: &mut InstallLayout, o: &InstallOpts) { + if let Some(bin_dir) = &o.bin_dir { + layout.bin_dir = PathBuf::from(bin_dir); + } + if let Some(libexec_dir) = &o.libexec_dir { + layout.libexec_dir = PathBuf::from(libexec_dir); + } + if let Some(share_dir) = &o.share_dir { + layout.share_dir = PathBuf::from(share_dir); + } +} + +fn validate_layout(layout: &InstallLayout) -> Result<()> { + layout.validate() +} + +fn effective_instance( + o: &InstallOpts, + layout: &InstallLayout, + explicit_prefix: bool, +) -> Option { + o.instance.clone().or_else(|| { + explicit_prefix + .then(|| layout.root.as_deref().map(prefix_instance)) + .flatten() + }) +} + +fn prefix_instance(prefix: &Path) -> String { + let base = prefix + .file_name() + .and_then(|name| name.to_str()) + .filter(|name| !name.is_empty()) + .unwrap_or("prefix"); + let slug = slugify_unit_part(base); + format!( + "{slug}-{:08x}", + fnv1a32(prefix.to_string_lossy().as_bytes()) + ) +} + +fn slugify_unit_part(input: &str) -> String { + let mut out = String::new(); + let mut last_dash = false; + for c in input.chars() { + let valid = c.is_ascii_alphanumeric(); + if valid { + out.push(c.to_ascii_lowercase()); + last_dash = false; + } else if !last_dash { + out.push('-'); + last_dash = true; + } + } + let slug = out.trim_matches('-'); + if slug.is_empty() { + "prefix".to_string() + } else { + slug.to_string() + } +} + +fn fnv1a32(bytes: &[u8]) -> u32 { + let mut hash = 0x811c9dc5u32; + for b in bytes { + hash ^= u32::from(*b); + hash = hash.wrapping_mul(0x01000193); + } + hash +} + +fn prepare_managed_binaries(o: &mut InstallOpts, layout: &InstallLayout) -> Result<()> { + if o.skip_managed_binaries { + println!(" [ok] using configured binaries (--skip-managed-binaries)"); + return Ok(()); + } + + if o.vmm_bin == DEFAULT_VMM_BIN { + o.vmm_bin = layout + .libexec_dir + .join(DEFAULT_VMM_BIN) + .display() + .to_string(); + } + if o.auth_bin == DEFAULT_AUTH_BIN { + o.auth_bin = layout + .libexec_dir + .join(DEFAULT_AUTH_BIN) + .display() + .to_string(); + } + if o.supervisor_bin == DEFAULT_SUPERVISOR_BIN { + o.supervisor_bin = layout + .libexec_dir + .join(DEFAULT_SUPERVISOR_BIN) + .display() + .to_string(); + } + + if o.no_start { + println!( + " [ok] managed binary targets {}, {} (--no-start: not building)", + layout.bin_dir.display(), + layout.libexec_dir.display() + ); + return Ok(()); + } + + let source = resolve_source_checkout(o, layout)?; + let target_dir = layout.cargo_target_dir(); + create_build_owned_dir(&target_dir)?; + build_managed_binaries(&source, &target_dir)?; + install_managed_binaries(&target_dir, layout)?; + install_share_assets(&source, layout)?; + println!( + " [ok] installed dstack binaries into {}, host daemons into {}", + layout.bin_dir.display(), + layout.libexec_dir.display() + ); + Ok(()) +} + +fn resolve_source_checkout(o: &InstallOpts, layout: &InstallLayout) -> Result { + if let Some(source) = o.source.as_deref() { + return checked_source_checkout(PathBuf::from(source)); + } + + let cwd = env::current_dir().context("resolving current directory")?; + if is_dstack_checkout(&cwd) { + return checked_source_checkout(cwd); + } + + let source = layout.source_dir(); + sync_source_cache(&source, &o.source_repo, &o.source_ref)?; + checked_source_checkout(source) +} + +fn checked_source_checkout(dir: PathBuf) -> Result { + if !is_dstack_checkout(&dir) { + bail!("{} is not a dstack source checkout", dir.display()); + } + dir.canonicalize() + .with_context(|| format!("canonicalizing {}", dir.display())) +} + +fn sync_source_cache(source: &Path, repo: &str, git_ref: &str) -> Result<()> { + let parent = source + .parent() + .with_context(|| format!("{} has no parent directory", source.display()))?; + create_build_owned_dir(parent)?; + + if source.exists() { + if !is_dstack_checkout(source) || !source.join(".git").is_dir() { + bail!( + "{} exists but is not a dstack git checkout; pass --source DIR or remove the cache", + source.display() + ); + } + println!(" [..] updating dstack source cache {}", source.display()); + run_git_at( + source, + ["fetch", "--tags", "origin"], + "fetching dstack source", + )?; + run_git_at( + source, + ["checkout", git_ref], + "checking out dstack source ref", + )?; + let remote_ref = format!("origin/{git_ref}"); + if git_status_at(source, ["rev-parse", "--verify", &remote_ref])? { + run_git_at( + source, + ["pull", "--ff-only", "origin", git_ref], + "fast-forwarding dstack source", + )?; + } + } else { + println!(" [..] cloning dstack source into {}", source.display()); + let mut cmd = git_command(); + let status = cmd + .arg("clone") + .arg(repo) + .arg(source) + .status() + .context("cloning dstack source")?; + if !status.success() { + bail!("failed to clone dstack source from {repo}"); + } + run_git_at( + source, + ["fetch", "--tags", "origin"], + "fetching dstack tags", + )?; + run_git_at( + source, + ["checkout", git_ref], + "checking out dstack source ref", + )?; + } + Ok(()) +} + +fn run_git_at(dir: &Path, args: [&str; N], what: &str) -> Result<()> { + let mut cmd = git_command(); + let status = cmd + .arg("-C") + .arg(dir) + .args(args) + .status() + .with_context(|| format!("{what} in {}", dir.display()))?; + if !status.success() { + bail!("{what} failed in {}", dir.display()); + } + Ok(()) +} + +fn git_status_at(dir: &Path, args: [&str; N]) -> Result { + Ok(git_command() + .arg("-C") + .arg(dir) + .args(args) + .status() + .with_context(|| format!("running git in {}", dir.display()))? + .success()) +} + +fn is_dstack_checkout(dir: &Path) -> bool { + dir.join("Cargo.toml").is_file() + && dir.join("crates/dstack-cli").is_dir() + && dir.join("crates/dstack-auth").is_dir() + && dir.join("vmm").is_dir() + && dir.join("supervisor").is_dir() +} + +fn build_managed_binaries(source: &Path, target_dir: &Path) -> Result<()> { + let mut cmd = cargo_build_command(target_dir)?; + let target_dir_arg = path_string(target_dir); + cmd.current_dir(source).args([ + "build", + "--release", + "--target-dir", + &target_dir_arg, + "-p", + "dstack-cli", + "-p", + "dstack-auth", + "-p", + "dstack-vmm", + "-p", + "supervisor", + ]); + let status = cmd.status().context("building managed dstack binaries")?; + if !status.success() { + bail!("failed to build managed dstack binaries"); + } + Ok(()) +} + +fn cargo_build_command(target_dir: &Path) -> Result { + if let Some((user, home)) = sudo_build_user() { + let cargo_home = home.join(".cargo/bin"); + let mut cmd = tool("sudo"); + cmd.args(["-H", "-u", &user, "env"]); + cmd.arg(format!( + "PATH={}:{}", + cargo_home.display(), + "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + )); + cmd.arg(format!("CARGO_TARGET_DIR={}", target_dir.display())); + cmd.arg("cargo"); + return Ok(cmd); + } + + let cargo = + find_cargo().context("could not find cargo; install Rust before running install")?; + let mut cmd = PCommand::new(cargo); + if let Some(path) = env::var_os("PATH") { + cmd.env("PATH", path); + } + cmd.env("CARGO_TARGET_DIR", target_dir); + Ok(cmd) +} + +fn git_command() -> PCommand { + if let Some((user, home)) = sudo_build_user() { + let cargo_home = home.join(".cargo/bin"); + let mut cmd = tool("sudo"); + cmd.args(["-H", "-u", &user, "env"]); + cmd.arg(format!( + "PATH={}:{}", + cargo_home.display(), + "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + )); + cmd.arg("git"); + return cmd; + } + tool("git") +} + +fn sudo_build_user() -> Option<(String, PathBuf)> { + let user = env::var("SUDO_USER").ok()?; + if user.is_empty() || user == "root" { + return None; + } + let home = user_home(&user)?; + if !home.join(".cargo/bin/cargo").is_file() { + return None; + } + Some((user, home)) +} + +fn user_home(user: &str) -> Option { + let out = tool("getent").args(["passwd", user]).output().ok()?; + if !out.status.success() { + return None; + } + let body = String::from_utf8(out.stdout).ok()?; + let home = body.lines().next()?.split(':').nth(5)?; + Some(PathBuf::from(home)) +} + +fn find_cargo() -> Option { + if let Some(cargo) = env::var_os("CARGO").map(PathBuf::from) { + if cargo.is_file() { + return Some(cargo); + } + } + if let Some(paths) = env::var_os("PATH") { + for dir in env::split_paths(&paths) { + let candidate = dir.join("cargo"); + if candidate.is_file() { + return Some(candidate); + } + } + } + let home = env::var_os("HOME").map(PathBuf::from)?; + let cargo = home.join(".cargo/bin/cargo"); + cargo.is_file().then_some(cargo) +} + +fn create_build_owned_dir(dir: &Path) -> Result<()> { + if let Some((user, _)) = sudo_build_user() { + let status = tool("install") + .args(["-d", "-m", "0755", "-o", &user]) + .arg(dir) + .status() + .with_context(|| format!("creating {}", dir.display()))?; + if !status.success() { + bail!("failed to create {}", dir.display()); + } + return Ok(()); + } + fs::create_dir_all(dir).with_context(|| format!("creating {}", dir.display())) +} + +fn create_install_dir(dir: &Path) -> Result<()> { + let status = tool("install") + .args(["-d", "-m", "0755"]) + .arg(dir) + .status() + .with_context(|| format!("creating {}", dir.display()))?; + if !status.success() { + bail!("failed to create {}", dir.display()); + } + Ok(()) +} + +fn install_managed_binaries(target_dir: &Path, layout: &InstallLayout) -> Result<()> { + create_install_dir(&layout.bin_dir)?; + create_install_dir(&layout.libexec_dir)?; + + for (built, installed, dest_dir) in USER_BINARIES + .iter() + .map(|(built, installed)| (*built, *installed, &layout.bin_dir)) + .chain( + DAEMON_BINARIES + .iter() + .map(|(built, installed)| (*built, *installed, &layout.libexec_dir)), + ) + { + let src = target_dir.join("release").join(built); + if !src.is_file() { + bail!("expected built binary {}", src.display()); + } + let dest = dest_dir.join(installed); + let status = tool("install") + .args(["-m", "0755"]) + .arg(&src) + .arg(&dest) + .status() + .with_context(|| format!("installing {}", dest.display()))?; + if !status.success() { + bail!("failed to install {}", dest.display()); + } + } + Ok(()) +} + +fn install_share_assets(source: &Path, layout: &InstallLayout) -> Result<()> { + fs::create_dir_all(&layout.share_dir) + .with_context(|| format!("creating {}", layout.share_dir.display()))?; + copy_dir_exact( + &source.join("key-provider-build"), + &layout.key_provider_dir(), + )?; + copy_dir_exact(&source.join("examples"), &layout.share_dir.join("examples"))?; + println!( + " [ok] installed assets into {}", + layout.share_dir.display() + ); + Ok(()) +} + +fn copy_dir_exact(src: &Path, dest: &Path) -> Result<()> { + if !src.is_dir() { + bail!("required asset directory missing: {}", src.display()); + } + if dest.exists() { + fs::remove_dir_all(dest).with_context(|| format!("removing {}", dest.display()))?; + } + copy_dir_all(src, dest) +} + +fn copy_dir_all(src: &Path, dest: &Path) -> Result<()> { + fs::create_dir_all(dest).with_context(|| format!("creating {}", dest.display()))?; + for entry in fs::read_dir(src).with_context(|| format!("reading {}", src.display()))? { + let entry = entry.with_context(|| format!("reading {}", src.display()))?; + let src_path = entry.path(); + let dest_path = dest.join(entry.file_name()); + let metadata = entry + .metadata() + .with_context(|| format!("reading metadata for {}", src_path.display()))?; + if metadata.is_dir() { + copy_dir_all(&src_path, &dest_path)?; + } else if metadata.is_file() { + fs::copy(&src_path, &dest_path).with_context(|| { + format!("copying {} to {}", src_path.display(), dest_path.display()) + })?; + fs::set_permissions(&dest_path, metadata.permissions()) + .with_context(|| format!("setting permissions on {}", dest_path.display()))?; + } + } + Ok(()) +} + +/// resolve the key provider for this install. Returns (addr, port, own_project). +fn resolve_key_provider( + o: &InstallOpts, + platform: Platform, + start: bool, + layout: &InstallLayout, +) -> Result<(String, u16, Option)> { + if let Some(ep) = &o.use_existing_key_provider { + let (addr, port) = split_addr_port(ep)?; + println!(" [ok] using existing key provider at {addr}:{port}"); + return Ok((addr, port, None)); + } + // AMD SEV-SNP has no SGX gramine key provider; the rendered [key_provider] + // block is unused (the KMS-in-CVM runs with local_key_provider_enabled = + // false), so don't require or start one. + if platform == Platform::AmdSevSnp { + println!(" [ok] no local key provider (sev-snp)"); + return Ok(("127.0.0.1".to_string(), o.key_provider_port, None)); + } + // TDX: run our own gramine provider from the installed static assets unless + // the operator points at an external provider or build directory. + let default_key_provider_src = layout.key_provider_dir(); + let src = match o.key_provider_src.as_deref() { + Some(src) => PathBuf::from(src), + None if !start + || default_key_provider_src + .join("docker-compose.yaml") + .exists() => + { + println!( + " [ok] using key provider source {}", + default_key_provider_src.display() + ); + default_key_provider_src + } + None => { + bail!( + "no key provider: pass --use-existing-key-provider ADDR:PORT, \ + or --key-provider-src DIR to run our own" + ) + } + }; + let project = format!("dstack-kp-{}", o.key_provider_port); + if !start { + println!( + " [ok] key provider source {} selected (not started because --no-start was passed)", + src.display() + ); + return Ok(("127.0.0.1".to_string(), o.key_provider_port, None)); + } + let status = tool("docker") + .args(["compose", "-p", &project, "-f"]) + .arg(src.join("docker-compose.yaml")) + .args(["up", "-d"]) + .status() + .context("running docker compose for the key provider")?; + if !status.success() { + bail!("failed to start our own key provider (docker compose up)"); + } + println!( + " [ok] started our own key provider (project {project}, :{})", + o.key_provider_port + ); + Ok(("127.0.0.1".to_string(), o.key_provider_port, Some(project))) +} + +fn split_addr_port(ep: &str) -> Result<(String, u16)> { + let (addr, port) = ep + .rsplit_once(':') + .with_context(|| format!("expected ADDR:PORT, got '{ep}'"))?; + Ok(( + addr.to_string(), + port.parse() + .with_context(|| format!("bad port in '{ep}'"))?, + )) +} + +fn os_image_digest_file(platform: Platform) -> &'static str { + match platform { + Platform::AmdSevSnp => "digest.sev.txt", + Platform::Tdx => "digest.txt", + } +} + +/// read the measured OS-image hash from the guest image's platform-specific +/// digest file, used to pin which image apps may boot. +/// Returns None when there's no image selected or no readable digest. +fn resolve_os_image_hash(images: &str, image: Option<&str>, platform: Platform) -> Option { + let img = image?; + let digest_file = os_image_digest_file(platform); + let path = Path::new(images).join(img).join(digest_file); + let hash = fs::read_to_string(path).ok()?.trim().to_string(); + (!hash.is_empty()).then_some(hash) +} + +/// resolve the OS-image pin, failing CLOSED: in KMS mode a missing/empty +/// platform digest is a hard error (an unpinned app could boot any unmeasured +/// image and still get keys), unless the operator opts out with +/// `--allow-unpinned-image`. Returns Some(hash) to pin, or None when pinning +/// is deliberately off (`--no-kms`, or the explicit opt-out). +fn resolve_image_pin(o: &InstallOpts, images: &str, platform: Platform) -> Result> { + let hash = resolve_os_image_hash(images, o.image.as_deref(), platform); + match &hash { + Some(h) => println!(" [ok] pinning app os image {h}"), + None if o.no_kms => {} + None if o.allow_unpinned_image => { + println!(" [!] app os image not pinned (--allow-unpinned-image) - apps' image is unchecked") + } + None => bail!( + "no os-image pin: could not read {digest_file} for image {:?} under {images}. \ + {} apps must be pinned to the measured OS image before they can receive keys. \ + use --image/--image-path with an image that contains {digest_file}, or pass \ + --allow-unpinned-image to proceed unpinned (not recommended)", + o.image.as_deref().unwrap_or(""), + platform.vmm_str(), + digest_file = os_image_digest_file(platform), + ), + } + Ok(hash) +} + +fn resolve_kms_port(o: &InstallOpts, st: &State) -> Result { + if o.no_kms { + return Ok(0); + } + if let Some(p) = o.kms_port { + return Ok(p); + } + if let Some(p) = state_kms_port(st) { + return Ok(p); + } + ports::free_local_port() +} + +fn state_kms_port(st: &State) -> Option { + st.kms_url.rsplit(':').next().and_then(|s| s.parse().ok()) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct TcpPortCheck { + what: &'static str, + flag: &'static str, + addr: String, + port: u16, + check_free: bool, +} + +fn tcp_port_plan( + o: &InstallOpts, + st: &State, + platform: Platform, + bind: &str, + client_url: &str, + kms_port: u16, + kms_owned: bool, +) -> Vec { + let auth_owned = + st.auth_port == o.auth_port && !st.auth_unit.is_empty() && unit_active(&st.auth_unit); + let vmm_owned = + st.client_url == client_url && !st.vmm_unit.is_empty() && unit_active(&st.vmm_unit); + + let mut ports = vec![ + TcpPortCheck { + what: "dashboard", + flag: "--dashboard-port", + addr: bind.to_string(), + port: o.dashboard_port, + check_free: !vmm_owned, + }, + TcpPortCheck { + what: "auth webhook", + flag: "--auth-port", + addr: "127.0.0.1".to_string(), + port: o.auth_port, + check_free: !auth_owned, + }, + ]; + if !o.no_kms { + ports.push(TcpPortCheck { + what: "kms", + flag: "--kms-port", + addr: "127.0.0.1".to_string(), + port: kms_port, + check_free: !kms_owned, + }); + } + if platform == Platform::Tdx && o.use_existing_key_provider.is_none() { + let expected_project = format!("dstack-kp-{}", o.key_provider_port); + let key_provider_owned = st.kp_own_project.as_deref() == Some(expected_project.as_str()); + ports.push(TcpPortCheck { + what: "key provider", + flag: "--key-provider-port", + addr: "127.0.0.1".to_string(), + port: o.key_provider_port, + check_free: !key_provider_owned, + }); + } + ports +} + +async fn kms_port_owned(st: &State, client_url: &str, kms_port: u16, no_kms: bool) -> bool { + if no_kms + || st.client_url != client_url + || state_kms_port(st) != Some(kms_port) + || st.vmm_unit.is_empty() + || !unit_active(&st.vmm_unit) + { + return false; + } + let Some(kms_vm_id) = &st.kms_vm_id else { + return false; + }; + match Vmm::connect(client_url) { + Ok(vmm) => vmm.has_vm(kms_vm_id).await, + Err(_) => false, + } +} + +/// fail BEFORE any side effect if a port we need is already taken, so a clash +/// refuses cleanly instead of half-installing. CIDs auto-offset (see +/// `pick_cid_start`); ports are user-facing, so we refuse with guidance rather +/// than silently moving the address the operator will connect to. +fn preflight_ports(ports: &[TcpPortCheck]) -> Result<()> { + for (idx, a) in ports.iter().enumerate() { + for b in ports.iter().skip(idx + 1) { + if a.port == b.port && listener_addrs_overlap(&a.addr, &b.addr) { + bail!( + "{} and {} both use {}:{}; choose distinct ports", + a.flag, + b.flag, + common_listener_addr(&a.addr, &b.addr), + a.port + ); + } + } + } + for port in ports { + if port.port == 0 { + bail!("{} must be between 1 and 65535", port.flag); + } + if port.check_free && !ports::tcp_port_free(&port.addr, port.port) { + bail!( + "{} port {}:{} is already in use; pass {} ", + port.what, + port.addr, + port.port, + port.flag + ); + } + } + Ok(()) +} + +fn listener_addrs_overlap(a: &str, b: &str) -> bool { + a == b || a == "0.0.0.0" || b == "0.0.0.0" || a == "::" || b == "::" +} + +fn common_listener_addr(a: &str, b: &str) -> String { + if a == b { + a.to_string() + } else { + format!("{a}/{b}") + } +} + +fn validate_instance(instance: &str) -> Result<()> { + if instance.is_empty() { + bail!("--instance must not be empty"); + } + if !instance + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.')) + { + bail!("--instance may contain only ascii letters, digits, '-', '_', and '.'"); + } + Ok(()) +} + +fn validate_install_opts(o: &InstallOpts) -> Result<()> { + if let Some(instance) = o.instance.as_deref() { + validate_instance(instance)?; + } + if o.key_provider_port == 0 { + bail!("--key-provider-port must be between 1 and 65535"); + } + if !o.no_kms && host::other_vmm_host_api_ports().contains(&o.host_api_port) { + bail!( + "host-api vsock port {} is already reserved by another dstack-vmm; pass --host-api-port ", + o.host_api_port + ); + } + Ok(()) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum KmsVmBootState { + Pending, + Ready, + Failed(String), +} + +/// poll VMM-reported guest boot state until the KMS CVM has completed the guest +/// boot script. This avoids probing the KMS HTTPS endpoint before its CA is +/// available, which would require disabling TLS certificate validation. +async fn wait_kms_vm_booted(vmm: &Vmm, vm_id: &str, timeout: Duration) -> KmsVmBootState { + let deadline = tokio::time::Instant::now() + timeout; + loop { + if let Ok(status) = vmm.status().await { + if let Some(vm) = status.vms.iter().find(|vm| vm.id == vm_id) { + let state = kms_vm_boot_state( + &vm.status, + &vm.boot_progress, + &vm.boot_error, + vm.instance_id.as_deref(), + ); + if state != KmsVmBootState::Pending { + return state; + } + } + } + if tokio::time::Instant::now() >= deadline { + return KmsVmBootState::Pending; + } + tokio::time::sleep(Duration::from_secs(2)).await; + } +} + +fn kms_vm_boot_state( + status: &str, + boot_progress: &str, + boot_error: &str, + instance_id: Option<&str>, +) -> KmsVmBootState { + let boot_error = boot_error.trim(); + if !boot_error.is_empty() { + return KmsVmBootState::Failed(format!("boot error: {boot_error}")); + } + match status { + "exited" | "stopped" | "removing" => { + return KmsVmBootState::Failed(format!("vm status {status}")); + } + _ => {} + } + let has_instance = instance_id.is_some_and(|id| !id.trim().is_empty()); + if status == "running" && boot_progress.trim() == "done" && has_instance { + return KmsVmBootState::Ready; + } + KmsVmBootState::Pending +} + +/// one-shot liveness probe of the VMM. +async fn vmm_reachable(client_url: &str) -> bool { + match Vmm::connect(client_url) { + Ok(vmm) => vmm.status().await.is_ok(), + Err(_) => false, + } +} + +/// poll the VMM `Status` RPC until it succeeds or the deadline passes. +async fn wait_ready(client_url: &str, timeout: Duration) -> bool { + let deadline = tokio::time::Instant::now() + timeout; + loop { + if let Ok(vmm) = Vmm::connect(client_url) { + if vmm.status().await.is_ok() { + return true; + } + } + if tokio::time::Instant::now() >= deadline { + return false; + } + tokio::time::sleep(Duration::from_millis(500)).await; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tcp_check(what: &'static str, flag: &'static str, port: u16) -> TcpPortCheck { + TcpPortCheck { + what, + flag, + addr: "127.0.0.1".to_string(), + port, + check_free: false, + } + } + + #[test] + fn preflight_rejects_duplicate_requested_ports() { + let checks = vec![ + tcp_check("dashboard", "--dashboard-port", 19080), + tcp_check("kms", "--kms-port", 19080), + ]; + let err = preflight_ports(&checks).unwrap_err().to_string(); + assert!(err.contains("--dashboard-port")); + assert!(err.contains("--kms-port")); + } + + #[test] + fn preflight_rejects_zero_port() { + let err = preflight_ports(&[tcp_check("kms", "--kms-port", 0)]) + .unwrap_err() + .to_string(); + assert!(err.contains("--kms-port")); + assert!(err.contains("between 1 and 65535")); + } + + #[test] + fn instance_rejects_systemd_unsafe_characters() { + for bad in ["", "a/b", "a b", "a%b"] { + assert!( + validate_instance(bad).is_err(), + "{bad:?} should be rejected" + ); + } + validate_instance("dstack-a_1.2").unwrap(); + } + + #[test] + fn kms_vm_boot_state_requires_running_done_instance() { + assert_eq!( + kms_vm_boot_state("running", "done", "", Some("abc")), + KmsVmBootState::Ready + ); + assert_eq!( + kms_vm_boot_state("running", "setting up docker", "", Some("abc")), + KmsVmBootState::Pending + ); + assert_eq!( + kms_vm_boot_state("running", "done", "failed to start containers", Some("abc")), + KmsVmBootState::Failed("boot error: failed to start containers".to_string()) + ); + } +} diff --git a/crates/dstackup/src/main.rs b/crates/dstackup/src/main.rs new file mode 100644 index 00000000..42082685 --- /dev/null +++ b/crates/dstackup/src/main.rs @@ -0,0 +1,116 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! `dstackup` — host setup and lifecycle for a dstack host. +//! +//! Local + privileged only (touches `/dev/sgx`, systemd, local files, the local +//! VMM socket). Day-to-day app operations live in the separate `dstack` binary. +//! +//! Modules: `cli` (arg parsing), `install`/`destroy` (the commands), `state` +//! (install-state persistence), `systemd` (unit management), `cid` (CID-window +//! allocation). + +mod cid; +mod cli; +mod destroy; +mod image; +mod install; +mod state; +mod systemd; + +use anyhow::Result; +use clap::Parser; +use cli::{Cli, Command}; +use dstack_cli_core::host::{self, Platform}; +use dstack_cli_core::layout::InstallLayout; +use dstack_cli_core::vmm::{Vmm, DEFAULT_HOST}; + +#[tokio::main] +async fn main() -> Result<()> { + let cli = Cli::parse(); + match cli.command { + Command::Status { prefix } => { + let host = cli + .host + .clone() + .unwrap_or_else(|| default_host(prefix.as_deref())); + let platform = default_platform(prefix.as_deref()).or_else(host::Platform::detect); + cmd_status(&host, platform).await + } + Command::Install(opts) => install::cmd_install(opts).await, + Command::Image(cmd) => image::cmd_image(cmd).await, + Command::Destroy { prefix, purge } => destroy::cmd_destroy(prefix.as_deref(), purge).await, + } +} + +fn default_host(prefix: Option<&str>) -> String { + state::read_state(&InstallLayout::new(prefix).state_dir) + .and_then(|s| (!s.client_url.is_empty()).then_some(s.client_url)) + .unwrap_or_else(|| DEFAULT_HOST.to_string()) +} + +fn default_platform(prefix: Option<&str>) -> Option { + state::read_state(&InstallLayout::new(prefix).state_dir) + .and_then(|s| host::Platform::parse_opt(&s.platform).ok().flatten()) +} + +async fn cmd_status(host: &str, platform: Option) -> Result<()> { + match platform { + Some(Platform::Tdx) => { + let sgx = host::check_sgx(); + println!("platform: tdx"); + println!( + "sgx: enclave={} provision={} => {}", + sgx.enclave, + sgx.provision, + if sgx.ok() { "ok" } else { "missing" } + ); + } + Some(Platform::AmdSevSnp) => { + let sev = host::check_sev(); + println!("platform: amd-sev-snp"); + println!( + "sev: /dev/sev={} => {}", + sev, + if sev { "ok" } else { "missing" } + ); + } + None => { + println!("platform: undetected"); + let sgx = host::check_sgx(); + println!( + "sgx: enclave={} provision={} => {}", + sgx.enclave, + sgx.provision, + if sgx.ok() { "ok" } else { "missing" } + ); + let sev = host::check_sev(); + println!( + "sev: /dev/sev={} => {}", + sev, + if sev { "ok" } else { "missing" } + ); + } + } + match host::detect_host_ip() { + Ok(ip) => { + let note = if host::is_link_local(&ip) { + " (link-local)" + } else { + "" + }; + println!("host ip: {ip}{note}"); + } + Err(e) => println!("host ip: (undetected: {e})"), + } + print!("vmm: {host} => "); + match Vmm::connect(host) { + Ok(vmm) => match vmm.status().await { + Ok(s) => println!("reachable ({} vms)", s.vms.len()), + Err(e) => println!("unreachable ({e})"), + }, + Err(e) => println!("invalid endpoint ({e})"), + } + Ok(()) +} diff --git a/crates/dstackup/src/state.rs b/crates/dstackup/src/state.rs new file mode 100644 index 00000000..3414ac81 --- /dev/null +++ b/crates/dstackup/src/state.rs @@ -0,0 +1,69 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! install-state persistence: what an install put in place, so re-runs are +//! idempotent and `destroy` can reverse it cleanly. + +use anyhow::{Context, Result}; +use serde::{Deserialize, Serialize}; +use std::fs; +use std::path::{Path, PathBuf}; + +#[derive(Serialize, Deserialize, Default)] +pub(crate) struct State { + /// Backward-compatible data prefix. New clients should use the explicit + /// directory fields below. + pub(crate) prefix: String, + #[serde(default)] + pub(crate) install_prefix: Option, + #[serde(default)] + pub(crate) config_dir: String, + #[serde(default)] + pub(crate) state_dir: String, + #[serde(default)] + pub(crate) cache_dir: String, + #[serde(default)] + pub(crate) run_dir: String, + #[serde(default)] + pub(crate) allowlist_path: String, + #[serde(default)] + pub(crate) platform: String, + pub(crate) client_url: String, + pub(crate) auth_port: u16, + /// systemd unit names (without the `.service` suffix). + #[serde(default)] + pub(crate) vmm_unit: String, + #[serde(default)] + pub(crate) auth_unit: String, + #[serde(default)] + pub(crate) kms_vm_id: Option, + #[serde(default)] + pub(crate) kms_url: String, + /// guest image selected by install for KMS and app deployments. + #[serde(default)] + pub(crate) image: Option, + /// docker-compose project for a key provider we started ourselves. + #[serde(default)] + pub(crate) kp_own_project: Option, +} + +pub(crate) fn state_path(prefix: &Path) -> PathBuf { + prefix.join("dstackup-state.json") +} + +pub(crate) fn read_state(prefix: &Path) -> Option { + let body = fs::read_to_string(state_path(prefix)).ok()?; + serde_json::from_str(&body).ok() +} + +pub(crate) fn write_state(prefix: &Path, st: &State) -> Result<()> { + write(&state_path(prefix), &serde_json::to_string_pretty(st)?) +} + +/// write a file atomically (temp + rename), so a crash mid-write never leaves +/// a torn config or state file. +pub(crate) fn write(path: &Path, body: &str) -> Result<()> { + dstack_cli_core::fsutil::write_atomic(path, body) + .with_context(|| format!("writing {}", path.display())) +} diff --git a/crates/dstackup/src/systemd.rs b/crates/dstackup/src/systemd.rs new file mode 100644 index 00000000..974521e4 --- /dev/null +++ b/crates/dstackup/src/systemd.rs @@ -0,0 +1,123 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! systemd unit management + the sanitized external-tool spawner. + +use anyhow::{bail, Context, Result}; +use std::fs; +use std::path::Path; +use std::process::Command as PCommand; + +/// spawn an external tool (systemctl/docker/curl) with a sanitized `PATH`, so a +/// hijacked environment can't substitute a different binary while we run as root. +pub(crate) fn tool(bin: &str) -> PCommand { + let mut c = PCommand::new(bin); + c.env("PATH", "/usr/sbin:/usr/bin:/sbin:/bin"); + c +} + +pub(crate) fn systemctl(args: &[&str]) -> bool { + tool("systemctl") + .args(args) + .status() + .map(|s| s.success()) + .unwrap_or(false) +} + +/// systemd unit name (no `.service` suffix): `dstack-` or, with an +/// instance, `dstack--` (so a fresh install coexists with an +/// existing `dstack-vmm.service`). +pub(crate) fn unit_name(base: &str, instance: &Option) -> String { + match instance { + Some(i) if !i.is_empty() => format!("dstack-{base}-{i}"), + _ => format!("dstack-{base}"), + } +} + +pub(crate) fn unit_active(unit: &str) -> bool { + systemctl(&["is-active", "--quiet", &format!("{unit}.service")]) +} + +/// write a unit file, reload systemd, and enable+start it (idempotent). +pub(crate) fn install_unit(unit: &str, contents: &str) -> Result<()> { + let path = format!("/etc/systemd/system/{unit}.service"); + fs::write(&path, contents).with_context(|| format!("writing {path}"))?; + systemctl(&["daemon-reload"]); + if !systemctl(&["enable", "--now", &format!("{unit}.service")]) { + bail!("failed to enable+start {unit}.service"); + } + Ok(()) +} + +/// stop, disable, and remove a unit (idempotent — missing unit is fine). +pub(crate) fn remove_unit(unit: &str) { + let svc = format!("{unit}.service"); + let _ = systemctl(&["disable", "--now", &svc]); + let _ = fs::remove_file(format!("/etc/systemd/system/{svc}")); +} + +pub(crate) fn auth_unit_file(bin: &str, allowlist: &Path, port: u16, prefix: &Path) -> String { + // bind 127.0.0.1 deliberately: the webhook decides key release, so it must + // never be reachable off-host. CVMs still reach it at 10.0.2.2: via + // user-mode networking (NAT), which maps to the host loopback. + format!( + "[Unit]\nDescription=dstack auth webhook\nAfter=network.target\n\n[Service]\n\ + ExecStart={bin} --config {cfg} --address 127.0.0.1 --port {port}\n\ + Restart=always\nRestartSec=2\nWorkingDirectory={wd}\n\n\ + [Install]\nWantedBy=multi-user.target\n", + bin = systemd_arg(bin), + cfg = systemd_arg(&allowlist.display().to_string()), + wd = systemd_arg(&prefix.display().to_string()), + ) +} + +pub(crate) fn vmm_unit_file(bin: &str, config: &Path, prefix: &Path, auth_unit: &str) -> String { + // KillMode defaults to control-group, so `systemctl stop` tears down the + // VMM + supervisor + CVM qemus together (deterministic teardown). + format!( + "[Unit]\nDescription=dstack VMM\nAfter=network.target docker.service {auth}.service\nWants={auth}.service\n\n\ + [Service]\nExecStart={bin} -c {cfg}\nRestart=always\nRestartSec=2\n\ + TimeoutStopSec=120\nWorkingDirectory={wd}\n\n\ + [Install]\nWantedBy=multi-user.target\n", + auth = auth_unit, + bin = systemd_arg(bin), + cfg = systemd_arg(&config.display().to_string()), + wd = systemd_arg(&prefix.display().to_string()), + ) +} + +fn systemd_arg(value: &str) -> String { + let mut out = String::with_capacity(value.len() + 2); + out.push('"'); + for ch in value.chars() { + match ch { + '%' => out.push_str("%%"), + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + ch => out.push(ch), + } + } + out.push('"'); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn systemd_arg_quotes_paths_and_escapes_specifiers() { + assert_eq!( + systemd_arg("/opt/dstack/bin/vmm"), + "\"/opt/dstack/bin/vmm\"" + ); + assert_eq!( + systemd_arg("/opt/dstack %/bin/\"vmm\""), + "\"/opt/dstack %%/bin/\\\"vmm\\\"\"" + ); + } +} diff --git a/docs/deployment.md b/docs/deployment.md index 644b1282..532ec3f8 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -1,20 +1,22 @@ # Deploying dstack -> **This guide is for self-hosted deployments** on your own TDX hardware. For cloud deployments, see [Quickstart](./quickstart.md). +> **This guide is for self-hosted deployments** on your own TDX or AMD SEV-SNP hardware. For cloud deployments, see [Quickstart](./quickstart.md). -This guide covers deploying dstack on bare metal TDX hosts. +This guide covers the full manual deployment path for self-hosted dstack. Before deploying, prepare the host with [Hardware enablement](./hardware-enablement.md). If you want the shortest path to a first app on one host, start with [Self-hosted quick onboarding](./onboarding.md). ## Overview dstack can be deployed in two ways: -- **Dev Deployment**: All components run directly on the host. For local development and testing only - no security guarantees. -- **Production Deployment**: KMS and Gateway run as CVMs with hardware-rooted security. Uses auth server for authorization and OS image whitelisting. Required for any deployment where security matters. +- **Single-node deployment**: `dstackup` downloads a verified guest image, renders host config, starts the VMM and auth webhook, bootstraps a KMS CVM, and deploys apps through direct port mappings. Use [Self-hosted quick onboarding](./onboarding.md) for the first-app path. +- **Production deployment**: KMS and Gateway run as CVMs with hardware-rooted security. Uses an auth server for authorization and OS image allowlisting. Required for multi-node deployments, Gateway routing, custom domains, or on-chain governance. + +For local development and contribution workflows, see [Contributing](../CONTRIBUTING.md). ## Prerequisites **Hardware:** -- Bare metal TDX server ([setup guide](https://github.com/canonical/tdx)) +- Bare metal TDX or AMD SEV-SNP server. See [Hardware enablement](./hardware-enablement.md). - At least 16GB RAM, 100GB free disk space - Public IPv4 address - Optional: NVIDIA H100 or Blackwell GPU for [Confidential Computing](https://www.nvidia.com/en-us/data-center/solutions/confidential-computing/) workloads @@ -24,77 +26,6 @@ dstack can be deployed in two ways: > **Note:** See [Hardware Requirements](https://docs.phala.network/dstack/hardware-requirements) for server recommendations. ---- - -## Dev Deployment - -This approach runs all components directly on the host for local development and testing. - -> **Warning:** Dev deployment uses KMS in dev mode with no security guarantees. Do NOT use for production. - -### Install Dependencies - -```bash -# Ubuntu 24.04 -sudo apt install -y build-essential chrpath diffstat lz4 wireguard-tools xorriso - -# Install Rust -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -``` - -### Build Configuration - -```bash -git clone https://github.com/Dstack-TEE/meta-dstack.git --recursive -cd meta-dstack/ -mkdir build && cd build -../build.sh hostcfg -``` - -Edit the generated `build-config.sh` for your environment. The minimal required changes are: - -| Variable | Description | -|----------|-------------| -| `KMS_DOMAIN` | DNS domain for KMS RPC (e.g., `kms.example.com`) | -| `GATEWAY_DOMAIN` | DNS domain for Gateway RPC (e.g., `gateway.example.com`) | -| `GATEWAY_PUBLIC_DOMAIN` | Public base domain for app routing (e.g., `apps.example.com`) | - -**TLS Certificates:** - -The Gateway requires TLS certificates. Configure Certbot with Cloudflare: - -```bash -CERTBOT_ENABLED=true -CF_API_TOKEN= -``` - -The certificates will be obtained automatically via ACME DNS-01 challenge. The KMS auto-generates its own certificates during bootstrap. - -Other variables like ports and CID pool settings have sensible defaults. - -```bash -vim ./build-config.sh -../build.sh hostcfg -``` - -### Download Guest Image - -```bash -../build.sh dl 0.5.5 -``` - -### Run Components - -Start in separate terminals: - -1. **KMS**: `./dstack-kms -c kms.toml` -2. **Gateway**: `sudo ./dstack-gateway -c gateway.toml` -3. **VMM**: `./dstack-vmm -c vmm.toml` - -> **Note:** This deployment uses KMS in dev mode without an auth server. For production deployments with proper security, see [Production Deployment](#production-deployment) below. - ---- - ## Production Deployment For production, deploy KMS and Gateway as CVMs with hardware-rooted security. Production deployments require: diff --git a/docs/hardware-enablement.md b/docs/hardware-enablement.md new file mode 100644 index 00000000..169c2899 --- /dev/null +++ b/docs/hardware-enablement.md @@ -0,0 +1,61 @@ +# Hardware enablement + +Use this page to prepare a bare-metal host before running the [self-hosted quick onboarding guide](./onboarding.md). + +dstack does not enable confidential-computing hardware by itself. The host firmware, kernel, device nodes, and QEMU build must already support the target platform. + +## Intel TDX hosts + +Use the [Canonical TDX setup guide](https://github.com/canonical/tdx) for Ubuntu hosts. The Canonical guide covers supported processors, host OS setup, BIOS settings, reboot, and host verification. + +For dstack, the host must have: + +- Intel TDX enabled in firmware and the host OS. +- SGX enabled in firmware and exposed to Linux. +- A TDX-capable QEMU available at `/usr/bin/qemu-system-x86_64`. +- SGX device nodes for the local key provider: + - `/dev/sgx_enclave` + - `/dev/sgx_provision` + +Check the host after you complete the platform setup: + +```bash +sudo dmesg | grep -i tdx +test -e /dev/sgx_enclave && test -e /dev/sgx_provision +/usr/bin/qemu-system-x86_64 --version +``` + +The Canonical guide's TDX verification expects `dmesg` to show that the TDX module initialized. If the SGX device nodes are missing, `dstackup install` cannot start the default local key provider. + +Do not install a generic QEMU package as a substitute for TDX host setup. Use the QEMU and kernel stack from your TDX host enablement path. + +## AMD SEV-SNP hosts + +Use your vendor or distribution's SEV-SNP enablement path. The [AMDSEV project](https://github.com/AMDESE/AMDSEV) documents CPU, BIOS, firmware, kernel, QEMU, OVMF, and verification requirements for SEV-SNP hosts. Confidential Containers also keeps platform setup separate from its [quickstart](https://github.com/confidential-containers/documentation/blob/main/quickstart.md) and points SEV users to AMD host preparation from its [SEV guide](https://github.com/confidential-containers/documentation/blob/main/guides/sev.md). + +For dstack, the host must have: + +- AMD SEV-SNP enabled in firmware and the host OS. +- `/dev/sev` exposed to Linux. +- A QEMU and OVMF stack that supports SEV-SNP. + +Check the host after you complete the platform setup: + +```bash +test -e /dev/sev +sudo dmesg | grep -e SEV-SNP -e RMP +cat /sys/module/kvm_amd/parameters/sev_snp +``` + +The AMDSEV verification path expects `dmesg` to show SEV-SNP and RMP initialization, and `sev_snp` to read `Y`. + +Host enablement is necessary but not sufficient for onboarding with KMS. The selected guest image must also contain `digest.sev.txt`, which `dstackup install` uses to pin apps to the measured SNP OS image. + +## What dstackup checks + +`dstackup install` does a local platform preflight before it writes host config: + +- For TDX, it checks the SGX device nodes used by the local key provider. +- For AMD SEV-SNP, it checks `/dev/sev`. + +These checks catch missing runtime devices. They do not replace the host enablement process above. diff --git a/docs/onboarding.md b/docs/onboarding.md new file mode 100644 index 00000000..f71f4ba0 --- /dev/null +++ b/docs/onboarding.md @@ -0,0 +1,299 @@ +# Self-hosted quick onboarding + +Use this guide to get a first dstack app running on one Intel TDX host. The workflow uses `dstackup` for host setup and `dstack` for app deployment: + +```bash +curl -fsSL https://raw.githubusercontent.com/Dstack-TEE/dstack/master/scripts/install.sh | sh +sudo dstackup install +sudo dstack deploy \ + -n hello-nginx \ + -c /usr/local/share/dstack/examples/hello-nginx/docker-compose.yaml \ + --port 8080:80 +curl http://127.0.0.1:8080/ +``` + +AMD SEV-SNP hosts use the same `dstackup` and `dstack` commands after you provide a guest image that contains the SNP image digest (`digest.sev.txt`). As of June 28, 2026, the latest stable CPU image from `meta-dstack` is TDX-pinned only, so the copy-paste path below is not the AMD happy path yet. + +For multi-node production, Gateway TLS, custom domains, or on-chain governance, use the full [deployment guide](./deployment.md). + +## What this workflow creates + +- User commands under `/usr/local/bin`. +- Host daemon binaries under `/usr/local/libexec/dstack`. +- Static assets and examples under `/usr/local/share/dstack`. +- Generated host config under `/etc/dstack`. +- Host state, KMS keys, VMs, and verified guest images under `/var/lib/dstack`. +- Source and build cache under `/var/cache/dstack`. +- Runtime sockets and process state under `/run/dstack`. +- A localhost-only VMM dashboard on port `9080`. +- A local `dstack-auth` webhook and `dstack-vmm` systemd unit. +- A single KMS CVM unless you pass `--no-kms`. +- A direct host port mapping for your app. + +## Prerequisites + +Run these commands on the self-hosted dstack machine. + +- Root or sudo access. +- A TDX host that satisfies [Hardware enablement](./hardware-enablement.md). +- Outbound HTTPS access to GitHub. + +If the host is not enabled yet, start with [Hardware enablement](./hardware-enablement.md). + +Install the build packages used by the onboarding flow: + +```bash +sudo apt update +sudo apt install -y \ + build-essential \ + ca-certificates \ + curl \ + git \ + libssl-dev \ + pkg-config \ + tar +``` + +Install and start Docker if it is not already available. The default TDX key provider uses Docker. Use your normal Docker installation process, or on Ubuntu: + +```bash +sudo apt install -y docker.io docker-compose-v2 +sudo systemctl enable --now docker +``` + +Install Rust and load Cargo into your shell: + +```bash +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +. "$HOME/.cargo/env" +``` + +Build and install the `dstackup` bootstrap command: + +```bash +curl -fsSL https://raw.githubusercontent.com/Dstack-TEE/dstack/master/scripts/install.sh | sh +``` + +The bootstrap installer builds `dstackup` from a temporary source checkout and installs it under `/usr/local/bin`. The `dstackup install` command then builds and installs `dstack`, `dstack-auth`, `dstack-vmm`, `supervisor`, static assets, and host config into the system layout. + +## 1. Install the host stack + +Run: + +```bash +sudo dstackup install +``` + +`dstackup install` auto-detects TDX or AMD SEV-SNP. If no local guest image exists, it downloads the latest CPU image from [meta-dstack releases](https://github.com/Dstack-TEE/meta-dstack/releases), requires the release SHA-256 digest by default, verifies the tarball, stages the unpack, and only then adopts the image. + +On TDX, `dstackup install` starts the SGX key provider automatically from `/usr/local/share/dstack/key-provider-build`. To use a different provider, pass one of: + +```bash +sudo dstackup install --key-provider-src /path/to/key-provider-build +sudo dstackup install --use-existing-key-provider 127.0.0.1:3443 +``` + +On AMD SEV-SNP, no SGX key provider is needed. The selected guest image must include `digest.sev.txt`; otherwise, `dstackup install` fails before it starts the host units because apps could not be pinned to the measured SNP OS image. + +To use a GPU image, pull it before install: + +```bash +sudo dstackup image pull --gpu +sudo dstackup install +``` + +If multiple images are present, pass the image name or release version to `--image`, such as `dstack-0.5.11`, `dstack-nvidia-0.5.11`, or `0.5.11`. If the requested release-shaped image is not local, `dstackup install` downloads it. + +When install succeeds, it prints the dashboard URL, the KMS address, and a `dstack deploy` command template. The default dashboard URL is: + +```text +http://127.0.0.1:9080 +``` + +If you connect from your laptop, open an SSH tunnel first: + +```bash +ssh -L 9080:127.0.0.1:9080 @ +``` + +Then open `http://127.0.0.1:9080` locally. `dstackup install --expose` is intentionally disabled until the remote TLS and token transport exists. + +## 2. Deploy a first app + +Deploy the checked-in nginx example: + +```bash +sudo dstack deploy \ + -n hello-nginx \ + -c /usr/local/share/dstack/examples/hello-nginx/docker-compose.yaml \ + --port 8080:80 +``` + +The deploy command: + +- converts the Docker Compose file into a dstack app-compose manifest, +- computes the compose hash and app ID, +- uses the VMM endpoint, guest image, and auth allowlist from `dstackup install`, +- registers the compose hash in the single-node auth allowlist, +- creates the CVM, and +- maps `http://127.0.0.1:8080/` on the host to port `80` in the CVM. + +Open the app from the host: + +```bash +curl http://127.0.0.1:8080/ +``` + +If you are connecting from your laptop, tunnel the app port too: + +```bash +ssh -L 8080:127.0.0.1:8080 @ +``` + +## Common operations + +Check deployed apps: + +```bash +dstack apps +``` + +Show recent app logs: + +```bash +dstack logs +``` + +Remove a local image: + +```bash +sudo dstackup image rm +``` + +List local images: + +```bash +sudo dstackup image list +``` + +Tear down the host units and KMS CVM: + +```bash +sudo dstackup destroy +``` + +Add `--purge` only when you also want to delete generated config, state, cached source, runtime files, and KMS keys for that install. For a custom `--prefix`, purge also removes dstack-owned installed files under that prefix. + +## Install with a custom prefix + +Use `--prefix` when you want a second isolated install on the same host. A custom prefix relocates the install layout under that directory: + +| Purpose | Example path for `--prefix /opt/dstack-test` | +| --- | --- | +| User commands | `/opt/dstack-test/bin` | +| Host daemons | `/opt/dstack-test/libexec/dstack` | +| Static assets | `/opt/dstack-test/share/dstack` | +| Config | `/opt/dstack-test/etc/dstack` | +| State and images | `/opt/dstack-test/var/lib/dstack` | +| Source and build cache | `/opt/dstack-test/var/cache/dstack` | +| Runtime files | `/opt/dstack-test/run/dstack` | + +Install `dstackup` into the prefix, then use the same prefix for `dstackup` and `dstack`: + +```bash +curl -fsSL https://raw.githubusercontent.com/Dstack-TEE/dstack/master/scripts/install.sh | sh -s -- --prefix /opt/dstack-test + +sudo /opt/dstack-test/bin/dstackup install \ + --prefix /opt/dstack-test \ + --dashboard-port 19080 \ + --auth-port 18001 \ + --host-api-port 10001 + +sudo /opt/dstack-test/bin/dstack \ + --prefix /opt/dstack-test \ + deploy \ + -n hello-nginx \ + -c /opt/dstack-test/share/dstack/examples/hello-nginx/docker-compose.yaml \ + --port 18080:80 +``` + +For a custom prefix, `dstackup` derives distinct systemd unit names from the prefix unless you pass `--instance`. You still need distinct TCP and vsock ports for each running install. + +Remove a custom-prefix install with the same prefix: + +```bash +sudo /opt/dstack-test/bin/dstackup destroy --prefix /opt/dstack-test --purge +``` + +## Security boundaries + +This onboarding path is designed for one operator on one host. + +- The VMM dashboard and management API bind to `127.0.0.1` by default. Use SSH tunneling for remote access. +- `dstackup install` pins the app OS image hash from the selected guest image (`digest.txt` for TDX, `digest.sev.txt` for SEV-SNP). If the digest cannot be read, install fails unless you pass `--allow-unpinned-image`. +- `dstack deploy` registers the app compose hash in the local auth allowlist from `dstackup install`. Without that allowlist update, a KMS-mode app can boot but will not receive keys. +- Gateway is not part of this flow. Apps are exposed through direct host port mappings. + +Use the [deployment guide](./deployment.md) when you need domain routing, Gateway certificates, on-chain authorization, KMS replicas, or multi-node operation. + +## Troubleshooting + +### Image download fails + +`dstackup install` downloads the latest CPU image when KMS mode needs an image and none exists locally. If the download fails, check network access to GitHub and the meta-dstack release: + +```bash +sudo dstackup image pull +``` + +If a release does not publish a SHA-256 digest, `dstackup image pull` and `dstackup install` fail before unpacking it. Use `--insecure` only when you intentionally accept an unverified image download. + +If you use a custom prefix or image directory, pass the same `--prefix` or `--image-path` to `install` and `image` commands. + +### Missing `digest.sev.txt` on AMD SEV-SNP + +TDX images pin apps with `digest.txt`. AMD SEV-SNP images pin apps with `digest.sev.txt`. If the selected image does not contain `digest.sev.txt`, install fails with: + +```text +no os-image pin: could not read digest.sev.txt +``` + +Use `--image` or `--image-path` with an SNP-capable image that contains `digest.sev.txt`. Do not use `--allow-unpinned-image` for onboarding unless you intentionally want apps to boot without OS-image pinning. + +### No key provider on TDX + +TDX uses an SGX-backed key provider for KMS sealing. `dstackup install` uses the installed key provider assets by default. To override that, pass one of: + +```bash +sudo dstackup install --use-existing-key-provider 127.0.0.1:3443 +sudo dstackup install --key-provider-src /path/to/key-provider-build +``` + +AMD SEV-SNP does not use this key provider. + +### Port already in use + +Move the conflicting port explicitly: + +```bash +sudo dstackup install --dashboard-port 19080 --auth-port 18001 --host-api-port 10001 +``` + +For app ports, change the `--port` mapping: + +```bash +sudo dstack deploy \ + -c /usr/local/share/dstack/examples/hello-nginx/docker-compose.yaml \ + --port 18080:80 +``` + +### KMS bootstrap does not finish + +Check the VMM service and the KMS CVM logs: + +```bash +sudo journalctl -u dstack-vmm -n 200 --no-pager +dstack logs +``` + +The KMS VM ID is printed by `dstackup install` when it creates or reuses the KMS CVM. diff --git a/examples/hello-nginx/docker-compose.yaml b/examples/hello-nginx/docker-compose.yaml new file mode 100644 index 00000000..8d2fcc53 --- /dev/null +++ b/examples/hello-nginx/docker-compose.yaml @@ -0,0 +1,5 @@ +services: + web: + image: nginx:alpine + ports: + - "80:80" diff --git a/scripts/install.sh b/scripts/install.sh new file mode 100755 index 00000000..ab2fcc19 --- /dev/null +++ b/scripts/install.sh @@ -0,0 +1,280 @@ +#!/bin/sh + +# SPDX-FileCopyrightText: 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +set -eu + +DEFAULT_REPO="https://github.com/Dstack-TEE/dstack" +DEFAULT_REF="master" +DEFAULT_PREFIX="/usr/local" + +usage() { + cat <<'EOF' +Install dstackup from source. + +Usage: + scripts/install.sh [options] + curl -fsSL https://raw.githubusercontent.com/Dstack-TEE/dstack/master/scripts/install.sh | sh + +Options: + --repo URL Git repository to clone when not run from a checkout. + Default: https://github.com/Dstack-TEE/dstack + --ref REF Git ref to checkout when cloning or updating DSTACK_SRC. + Default: master + --src DIR Persistent source checkout to build from. + Default: a temporary checkout + --prefix DIR Install dstackup under DIR/bin. Use the same DIR with + dstackup install --prefix for a self-contained install. + Default: /usr/local + --no-sudo Do not use sudo for creating DIR/bin or installing binaries. + -h, --help Show this help. + +Environment: + DSTACK_REPO Same as --repo. + DSTACK_REF Same as --ref. + DSTACK_SRC Same as --src. + DSTACK_INSTALL_PREFIX Same as --prefix. +EOF +} + +repo=${DSTACK_REPO:-$DEFAULT_REPO} +ref=${DSTACK_REF:-$DEFAULT_REF} +src=${DSTACK_SRC:-} +prefix=${DSTACK_INSTALL_PREFIX:-$DEFAULT_PREFIX} +prefix_set=0 +no_sudo=0 +tmp_src= + +if [ "${DSTACK_INSTALL_PREFIX+x}" = x ]; then + prefix_set=1 +fi + +cleanup() { + if [ -n "$tmp_src" ]; then + rm -rf "$tmp_src" + fi +} +trap cleanup EXIT INT TERM + +while [ "$#" -gt 0 ]; do + case "$1" in + --repo) + if [ "$#" -lt 2 ]; then + echo "error: --repo requires a URL" >&2 + exit 1 + fi + repo=$2 + shift 2 + ;; + --ref) + if [ "$#" -lt 2 ]; then + echo "error: --ref requires a ref" >&2 + exit 1 + fi + ref=$2 + shift 2 + ;; + --src) + if [ "$#" -lt 2 ]; then + echo "error: --src requires a directory" >&2 + exit 1 + fi + src=$2 + shift 2 + ;; + --prefix|--root) + if [ "$#" -lt 2 ]; then + echo "error: --prefix requires a directory" >&2 + exit 1 + fi + prefix=$2 + prefix_set=1 + shift 2 + ;; + --no-sudo) + no_sudo=1 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "error: unknown option: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +need_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "error: required command not found: $1" >&2 + exit 1 + fi +} + +is_checkout() { + [ -f "$1/Cargo.toml" ] && + [ -d "$1/crates/dstackup" ] && + [ -d "$1/crates/dstack-cli" ] && + [ -d "$1/vmm" ] && + [ -d "$1/supervisor" ] +} + +abs_dir() { + (cd "$1" && pwd) +} + +script_checkout() { + case "$0" in + */*) + script_dir=$(dirname "$0") + if [ -d "$script_dir/.." ] && is_checkout "$script_dir/.."; then + abs_dir "$script_dir/.." + return 0 + fi + ;; + esac + return 1 +} + +resolve_source() { + if is_checkout "."; then + abs_dir "." + return 0 + fi + + if checkout=$(script_checkout); then + echo "$checkout" + return 0 + fi + + need_cmd git + + if [ -n "$src" ] && [ -e "$src" ]; then + if ! is_checkout "$src" || [ ! -d "$src/.git" ]; then + echo "error: $src exists but is not a dstack git checkout" >&2 + exit 1 + fi + echo "updating dstack source in $src" + ( + cd "$src" + git fetch --tags origin + git checkout "$ref" + if git rev-parse --verify "origin/$ref" >/dev/null 2>&1; then + git pull --ff-only origin "$ref" + fi + ) + elif [ -n "$src" ]; then + echo "cloning dstack source into $src" + git clone "$repo" "$src" + ( + cd "$src" + git fetch --tags origin + git checkout "$ref" + ) + else + need_cmd mktemp + tmp_src=$(mktemp -d "${TMPDIR:-/tmp}/dstack-install.XXXXXX") + src="$tmp_src/source" + echo "cloning dstack source into a temporary checkout" + git clone "$repo" "$src" + ( + cd "$src" + git fetch --tags origin + git checkout "$ref" + ) + fi + + abs_dir "$src" +} + +validate_prefix() { + case "$prefix" in + /*) ;; + *) + echo "error: --prefix must be an absolute path" >&2 + exit 1 + ;; + esac + if [ "$prefix" = "/" ]; then + echo "error: --prefix must not be /" >&2 + exit 1 + fi + case "$prefix" in + *"/../"*|*"/.."|*"/./"*|*"/.") + echo "error: --prefix must not contain . or .. path components" >&2 + exit 1 + ;; + esac +} + +validate_prefix + +if ! command -v cargo >/dev/null 2>&1 && [ -n "${HOME:-}" ] && [ -f "$HOME/.cargo/env" ]; then + # Mirrors rustup's post-install shell setup when the current shell has not + # loaded Cargo yet. + . "$HOME/.cargo/env" +fi + +need_cmd cargo +need_cmd install + +checkout=$(resolve_source) +bin_dir="$prefix/bin" + +if [ "$no_sudo" -eq 0 ] && [ "$(id -u)" -ne 0 ]; then + sudo_cmd=sudo +else + sudo_cmd= +fi + +if [ -n "$sudo_cmd" ]; then + need_cmd sudo + $sudo_cmd install -d -m 0755 "$bin_dir" +else + install -d -m 0755 "$bin_dir" +fi + +echo "building dstackup from $checkout" +( + cd "$checkout" + cargo build --release \ + -p dstackup +) + +install_bin() { + src_bin="$checkout/target/release/$1" + dest_bin="$bin_dir/$2" + if [ ! -f "$src_bin" ]; then + echo "error: expected binary not found: $src_bin" >&2 + exit 1 + fi + if [ -n "$sudo_cmd" ]; then + $sudo_cmd install -m 0755 "$src_bin" "$dest_bin" + else + install -m 0755 "$src_bin" "$dest_bin" + fi +} + +install_bin dstackup dstackup + +if [ "$prefix_set" -eq 1 ]; then + next_install="sudo $bin_dir/dstackup install --prefix $prefix" +else + next_install="sudo $bin_dir/dstackup install" +fi + +cat <