From d1a797acbde8178ed1de0ad9c55b050d4cb89d30 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Thu, 2 Jul 2026 14:11:53 -0700 Subject: [PATCH 01/24] feat(kubernetes): add sidecar supervisor topology Add the Kubernetes sidecar supervisor topology, its Helm/Skaffold configuration, topology documentation, and sidecar e2e matrix coverage. Skip root-only sandbox identity rewriting when process enforcement is network-only so the low-permission sidecar process container can start successfully. Signed-off-by: Taylor Mutch --- .../skills/debug-openshell-cluster/SKILL.md | 27 + .agents/skills/helm-dev-environment/SKILL.md | 19 +- .github/workflows/branch-e2e.yml | 11 +- Cargo.lock | 1 + architecture/build.md | 9 +- architecture/compute-runtimes.md | 16 +- crates/openshell-core/src/grpc_client.rs | 7 +- crates/openshell-core/src/sandbox_env.rs | 41 + crates/openshell-driver-kubernetes/README.md | 26 +- .../openshell-driver-kubernetes/src/config.rs | 142 ++- .../openshell-driver-kubernetes/src/driver.rs | 920 +++++++++++++++++- crates/openshell-driver-kubernetes/src/lib.rs | 5 +- .../openshell-driver-kubernetes/src/main.rs | 17 +- crates/openshell-driver-podman/README.md | 4 +- .../openshell-driver-podman/src/container.rs | 5 +- crates/openshell-sandbox/Cargo.toml | 3 + crates/openshell-sandbox/src/lib.rs | 356 ++++++- crates/openshell-sandbox/src/main.rs | 202 +++- .../data/sandbox-policy.rego | 14 + .../src/identity.rs | 69 +- .../openshell-supervisor-network/src/opa.rs | 202 +++- .../openshell-supervisor-network/src/proxy.rs | 222 ++++- .../openshell-supervisor-network/src/run.rs | 4 +- .../src/netns/mod.rs | 194 ++++ .../src/netns/nft_ruleset.rs | 63 ++ .../src/process.rs | 74 +- .../openshell-supervisor-process/src/run.rs | 139 ++- .../openshell-supervisor-process/src/ssh.rs | 157 ++- deploy/docker/Dockerfile.supervisor | 25 +- deploy/helm/openshell/README.md | 4 +- deploy/helm/openshell/ci/values-sidecar.yaml | 13 + deploy/helm/openshell/skaffold.yaml | 8 + .../openshell/templates/gateway-config.yaml | 2 + .../openshell/tests/gateway_config_test.yaml | 28 +- deploy/helm/openshell/values.yaml | 11 +- docs/kubernetes/access-control.mdx | 2 +- docs/kubernetes/ingress.mdx | 2 +- docs/kubernetes/managing-certificates.mdx | 2 +- docs/kubernetes/openshift.mdx | 2 +- docs/kubernetes/setup.mdx | 3 +- docs/kubernetes/topology.mdx | 159 ++- docs/reference/gateway-config.mdx | 12 +- docs/reference/sandbox-compute-drivers.mdx | 20 + e2e/with-kube-gateway.sh | 42 + tasks/helm.toml | 15 + tasks/test.toml | 5 + 46 files changed, 3040 insertions(+), 264 deletions(-) create mode 100644 deploy/helm/openshell/ci/values-sidecar.yaml diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index 5bc04beb39..08e4230139 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -268,6 +268,33 @@ kubectl -n openshell get configmap openshell-config -o jsonpath='{.data.gateway\ kubectl -n get sandbox -o jsonpath='{.spec.template.spec.serviceAccountName}{"\n"}' ``` +If `supervisor_topology = "sidecar"` is rendered, sandbox pods should have an +`openshell-network-init` init container running `--mode=network-init`, an +`agent` container running `openshell-sandbox --mode=process`, and an +`openshell-supervisor-network` container running `--mode=network`. The init +container owns nftables setup and should be the only sidecar topology container +with `NET_ADMIN`. It also needs `CHOWN`/`FOWNER` to hand shared emptyDir state +to `sidecar_proxy_uid`. The long-running network sidecar runs as +`sidecar_proxy_uid` with primary GID `0` so it can read the root-owned, +group-readable projected service-account token. In sidecar topology the +`openshell-sa-token` projected volume should render `defaultMode: 288` (`0440`); +if the proxy logs `failed to read K8s SA token`, verify this token mode and the +network sidecar security context. The process container should also publish the +workload entrypoint PID to `OPENSHELL_ENTRYPOINT_PID_FILE` +(`/run/openshell-sidecar/entrypoint.pid` by default), and the network sidecar +should read it for binary-scoped policy decisions; if allowed network rules are +all denied, inspect that file and the network sidecar logs. +Inspect all three when sandbox registration or egress enforcement fails: + +```bash +kubectl -n openshell get configmap openshell-config -o jsonpath='{.data.gateway\.toml}' | grep supervisor_topology +kubectl -n get pod -o jsonpath='{range .spec.initContainers[*]}{.name}{" "}{.command}{"\n"}{end}' +kubectl -n get pod -o jsonpath='{range .spec.containers[*]}{.name}{" "}{.command}{"\n"}{end}' +kubectl -n logs -c openshell-network-init --tail=200 +kubectl -n logs -c openshell-supervisor-network --tail=200 +kubectl -n logs -c agent --tail=200 +``` + ### Step 6: Check VM-Backed Gateways Use the VM driver logs and host diagnostics available in the user's environment. Verify: diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index bffa4e2e86..7d6ad7cd5b 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -60,9 +60,17 @@ mise run helm:skaffold:dev mise run helm:skaffold:run ``` +**Supervisor sidecar topology** (build once and leave running): +```bash +mise run helm:skaffold:run:sidecar +``` + Both commands build the `gateway` and `supervisor` images and deploy the OpenShell Helm -chart. The `pkiInitJob` hook (a pre-install Job that runs `openshell-gateway generate-certs`) -generates mTLS secrets on first install. Envoy Gateway opt-in; see the Optional Add-ons section below. +chart. The sidecar profile renders an `openshell-network-init` init container for +nftables setup and a non-root `openshell-supervisor-network` runtime sidecar for +proxying. The `pkiInitJob` hook (a pre-install Job that runs `openshell-gateway +generate-certs`) generates mTLS secrets on first install. Envoy Gateway opt-in; +see the Optional Add-ons section below. The gateway Service uses ClusterIP. Access is via Envoy Gateway (port `8080`) or `kubectl port-forward`. @@ -126,6 +134,12 @@ openshell sandbox list --gateway-endpoint https://localhost:8090 mise run helm:skaffold:delete ``` +For a sidecar-profile deployment: + +```bash +mise run helm:skaffold:delete:sidecar +``` + ### Delete the cluster entirely ```bash @@ -250,6 +264,7 @@ for dependencies still declared in `Chart.yaml`. | `deploy/helm/openshell/ci/values-gateway.yaml` | Envoy Gateway GRPCRoute + Gateway overlay | | `deploy/helm/openshell/ci/values-high-availability.yaml` | HA test overlay (`replicaCount: 2` with external PostgreSQL Secret) | | `deploy/helm/openshell/ci/values-keycloak.yaml` | Keycloak OIDC overlay | +| `deploy/helm/openshell/ci/values-sidecar.yaml` | Supervisor sidecar topology overlay for Kubernetes e2e/dev | | `deploy/helm/openshell/ci/values-spire.yaml` | SPIFFE/SPIRE provider token grant overlay | | `deploy/helm/openshell/ci/values-spire-stack.yaml` | SPIRE hardened chart values for local dev | | `deploy/helm/openshell/ci/values-tls-disabled.yaml` | Lint-only: TLS + auth disabled (reverse-proxy edge termination) | diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index ebe7834068..859f22ef95 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -121,16 +121,25 @@ jobs: include: - agent_sandbox_api: v1beta1 agent_sandbox_version: v0.5.0 + topology: combined + extra_helm_values: "" - agent_sandbox_api: v1alpha1 agent_sandbox_version: v0.4.6 + topology: combined + extra_helm_values: "" + - agent_sandbox_api: v1beta1 + agent_sandbox_version: v0.5.0 + topology: sidecar + extra_helm_values: deploy/helm/openshell/ci/values-sidecar.yaml permissions: contents: read packages: read uses: ./.github/workflows/e2e-kubernetes-test.yml with: image-tag: ${{ github.sha }} - job-name: Kubernetes E2E (Rust smoke, Agent Sandbox ${{ matrix.agent_sandbox_api }}) + job-name: Kubernetes E2E (Rust smoke, ${{ matrix.topology }}, Agent Sandbox ${{ matrix.agent_sandbox_api }}) agent-sandbox-version: ${{ matrix.agent_sandbox_version }} + extra-helm-values: ${{ matrix.extra_helm_values }} kubernetes-ha-e2e: needs: [pr_metadata, build-gateway, build-supervisor] diff --git a/Cargo.lock b/Cargo.lock index 13b670f558..e94eb56f08 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3827,6 +3827,7 @@ dependencies = [ "clap", "futures", "miette", + "nix", "openshell-core", "openshell-ocsf", "openshell-policy", diff --git a/architecture/build.md b/architecture/build.md index a3cb2e25fb..633efa72d2 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -91,10 +91,11 @@ Runtime layout: as a release artifact. Linux GNU VM driver binaries must not reference `GLIBC_*` symbols newer than `GLIBC_2.28`; release workflows verify this before publishing artifacts. -- **Supervisor**: `scratch` base, static musl binary at `/openshell-sandbox`. - Static linkage is required because the image is mounted/extracted into - sandbox environments (Docker extraction, Podman image volumes, Kubernetes - init-container copy-self) and cannot rely on a dynamic loader. +- **Supervisor**: Alpine base with `nftables`, static musl binary at + `/openshell-sandbox`. Static linkage keeps the binary usable when the image + is mounted/extracted into sandbox environments (Docker extraction, Podman + image volumes, Kubernetes init-container copy-self), while `nftables` supports + Kubernetes supervisor sidecar egress enforcement. Gateway image builds bake the corresponding supervisor image tag into the gateway binary so Docker sandboxes do not depend on `:latest` by default. diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index f122fda5d7..ac239bfb8c 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -81,7 +81,7 @@ The supervisor must be available inside each sandbox workload: |---|---| | Docker | Bind-mounted local supervisor binary, or a binary extracted from the configured supervisor image. | | Podman | Read-only OCI image volume containing the supervisor binary. | -| Kubernetes | Sandbox pod image or pod template configuration. | +| Kubernetes | Supervisor image side-loaded into the sandbox pod by image volume or init container. | | VM | Embedded in the guest rootfs bundle. | | Extension | Defined by the out-of-tree driver. | @@ -89,6 +89,20 @@ Driver-controlled environment variables must override sandbox image or template values for sandbox ID, sandbox name, gateway endpoint, relay socket path, TLS paths, and command metadata. +Kubernetes can run the supervisor in the default combined topology or in a +sidecar topology. Combined mode keeps network and process supervision in the +agent container. Sidecar mode runs network enforcement, the proxy, and gateway +loopback forwarding in a dedicated sidecar, while the agent container runs only +the process-supervision leaf and launches the user workload after the sidecar +signals readiness. In sidecar mode, an init container performs the privileged +pod-network nftables setup with `NET_ADMIN` and hands shared state ownership to +the configured proxy UID; the long-running network sidecar runs as that UID and +does not keep `NET_ADMIN`. The agent container runs as the resolved sandbox +UID/GID with no added Linux capabilities. Sidecar mode preserves gateway session +and SSH behavior, but treats the process leaf as network-only: Landlock +filesystem policy, process privilege dropping, and process/binary identity +checks are not applied there. + ## Images The gateway image and Helm chart are built from this repository. Sandbox images diff --git a/crates/openshell-core/src/grpc_client.rs b/crates/openshell-core/src/grpc_client.rs index 96158a1d10..4f2477c25f 100644 --- a/crates/openshell-core/src/grpc_client.rs +++ b/crates/openshell-core/src/grpc_client.rs @@ -167,9 +167,14 @@ async fn build_plain_channel(endpoint: &str) -> Result { .into_diagnostic() .wrap_err_with(|| format!("failed to read client key from {key_path}"))?; - let tls_config = ClientTlsConfig::new() + let mut tls_config = ClientTlsConfig::new() .ca_certificate(Certificate::from_pem(ca_pem)) .identity(Identity::from_pem(cert_pem, key_pem)); + if let Ok(server_name) = std::env::var(sandbox_env::GATEWAY_TLS_SERVER_NAME) + && !server_name.is_empty() + { + tls_config = tls_config.domain_name(server_name); + } ep = ep .tls_config(tls_config) diff --git a/crates/openshell-core/src/sandbox_env.rs b/crates/openshell-core/src/sandbox_env.rs index c56a1c889d..ae3a21787e 100644 --- a/crates/openshell-core/src/sandbox_env.rs +++ b/crates/openshell-core/src/sandbox_env.rs @@ -29,6 +29,47 @@ pub const SANDBOX_COMMAND: &str = "OPENSHELL_SANDBOX_COMMAND"; /// Deployment-controlled telemetry toggle propagated to the sandbox supervisor. pub const TELEMETRY_ENABLED: &str = "OPENSHELL_TELEMETRY_ENABLED"; +/// Supervisor pod/runtime topology. Kubernetes sidecar mode sets this to +/// `"sidecar"`; the default combined supervisor path omits it. +pub const SUPERVISOR_TOPOLOGY: &str = "OPENSHELL_SUPERVISOR_TOPOLOGY"; + +/// Network enforcement backend selected by the compute driver. +pub const NETWORK_ENFORCEMENT_MODE: &str = "OPENSHELL_NETWORK_ENFORCEMENT_MODE"; + +/// Process enforcement mode selected by the compute driver. +/// +/// The default when unset is `"full"`, where the process supervisor enforces +/// filesystem/process policy before spawning workloads. Kubernetes sidecar +/// topology sets this to `"network-only"` so the process wrapper can run as +/// the sandbox UID without Linux capabilities while preserving SSH/session +/// behavior. +pub const PROCESS_ENFORCEMENT_MODE: &str = "OPENSHELL_PROCESS_ENFORCEMENT_MODE"; + +/// Whether network policy evaluation must bind requests to the peer binary. +/// +/// The default when unset is `"required"`. Kubernetes sidecar experiments may +/// set this to `"relaxed"` to enforce endpoint and L7 policy without per-binary +/// `/proc` identity binding. +pub const NETWORK_BINARY_IDENTITY: &str = "OPENSHELL_NETWORK_BINARY_IDENTITY"; + +/// File written by the network supervisor when sidecar networking is ready. +pub const SUPERVISOR_READY_FILE: &str = "OPENSHELL_SUPERVISOR_READY_FILE"; + +/// File written by the process supervisor with the workload entrypoint PID and +/// read by the network sidecar for process/binary-bound network policy checks. +pub const ENTRYPOINT_PID_FILE: &str = "OPENSHELL_ENTRYPOINT_PID_FILE"; + +/// Loopback address where the network sidecar forwards gateway gRPC traffic. +pub const GATEWAY_FORWARD_ADDR: &str = "OPENSHELL_GATEWAY_FORWARD_ADDR"; + +/// Optional TLS server name used when the process supervisor reaches the +/// gateway through a loopback TCP forward. +pub const GATEWAY_TLS_SERVER_NAME: &str = "OPENSHELL_GATEWAY_TLS_SERVER_NAME"; + +/// Directory where the network supervisor writes the proxy CA files consumed +/// by workload child processes. +pub const PROXY_TLS_DIR: &str = "OPENSHELL_PROXY_TLS_DIR"; + /// Path to the CA certificate for mTLS communication with the gateway. pub const TLS_CA: &str = "OPENSHELL_TLS_CA"; diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index 48ddfb8f11..9e8c26c9d3 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -53,9 +53,29 @@ pods do not need direct external ingress for SSH. ## Container Security Context -The driver grants the sandbox agent container the Linux capabilities the -supervisor needs for namespace setup and policy enforcement. It can also request -a Kubernetes AppArmor profile through `app_armor_profile`. +The default `combined` supervisor topology grants the sandbox agent container +the Linux capabilities the supervisor needs for namespace setup and process, +filesystem, and network policy enforcement. + +The `sidecar` supervisor topology moves pod-level network setup into a root init +container and runs the long-lived network sidecar as a non-root UID with no +added Linux capabilities. The agent container also runs as the resolved sandbox +UID/GID with `allowPrivilegeEscalation: false` and `capabilities.drop: ["ALL"]`. +In this mode OpenShell preserves gateway session and SSH behavior, but the +process supervisor defaults to network-only mode and does not apply Landlock +filesystem policy, process privilege dropping, or process/binary identity +checks. Network endpoint and L7 policy remain enforced by the network sidecar. +Set `process_enforcement = "full"` only when you want combined-mode +process/filesystem guards and accept the added agent-container permissions. + +Sidecar mode uses the pod `fsGroup` to make the projected service-account token +and sandbox client TLS secret group-readable so the non-root process supervisor +can authenticate to the gateway. Treat the agent container as trusted with +respect to those in-pod gateway credentials until a narrower credential handoff +exists. + +The driver can request a Kubernetes AppArmor profile through +`app_armor_profile`. Supported values are `Unconfined`, `RuntimeDefault`, and `Localhost/`. An empty or unset value omits diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 292563c2e6..a0d3920cd4 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -15,6 +15,9 @@ pub const DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME: &str = "default"; /// Default storage size for the workspace PVC. pub const DEFAULT_WORKSPACE_STORAGE_SIZE: &str = "2Gi"; +/// Default UID for the long-running Kubernetes network supervisor sidecar. +pub const DEFAULT_PROXY_UID: u32 = 1337; + /// How the supervisor binary is delivered into sandbox pods. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] @@ -59,12 +62,16 @@ pub enum SupervisorTopology { /// Run networking and process supervision in the agent container. #[default] Combined, + /// Run network supervision in a privileged sidecar and process supervision + /// as a low-capability wrapper in the agent container. + Sidecar, } impl std::fmt::Display for SupervisorTopology { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Self::Combined => f.write_str("combined"), + Self::Sidecar => f.write_str("sidecar"), } } } @@ -75,11 +82,49 @@ impl FromStr for SupervisorTopology { fn from_str(s: &str) -> Result { match s { "combined" => Ok(Self::Combined), + "sidecar" => Ok(Self::Sidecar), other => Err(format!("unknown supervisor topology '{other}'")), } } } +/// Process/filesystem controls applied by the process supervisor in split +/// Kubernetes topologies. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ProcessEnforcementMode { + /// Preserve process launch and session relay behavior, but leave + /// filesystem/process guards to the network supervisor topology. + #[default] + NetworkOnly, + /// Run the process supervisor with the same process/filesystem controls as + /// combined topology. + Full, +} + +impl std::fmt::Display for ProcessEnforcementMode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::NetworkOnly => f.write_str("network-only"), + Self::Full => f.write_str("full"), + } + } +} + +impl FromStr for ProcessEnforcementMode { + type Err = String; + + fn from_str(s: &str) -> Result { + match s { + "network-only" => Ok(Self::NetworkOnly), + "full" => Ok(Self::Full), + other => Err(format!( + "unknown process enforcement mode '{other}'; expected 'network-only' or 'full'" + )), + } + } +} + /// Kubernetes `AppArmor` profile requested for the sandbox agent container. #[derive(Debug, Clone, PartialEq, Eq)] pub enum AppArmorProfile { @@ -206,6 +251,14 @@ pub struct KubernetesComputeConfig { pub supervisor_sideload_method: SupervisorSideloadMethod, /// How the supervisor is arranged for Kubernetes sandbox pods. pub supervisor_topology: SupervisorTopology, + /// Process/filesystem enforcement mode used by the agent container in + /// non-combined topologies. `network-only` keeps the low-permission agent + /// shape; `full` grants the agent supervisor combined-mode controls. + pub process_enforcement: ProcessEnforcementMode, + /// UID used by the long-running network sidecar in `sidecar` topology. + /// The network init container installs nftables rules that exempt this + /// UID, so it must not match the sandbox workload UID. + pub proxy_uid: u32, pub grpc_endpoint: String, pub ssh_socket_path: String, pub client_tls_secret_name: String, @@ -292,6 +345,8 @@ impl Default for KubernetesComputeConfig { supervisor_image_pull_policy: String::new(), supervisor_sideload_method: SupervisorSideloadMethod::default(), supervisor_topology: SupervisorTopology::default(), + process_enforcement: ProcessEnforcementMode::default(), + proxy_uid: DEFAULT_PROXY_UID, grpc_endpoint: String::new(), ssh_socket_path: "/run/openshell/ssh.sock".to_string(), client_tls_secret_name: String::new(), @@ -336,6 +391,16 @@ impl KubernetesComputeConfig { ) } + pub fn validate_proxy_uid(&self) -> Result<(), String> { + if self.proxy_uid < openshell_policy::MIN_SANDBOX_UID { + return Err(format!( + "proxy_uid must be at least {}", + openshell_policy::MIN_SANDBOX_UID + )); + } + Ok(()) + } + /// Resolve the sandbox UID/GID pair. /// /// Resolution order: @@ -351,6 +416,7 @@ impl KubernetesComputeConfig { if let Some(uid) = self.sandbox_uid { return uid; } + // Try OpenShift SCC annotation. if let Some(anns) = namespace_annotations && let Some(range) = anns.get(ANNOTATION_SCC_UID_RANGE) && let Some(uid) = Self::from_open_shift_uid_range(range) @@ -462,19 +528,32 @@ mod tests { } #[test] - fn default_service_account_name_is_default() { + fn default_supervisor_topology_is_combined() { let cfg = KubernetesComputeConfig::default(); - assert_eq!( - cfg.service_account_name, - DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME - ); + assert_eq!(cfg.supervisor_topology, SupervisorTopology::Combined); + assert_eq!(cfg.supervisor_topology.to_string(), "combined"); } #[test] - fn default_supervisor_topology_is_combined() { + fn default_proxy_uid_is_dedicated_non_root_uid() { let cfg = KubernetesComputeConfig::default(); - assert_eq!(cfg.supervisor_topology, SupervisorTopology::Combined); - assert_eq!(cfg.supervisor_topology.to_string(), "combined"); + assert_eq!(cfg.proxy_uid, DEFAULT_PROXY_UID); + } + + #[test] + fn default_process_enforcement_is_network_only() { + let cfg = KubernetesComputeConfig::default(); + assert_eq!(cfg.process_enforcement, ProcessEnforcementMode::NetworkOnly); + assert_eq!(cfg.process_enforcement.to_string(), "network-only"); + } + + #[test] + fn serde_override_supervisor_topology_sidecar() { + let json = serde_json::json!({ + "supervisor_topology": "sidecar" + }); + let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); + assert_eq!(cfg.supervisor_topology, SupervisorTopology::Sidecar); } #[test] @@ -486,6 +565,35 @@ mod tests { assert_eq!(cfg.supervisor_topology, SupervisorTopology::Combined); } + #[test] + fn serde_override_process_enforcement_full() { + let json = serde_json::json!({ + "process_enforcement": "full" + }); + let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); + assert_eq!(cfg.process_enforcement, ProcessEnforcementMode::Full); + } + + #[test] + fn serde_override_proxy_uid() { + let json = serde_json::json!({ + "proxy_uid": 2000 + }); + let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); + assert_eq!(cfg.proxy_uid, 2000); + cfg.validate_proxy_uid().unwrap(); + } + + #[test] + fn validate_proxy_uid_rejects_privileged_uid() { + let cfg = KubernetesComputeConfig { + proxy_uid: 999, + ..KubernetesComputeConfig::default() + }; + let err = cfg.validate_proxy_uid().unwrap_err(); + assert!(err.contains("proxy_uid")); + } + #[test] fn serde_rejects_invalid_supervisor_topology() { let json = serde_json::json!({ @@ -495,6 +603,24 @@ mod tests { assert!(err.to_string().contains("unknown variant")); } + #[test] + fn serde_rejects_invalid_process_enforcement() { + let json = serde_json::json!({ + "process_enforcement": "privileged" + }); + let err = serde_json::from_value::(json).unwrap_err(); + assert!(err.to_string().contains("unknown variant")); + } + + #[test] + fn default_service_account_name_is_default() { + let cfg = KubernetesComputeConfig::default(); + assert_eq!( + cfg.service_account_name, + DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME + ); + } + #[test] fn serde_override_workspace_storage_size() { let json = serde_json::json!({ diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 166f18b1cc..ae45b3f50e 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -5,8 +5,9 @@ use super::AppArmorProfile; use crate::config::{ - DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_SANDBOX_UID, DEFAULT_WORKSPACE_STORAGE_SIZE, - KubernetesComputeConfig, SupervisorSideloadMethod, + DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_SANDBOX_UID, + DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, ProcessEnforcementMode, + SupervisorSideloadMethod, SupervisorTopology, }; use futures::{Stream, StreamExt, TryStreamExt}; use k8s_openapi::api::core::v1::{Event as KubeEventObj, Namespace, Node}; @@ -221,6 +222,9 @@ impl KubernetesComputeDriver { config .validate_sandbox_identity_config() .map_err(KubernetesDriverError::Precondition)?; + config + .validate_proxy_uid() + .map_err(KubernetesDriverError::Precondition)?; let base_config = match kube::Config::incluster() { Ok(c) => c, Err(_) => kube::Config::infer() @@ -549,7 +553,8 @@ impl KubernetesComputeDriver { .map_err(KubernetesDriverError::Message)?; // Resolve sandbox UID/GID from config or OpenShift SCC namespace annotations. - let (resolved_uid, resolved_gid, ns_annotations) = self.resolve_sandbox_identity().await; + let (resolved_user_id, resolved_group_id, ns_annotations) = + self.resolve_sandbox_identity().await; let params = SandboxPodParams { default_image: &self.config.default_image, @@ -558,6 +563,9 @@ impl KubernetesComputeDriver { supervisor_image: &self.config.supervisor_image, supervisor_image_pull_policy: &self.config.supervisor_image_pull_policy, supervisor_sideload_method: self.config.supervisor_sideload_method, + supervisor_topology: self.config.supervisor_topology, + process_enforcement: self.config.process_enforcement, + proxy_uid: self.config.proxy_uid, service_account_name: &self.config.service_account_name, sandbox_id: &sandbox.id, sandbox_name: &sandbox.name, @@ -574,9 +582,10 @@ impl KubernetesComputeDriver { provider_spiffe_workload_api_socket_path: &self .config .provider_spiffe_workload_api_socket_path, - sandbox_uid: resolved_uid, - sandbox_gid: resolved_gid, + sandbox_uid: resolved_user_id, + sandbox_gid: resolved_group_id, }; + validate_sidecar_proxy_identity(¶ms)?; let mut obj = DynamicObject::new(name, &agent_sandbox_api.resource); // Copy only the SCC-related annotations onto the Sandbox CR for @@ -1035,6 +1044,31 @@ const SUPERVISOR_VOLUME_NAME: &str = "openshell-supervisor-bin"; /// Name of the init container that installs the supervisor binary. const SUPERVISOR_INIT_CONTAINER_NAME: &str = "openshell-supervisor-install"; +/// Name of the init container that prepares pod-level sidecar networking. +const SUPERVISOR_NETWORK_INIT_CONTAINER_NAME: &str = "openshell-network-init"; + +/// Container name for the network-only supervisor sidecar. +const SUPERVISOR_NETWORK_SIDECAR_NAME: &str = "openshell-supervisor-network"; + +/// Shared volume used by the network sidecar to signal readiness to the +/// process-only supervisor in the agent container. +const SIDECAR_STATE_VOLUME_NAME: &str = "openshell-sidecar-state"; +const SIDECAR_STATE_MOUNT_PATH: &str = "/run/openshell-sidecar"; +const SIDECAR_READY_FILE: &str = "/run/openshell-sidecar/supervisor.ready"; +const SIDECAR_ENTRYPOINT_PID_FILE: &str = "/run/openshell-sidecar/entrypoint.pid"; +const SIDECAR_SSH_SOCKET_FILE: &str = "/run/openshell-sidecar/ssh.sock"; + +/// Shared TLS work directory. The network sidecar writes the proxy CA bundle +/// here, while the agent container consumes it after the readiness file exists. +const SIDECAR_TLS_VOLUME_NAME: &str = "openshell-supervisor-tls"; +const SIDECAR_TLS_MOUNT_PATH: &str = "/etc/openshell-tls/proxy"; +const SIDECAR_CLIENT_TLS_MOUNT_PATH: &str = "/etc/openshell-tls/proxy/client"; + +/// Loopback listener owned by the network sidecar. The process-only supervisor +/// connects here for gateway gRPC, and the sidecar forwards bytes to the real +/// gateway endpoint using its own network privileges. +const SIDECAR_GATEWAY_FORWARD_ADDR: &str = "127.0.0.1:18080"; + /// Build the emptyDir volume that holds the supervisor binary. /// /// The init container writes the binary here; the agent container reads it. @@ -1109,31 +1143,12 @@ fn supervisor_init_container( spec } -/// Apply supervisor side-load transforms to an already-built pod template JSON. -/// -/// Depending on the sideload method: -/// - **`ImageVolume`**: mounts the supervisor OCI image directly as a read-only -/// volume (no init container needed, requires K8s >= v1.33). -/// - **`InitContainer`**: injects an emptyDir volume and an init container that -/// copies the supervisor binary from the supervisor image into that volume. -/// -/// In both cases, the agent container gets a command override to run the -/// side-loaded binary and `runAsUser: 0` so it can create network namespaces, -/// set up the proxy, and configure Landlock/seccomp. -#[allow(clippy::similar_names)] -fn apply_supervisor_sideload( - pod_template: &mut serde_json::Value, +fn apply_supervisor_binary_source( + spec: &mut serde_json::Map, supervisor_image: &str, supervisor_image_pull_policy: &str, method: SupervisorSideloadMethod, - sandbox_uid: u32, - sandbox_gid: u32, ) { - let Some(spec) = pod_template.get_mut("spec").and_then(|v| v.as_object_mut()) else { - return; - }; - - // 1. Add the volume (image source or emptyDir depending on method) let volumes = spec .entry("volumes") .or_insert_with(|| serde_json::json!([])) @@ -1152,7 +1167,6 @@ fn apply_supervisor_sideload( } } - // 2. Add the init container only for the init-container method if method == SupervisorSideloadMethod::InitContainer { let init_containers = spec .entry("initContainers") @@ -1165,8 +1179,35 @@ fn apply_supervisor_sideload( )); } } +} - // 3. Find the agent container and add volume mount + command override +/// Apply supervisor side-load transforms to an already-built pod template JSON. +/// +/// Depending on the sideload method: +/// - **`ImageVolume`**: mounts the supervisor OCI image directly as a read-only +/// volume (no init container needed, requires K8s >= v1.33). +/// - **`InitContainer`**: injects an emptyDir volume and an init container that +/// copies the supervisor binary from the supervisor image into that volume. +/// +/// In both cases, the agent container gets a command override to run the +/// side-loaded binary as root so it can create network namespaces, set up the +/// proxy, and configure Landlock/seccomp. +#[allow(clippy::similar_names)] +fn apply_supervisor_sideload( + pod_template: &mut serde_json::Value, + supervisor_image: &str, + supervisor_image_pull_policy: &str, + method: SupervisorSideloadMethod, + sandbox_uid: u32, + sandbox_gid: u32, +) { + let Some(spec) = pod_template.get_mut("spec").and_then(|v| v.as_object_mut()) else { + return; + }; + + apply_supervisor_binary_source(spec, supervisor_image, supervisor_image_pull_policy, method); + + // Find the agent container and add volume mount + command override let Some(containers) = spec.get_mut("containers").and_then(|v| v.as_array_mut()) else { return; }; @@ -1227,6 +1268,420 @@ fn apply_supervisor_sideload( } } +fn sidecar_state_volume_mount() -> serde_json::Value { + serde_json::json!({ + "name": SIDECAR_STATE_VOLUME_NAME, + "mountPath": SIDECAR_STATE_MOUNT_PATH, + }) +} + +fn sidecar_tls_volume_mount() -> serde_json::Value { + serde_json::json!({ + "name": SIDECAR_TLS_VOLUME_NAME, + "mountPath": SIDECAR_TLS_MOUNT_PATH, + }) +} + +fn sidecar_process_gateway_endpoint(grpc_endpoint: &str) -> String { + if grpc_endpoint.is_empty() { + String::new() + } else if grpc_endpoint.starts_with("https://") { + format!("https://{SIDECAR_GATEWAY_FORWARD_ADDR}") + } else { + format!("http://{SIDECAR_GATEWAY_FORWARD_ADDR}") + } +} + +fn gateway_tls_server_name(grpc_endpoint: &str) -> Option { + let rest = grpc_endpoint.strip_prefix("https://")?; + let authority = rest.split('/').next().unwrap_or(rest); + if authority.is_empty() { + return None; + } + if let Some(bracketed) = authority.strip_prefix('[') { + return bracketed.split(']').next().map(str::to_string); + } + authority + .split(':') + .next() + .filter(|host| !host.is_empty()) + .map(str::to_string) +} + +fn copy_log_level_env( + env: &mut Vec, + template_environment: &std::collections::HashMap, + spec_environment: &std::collections::HashMap, +) { + if let Some(value) = spec_environment + .get(openshell_core::sandbox_env::LOG_LEVEL) + .or_else(|| template_environment.get(openshell_core::sandbox_env::LOG_LEVEL)) + { + upsert_env(env, openshell_core::sandbox_env::LOG_LEVEL, value); + } +} + +fn supervisor_sidecar_env( + template_environment: &std::collections::HashMap, + spec_environment: &std::collections::HashMap, + params: &SandboxPodParams<'_>, +) -> Vec { + let mut env = Vec::new(); + apply_required_env( + &mut env, + params.sandbox_id, + params.sandbox_name, + params.grpc_endpoint, + "", + !params.client_tls_secret_name.is_empty(), + provider_spiffe_socket_path(params), + ); + if !params.client_tls_secret_name.is_empty() { + upsert_env( + &mut env, + openshell_core::sandbox_env::TLS_CA, + &format!("{SIDECAR_CLIENT_TLS_MOUNT_PATH}/ca.crt"), + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::TLS_CERT, + &format!("{SIDECAR_CLIENT_TLS_MOUNT_PATH}/tls.crt"), + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::TLS_KEY, + &format!("{SIDECAR_CLIENT_TLS_MOUNT_PATH}/tls.key"), + ); + } + copy_log_level_env(&mut env, template_environment, spec_environment); + upsert_env( + &mut env, + openshell_core::sandbox_env::SUPERVISOR_TOPOLOGY, + "sidecar", + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::NETWORK_ENFORCEMENT_MODE, + "sidecar-nftables", + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::NETWORK_BINARY_IDENTITY, + "relaxed", + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::SUPERVISOR_READY_FILE, + SIDECAR_READY_FILE, + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::ENTRYPOINT_PID_FILE, + SIDECAR_ENTRYPOINT_PID_FILE, + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::GATEWAY_FORWARD_ADDR, + SIDECAR_GATEWAY_FORWARD_ADDR, + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::PROXY_TLS_DIR, + SIDECAR_TLS_MOUNT_PATH, + ); + env +} + +fn supervisor_sidecar_container( + template_environment: &std::collections::HashMap, + spec_environment: &std::collections::HashMap, + params: &SandboxPodParams<'_>, +) -> serde_json::Value { + let mut container = serde_json::json!({ + "name": SUPERVISOR_NETWORK_SIDECAR_NAME, + "image": params.supervisor_image, + "command": [ + SUPERVISOR_IMAGE_BINARY_PATH, + "--mode=network", + ], + "env": supervisor_sidecar_env(template_environment, spec_environment, params), + "securityContext": { + "runAsUser": params.proxy_uid, + "runAsGroup": params.sandbox_gid, + "runAsNonRoot": true, + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": ["ALL"] + } + }, + "volumeMounts": [ + sidecar_state_volume_mount(), + sidecar_tls_volume_mount(), + { + "name": "openshell-sa-token", + "mountPath": "/var/run/secrets/openshell", + "readOnly": true + } + ] + }); + if !params.supervisor_image_pull_policy.is_empty() { + container["imagePullPolicy"] = serde_json::json!(params.supervisor_image_pull_policy); + } + if params.provider_spiffe_enabled { + container["volumeMounts"] + .as_array_mut() + .expect("volumeMounts is an array") + .push(serde_json::json!({ + "name": SPIFFE_WORKLOAD_API_VOLUME_NAME, + "mountPath": spiffe_socket_mount_path(params.provider_spiffe_workload_api_socket_path), + "readOnly": true, + })); + } + if let Some(profile) = params.app_armor_profile { + container["securityContext"]["appArmorProfile"] = app_armor_profile_to_k8s(profile); + } + container +} + +fn supervisor_network_init_container(params: &SandboxPodParams<'_>) -> serde_json::Value { + let mut container = serde_json::json!({ + "name": SUPERVISOR_NETWORK_INIT_CONTAINER_NAME, + "image": params.supervisor_image, + "command": [ + SUPERVISOR_IMAGE_BINARY_PATH, + "--mode=network-init", + "--proxy-uid", + params.proxy_uid.to_string(), + "--proxy-gid", + params.sandbox_gid.to_string(), + "--sidecar-state-dir", + SIDECAR_STATE_MOUNT_PATH, + "--sidecar-tls-dir", + SIDECAR_TLS_MOUNT_PATH, + ], + "securityContext": { + "runAsUser": 0, + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": ["ALL"], + "add": ["NET_ADMIN", "NET_RAW", "CHOWN", "FOWNER"] + } + }, + "volumeMounts": [ + sidecar_state_volume_mount(), + sidecar_tls_volume_mount(), + ] + }); + if !params.supervisor_image_pull_policy.is_empty() { + container["imagePullPolicy"] = serde_json::json!(params.supervisor_image_pull_policy); + } + if !params.client_tls_secret_name.is_empty() { + container["volumeMounts"] + .as_array_mut() + .expect("volumeMounts is an array") + .push(serde_json::json!({ + "name": "openshell-client-tls", + "mountPath": "/etc/openshell-tls/client", + "readOnly": true + })); + } + if let Some(profile) = params.app_armor_profile { + container["securityContext"]["appArmorProfile"] = app_armor_profile_to_k8s(profile); + } + container +} + +fn apply_supervisor_sidecar_topology( + pod_template: &mut serde_json::Value, + template_environment: &std::collections::HashMap, + spec_environment: &std::collections::HashMap, + params: &SandboxPodParams<'_>, +) { + let Some(spec) = pod_template.get_mut("spec").and_then(|v| v.as_object_mut()) else { + return; + }; + + let pod_security_context = spec + .entry("securityContext") + .or_insert_with(|| serde_json::json!({})); + if let Some(sc) = pod_security_context.as_object_mut() { + sc.insert("fsGroup".to_string(), serde_json::json!(params.sandbox_gid)); + } + + apply_supervisor_binary_source( + spec, + params.supervisor_image, + params.supervisor_image_pull_policy, + params.supervisor_sideload_method, + ); + + let volumes = spec + .entry("volumes") + .or_insert_with(|| serde_json::json!([])) + .as_array_mut(); + if let Some(volumes) = volumes { + volumes.push(serde_json::json!({ + "name": SIDECAR_STATE_VOLUME_NAME, + "emptyDir": {} + })); + volumes.push(serde_json::json!({ + "name": SIDECAR_TLS_VOLUME_NAME, + "emptyDir": {} + })); + } + + let init_containers = spec + .entry("initContainers") + .or_insert_with(|| serde_json::json!([])) + .as_array_mut(); + if let Some(init_containers) = init_containers { + init_containers.push(supervisor_network_init_container(params)); + } + + let Some(containers) = spec.get_mut("containers").and_then(|v| v.as_array_mut()) else { + return; + }; + + let target_index = containers + .iter() + .position(|c| c.get("name").and_then(|v| v.as_str()) == Some("agent")) + .unwrap_or(0); + + if let Some(container) = containers + .get_mut(target_index) + .and_then(|v| v.as_object_mut()) + { + container.insert( + "command".to_string(), + serde_json::json!([ + format!("{}/openshell-sandbox", SUPERVISOR_MOUNT_PATH), + "--mode=process" + ]), + ); + + let security_context = container + .entry("securityContext") + .or_insert_with(|| serde_json::json!({})); + if let Some(sc) = security_context.as_object_mut() { + match params.process_enforcement { + ProcessEnforcementMode::NetworkOnly => { + sc.insert( + "runAsUser".to_string(), + serde_json::json!(params.sandbox_uid), + ); + sc.insert( + "runAsGroup".to_string(), + serde_json::json!(params.sandbox_gid), + ); + sc.insert("runAsNonRoot".to_string(), serde_json::json!(true)); + sc.insert( + "allowPrivilegeEscalation".to_string(), + serde_json::json!(false), + ); + sc.insert( + "capabilities".to_string(), + serde_json::json!({ + "drop": ["ALL"] + }), + ); + } + ProcessEnforcementMode::Full => { + sc.insert("runAsUser".to_string(), serde_json::json!(0)); + sc.remove("runAsGroup"); + sc.remove("runAsNonRoot"); + sc.remove("allowPrivilegeEscalation"); + sc.entry("capabilities".to_string()).or_insert_with(|| { + serde_json::json!({ + "add": ["SYS_ADMIN", "NET_ADMIN", "SYS_PTRACE", "SYSLOG"] + }) + }); + } + } + } + + let volume_mounts = container + .entry("volumeMounts") + .or_insert_with(|| serde_json::json!([])) + .as_array_mut(); + if let Some(volume_mounts) = volume_mounts { + volume_mounts.push(supervisor_volume_mount()); + volume_mounts.push(sidecar_state_volume_mount()); + volume_mounts.push(sidecar_tls_volume_mount()); + } + + let env = container + .entry("env") + .or_insert_with(|| serde_json::json!([])) + .as_array_mut(); + if let Some(env) = env { + let process_endpoint = sidecar_process_gateway_endpoint(params.grpc_endpoint); + upsert_env( + env, + openshell_core::sandbox_env::ENDPOINT, + &process_endpoint, + ); + if let Some(server_name) = gateway_tls_server_name(params.grpc_endpoint) { + upsert_env( + env, + openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME, + &server_name, + ); + } + upsert_env( + env, + openshell_core::sandbox_env::SUPERVISOR_TOPOLOGY, + "sidecar", + ); + upsert_env( + env, + openshell_core::sandbox_env::NETWORK_ENFORCEMENT_MODE, + "sidecar-nftables", + ); + upsert_env( + env, + openshell_core::sandbox_env::PROCESS_ENFORCEMENT_MODE, + ¶ms.process_enforcement.to_string(), + ); + upsert_env( + env, + openshell_core::sandbox_env::SSH_SOCKET_PATH, + SIDECAR_SSH_SOCKET_FILE, + ); + upsert_env( + env, + openshell_core::sandbox_env::SUPERVISOR_READY_FILE, + SIDECAR_READY_FILE, + ); + upsert_env( + env, + openshell_core::sandbox_env::ENTRYPOINT_PID_FILE, + SIDECAR_ENTRYPOINT_PID_FILE, + ); + upsert_env( + env, + openshell_core::sandbox_env::PROXY_TLS_DIR, + SIDECAR_TLS_MOUNT_PATH, + ); + upsert_env( + env, + openshell_core::sandbox_env::SANDBOX_UID, + ¶ms.sandbox_uid.to_string(), + ); + upsert_env( + env, + openshell_core::sandbox_env::SANDBOX_GID, + ¶ms.sandbox_gid.to_string(), + ); + } + } + + containers.push(supervisor_sidecar_container( + template_environment, + spec_environment, + params, + )); +} + /// Apply workspace persistence transforms to an already-built pod template. /// /// This injects: @@ -1242,6 +1697,7 @@ fn apply_supervisor_sideload( /// The init container mounts the PVC at a temporary path so it can still see /// the image's `/sandbox` directory. It checks for a sentinel file and skips /// the copy if the PVC was already initialised. +#[allow(clippy::similar_names)] fn apply_workspace_persistence( pod_template: &mut serde_json::Value, image: &str, @@ -1301,6 +1757,10 @@ fn apply_workspace_persistence( // self-referential symlinks under `/sandbox/.uv`, and GNU cp can // fail while seeding the PVC even though preserving the symlink as-is // is valid. `tar` copies the tree without dereferencing those links. + // Archive only the contents, not the `/sandbox` directory entry + // itself, so extraction never tries to chmod the PVC mount root. + // Extract without restoring owner, mode, or timestamps so the + // non-root init container can seed kubelet-owned PVCs. // // The inner `[ -d ... ]` guard handles custom images that don't have // a /sandbox directory — the copy is skipped but the sentinel is @@ -1308,7 +1768,12 @@ fn apply_workspace_persistence( let copy_cmd = format!( "if [ ! -f {WORKSPACE_INIT_MOUNT_PATH}/{WORKSPACE_SENTINEL} ]; then \ if [ -d {WORKSPACE_MOUNT_PATH} ]; then \ - tar -C {WORKSPACE_MOUNT_PATH} -cf - . | tar -C {WORKSPACE_INIT_MOUNT_PATH} -xpf -; \ + tmp=$(mktemp) && rm -f \"$tmp\" && \ + (cd {WORKSPACE_MOUNT_PATH} && find . -mindepth 1 -maxdepth 1 -exec tar -cf \"$tmp\" {{}} +) && \ + if [ -f \"$tmp\" ]; then \ + tar -C {WORKSPACE_INIT_MOUNT_PATH} --no-same-owner --no-same-permissions --touch -xf \"$tmp\" && \ + rm -f \"$tmp\"; \ + fi; \ fi && \ touch {WORKSPACE_INIT_MOUNT_PATH}/{WORKSPACE_SENTINEL}; \ fi" @@ -1366,6 +1831,9 @@ struct SandboxPodParams<'a> { supervisor_image: &'a str, supervisor_image_pull_policy: &'a str, supervisor_sideload_method: SupervisorSideloadMethod, + supervisor_topology: SupervisorTopology, + process_enforcement: ProcessEnforcementMode, + proxy_uid: u32, service_account_name: &'a str, sandbox_id: &'a str, sandbox_name: &'a str, @@ -1397,6 +1865,9 @@ impl Default for SandboxPodParams<'_> { supervisor_image: "", supervisor_image_pull_policy: "", supervisor_sideload_method: SupervisorSideloadMethod::default(), + supervisor_topology: SupervisorTopology::default(), + process_enforcement: ProcessEnforcementMode::default(), + proxy_uid: DEFAULT_PROXY_UID, service_account_name: DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, sandbox_id: "", sandbox_name: "", @@ -1417,6 +1888,20 @@ impl Default for SandboxPodParams<'_> { } } +fn validate_sidecar_proxy_identity( + params: &SandboxPodParams<'_>, +) -> Result<(), KubernetesDriverError> { + if params.supervisor_topology == SupervisorTopology::Sidecar + && params.proxy_uid == params.sandbox_uid + { + return Err(KubernetesDriverError::Precondition(format!( + "proxy_uid ({}) must not match sandbox_uid ({}) in sidecar topology", + params.proxy_uid, params.sandbox_uid + ))); + } + Ok(()) +} + fn spec_pod_env(spec: Option<&SandboxSpec>) -> std::collections::HashMap { let mut env = spec.map_or_else(Default::default, |s| s.environment.clone()); if let Some(s) = spec.filter(|s| !s.log_level.is_empty()) { @@ -1707,13 +2192,22 @@ fn sandbox_template_to_k8s_with_gpu_requirements( serde_json::Value::Array(vec![serde_json::Value::Object(container)]), ); - // Add TLS secret volume. Mode 0400 (owner-read) prevents the - // unprivileged sandbox user from reading the mTLS private key. + // Add TLS secret volume. Combined mode uses mode 0400 because the + // supervisor starts as root and drops privileges before running workload + // children. Sidecar mode keeps the process supervisor non-root, so it uses + // pod fsGroup + 0440 to preserve gateway session and SSH control behavior. let mut volumes: Vec = Vec::new(); if !params.client_tls_secret_name.is_empty() { + let client_tls_default_mode = match params.supervisor_topology { + SupervisorTopology::Combined => 0o400, + SupervisorTopology::Sidecar => 0o440, + }; volumes.push(serde_json::json!({ "name": "openshell-client-tls", - "secret": { "secretName": params.client_tls_secret_name, "defaultMode": 256 } + "secret": { + "secretName": params.client_tls_secret_name, + "defaultMode": client_tls_default_mode + } })); } if params.provider_spiffe_enabled { @@ -1728,7 +2222,12 @@ fn sandbox_template_to_k8s_with_gpu_requirements( // Projected ServiceAccountToken volume — kubelet writes a short-lived // audience-bound JWT into /var/run/secrets/openshell/token and rotates // it automatically. The supervisor exchanges this for a gateway-minted - // JWT via `IssueSandboxToken` once at startup. + // JWT via `IssueSandboxToken` once at startup. In sidecar topology both + // supervisor containers run with the sandbox GID and need group-read access. + let sa_token_default_mode = match params.supervisor_topology { + SupervisorTopology::Combined => 0o400, + SupervisorTopology::Sidecar => 0o440, + }; volumes.push(serde_json::json!({ "name": "openshell-sa-token", "projected": { @@ -1739,7 +2238,7 @@ fn sandbox_template_to_k8s_with_gpu_requirements( "path": "token" } }], - "defaultMode": 256 + "defaultMode": sa_token_default_mode } })); spec.insert("volumes".to_string(), serde_json::Value::Array(volumes)); @@ -1763,14 +2262,26 @@ fn sandbox_template_to_k8s_with_gpu_requirements( let mut result = serde_json::Value::Object(template_value); - apply_supervisor_sideload( - &mut result, - params.supervisor_image, - params.supervisor_image_pull_policy, - params.supervisor_sideload_method, - params.sandbox_uid, - params.sandbox_gid, - ); + match params.supervisor_topology { + SupervisorTopology::Combined => { + apply_supervisor_sideload( + &mut result, + params.supervisor_image, + params.supervisor_image_pull_policy, + params.supervisor_sideload_method, + params.sandbox_uid, + params.sandbox_gid, + ); + } + SupervisorTopology::Sidecar => { + apply_supervisor_sidecar_topology( + &mut result, + &template.environment, + spec_environment, + params, + ); + } + } // Inject workspace persistence (init container + PVC volume mount) so // that /sandbox data survives pod rescheduling. Skipped when the user @@ -2262,6 +2773,15 @@ mod tests { assert!(!should_try_next_sandbox_api_version(&err)); } + fn rendered_env<'a>(container: &'a serde_json::Value, name: &str) -> Option<&'a str> { + container["env"] + .as_array()? + .iter() + .find(|item| item.get("name").and_then(|value| value.as_str()) == Some(name))? + .get("value")? + .as_str() + } + #[test] fn driver_config_rejects_invalid_shape() { let template = SandboxTemplate { @@ -2601,6 +3121,312 @@ mod tests { ); } + #[test] + fn sidecar_topology_renders_process_agent_and_network_sidecar() { + let params = SandboxPodParams { + supervisor_topology: SupervisorTopology::Sidecar, + supervisor_sideload_method: SupervisorSideloadMethod::InitContainer, + supervisor_image: "supervisor-image:latest", + supervisor_image_pull_policy: "IfNotPresent", + grpc_endpoint: "https://openshell-gateway.openshell.svc:8080", + client_tls_secret_name: "openshell-client-tls", + proxy_uid: 2200, + sandbox_uid: 1500, + sandbox_gid: 1500, + ..SandboxPodParams::default() + }; + let pod_template = sandbox_template_to_k8s( + &SandboxTemplate { + image: "agent-image:latest".to_string(), + ..SandboxTemplate::default() + }, + false, + &std::collections::HashMap::new(), + false, + ¶ms, + ); + + assert!( + pod_template["spec"]["shareProcessNamespace"].is_null(), + "sidecar mode no longer needs a shared process namespace when binary identity is relaxed" + ); + assert_eq!(pod_template["spec"]["securityContext"]["fsGroup"], 1500); + let containers = pod_template["spec"]["containers"].as_array().unwrap(); + assert_eq!(containers.len(), 2); + + let agent = containers + .iter() + .find(|container| container["name"] == "agent") + .unwrap(); + assert_eq!( + agent["command"], + serde_json::json!([ + format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox"), + "--mode=process" + ]) + ); + assert_eq!(agent["securityContext"]["runAsUser"], 1500); + assert_eq!(agent["securityContext"]["runAsGroup"], 1500); + assert_eq!(agent["securityContext"]["runAsNonRoot"], true); + assert_eq!( + agent["securityContext"]["capabilities"], + serde_json::json!({ + "drop": ["ALL"] + }) + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::ENDPOINT), + Some("https://127.0.0.1:18080") + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME), + Some("openshell-gateway.openshell.svc") + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::PROCESS_ENFORCEMENT_MODE), + Some("network-only") + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::SSH_SOCKET_PATH), + Some(SIDECAR_SSH_SOCKET_FILE) + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::SUPERVISOR_READY_FILE), + Some(SIDECAR_READY_FILE) + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::ENTRYPOINT_PID_FILE), + Some(SIDECAR_ENTRYPOINT_PID_FILE) + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::PROXY_TLS_DIR), + Some(SIDECAR_TLS_MOUNT_PATH) + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::SANDBOX_UID), + Some("1500") + ); + + let sidecar = containers + .iter() + .find(|container| container["name"] == SUPERVISOR_NETWORK_SIDECAR_NAME) + .unwrap(); + assert_eq!(sidecar["image"], "supervisor-image:latest"); + assert_eq!(sidecar["imagePullPolicy"], "IfNotPresent"); + assert_eq!( + sidecar["command"], + serde_json::json!([SUPERVISOR_IMAGE_BINARY_PATH, "--mode=network"]) + ); + assert_eq!(sidecar["securityContext"]["runAsUser"], 2200); + assert_eq!(sidecar["securityContext"]["runAsGroup"], 1500); + assert_eq!(sidecar["securityContext"]["runAsNonRoot"], true); + assert_eq!( + sidecar["securityContext"]["capabilities"], + serde_json::json!({ + "drop": ["ALL"] + }) + ); + assert_eq!( + rendered_env(sidecar, openshell_core::sandbox_env::ENDPOINT), + Some("https://openshell-gateway.openshell.svc:8080") + ); + assert_eq!( + rendered_env(sidecar, openshell_core::sandbox_env::GATEWAY_FORWARD_ADDR), + Some(SIDECAR_GATEWAY_FORWARD_ADDR) + ); + assert_eq!( + rendered_env( + sidecar, + openshell_core::sandbox_env::NETWORK_BINARY_IDENTITY + ), + Some("relaxed") + ); + assert_eq!( + rendered_env(sidecar, openshell_core::sandbox_env::ENTRYPOINT_PID_FILE), + Some(SIDECAR_ENTRYPOINT_PID_FILE) + ); + assert_eq!( + rendered_env(sidecar, openshell_core::sandbox_env::PROXY_TLS_DIR), + Some(SIDECAR_TLS_MOUNT_PATH) + ); + assert_eq!( + rendered_env(sidecar, openshell_core::sandbox_env::TLS_CA), + Some("/etc/openshell-tls/proxy/client/ca.crt") + ); + let sidecar_mounts = sidecar["volumeMounts"].as_array().unwrap(); + assert!( + !sidecar_mounts + .iter() + .any(|mount| mount["name"] == "openshell-client-tls"), + "runtime sidecar should use the init-copied TLS files, not the root-owned Secret mount" + ); + let volumes = pod_template["spec"]["volumes"].as_array().unwrap(); + let sa_token = volumes + .iter() + .find(|volume| volume["name"] == "openshell-sa-token") + .unwrap(); + assert_eq!(sa_token["projected"]["defaultMode"], 0o440); + let client_tls = volumes + .iter() + .find(|volume| volume["name"] == "openshell-client-tls") + .unwrap(); + assert_eq!(client_tls["secret"]["defaultMode"], 0o440); + + let init_containers = pod_template["spec"]["initContainers"].as_array().unwrap(); + let network_init = init_containers + .iter() + .find(|container| container["name"] == SUPERVISOR_NETWORK_INIT_CONTAINER_NAME) + .unwrap(); + assert_eq!(network_init["image"], "supervisor-image:latest"); + assert_eq!(network_init["imagePullPolicy"], "IfNotPresent"); + assert_eq!( + network_init["command"], + serde_json::json!([ + SUPERVISOR_IMAGE_BINARY_PATH, + "--mode=network-init", + "--proxy-uid", + "2200", + "--proxy-gid", + "1500", + "--sidecar-state-dir", + SIDECAR_STATE_MOUNT_PATH, + "--sidecar-tls-dir", + SIDECAR_TLS_MOUNT_PATH + ]) + ); + assert_eq!( + network_init["securityContext"]["capabilities"], + serde_json::json!({ + "drop": ["ALL"], + "add": ["NET_ADMIN", "NET_RAW", "CHOWN", "FOWNER"] + }) + ); + let network_init_mounts = network_init["volumeMounts"].as_array().unwrap(); + assert!(network_init_mounts.iter().any(|mount| { + mount["name"] == "openshell-client-tls" + && mount["mountPath"] == "/etc/openshell-tls/client" + })); + } + + #[test] + fn sidecar_topology_adds_shared_state_and_tls_volumes() { + let params = SandboxPodParams { + supervisor_topology: SupervisorTopology::Sidecar, + supervisor_sideload_method: SupervisorSideloadMethod::ImageVolume, + supervisor_image: "supervisor-image:latest", + grpc_endpoint: "http://openshell-gateway.openshell.svc:8080", + ..SandboxPodParams::default() + }; + let pod_template = sandbox_template_to_k8s( + &SandboxTemplate::default(), + false, + &std::collections::HashMap::new(), + false, + ¶ms, + ); + + let volumes = pod_template["spec"]["volumes"].as_array().unwrap(); + assert!( + volumes + .iter() + .any(|volume| volume["name"] == SIDECAR_STATE_VOLUME_NAME) + ); + assert!( + volumes + .iter() + .any(|volume| volume["name"] == SIDECAR_TLS_VOLUME_NAME) + ); + assert!(volumes.iter().any(|volume| { + volume["name"] == SUPERVISOR_VOLUME_NAME && volume["image"].is_object() + })); + + let containers = pod_template["spec"]["containers"].as_array().unwrap(); + for container_name in ["agent", SUPERVISOR_NETWORK_SIDECAR_NAME] { + let container = containers + .iter() + .find(|container| container["name"] == container_name) + .unwrap(); + let mounts = container["volumeMounts"].as_array().unwrap(); + assert!(mounts.iter().any(|mount| { + mount["name"] == SIDECAR_STATE_VOLUME_NAME + && mount["mountPath"] == SIDECAR_STATE_MOUNT_PATH + })); + assert!(mounts.iter().any(|mount| { + mount["name"] == SIDECAR_TLS_VOLUME_NAME + && mount["mountPath"] == SIDECAR_TLS_MOUNT_PATH + })); + } + } + + #[test] + fn sidecar_topology_full_process_enforcement_keeps_combined_agent_permissions() { + let params = SandboxPodParams { + supervisor_topology: SupervisorTopology::Sidecar, + process_enforcement: ProcessEnforcementMode::Full, + supervisor_sideload_method: SupervisorSideloadMethod::InitContainer, + supervisor_image: "supervisor-image:latest", + grpc_endpoint: "https://openshell-gateway.openshell.svc:8080", + sandbox_uid: 1500, + sandbox_gid: 1500, + proxy_uid: 2200, + ..SandboxPodParams::default() + }; + let pod_template = sandbox_template_to_k8s( + &SandboxTemplate::default(), + false, + &std::collections::HashMap::new(), + false, + ¶ms, + ); + + let containers = pod_template["spec"]["containers"].as_array().unwrap(); + let agent = containers + .iter() + .find(|container| container["name"] == "agent") + .unwrap(); + let sc = &agent["securityContext"]; + assert_eq!(sc["runAsUser"], 0); + assert!(sc.get("runAsGroup").is_none()); + assert!(sc.get("runAsNonRoot").is_none()); + assert!(sc.get("allowPrivilegeEscalation").is_none()); + assert_eq!( + sc["capabilities"], + serde_json::json!({ + "add": ["SYS_ADMIN", "NET_ADMIN", "SYS_PTRACE", "SYSLOG"] + }) + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::PROCESS_ENFORCEMENT_MODE), + Some("full") + ); + + let sidecar = containers + .iter() + .find(|container| container["name"] == SUPERVISOR_NETWORK_SIDECAR_NAME) + .unwrap(); + assert_eq!(sidecar["securityContext"]["runAsUser"], 2200); + assert_eq!( + sidecar["securityContext"]["capabilities"], + serde_json::json!({ + "drop": ["ALL"] + }) + ); + } + + #[test] + fn sidecar_topology_rejects_proxy_uid_matching_sandbox_uid() { + let params = SandboxPodParams { + supervisor_topology: SupervisorTopology::Sidecar, + proxy_uid: 1500, + sandbox_uid: 1500, + ..SandboxPodParams::default() + }; + + let err = validate_sidecar_proxy_identity(¶ms).unwrap_err(); + assert!(matches!(err, KubernetesDriverError::Precondition(_))); + assert!(err.to_string().contains("proxy_uid")); + } + /// Regression test: TLS mount path must match env var paths. /// The volume is mounted at a specific path and the env vars must point to /// files within that same path, otherwise the sandbox will fail to start @@ -3179,6 +4005,16 @@ mod tests { script.contains("tar -C"), "init script must seed image contents with a tar stream" ); + assert!( + script.contains("find . -mindepth 1 -maxdepth 1"), + "init script must archive sandbox contents without the mount root entry" + ); + assert!( + script.contains("--no-same-owner") + && script.contains("--no-same-permissions") + && script.contains("--touch"), + "init script must avoid restoring metadata onto the PVC root" + ); } #[test] diff --git a/crates/openshell-driver-kubernetes/src/lib.rs b/crates/openshell-driver-kubernetes/src/lib.rs index 953ed4abdf..114055bcc5 100644 --- a/crates/openshell-driver-kubernetes/src/lib.rs +++ b/crates/openshell-driver-kubernetes/src/lib.rs @@ -6,8 +6,9 @@ pub mod driver; pub mod grpc; pub use config::{ - AppArmorProfile, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_WORKSPACE_STORAGE_SIZE, - KubernetesComputeConfig, SupervisorSideloadMethod, SupervisorTopology, + AppArmorProfile, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, + DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, ProcessEnforcementMode, + SupervisorSideloadMethod, SupervisorTopology, }; pub use driver::{KubernetesComputeDriver, KubernetesDriverError}; pub use grpc::ComputeDriverService; diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 77f671dcb2..34ec1b55d7 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -10,8 +10,9 @@ use tracing_subscriber::EnvFilter; use openshell_core::VERSION; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; use openshell_driver_kubernetes::{ - AppArmorProfile, ComputeDriverService, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, - KubernetesComputeConfig, KubernetesComputeDriver, SupervisorSideloadMethod, SupervisorTopology, + AppArmorProfile, ComputeDriverService, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, + KubernetesComputeConfig, KubernetesComputeDriver, ProcessEnforcementMode, + SupervisorSideloadMethod, SupervisorTopology, }; #[derive(Parser, Debug)] @@ -87,6 +88,16 @@ struct Args { )] supervisor_topology: SupervisorTopology, + #[arg( + long, + env = "OPENSHELL_PROCESS_ENFORCEMENT", + default_value = "network-only" + )] + process_enforcement: ProcessEnforcementMode, + + #[arg(long, env = "OPENSHELL_PROXY_UID", default_value_t = DEFAULT_PROXY_UID)] + proxy_uid: u32, + #[arg(long, env = "OPENSHELL_ENABLE_USER_NAMESPACES")] enable_user_namespaces: bool, @@ -131,6 +142,8 @@ async fn main() -> Result<()> { supervisor_image_pull_policy: args.supervisor_image_pull_policy.unwrap_or_default(), supervisor_sideload_method: args.supervisor_sideload_method, supervisor_topology: args.supervisor_topology, + process_enforcement: args.process_enforcement, + proxy_uid: args.proxy_uid, grpc_endpoint: args.grpc_endpoint.unwrap_or_default(), ssh_socket_path: args.sandbox_ssh_socket_path, client_tls_secret_name: args.client_tls_secret_name.unwrap_or_default(), diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 54501cb8c9..c6a619f3d1 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -130,8 +130,8 @@ sequenceDiagram C->>C: entrypoint: /opt/openshell/bin/openshell-sandbox ``` -The supervisor image from `deploy/docker/Dockerfile.supervisor` copies the static -`openshell-sandbox` binary to `/openshell-sandbox`. +The supervisor image from `deploy/docker/Dockerfile.supervisor` provides the +static `openshell-sandbox` binary at `/openshell-sandbox`. Mounting that image at `/opt/openshell/bin` makes the binary available as `/opt/openshell/bin/openshell-sandbox`. diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 7514fbc510..b4f7c176be 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -887,9 +887,8 @@ pub fn build_container_spec_with_token_and_gpu_devices( // Side-load the supervisor binary from a standalone OCI image. // Podman resolves image_volumes at the libpod layer, mounting the // image's filesystem at the destination path without starting a - // container from it. The supervisor image is FROM scratch with just - // the binary at /openshell-sandbox, so it appears at - // /opt/openshell/bin/openshell-sandbox. + // container from it. The supervisor image exposes the binary at + // /openshell-sandbox, so it appears at /opt/openshell/bin/openshell-sandbox. image_volumes, hostname: format!("sandbox-{}", sandbox.name), // Override the image's ENTRYPOINT so the supervisor binary runs diff --git a/crates/openshell-sandbox/Cargo.toml b/crates/openshell-sandbox/Cargo.toml index 086dbe02c3..a5d3449106 100644 --- a/crates/openshell-sandbox/Cargo.toml +++ b/crates/openshell-sandbox/Cargo.toml @@ -33,6 +33,9 @@ clap = { workspace = true } # Error handling miette = { workspace = true } +# Unix ownership for Kubernetes sidecar init setup +nix = { workspace = true } + # TLS crypto provider install (main.rs) rustls = { workspace = true } diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 53b1eba582..3ff260c7c3 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -13,10 +13,10 @@ mod mechanistic_mapper; #[cfg_attr(not(target_os = "linux"), allow(dead_code))] mod metadata_server; -use miette::Result; +use miette::{IntoDiagnostic, Result}; use std::future::Future; use std::sync::Arc; -use std::sync::atomic::AtomicU32; +use std::sync::atomic::{AtomicU32, Ordering}; use std::time::Duration; use tracing::{debug, info, warn}; @@ -64,12 +64,22 @@ use openshell_core::denial::DenialEvent; use openshell_core::policy::{NetworkMode, NetworkPolicy, ProxyPolicy, SandboxPolicy}; use openshell_core::provider_credentials::ProviderCredentialState; use openshell_supervisor_network::opa::OpaEngine; +use openshell_supervisor_process::process::ProcessEnforcementMode; pub use openshell_supervisor_process::process::{ProcessHandle, ProcessStatus}; use openshell_supervisor_process::skills; +use tokio::io::copy_bidirectional; +use tokio::net::{TcpListener, TcpStream}; use tokio::sync::mpsc::UnboundedSender; #[cfg(target_os = "linux")] use tokio::time::timeout; +const SIDECAR_NETWORK_ENFORCEMENT_MODE: &str = "sidecar-nftables"; +const SIDECAR_TLS_DIR: &str = "/etc/openshell-tls/proxy"; +const SIDECAR_CA_CERT: &str = "openshell-ca.pem"; +const SIDECAR_CA_BUNDLE: &str = "ca-bundle.pem"; +const SIDECAR_PROCESS_PROXY_ADDR: &str = "127.0.0.1:3128"; +const SIDECAR_READY_TIMEOUT_SECS: u64 = 120; + /// Run a command in the sandbox. /// /// # Errors @@ -125,6 +135,16 @@ pub async fn run_sandbox( } } + let sidecar_network_enforcement = sidecar_network_enforcement_enabled(); + let process_enforcement_mode = process_enforcement_mode(); + let sidecar_ready_file = supervisor_ready_file(); + if process_enabled + && !network_enabled + && let Some(path) = sidecar_ready_file.as_deref() + { + wait_for_supervisor_ready(path).await?; + } + // Load policy and initialize OPA engine let openshell_endpoint_for_proxy = openshell_endpoint.clone(); let sandbox_name_for_agg = sandbox.clone(); @@ -251,6 +271,12 @@ pub async fn run_sandbox( // Shared PID: set after process spawn so the proxy can look up // the entrypoint process's /proc/net/tcp for identity binding. let entrypoint_pid = Arc::new(AtomicU32::new(0)); + if network_enabled + && !process_enabled + && let Some(path) = entrypoint_pid_file() + { + spawn_entrypoint_pid_file_watcher(path, entrypoint_pid.clone()); + } // Create the workload's network namespace. It is shared infrastructure: // the proxy binds to its host-side veth IP, the bypass monitor reads @@ -258,7 +284,7 @@ pub async fn run_sandbox( // it via setns(). The RAII handle lives in this frame for the duration // of the sandbox. #[cfg(target_os = "linux")] - let netns = if network_enabled { + let netns = if network_enabled && !sidecar_network_enforcement { openshell_supervisor_process::netns::create_netns_for_proxy(&policy)? } else { None @@ -328,6 +354,34 @@ pub async fn run_sandbox( None }; + let _gateway_forward = if network_enabled && sidecar_network_enforcement { + let endpoint = openshell_endpoint_for_proxy.as_deref().ok_or_else(|| { + miette::miette!("sidecar network enforcement requires an OpenShell gateway endpoint") + })?; + Some(start_gateway_forward_from_env(endpoint).await?) + } else { + None + }; + + #[cfg(target_os = "linux")] + if network_enabled && sidecar_network_enforcement { + if !matches!(policy.network.mode, NetworkMode::Proxy) { + return Err(miette::miette!( + "sidecar network enforcement requires proxy network mode" + )); + } + if let Some(path) = sidecar_ready_file.as_deref() { + write_supervisor_ready(path)?; + } + } + + #[cfg(not(target_os = "linux"))] + if network_enabled && sidecar_network_enforcement { + return Err(miette::miette!( + "sidecar network enforcement is only supported on Linux" + )); + } + // Spawn the denial-aggregator flush task. The aggregator drains denial // events from the proxy + bypass monitor, batches them, and ships // summaries to the gateway via `SubmitPolicyAnalysis`. @@ -478,8 +532,17 @@ pub async fn run_sandbox( } } + let process_policy = process_policy_for_topology(&policy, sidecar_network_enforcement)?; + let exit_code = if process_enabled { - let ca_file_paths = networking.as_ref().and_then(|n| n.ca_file_paths.clone()); + let ca_file_paths = networking + .as_ref() + .and_then(|n| n.ca_file_paths.clone()) + .or_else(|| { + sidecar_network_enforcement + .then(sidecar_ca_file_paths) + .flatten() + }); openshell_supervisor_process::run::run_process( program, @@ -490,7 +553,8 @@ pub async fn run_sandbox( sandbox_id.as_deref(), openshell_endpoint.as_deref(), ssh_socket_path, - &policy, + &process_policy, + process_enforcement_mode, entrypoint_pid, provider_credentials, provider_env, @@ -551,6 +615,205 @@ async fn wait_for_shutdown_signal() { } } +fn sidecar_network_enforcement_enabled() -> bool { + std::env::var(openshell_core::sandbox_env::NETWORK_ENFORCEMENT_MODE) + .is_ok_and(|value| value == SIDECAR_NETWORK_ENFORCEMENT_MODE) +} + +fn process_enforcement_mode() -> ProcessEnforcementMode { + match std::env::var(openshell_core::sandbox_env::PROCESS_ENFORCEMENT_MODE) + .unwrap_or_else(|_| "full".to_string()) + .as_str() + { + "network-only" => ProcessEnforcementMode::NetworkOnly, + _ => ProcessEnforcementMode::Full, + } +} + +fn supervisor_ready_file() -> Option { + std::env::var(openshell_core::sandbox_env::SUPERVISOR_READY_FILE) + .ok() + .filter(|value| !value.is_empty()) +} + +fn entrypoint_pid_file() -> Option { + std::env::var(openshell_core::sandbox_env::ENTRYPOINT_PID_FILE) + .ok() + .filter(|value| !value.is_empty()) +} + +fn spawn_entrypoint_pid_file_watcher(path: String, entrypoint_pid: Arc) { + tokio::spawn(async move { + let pid_path = std::path::PathBuf::from(&path); + loop { + match std::fs::read_to_string(&pid_path) { + Ok(contents) => match contents.trim().parse::() { + Ok(pid) if pid > 0 => { + entrypoint_pid.store(pid, Ordering::Release); + info!(path, pid, "Loaded sidecar workload entrypoint PID"); + return; + } + Ok(_) | Err(_) => { + debug!(path, contents = %contents.trim(), "Ignoring invalid entrypoint PID file contents"); + } + }, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} + Err(err) => { + debug!(path, error = %err, "Failed to read entrypoint PID file"); + } + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + }); +} + +async fn wait_for_supervisor_ready(path: &str) -> Result<()> { + let ready_path = std::path::Path::new(path); + let deadline = tokio::time::Instant::now() + Duration::from_secs(SIDECAR_READY_TIMEOUT_SECS); + loop { + if ready_path.exists() { + info!(path, "Network supervisor sidecar is ready"); + return Ok(()); + } + if tokio::time::Instant::now() >= deadline { + return Err(miette::miette!( + "timed out waiting for network supervisor sidecar readiness file {path}" + )); + } + tokio::time::sleep(Duration::from_millis(250)).await; + } +} + +#[cfg(target_os = "linux")] +fn write_supervisor_ready(path: &str) -> Result<()> { + let ready_path = std::path::Path::new(path); + if let Some(parent) = ready_path.parent() { + std::fs::create_dir_all(parent).into_diagnostic()?; + } + std::fs::write(ready_path, b"ready\n").into_diagnostic()?; + info!(path, "Network supervisor sidecar readiness file written"); + Ok(()) +} + +fn sidecar_ca_file_paths() -> Option<(std::path::PathBuf, std::path::PathBuf)> { + let tls_dir = std::env::var(openshell_core::sandbox_env::PROXY_TLS_DIR) + .unwrap_or_else(|_| SIDECAR_TLS_DIR.to_string()); + let cert = std::path::Path::new(&tls_dir).join(SIDECAR_CA_CERT); + let bundle = std::path::Path::new(&tls_dir).join(SIDECAR_CA_BUNDLE); + (cert.exists() && bundle.exists()).then_some((cert, bundle)) +} + +fn process_policy_for_topology( + policy: &SandboxPolicy, + sidecar_network_enforcement: bool, +) -> Result { + let mut process_policy = policy.clone(); + if sidecar_network_enforcement && matches!(process_policy.network.mode, NetworkMode::Proxy) { + let proxy = process_policy + .network + .proxy + .get_or_insert(ProxyPolicy { http_addr: None }); + if proxy.http_addr.is_none() { + proxy.http_addr = Some(SIDECAR_PROCESS_PROXY_ADDR.parse().into_diagnostic()?); + } + } + Ok(process_policy) +} + +struct GatewayForwardHandle { + task: tokio::task::JoinHandle<()>, +} + +impl Drop for GatewayForwardHandle { + fn drop(&mut self) { + self.task.abort(); + } +} + +async fn start_gateway_forward_from_env(endpoint: &str) -> Result { + let listen_addr = + std::env::var(openshell_core::sandbox_env::GATEWAY_FORWARD_ADDR).map_err(|_| { + miette::miette!( + "{} is required for sidecar gateway forwarding", + openshell_core::sandbox_env::GATEWAY_FORWARD_ADDR + ) + })?; + start_gateway_forward(&listen_addr, endpoint).await +} + +async fn start_gateway_forward(listen_addr: &str, endpoint: &str) -> Result { + let upstream = gateway_tcp_addr(endpoint)?; + let listener = TcpListener::bind(listen_addr).await.into_diagnostic()?; + info!( + listen_addr, + upstream, "Gateway loopback TCP forward started for sidecar topology" + ); + + let task = tokio::spawn(async move { + loop { + let (mut inbound, peer) = match listener.accept().await { + Ok(accepted) => accepted, + Err(e) => { + warn!(error = %e, "Gateway forward accept failed"); + continue; + } + }; + let upstream = upstream.clone(); + tokio::spawn(async move { + let mut outbound = match TcpStream::connect(&upstream).await { + Ok(stream) => stream, + Err(e) => { + warn!(peer = %peer, upstream, error = %e, "Gateway forward connect failed"); + return; + } + }; + if let Err(e) = copy_bidirectional(&mut inbound, &mut outbound).await { + debug!(peer = %peer, error = %e, "Gateway forward connection closed with error"); + } + }); + } + }); + + Ok(GatewayForwardHandle { task }) +} + +fn gateway_tcp_addr(endpoint: &str) -> Result { + let (scheme, rest) = endpoint + .split_once("://") + .ok_or_else(|| miette::miette!("gateway endpoint must include a URL scheme"))?; + let default_port = match scheme { + "http" => 80, + "https" => 443, + other => { + return Err(miette::miette!( + "unsupported gateway endpoint scheme '{other}' for sidecar forwarding" + )); + } + }; + let authority = rest.split('/').next().unwrap_or(rest); + if authority.is_empty() { + return Err(miette::miette!("gateway endpoint is missing a host")); + } + if authority.starts_with('[') { + let closing = authority + .find(']') + .ok_or_else(|| miette::miette!("invalid bracketed IPv6 gateway endpoint"))?; + let host = &authority[..=closing]; + let port = authority[closing + 1..] + .strip_prefix(':') + .and_then(|value| value.parse::().ok()) + .unwrap_or(default_port); + return Ok(format!("{host}:{port}")); + } + let (host, port) = match authority.rsplit_once(':') { + Some((host, port)) if !host.is_empty() => { + (host, port.parse::().unwrap_or(default_port)) + } + _ => (authority, default_port), + }; + Ok(format!("{host}:{port}")) +} + /// Flush aggregated denial summaries to the gateway via `SubmitPolicyAnalysis`. async fn flush_proposals_to_gateway( endpoint: &str, @@ -1960,8 +2223,24 @@ fn format_setting_value(es: &openshell_core::proto::EffectiveSetting) -> String )] mod tests { use super::*; + use openshell_core::policy::{ + FilesystemPolicy, LandlockPolicy, NetworkMode, NetworkPolicy, ProcessPolicy, ProxyPolicy, + }; use std::sync::atomic::{AtomicBool, Ordering}; + fn proxy_policy(http_addr: Option) -> SandboxPolicy { + SandboxPolicy { + version: 1, + filesystem: FilesystemPolicy::default(), + network: NetworkPolicy { + mode: NetworkMode::Proxy, + proxy: Some(ProxyPolicy { http_addr }), + }, + landlock: LandlockPolicy::default(), + process: ProcessPolicy::default(), + } + } + fn effective_bool(value: bool) -> openshell_core::proto::EffectiveSetting { openshell_core::proto::EffectiveSetting { value: Some(openshell_core::proto::SettingValue { @@ -1973,6 +2252,73 @@ mod tests { } } + #[test] + fn sidecar_process_policy_sets_loopback_proxy_addr() { + let policy = proxy_policy(None); + + let process_policy = process_policy_for_topology(&policy, true).unwrap(); + + let http_addr = process_policy + .network + .proxy + .and_then(|proxy| proxy.http_addr) + .expect("sidecar process policy should set proxy address"); + assert_eq!(http_addr.to_string(), SIDECAR_PROCESS_PROXY_ADDR); + assert!( + policy + .network + .proxy + .as_ref() + .expect("original policy should keep proxy config") + .http_addr + .is_none(), + "process policy normalization must not mutate the network policy" + ); + } + + #[test] + fn non_sidecar_process_policy_preserves_proxy_addr() { + let policy = proxy_policy(None); + + let process_policy = process_policy_for_topology(&policy, false).unwrap(); + + assert!( + process_policy + .network + .proxy + .and_then(|proxy| proxy.http_addr) + .is_none() + ); + } + + #[test] + fn gateway_tcp_addr_uses_explicit_port() { + assert_eq!( + gateway_tcp_addr("https://openshell-gateway.openshell.svc:8080").unwrap(), + "openshell-gateway.openshell.svc:8080" + ); + } + + #[test] + fn gateway_tcp_addr_uses_scheme_default_port() { + assert_eq!( + gateway_tcp_addr("https://openshell-gateway.openshell.svc").unwrap(), + "openshell-gateway.openshell.svc:443" + ); + assert_eq!( + gateway_tcp_addr("http://openshell-gateway.openshell.svc").unwrap(), + "openshell-gateway.openshell.svc:80" + ); + } + + #[test] + fn gateway_tcp_addr_preserves_ipv6_brackets() { + assert_eq!( + gateway_tcp_addr("https://[fd00::1]:8443").unwrap(), + "[fd00::1]:8443" + ); + } + #[test] fn apply_ocsf_json_setting_enables_from_initial_settings_snapshot() { let enabled = AtomicBool::new(false); diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index 91b145c2e0..9a165c643c 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -35,15 +35,26 @@ const DEBUG_RPC_SUBCOMMAND: &str = "debug-rpc"; /// Default `--mode` value: run both supervisor leaves in a single binary. const DEFAULT_MODE: &str = "network,process"; +const SIDECAR_STATE_DIR: &str = "/run/openshell-sidecar"; +const SIDECAR_TLS_DIR: &str = "/etc/openshell-tls/proxy"; +#[cfg(target_os = "linux")] +const CLIENT_TLS_DIR: &str = "/etc/openshell-tls/client"; +#[cfg(target_os = "linux")] +const SIDECAR_CLIENT_TLS_SUBDIR: &str = "client"; +#[cfg(target_os = "linux")] +const CLIENT_TLS_FILES: [&str; 3] = ["ca.crt", "tls.crt", "tls.key"]; /// Which supervisor leaves are enabled in this process. /// /// Parsed from a comma-separated `--mode` value, e.g. `network`, -/// `process`, or `network,process`. At least one must be set. +/// `process`, or `network,process`. `network-init` is a one-shot setup mode +/// used by the Kubernetes sidecar topology and cannot be combined with other +/// mode components. At least one must be set. #[derive(Clone, Copy, Debug)] struct Mode { network: bool, process: bool, + network_init: bool, } impl std::str::FromStr for Mode { @@ -53,20 +64,27 @@ impl std::str::FromStr for Mode { let mut mode = Self { network: false, process: false, + network_init: false, }; for part in s.split(',').map(str::trim).filter(|p| !p.is_empty()) { match part { "network" => mode.network = true, "process" => mode.process = true, + "network-init" => mode.network_init = true, other => { return Err(format!( - "unknown mode component '{other}' (expected 'network' and/or 'process')" + "unknown mode component '{other}' (expected 'network', 'process', or 'network-init')" )); } } } - if !mode.network && !mode.process { - return Err("--mode must enable at least one of: network, process".into()); + if mode.network_init && (mode.network || mode.process) { + return Err("--mode=network-init cannot be combined with other components".into()); + } + if !mode.network && !mode.process && !mode.network_init { + return Err( + "--mode must enable at least one of: network, process, network-init".into(), + ); } Ok(mode) } @@ -149,9 +167,28 @@ struct Args { /// "network" and/or "process". Defaults to both (single-binary /// topology). Use --mode=network for a network-only sidecar, or /// --mode=process for a process-only supervisor when network - /// enforcement runs in another pod. + /// enforcement runs in another pod. Use --mode=network-init only in + /// the Kubernetes init container that prepares sidecar nftables. #[arg(long, default_value = DEFAULT_MODE)] mode: Mode, + + /// UID that the long-running Kubernetes network sidecar will run as. + /// `--mode=network-init` installs nftables rules that exempt this UID. + #[arg(long, env = "OPENSHELL_PROXY_UID", default_value_t = 1337)] + proxy_uid: u32, + + /// GID assigned to shared sidecar state directories. Defaults to + /// `--proxy-uid` when omitted. + #[arg(long, env = "OPENSHELL_PROXY_GID")] + proxy_gid: Option, + + /// Shared state directory between the network init container and sidecar. + #[arg(long, env = "OPENSHELL_SIDECAR_STATE_DIR", default_value = SIDECAR_STATE_DIR)] + sidecar_state_dir: String, + + /// Shared TLS work directory between the network init container and sidecar. + #[arg(long, env = "OPENSHELL_PROXY_TLS_DIR", default_value = SIDECAR_TLS_DIR)] + sidecar_tls_dir: String, } /// Copy the running executable to `dest`, creating parent directories as @@ -194,6 +231,131 @@ fn copy_self(dest: &str) -> Result<()> { Ok(()) } +#[cfg(target_os = "linux")] +fn prepare_sidecar_directory(path: &Path, uid: u32, gid: u32, mode: u32) -> Result<()> { + use miette::Context as _; + use nix::unistd::{Gid, Uid, chown}; + use std::os::unix::fs::PermissionsExt; + + std::fs::create_dir_all(path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create sidecar directory {}", path.display()))?; + let mut perms = std::fs::metadata(path).into_diagnostic()?.permissions(); + perms.set_mode(mode); + std::fs::set_permissions(path, perms) + .into_diagnostic() + .wrap_err_with(|| format!("failed to chmod sidecar directory {}", path.display()))?; + chown(path, Some(Uid::from_raw(uid)), Some(Gid::from_raw(gid))) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to chown sidecar directory {} to {uid}:{gid}", + path.display() + ) + })?; + Ok(()) +} + +#[cfg(target_os = "linux")] +fn copy_sidecar_client_tls_if_present( + source_dir: &Path, + sidecar_tls_dir: &Path, + uid: u32, + gid: u32, +) -> Result<()> { + use miette::Context as _; + use nix::unistd::{Gid, Uid, chown}; + use std::os::unix::fs::PermissionsExt; + + if !source_dir.exists() { + return Ok(()); + } + + let dest_dir = sidecar_tls_dir.join(SIDECAR_CLIENT_TLS_SUBDIR); + prepare_sidecar_directory(&dest_dir, uid, gid, 0o750)?; + for file_name in CLIENT_TLS_FILES { + let source = source_dir.join(file_name); + if !source.exists() { + return Err(miette::miette!( + "client TLS source file is missing: {}", + source.display() + )); + } + let dest = dest_dir.join(file_name); + std::fs::copy(&source, &dest) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to copy client TLS file {} to {}", + source.display(), + dest.display() + ) + })?; + let mut perms = std::fs::metadata(&dest).into_diagnostic()?.permissions(); + perms.set_mode(0o400); + std::fs::set_permissions(&dest, perms) + .into_diagnostic() + .wrap_err_with(|| { + format!("failed to chmod copied client TLS file {}", dest.display()) + })?; + chown(&dest, Some(Uid::from_raw(uid)), Some(Gid::from_raw(gid))) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to chown copied client TLS file {} to {uid}:{gid}", + dest.display() + ) + })?; + } + + Ok(()) +} + +#[cfg(target_os = "linux")] +fn run_network_init( + proxy_uid: u32, + proxy_gid: u32, + sidecar_state_dir: &str, + sidecar_tls_dir: &str, +) -> Result<()> { + if proxy_uid < openshell_policy::MIN_SANDBOX_UID { + return Err(miette::miette!( + "--proxy-uid must be at least {}", + openshell_policy::MIN_SANDBOX_UID + )); + } + if proxy_gid < openshell_policy::MIN_SANDBOX_UID { + return Err(miette::miette!( + "--proxy-gid must be at least {}", + openshell_policy::MIN_SANDBOX_UID + )); + } + + let sidecar_state_dir = Path::new(sidecar_state_dir); + let sidecar_tls_dir = Path::new(sidecar_tls_dir); + prepare_sidecar_directory(sidecar_state_dir, proxy_uid, proxy_gid, 0o775)?; + prepare_sidecar_directory(sidecar_tls_dir, proxy_uid, proxy_gid, 0o755)?; + copy_sidecar_client_tls_if_present( + Path::new(CLIENT_TLS_DIR), + sidecar_tls_dir, + proxy_uid, + proxy_gid, + )?; + openshell_supervisor_process::netns::install_sidecar_bypass_rules(proxy_uid) +} + +#[cfg(not(target_os = "linux"))] +fn run_network_init( + _proxy_uid: u32, + _proxy_gid: u32, + _sidecar_state_dir: &str, + _sidecar_tls_dir: &str, +) -> Result<()> { + Err(miette::miette!( + "--mode=network-init is only supported on Linux" + )) +} + fn main() -> Result<()> { // Handle `copy-self ` before clap so it works without any of the // sandbox flags. Kubernetes init containers invoke this path to seed an @@ -222,6 +384,16 @@ fn main() -> Result<()> { let args = Args::parse(); + if args.mode.network_init { + let proxy_gid = args.proxy_gid.unwrap_or(args.proxy_uid); + return run_network_init( + args.proxy_uid, + proxy_gid, + &args.sidecar_state_dir, + &args.sidecar_tls_dir, + ); + } + // Try to open a rolling log file; fall back to stderr-only logging if it fails // (e.g., /var/log is not writable in custom workload images). // Rotates daily, keeps the 3 most recent files to bound disk usage. @@ -421,4 +593,24 @@ mod tests { let final_path = dest_dir.join("openshell-sandbox"); assert!(final_path.exists(), "binary should land inside dest dir"); } + + #[test] + fn mode_parses_network_init_standalone() { + let mode = "network-init".parse::().unwrap(); + assert!(mode.network_init); + assert!(!mode.network); + assert!(!mode.process); + } + + #[test] + fn mode_rejects_combined_network_init() { + let err = "network-init,network".parse::().unwrap_err(); + assert!(err.contains("cannot be combined")); + } + + #[test] + fn mode_rejects_empty_value() { + let err = "".parse::().unwrap_err(); + assert!(err.contains("at least one")); + } } diff --git a/crates/openshell-supervisor-network/data/sandbox-policy.rego b/crates/openshell-supervisor-network/data/sandbox-policy.rego index efcdf07320..d70c69b748 100644 --- a/crates/openshell-supervisor-network/data/sandbox-policy.rego +++ b/crates/openshell-supervisor-network/data/sandbox-policy.rego @@ -19,6 +19,10 @@ allow_network if { network_policy_for_request } +binary_identity_required if { + object.get(object.get(data, "runtime", {}), "require_binary_identity", true) +} + # --- Deny reasons (specific diagnostics for debugging policy denials) --- deny_reason := "missing input.network" if { @@ -131,6 +135,12 @@ endpoint_allowed(policy, network) if { endpoint.ports[_] == network.port } +# Binary matching can be relaxed by trusted runtime configuration. In that +# mode, network policies are endpoint/L7 scoped and ignore policy.binaries. +binary_allowed(_, _) if { + not binary_identity_required +} + # Binary matching: exact path. # SHA256 integrity is enforced in Rust via trust-on-first-use (TOFU) cache, # not in Rego. The proxy computes and caches binary hashes at runtime. @@ -161,6 +171,10 @@ binary_allowed(policy, exec) if { glob.match(b.path, ["/"], p) } +user_declared_binary_allowed(_, _) if { + not binary_identity_required +} + user_declared_binary_allowed(policy, exec) if { some b b := policy.binaries[_] diff --git a/crates/openshell-supervisor-network/src/identity.rs b/crates/openshell-supervisor-network/src/identity.rs index fce568f41e..5e89c35031 100644 --- a/crates/openshell-supervisor-network/src/identity.rs +++ b/crates/openshell-supervisor-network/src/identity.rs @@ -100,23 +100,34 @@ impl BinaryIdentityCache { /// Returns `Ok(hash)` if it matches, `Err` if the hash changed (binary tampered). #[cfg_attr(not(target_os = "linux"), allow(dead_code))] pub fn verify_or_cache(&self, path: &Path) -> Result { - self.verify_or_cache_with_hasher(path, procfs::file_sha256) + self.verify_or_cache_with_paths(path, path, procfs::file_sha256) } - fn verify_or_cache_with_hasher(&self, path: &Path, mut hash_file: F) -> Result + #[cfg(target_os = "linux")] + pub fn verify_or_cache_process_exe(&self, display_path: &Path, pid: u32) -> Result { + let proc_exe = PathBuf::from(format!("/proc/{pid}/exe")); + self.verify_or_cache_with_paths(display_path, &proc_exe, procfs::file_sha256) + } + + fn verify_or_cache_with_paths( + &self, + cache_path: &Path, + access_path: &Path, + mut hash_file: F, + ) -> Result where F: FnMut(&Path) -> Result, { let start = std::time::Instant::now(); - let metadata = std::fs::metadata(path) - .map_err(|error| miette::miette!("Failed to stat {}: {error}", path.display()))?; + let metadata = std::fs::metadata(access_path) + .map_err(|error| miette::miette!("Failed to stat {}: {error}", cache_path.display()))?; let fingerprint = FileFingerprint::from_metadata(&metadata); let cached = self .hashes .lock() .map_err(|_| miette::miette!("Binary identity cache lock poisoned"))? - .get(path) + .get(cache_path) .cloned(); if let Some(cached_binary) = &cached @@ -125,7 +136,7 @@ impl BinaryIdentityCache { debug!( " verify_or_cache: {}ms CACHE HIT path={}", start.elapsed().as_millis(), - path.display() + cache_path.display() ); return Ok(cached_binary.hash.clone()); } @@ -133,29 +144,29 @@ impl BinaryIdentityCache { debug!( " verify_or_cache: CACHE MISS size={} path={}", metadata.len(), - path.display() + cache_path.display() ); - let current_hash = hash_file(path)?; + let current_hash = hash_file(access_path)?; let mut hashes = self .hashes .lock() .map_err(|_| miette::miette!("Binary identity cache lock poisoned"))?; - if let Some(existing) = hashes.get(path) + if let Some(existing) = hashes.get(cache_path) && existing.hash != current_hash { return Err(miette::miette!( "Binary integrity violation: {} hash changed (cached: {}, current: {})", - path.display(), + cache_path.display(), existing.hash, current_hash )); } hashes.insert( - path.to_path_buf(), + cache_path.to_path_buf(), CachedBinary { hash: current_hash.clone(), fingerprint, @@ -165,7 +176,7 @@ impl BinaryIdentityCache { debug!( " verify_or_cache TOTAL (cold): {}ms path={}", start.elapsed().as_millis(), - path.display() + cache_path.display() ); Ok(current_hash) @@ -212,13 +223,13 @@ mod tests { let mut hash_calls = 0; let hash1 = cache - .verify_or_cache_with_hasher(tmp.path(), |path| { + .verify_or_cache_with_paths(tmp.path(), tmp.path(), |path| { hash_calls += 1; procfs::file_sha256(path) }) .unwrap(); let hash2 = cache - .verify_or_cache_with_hasher(tmp.path(), |path| { + .verify_or_cache_with_paths(tmp.path(), tmp.path(), |path| { hash_calls += 1; procfs::file_sha256(path) }) @@ -238,7 +249,7 @@ mod tests { let mut hash_calls = 0; let hash1 = cache - .verify_or_cache_with_hasher(tmp.path(), |path| { + .verify_or_cache_with_paths(tmp.path(), tmp.path(), |path| { hash_calls += 1; procfs::file_sha256(path) }) @@ -254,7 +265,7 @@ mod tests { .unwrap(); let hash2 = cache - .verify_or_cache_with_hasher(tmp.path(), |path| { + .verify_or_cache_with_paths(tmp.path(), tmp.path(), |path| { hash_calls += 1; procfs::file_sha256(path) }) @@ -275,7 +286,7 @@ mod tests { let mut hash_calls = 0; cache - .verify_or_cache_with_hasher(&path, |path| { + .verify_or_cache_with_paths(&path, &path, |path| { hash_calls += 1; procfs::file_sha256(path) }) @@ -292,7 +303,7 @@ mod tests { .set_modified(original_mtime) .unwrap(); - let result = cache.verify_or_cache_with_hasher(&path, |path| { + let result = cache.verify_or_cache_with_paths(&path, &path, |path| { hash_calls += 1; procfs::file_sha256(path) }); @@ -301,6 +312,28 @@ mod tests { assert_eq!(hash_calls, 2); } + #[test] + fn display_path_can_differ_from_access_path() { + let mut tmp = tempfile::NamedTempFile::new().unwrap(); + tmp.write_all(b"binary content").unwrap(); + tmp.flush().unwrap(); + let display_path = Path::new("/usr/bin/python3"); + + let cache = BinaryIdentityCache::new(); + let hash = cache + .verify_or_cache_with_paths(display_path, tmp.path(), procfs::file_sha256) + .unwrap(); + + assert!(!hash.is_empty()); + assert!( + cache + .hashes + .lock() + .unwrap() + .contains_key(Path::new("/usr/bin/python3")) + ); + } + #[test] fn hash_mismatch_returns_error() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/openshell-supervisor-network/src/opa.rs b/crates/openshell-supervisor-network/src/opa.rs index fbab5fedd7..850c383205 100644 --- a/crates/openshell-supervisor-network/src/opa.rs +++ b/crates/openshell-supervisor-network/src/opa.rs @@ -18,6 +18,7 @@ use std::sync::{ Arc, Mutex, atomic::{AtomicU64, Ordering}, }; +use tracing::info; /// Baked-in rego rules for OPA policy evaluation. /// These rules define the network access decision logic and static config @@ -55,6 +56,49 @@ pub struct NetworkInput { pub cmdline_paths: Vec, } +pub(crate) fn network_binary_identity_required() -> bool { + std::env::var(openshell_core::sandbox_env::NETWORK_BINARY_IDENTITY).map_or(true, |value| { + !matches!( + value.as_str(), + "relaxed" | "disabled" | "endpoint-only" | "false" | "0" + ) + }) +} + +fn inject_runtime_policy_data(data: &mut serde_json::Value, require_binary_identity: bool) { + let Some(obj) = data.as_object_mut() else { + return; + }; + obj.insert( + "runtime".to_string(), + serde_json::json!({ + "require_binary_identity": require_binary_identity, + }), + ); +} + +fn emit_binary_identity_mode(require_binary_identity: bool, source: &str) { + info!( + require_binary_identity, + source, "Configured OPA runtime binary identity mode" + ); + openshell_ocsf::ocsf_emit!( + openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(openshell_ocsf::SeverityId::Informational) + .status(openshell_ocsf::StatusId::Success) + .state(openshell_ocsf::StateId::Enabled, "configured") + .unmapped( + "require_binary_identity", + serde_json::json!(require_binary_identity) + ) + .unmapped("source", serde_json::json!(source)) + .message(format!( + "OPA runtime binary identity mode configured [source:{source} require_binary_identity:{require_binary_identity}]" + )) + .build() + ); +} + /// Sandbox configuration extracted from OPA data at startup. pub struct SandboxConfig { pub filesystem: FilesystemPolicy, @@ -146,7 +190,9 @@ impl OpaEngine { engine .add_policy_from_file(policy_path) .map_err(|e| miette::miette!("{e}"))?; - let data_json = preprocess_yaml_data(&yaml_str)?; + let require_binary_identity = network_binary_identity_required(); + emit_binary_identity_mode(require_binary_identity, "files"); + let data_json = preprocess_yaml_data(&yaml_str, require_binary_identity)?; engine .add_data_json(&data_json) .map_err(|e| miette::miette!("{e}"))?; @@ -160,11 +206,24 @@ impl OpaEngine { /// /// Preprocesses the YAML data to expand access presets and validate L7 config. pub fn from_strings(policy: &str, data_yaml: &str) -> Result { + Self::from_strings_with_binary_identity_required( + policy, + data_yaml, + network_binary_identity_required(), + ) + } + + pub(crate) fn from_strings_with_binary_identity_required( + policy: &str, + data_yaml: &str, + require_binary_identity: bool, + ) -> Result { let mut engine = regorus::Engine::new(); engine .add_policy("policy.rego".into(), policy.into()) .map_err(|e| miette::miette!("{e}"))?; - let data_json = preprocess_yaml_data(data_yaml)?; + emit_binary_identity_mode(require_binary_identity, "strings"); + let data_json = preprocess_yaml_data(data_yaml, require_binary_identity)?; engine .add_data_json(&data_json) .map_err(|e| miette::miette!("{e}"))?; @@ -193,11 +252,25 @@ impl OpaEngine { /// gap between user-specified symlink paths (e.g., `/usr/bin/python3`) and /// kernel-resolved canonical paths (e.g., `/usr/bin/python3.11`). pub fn from_proto_with_pid(proto: &ProtoSandboxPolicy, entrypoint_pid: u32) -> Result { + Self::from_proto_with_pid_and_binary_identity_required( + proto, + entrypoint_pid, + network_binary_identity_required(), + ) + } + + fn from_proto_with_pid_and_binary_identity_required( + proto: &ProtoSandboxPolicy, + entrypoint_pid: u32, + require_binary_identity: bool, + ) -> Result { + emit_binary_identity_mode(require_binary_identity, "proto"); let data_json_str = proto_to_opa_data_json(proto, entrypoint_pid); // Parse back to Value for preprocessing, then re-serialize let mut data: serde_json::Value = serde_json::from_str(&data_json_str) .map_err(|e| miette::miette!("internal: failed to parse proto JSON: {e}"))?; + inject_runtime_policy_data(&mut data, require_binary_identity); // Validate BEFORE expanding presets let (errors, warnings) = crate::l7::validate_l7_policies(&data); @@ -720,9 +793,10 @@ fn parse_process_policy(val: ®orus::Value) -> ProcessPolicy { } /// Preprocess YAML policy data: parse, normalize, validate, expand access presets, return JSON. -fn preprocess_yaml_data(yaml_str: &str) -> Result { +fn preprocess_yaml_data(yaml_str: &str, require_binary_identity: bool) -> Result { let mut data: serde_json::Value = serde_yml::from_str(yaml_str) .map_err(|e| miette::miette!("failed to parse YAML data: {e}"))?; + inject_runtime_policy_data(&mut data, require_binary_identity); // Normalize port → ports for all endpoints so Rego always sees "ports" array. normalize_endpoint_ports(&mut data); @@ -2264,6 +2338,88 @@ process: assert!(eval_l7(&engine, &input)); } + #[test] + fn l7_get_allowed_by_rules_when_binary_identity_relaxed() { + let engine = + OpaEngine::from_strings_with_binary_identity_required(TEST_POLICY, L7_TEST_DATA, false) + .expect("Failed to load relaxed L7 test data"); + let mut input = l7_input("api.example.com", 8080, "GET", "/repos/myorg/foo"); + input["exec"]["path"] = "".into(); + assert!(eval_l7(&engine, &input)); + } + + #[test] + fn relaxed_binary_identity_preserves_matched_policy_and_l7_for_proto() { + let mut network_policies = std::collections::HashMap::new(); + network_policies.insert( + "test_l7".to_string(), + NetworkPolicyRule { + name: "test_l7".to_string(), + endpoints: vec![NetworkEndpoint { + host: "host.k3d.internal".to_string(), + port: 56123, + protocol: "rest".to_string(), + enforcement: "enforce".to_string(), + rules: vec![L7Rule { + allow: Some(L7Allow { + method: "GET".to_string(), + path: "/allowed".to_string(), + command: String::new(), + query: std::collections::HashMap::new(), + operation_type: String::new(), + operation_name: String::new(), + fields: Vec::new(), + params: std::collections::HashMap::new(), + }), + }], + allowed_ips: vec!["192.168.0.0/16".to_string()], + ..Default::default() + }], + binaries: vec![NetworkBinary { + path: "/usr/bin/curl".to_string(), + ..Default::default() + }], + }, + ); + let proto = ProtoSandboxPolicy { + version: 1, + filesystem: Some(ProtoFs { + include_workdir: true, + read_only: vec![], + read_write: vec![], + }), + landlock: Some(openshell_core::proto::LandlockPolicy { + compatibility: "best_effort".to_string(), + }), + process: Some(ProtoProc { + run_as_user: "sandbox".to_string(), + run_as_group: "sandbox".to_string(), + }), + network_policies, + }; + let engine = OpaEngine::from_proto_with_pid_and_binary_identity_required(&proto, 0, false) + .expect("engine from relaxed proto"); + let network_input = NetworkInput { + host: "host.k3d.internal".into(), + port: 56123, + binary_path: PathBuf::new(), + binary_sha256: String::new(), + ancestors: vec![], + cmdline_paths: vec![], + }; + let action = engine.evaluate_network_action(&network_input).unwrap(); + assert_eq!( + action, + NetworkAction::Allow { + matched_policy: Some("test_l7".to_string()) + } + ); + + let mut input = l7_input("host.k3d.internal", 56123, "GET", "/allowed"); + input["exec"]["path"] = "".into(); + assert!(eval_l7(&engine, &input)); + } + #[test] fn l7_post_allowed_by_rules() { let engine = l7_engine(); @@ -4592,6 +4748,46 @@ process: ); } + #[test] + fn relaxed_binary_identity_allows_declared_endpoint_without_binary_match() { + let engine = OpaEngine::from_strings_with_binary_identity_required( + TEST_POLICY, + INFERENCE_TEST_DATA, + false, + ) + .expect("Failed to load relaxed binary identity test data"); + let input = NetworkInput { + host: "api.anthropic.com".into(), + port: 443, + binary_path: PathBuf::from("/tmp/unlisted-agent"), + binary_sha256: "unused".into(), + ancestors: vec![], + cmdline_paths: vec![], + }; + + let action = engine.evaluate_network_action(&input).unwrap(); + assert_eq!( + action, + NetworkAction::Allow { + matched_policy: Some("claude_code".to_string()) + }, + ); + assert!( + engine.query_exact_declared_endpoint_host(&input).unwrap(), + "relaxed identity should preserve exact declared endpoint handling" + ); + + let undeclared = NetworkInput { + host: "api.openai.com".into(), + ..input + }; + let action = engine.evaluate_network_action(&undeclared).unwrap(); + assert!( + matches!(action, NetworkAction::Deny { .. }), + "relaxed identity must not allow undeclared endpoints" + ); + } + #[test] fn unknown_endpoint_returns_deny() { let engine = inference_engine(); diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 0d2c8c0258..c38ecbd3a5 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -42,6 +42,8 @@ const TUNNEL_PROTOCOL_PEEK_POLL: std::time::Duration = std::time::Duration::from const TUNNEL_PROTOCOL_PEEK_POLL: std::time::Duration = std::time::Duration::from_millis(1); const INFERENCE_LOCAL_HOST: &str = "inference.local"; const INFERENCE_LOCAL_PORT: u16 = 443; +#[cfg(target_os = "linux")] +const SIDECAR_SUPERVISOR_TOPOLOGY: &str = "sidecar"; /// Hostnames injected by compute drivers as `/etc/hosts` aliases for the host /// machine. Traffic to these names is eligible for the trusted-gateway SSRF @@ -1426,7 +1428,7 @@ fn resolve_owner_identity( })?; let bin_hash = identity_cache - .verify_or_cache(&bin_path) + .verify_or_cache_process_exe(&bin_path, owner_pid) .map_err(|e| IdentityError { reason: format!("binary integrity check failed: {e}"), binary: Some(bin_path.clone()), @@ -1434,11 +1436,15 @@ fn resolve_owner_identity( ancestors: vec![], })?; - let ancestors = crate::procfs::collect_ancestor_binaries(owner_pid, entrypoint_pid); + let ancestor_identities = collect_ancestor_identities(owner_pid, entrypoint_pid); + let ancestors: Vec = ancestor_identities + .iter() + .map(|(_, path)| path.clone()) + .collect(); - for ancestor in &ancestors { + for (ancestor_pid, ancestor) in &ancestor_identities { identity_cache - .verify_or_cache(ancestor) + .verify_or_cache_process_exe(ancestor, *ancestor_pid) .map_err(|e| IdentityError { reason: format!( "ancestor integrity check failed for {}: {e}", @@ -1463,6 +1469,31 @@ fn resolve_owner_identity( }) } +#[cfg(target_os = "linux")] +fn collect_ancestor_identities(pid: u32, stop_pid: u32) -> Vec<(u32, PathBuf)> { + const MAX_DEPTH: usize = 64; + let mut ancestors = Vec::new(); + let mut current = pid; + + for _ in 0..MAX_DEPTH { + let ppid = match crate::procfs::read_ppid(current) { + Some(p) if p > 0 && p != current => p, + _ => break, + }; + + if let Ok(path) = crate::procfs::binary_path(ppid.cast_signed()) { + ancestors.push((ppid, path)); + } + + if ppid == stop_pid || ppid == 1 { + break; + } + current = ppid; + } + + ancestors +} + /// Resolve the identity of the process owning a TCP peer connection. /// /// Walks `/proc//net/tcp` to find the socket inode, locates @@ -1573,8 +1604,17 @@ fn evaluate_opa_tcp( } }; - let pid = entrypoint_pid.load(Ordering::Acquire); - if pid == 0 { + if !crate::opa::network_binary_identity_required() { + let result = evaluate_endpoint_only_opa(engine, host, port); + debug!( + "evaluate_opa_tcp endpoint-only: host={host} port={port} action={:?}", + result.action + ); + return result; + } + + let entrypoint_pid = entrypoint_pid.load(Ordering::Acquire); + let Some(proc_net_anchor_pid) = proc_net_anchor_pid(entrypoint_pid) else { return deny( "entrypoint process not yet spawned".into(), None, @@ -1582,12 +1622,12 @@ fn evaluate_opa_tcp( vec![], vec![], ); - } + }; let total_start = std::time::Instant::now(); let peer_port = peer_addr.port(); - let identity = match resolve_process_identity(pid, peer_port, identity_cache) { + let identity = match resolve_process_identity(proc_net_anchor_pid, peer_port, identity_cache) { Ok(id) => id, Err(err) => { return deny( @@ -1641,6 +1681,52 @@ fn evaluate_opa_tcp( result } +#[cfg(target_os = "linux")] +fn proc_net_anchor_pid(entrypoint_pid: u32) -> Option { + if entrypoint_pid != 0 { + return Some(entrypoint_pid); + } + sidecar_topology_enabled().then(std::process::id) +} + +#[cfg(target_os = "linux")] +fn sidecar_topology_enabled() -> bool { + std::env::var(openshell_core::sandbox_env::SUPERVISOR_TOPOLOGY) + .is_ok_and(|value| value == SIDECAR_SUPERVISOR_TOPOLOGY) +} + +fn evaluate_endpoint_only_opa(engine: &OpaEngine, host: &str, port: u16) -> ConnectDecision { + let input = crate::opa::NetworkInput { + host: host.to_string(), + port, + binary_path: PathBuf::new(), + binary_sha256: String::new(), + ancestors: vec![], + cmdline_paths: vec![], + }; + + match engine.evaluate_network_action_with_generation(&input) { + Ok((action, generation)) => ConnectDecision { + action, + generation, + binary: None, + binary_pid: None, + ancestors: vec![], + cmdline_paths: vec![], + }, + Err(e) => ConnectDecision { + action: NetworkAction::Deny { + reason: format!("policy evaluation error: {e}"), + }, + generation: engine.current_generation(), + binary: None, + binary_pid: None, + ancestors: vec![], + cmdline_paths: vec![], + }, + } +} + /// Non-Linux stub: OPA identity binding requires /proc. #[cfg(not(target_os = "linux"))] fn evaluate_opa_tcp( @@ -1648,9 +1734,13 @@ fn evaluate_opa_tcp( engine: &OpaEngine, _identity_cache: &BinaryIdentityCache, _entrypoint_pid: &AtomicU32, - _host: &str, - _port: u16, + host: &str, + port: u16, ) -> ConnectDecision { + if !crate::opa::network_binary_identity_required() { + return evaluate_endpoint_only_opa(engine, host, port); + } + ConnectDecision { action: NetworkAction::Deny { reason: "identity binding unavailable on this platform".into(), @@ -2152,14 +2242,24 @@ fn query_l7_route_snapshot( }; match engine.query_endpoint_configs_with_generation(&input) { - Ok((vals, generation)) => Some(L7RouteSnapshot { - configs: vals + Ok((vals, generation)) => { + let configs: Vec<_> = vals .into_iter() .filter_map(|val| crate::l7::parse_l7_config(&val)) .map(|config| L7ConfigSnapshot { config }) - .collect(), - generation, - }), + .collect(); + debug!( + host, + port, + generation, + config_count = configs.len(), + "Forward proxy L7 route lookup complete" + ); + Some(L7RouteSnapshot { + configs, + generation, + }) + } Err(e) => { let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Fail) @@ -3337,10 +3437,29 @@ async fn handle_forward_proxy( } }; let policy_str = matched_policy.as_deref().unwrap_or("-"); + debug!( + host = %host_lc, + port, + binary = %binary_str, + binary_pid = %pid_str, + matched_policy = %policy_str, + decision_generation = decision.generation, + current_generation = opa_engine.current_generation(), + action = ?decision.action, + "Forward proxy L4 policy decision" + ); let sandbox_entrypoint_pid = entrypoint_pid.load(Ordering::Acquire); let forward_generation_guard = match opa_engine.generation_guard(decision.generation) { Ok(guard) => guard, Err(e) => { + warn!( + host = %host_lc, + port, + decision_generation = decision.generation, + current_generation = opa_engine.current_generation(), + error = %e, + "Forward proxy rejected request because policy generation changed after L4 decision" + ); emit_l7_tunnel_close_after_policy_change(&host_lc, port, e); emit_activity_simple(activity_tx, true, "policy_stale"); respond( @@ -3401,6 +3520,15 @@ async fn handle_forward_proxy( && !route.configs.is_empty() { if route.generation != forward_generation_guard.captured_generation() { + warn!( + host = %host_lc, + port, + decision_generation = decision.generation, + guard_generation = forward_generation_guard.captured_generation(), + route_generation = route.generation, + current_generation = opa_engine.current_generation(), + "Forward proxy rejected request because L7 route lookup used a different policy generation" + ); emit_l7_tunnel_close_after_policy_change( &host_lc, port, @@ -3426,6 +3554,14 @@ async fn handle_forward_proxy( let tunnel_engine = match opa_engine.clone_engine_for_tunnel(route.generation) { Ok(engine) => engine, Err(e) => { + warn!( + host = %host_lc, + port, + route_generation = route.generation, + current_generation = opa_engine.current_generation(), + error = %e, + "Forward proxy rejected request because L7 tunnel engine could not be cloned" + ); emit_l7_tunnel_close_after_policy_change(&host_lc, port, e); emit_activity_simple(activity_tx, true, "policy_stale"); respond( @@ -4105,6 +4241,14 @@ async fn handle_forward_proxy( }; if let Err(e) = forward_generation_guard.ensure_current() { + warn!( + host = %host_lc, + port, + captured_generation = forward_generation_guard.captured_generation(), + current_generation = forward_generation_guard.current_generation(), + error = %e, + "Forward proxy rejected request because policy changed before upstream connect" + ); emit_l7_tunnel_close_after_policy_change(&host_lc, port, e); emit_activity_simple(activity_tx, true, "policy_stale"); respond( @@ -4243,6 +4387,14 @@ async fn handle_forward_proxy( }; if let Err(e) = forward_generation_guard.ensure_current() { + warn!( + host = %host_lc, + port, + captured_generation = forward_generation_guard.captured_generation(), + current_generation = forward_generation_guard.current_generation(), + error = %e, + "Forward proxy rejected request because policy changed before relay" + ); emit_l7_tunnel_close_after_policy_change(&host_lc, port, e); respond( client, @@ -4379,6 +4531,46 @@ mod tests { use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; + #[test] + fn endpoint_only_opa_allows_declared_endpoint_without_process_identity() { + let policy = include_str!("../data/sandbox-policy.rego"); + let data = r#" +version: 1 +network_policies: + test_l7: + name: test_l7 + endpoints: + - host: host.k3d.internal + port: 56123 + protocol: rest + enforcement: enforce + rules: + - allow: + method: GET + path: /allowed + binaries: + - path: /usr/bin/curl +"#; + let engine = OpaEngine::from_strings_with_binary_identity_required(policy, data, false) + .expect("relaxed engine"); + + let decision = evaluate_endpoint_only_opa(&engine, "host.k3d.internal", 56123); + assert_eq!( + decision.action, + NetworkAction::Allow { + matched_policy: Some("test_l7".to_string()), + } + ); + assert!(decision.binary.is_none()); + assert!(decision.ancestors.is_empty()); + + let denied = evaluate_endpoint_only_opa(&engine, "api.example.com", 443); + assert!( + matches!(denied.action, NetworkAction::Deny { .. }), + "endpoint-only mode must still deny undeclared endpoints" + ); + } + fn websocket_l7_config( protocol: crate::l7::L7Protocol, websocket_credential_rewrite: bool, diff --git a/crates/openshell-supervisor-network/src/run.rs b/crates/openshell-supervisor-network/src/run.rs index 9553e06736..8e17758bd6 100644 --- a/crates/openshell-supervisor-network/src/run.rs +++ b/crates/openshell-supervisor-network/src/run.rs @@ -201,7 +201,9 @@ pub async fn run_networking( let (tls_state, ca_file_paths) = if matches!(policy.network.mode, NetworkMode::Proxy) { match SandboxCa::generate() { Ok(ca) => { - let tls_dir = std::path::Path::new("/etc/openshell-tls"); + let tls_dir = std::env::var(openshell_core::sandbox_env::PROXY_TLS_DIR) + .unwrap_or_else(|_| "/etc/openshell-tls".to_string()); + let tls_dir = std::path::Path::new(&tls_dir); let system_ca_bundle = read_system_ca_bundle(); match write_ca_files(&ca, tls_dir, &system_ca_bundle) { Ok(paths) => { diff --git a/crates/openshell-supervisor-process/src/netns/mod.rs b/crates/openshell-supervisor-process/src/netns/mod.rs index cc7b1d84ce..86a5406ade 100644 --- a/crates/openshell-supervisor-process/src/netns/mod.rs +++ b/crates/openshell-supervisor-process/src/netns/mod.rs @@ -467,6 +467,123 @@ pub fn create_netns_for_proxy( } } +/// Install pod-network bypass enforcement for Kubernetes sidecar topology. +/// +/// This runs in the current network namespace, not in a per-workload netns. +/// The rules allow loopback and the sidecar proxy UID, then reject direct +/// TCP/UDP egress from other UIDs so traffic must use the sidecar's local +/// proxy. +/// +/// # Errors +/// +/// Returns an error when `nft` is unavailable or the ruleset cannot be loaded. +pub fn install_sidecar_bypass_rules(proxy_uid: u32) -> Result<()> { + match install_sidecar_nft_bypass_rules(proxy_uid) { + Ok(()) => Ok(()), + Err(nft_error) => { + warn!( + error = %nft_error, + "Failed to install nftables sidecar rules; trying iptables-legacy fallback" + ); + install_sidecar_iptables_legacy_bypass_rules(proxy_uid).map_err(|iptables_error| { + miette::miette!( + "sidecar nft ruleset load failed: {nft_error}; sidecar iptables-legacy fallback failed: {iptables_error}" + ) + }) + } + } +} + +fn install_sidecar_nft_bypass_rules(proxy_uid: u32) -> Result<()> { + let nft_cmd = find_nft().ok_or_else(|| { + miette::miette!( + "trusted nft helper not found; sidecar network enforcement requires nftables" + ) + })?; + let log_prefix = Some("openshell:sidecar-bypass:"); + let ruleset = nft_ruleset::generate_sidecar_bypass_ruleset(proxy_uid, log_prefix); + run_nft_current_namespace(&nft_cmd, &ruleset) +} + +const SIDECAR_IPTABLES_CHAIN: &str = "OPENSHELL_SIDECAR_BYPASS"; + +fn install_sidecar_iptables_legacy_bypass_rules(proxy_uid: u32) -> Result<()> { + let iptables_cmd = find_iptables_legacy().ok_or_else(|| { + miette::miette!( + "trusted iptables-legacy helper not found; sidecar network enforcement fallback unavailable" + ) + })?; + + cleanup_sidecar_iptables_legacy_rules(&iptables_cmd); + + let proxy_uid_arg = proxy_uid.to_string(); + let commands: Vec> = vec![ + vec!["-N", SIDECAR_IPTABLES_CHAIN], + vec!["-A", SIDECAR_IPTABLES_CHAIN, "-o", "lo", "-j", "ACCEPT"], + vec![ + "-A", + SIDECAR_IPTABLES_CHAIN, + "-m", + "conntrack", + "--ctstate", + "ESTABLISHED,RELATED", + "-j", + "ACCEPT", + ], + vec![ + "-A", + SIDECAR_IPTABLES_CHAIN, + "-m", + "owner", + "--uid-owner", + &proxy_uid_arg, + "-j", + "ACCEPT", + ], + vec![ + "-A", + SIDECAR_IPTABLES_CHAIN, + "-p", + "tcp", + "-j", + "REJECT", + "--reject-with", + "tcp-reset", + ], + vec![ + "-A", + SIDECAR_IPTABLES_CHAIN, + "-p", + "udp", + "-j", + "REJECT", + "--reject-with", + "icmp-port-unreachable", + ], + vec!["-A", "OUTPUT", "-j", SIDECAR_IPTABLES_CHAIN], + ]; + + for args in commands { + if let Err(e) = run_iptables_legacy_current_namespace(&iptables_cmd, &args) { + cleanup_sidecar_iptables_legacy_rules(&iptables_cmd); + return Err(e); + } + } + + Ok(()) +} + +fn cleanup_sidecar_iptables_legacy_rules(iptables_cmd: &str) { + while run_iptables_legacy_current_namespace( + iptables_cmd, + &["-D", "OUTPUT", "-j", SIDECAR_IPTABLES_CHAIN], + ) + .is_ok() + {} + let _ = run_iptables_legacy_current_namespace(iptables_cmd, &["-F", SIDECAR_IPTABLES_CHAIN]); + let _ = run_iptables_legacy_current_namespace(iptables_cmd, &["-X", SIDECAR_IPTABLES_CHAIN]); +} + /// Run an `ip` command on the host. fn run_ip(args: &[&str]) -> Result<()> { let ip_path = find_trusted_binary("ip", IP_SEARCH_PATHS)?; @@ -490,6 +607,62 @@ fn run_ip(args: &[&str]) -> Result<()> { Ok(()) } +fn run_iptables_legacy_current_namespace(iptables_cmd: &str, args: &[&str]) -> Result<()> { + debug!( + command = %format!("{iptables_cmd} {}", args.join(" ")), + "Running iptables-legacy sidecar command" + ); + + let output = Command::new(iptables_cmd) + .args(args) + .output() + .into_diagnostic()?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(miette::miette!( + "{iptables_cmd} {} failed: {}", + args.join(" "), + stderr.trim() + )); + } + + Ok(()) +} + +fn run_nft_current_namespace(nft_cmd: &str, ruleset: &str) -> Result<()> { + use std::io::Write; + let mut tmp = tempfile::Builder::new() + .prefix("openshell-sidecar-nft-") + .suffix(".conf") + .tempfile() + .into_diagnostic()?; + tmp.write_all(ruleset.as_bytes()).into_diagnostic()?; + let ruleset_path = tmp.path().to_string_lossy().to_string(); + + debug!( + command = %format!("{nft_cmd} -f {ruleset_path}"), + "Loading nftables sidecar ruleset" + ); + + let output = Command::new(nft_cmd) + .args(["-f", &ruleset_path]) + .output() + .into_diagnostic()?; + + drop(tmp); + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(miette::miette!( + "sidecar nft ruleset load failed: {}", + stderr.trim() + )); + } + + Ok(()) +} + /// Run an `ip` command inside a network namespace via `nsenter --net=`. /// /// We use `nsenter` instead of `ip netns exec` because `ip netns exec` @@ -605,6 +778,11 @@ fn enable_nf_log_all_netns() { /// Well-known paths where nft may be installed. const NFT_SEARCH_PATHS: &[&str] = &["/usr/sbin/nft", "/sbin/nft", "/usr/bin/nft"]; +const IPTABLES_LEGACY_SEARCH_PATHS: &[&str] = &[ + "/usr/sbin/iptables-legacy", + "/sbin/iptables-legacy", + "/usr/bin/iptables-legacy", +]; fn find_trusted_binary<'a>(name: &str, paths: &'a [&str]) -> Result<&'a str> { paths @@ -629,6 +807,12 @@ fn find_nft() -> Option { .map(String::from) } +fn find_iptables_legacy() -> Option { + find_trusted_binary("iptables-legacy", IPTABLES_LEGACY_SEARCH_PATHS) + .ok() + .map(String::from) +} + #[cfg(test)] mod tests { use super::*; @@ -668,6 +852,16 @@ mod tests { } } + #[test] + fn iptables_legacy_search_paths_are_absolute() { + for path in IPTABLES_LEGACY_SEARCH_PATHS { + assert!( + path.starts_with('/'), + "IPTABLES_LEGACY_SEARCH_PATHS entry must be absolute: {path}" + ); + } + } + #[test] #[ignore = "requires root privileges"] fn test_create_and_drop_namespace() { diff --git a/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs b/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs index ba7aeb9368..d7ec5132e2 100644 --- a/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs +++ b/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs @@ -53,6 +53,46 @@ pub fn generate_bypass_ruleset(host_ip: &str, proxy_port: u16, log_prefix: Optio ) } +/// Generate a pod-network ruleset for Kubernetes sidecar enforcement. +/// +/// The network sidecar and the process supervisor share a pod network +/// namespace. The sidecar runs as `proxy_uid` and owns external egress; +/// sandbox traffic must use loopback services hosted by that sidecar +/// (gateway forward and HTTP CONNECT proxy). +pub fn generate_sidecar_bypass_ruleset(proxy_uid: u32, log_prefix: Option<&str>) -> String { + let log_tcp = log_prefix + .map(|p| { + format!( + "\n tcp flags syn limit rate 5/second burst 10 packets log prefix \"{p}\" flags skuid" + ) + }) + .unwrap_or_default(); + let log_udp = log_prefix + .map(|p| { + format!( + "\n meta l4proto udp limit rate 5/second burst 10 packets log prefix \"{p}\" flags skuid" + ) + }) + .unwrap_or_default(); + + format!( + r#"table inet openshell_sidecar_bypass {{ + chain output {{ + type filter hook output priority 0; policy accept; + + oifname "lo" accept + ct state established,related accept + meta skuid {proxy_uid} accept{log_tcp} + meta nfproto ipv4 meta l4proto tcp reject with icmp type port-unreachable + meta nfproto ipv6 meta l4proto tcp reject with icmpv6 type port-unreachable{log_udp} + meta nfproto ipv4 meta l4proto udp reject with icmp type port-unreachable + meta nfproto ipv6 meta l4proto udp reject with icmpv6 type port-unreachable + }} +}} +"# + ) +} + #[cfg(test)] mod tests { use super::*; @@ -145,4 +185,27 @@ mod tests { "UDP log rule must come before UDP reject rule" ); } + + #[test] + fn sidecar_ruleset_allows_supervisor_uid_and_loopback() { + let ruleset = generate_sidecar_bypass_ruleset(1337, None); + assert!(ruleset.contains("table inet openshell_sidecar_bypass")); + assert!(ruleset.contains("oifname \"lo\" accept")); + assert!(ruleset.contains("meta skuid 1337 accept")); + } + + #[test] + fn sidecar_ruleset_rejects_tcp_and_udp_egress() { + let ruleset = generate_sidecar_bypass_ruleset(0, Some("openshell:sidecar:test:")); + assert!(ruleset.contains("meta nfproto ipv4 meta l4proto tcp reject")); + assert!(ruleset.contains("meta nfproto ipv6 meta l4proto tcp reject")); + assert!(ruleset.contains("meta nfproto ipv4 meta l4proto udp reject")); + assert!(ruleset.contains("meta nfproto ipv6 meta l4proto udp reject")); + assert_eq!( + ruleset + .matches("log prefix \"openshell:sidecar:test:\"") + .count(), + 2 + ); + } } diff --git a/crates/openshell-supervisor-process/src/process.rs b/crates/openshell-supervisor-process/src/process.rs index fcd7ae69c1..bd8be04c86 100644 --- a/crates/openshell-supervisor-process/src/process.rs +++ b/crates/openshell-supervisor-process/src/process.rs @@ -28,10 +28,32 @@ use std::sync::OnceLock; use tokio::process::{Child, Command}; use tracing::{debug, info}; +/// Process/filesystem enforcement performed by the process supervisor. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProcessEnforcementMode { + /// Preserve the existing supervisor behavior: prepare filesystem policy, + /// drop privileges, and apply Landlock/seccomp to workload processes. + Full, + /// Preserve process launch and SSH/session behavior, but skip controls + /// that require root or extra Linux capabilities. Kubernetes sidecar mode + /// uses this when network policy is enforced by the network sidecar. + NetworkOnly, +} + +impl ProcessEnforcementMode { + #[must_use] + pub const fn enforces_process_controls(self) -> bool { + matches!(self, Self::Full) + } +} + const SUPERVISOR_ONLY_ENV_VARS: &[&str] = &[ openshell_core::sandbox_env::SANDBOX_TOKEN, openshell_core::sandbox_env::SANDBOX_TOKEN_FILE, openshell_core::sandbox_env::K8S_SA_TOKEN_FILE, + openshell_core::sandbox_env::TLS_CA, + openshell_core::sandbox_env::TLS_CERT, + openshell_core::sandbox_env::TLS_KEY, openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, ]; @@ -443,6 +465,7 @@ impl ProcessHandle { workdir: Option<&str>, interactive: bool, policy: &SandboxPolicy, + enforcement_mode: ProcessEnforcementMode, netns: Option<&NetworkNamespace>, ca_paths: Option<&(PathBuf, PathBuf)>, provider_env: &HashMap, @@ -453,6 +476,7 @@ impl ProcessHandle { workdir, interactive, policy, + enforcement_mode, netns.and_then(NetworkNamespace::ns_fd), ca_paths, provider_env, @@ -465,12 +489,14 @@ impl ProcessHandle { /// /// Returns an error if the process fails to start. #[cfg(not(target_os = "linux"))] + #[allow(clippy::too_many_arguments)] pub fn spawn( program: &str, args: &[String], workdir: Option<&str>, interactive: bool, policy: &SandboxPolicy, + enforcement_mode: ProcessEnforcementMode, ca_paths: Option<&(PathBuf, PathBuf)>, provider_env: &HashMap, ) -> Result { @@ -480,6 +506,7 @@ impl ProcessHandle { workdir, interactive, policy, + enforcement_mode, ca_paths, provider_env, ) @@ -493,6 +520,7 @@ impl ProcessHandle { workdir: Option<&str>, interactive: bool, policy: &SandboxPolicy, + enforcement_mode: ProcessEnforcementMode, netns_fd: Option, ca_paths: Option<&(PathBuf, PathBuf)>, provider_env: &HashMap, @@ -552,18 +580,30 @@ impl ProcessHandle { // process where the tracing subscriber is functional. The child's // pre_exec context cannot reliably emit structured logs. #[cfg(target_os = "linux")] - sandbox::linux::log_sandbox_readiness(policy, workdir); + if enforcement_mode.enforces_process_controls() { + sandbox::linux::log_sandbox_readiness(policy, workdir); + } // Phase 1 (as root): Prepare Landlock ruleset by opening PathFds. // This MUST happen before drop_privileges() so that root-only paths // (e.g. mode 700 directories) can be opened. See issue #803. #[cfg(target_os = "linux")] - let prepared_sandbox = sandbox::linux::prepare(policy, workdir) - .map_err(|err| miette::miette!("Failed to prepare sandbox: {err}"))?; + let prepared_sandbox = if enforcement_mode.enforces_process_controls() { + Some( + sandbox::linux::prepare(policy, workdir) + .map_err(|err| miette::miette!("Failed to prepare sandbox: {err}"))?, + ) + } else { + None + }; #[cfg(target_os = "linux")] - let supervisor_identity_mount = supervisor_identity_mount_from_env().map_err(|err| { - miette::miette!("Failed to prepare supervisor identity isolation: {err}") - })?; + let supervisor_identity_mount = if enforcement_mode.enforces_process_controls() { + supervisor_identity_mount_from_env().map_err(|err| { + miette::miette!("Failed to prepare supervisor identity isolation: {err}") + })? + } else { + None + }; // Set up process group for signal handling (non-interactive mode only). // In interactive mode, we inherit the parent's process group to maintain @@ -575,7 +615,7 @@ impl ProcessHandle { // Wrap in Option so we can .take() it out of the FnMut closure. // pre_exec is only called once (after fork, before exec). #[cfg(target_os = "linux")] - let mut prepared_sandbox = Some(prepared_sandbox); + let mut prepared_sandbox = prepared_sandbox; #[allow(unsafe_code)] unsafe { cmd.pre_exec(move || { @@ -600,8 +640,10 @@ impl ProcessHandle { // Drop privileges. initgroups/setgid/setuid need access to // /etc/group and /etc/passwd which would be blocked if // Landlock were already enforced. - drop_privileges(&policy) - .map_err(|err| std::io::Error::other(err.to_string()))?; + if enforcement_mode.enforces_process_controls() { + drop_privileges(&policy) + .map_err(|err| std::io::Error::other(err.to_string()))?; + } harden_child_process().map_err(|err| std::io::Error::other(err.to_string()))?; @@ -629,12 +671,14 @@ impl ProcessHandle { } #[cfg(not(target_os = "linux"))] + #[allow(clippy::too_many_arguments)] fn spawn_impl( program: &str, args: &[String], workdir: Option<&str>, interactive: bool, policy: &SandboxPolicy, + enforcement_mode: ProcessEnforcementMode, ca_paths: Option<&(PathBuf, PathBuf)>, provider_env: &HashMap, ) -> Result { @@ -697,13 +741,17 @@ impl ProcessHandle { // Drop privileges before applying sandbox restrictions. // initgroups/setgid/setuid need access to /etc/group and /etc/passwd // which may be blocked by Landlock. - drop_privileges(&policy) - .map_err(|err| std::io::Error::other(err.to_string()))?; + if enforcement_mode.enforces_process_controls() { + drop_privileges(&policy) + .map_err(|err| std::io::Error::other(err.to_string()))?; + } harden_child_process().map_err(|err| std::io::Error::other(err.to_string()))?; - sandbox::apply(&policy, workdir.as_deref()) - .map_err(|err| std::io::Error::other(err.to_string()))?; + if enforcement_mode.enforces_process_controls() { + sandbox::apply(&policy, workdir.as_deref()) + .map_err(|err| std::io::Error::other(err.to_string()))?; + } Ok(()) }); diff --git a/crates/openshell-supervisor-process/src/run.rs b/crates/openshell-supervisor-process/src/run.rs index e16f118922..bd3fea91f1 100644 --- a/crates/openshell-supervisor-process/src/run.rs +++ b/crates/openshell-supervisor-process/src/run.rs @@ -33,7 +33,7 @@ use openshell_core::denial::DenialEvent; #[cfg(target_os = "linux")] use crate::managed_children; -use crate::process::ProcessHandle; +use crate::process::{ProcessEnforcementMode, ProcessHandle}; fn ocsf_ctx() -> &'static openshell_ocsf::SandboxContext { openshell_ocsf::ctx::ctx() @@ -57,6 +57,7 @@ pub async fn run_process( openshell_endpoint: Option<&str>, ssh_socket_path: Option, policy: &SandboxPolicy, + enforcement_mode: ProcessEnforcementMode, entrypoint_pid: Arc, provider_credentials: ProviderCredentialState, provider_env: std::collections::HashMap, @@ -71,21 +72,26 @@ pub async fn run_process( // /etc/group so the "sandbox" entry matches. Must run before // validate_sandbox_user so passwd lookups see the correct identity. #[cfg(unix)] - crate::process::update_sandbox_passwd_entries()?; + if enforcement_mode.enforces_process_controls() { + crate::process::update_sandbox_passwd_entries()?; + } // Validate that the sandbox user exists in the image. All sandbox images // must include a "sandbox" user for privilege dropping; failing fast here // beats silently running children as root. #[cfg(unix)] - crate::process::validate_sandbox_user(policy)?; - #[cfg(unix)] - crate::process::validate_sandbox_group(policy)?; + if enforcement_mode.enforces_process_controls() { + crate::process::validate_sandbox_user(policy)?; + crate::process::validate_sandbox_group(policy)?; + } // Create read_write directories and chown newly-created ones to the // sandbox user/group. Runs as the supervisor (root) before the child // is forked so the workload sees writable paths it owns. #[cfg(unix)] - crate::process::prepare_filesystem(policy)?; + if enforcement_mode.enforces_process_controls() { + crate::process::prepare_filesystem(policy)?; + } // Eagerly fetch initial settings and install the agent skill if the // proposals flag is on at startup, rather than waiting for the policy @@ -206,31 +212,10 @@ pub async fn run_process( // their env so cooperative tools (curl, npm, Node) route through the // CONNECT proxy. Linux uses the netns host_ip; on other targets fall back // to the policy-declared http_addr directly. - let ssh_proxy_url = if matches!(policy.network.mode, NetworkMode::Proxy) { - #[cfg(target_os = "linux")] - { - netns.map(|ns| { - let port = policy - .network - .proxy - .as_ref() - .and_then(|p| p.http_addr) - .map_or(3128, |addr| addr.port()); - format!("http://{}:{port}", ns.host_ip()) - }) - } - #[cfg(not(target_os = "linux"))] - { - policy - .network - .proxy - .as_ref() - .and_then(|p| p.http_addr) - .map(|addr| format!("http://{addr}")) - } - } else { - None - }; + #[cfg(target_os = "linux")] + let ssh_proxy_url = ssh_proxy_url_for_policy(policy, netns.map(NetworkNamespace::host_ip)); + #[cfg(not(target_os = "linux"))] + let ssh_proxy_url = ssh_proxy_url_for_policy(policy, None); let ssh_socket_path: Option = ssh_socket_path.map(std::path::PathBuf::from); if let Some(listen_path) = ssh_socket_path.clone() { @@ -259,6 +244,7 @@ pub async fn run_process( ca_paths, provider_credentials_clone, user_env_clone, + enforcement_mode, ) .await { @@ -325,6 +311,7 @@ pub async fn run_process( workdir, interactive, policy, + enforcement_mode, netns, ca_file_paths.as_ref(), &provider_env, @@ -337,12 +324,16 @@ pub async fn run_process( workdir, interactive, policy, + enforcement_mode, ca_file_paths.as_ref(), &provider_env, )?; // Store the entrypoint PID so the proxy can resolve TCP peer identity entrypoint_pid.store(handle.pid(), Ordering::Release); + if let Some(path) = entrypoint_pid_file() { + write_entrypoint_pid_file(&path, handle.pid())?; + } ocsf_emit!( ProcessActivityBuilder::new(ocsf_ctx()) .activity(ActivityId::Open) @@ -395,6 +386,42 @@ pub async fn run_process( Ok(status.code()) } +fn entrypoint_pid_file() -> Option { + std::env::var(openshell_core::sandbox_env::ENTRYPOINT_PID_FILE) + .ok() + .filter(|value| !value.is_empty()) +} + +fn write_entrypoint_pid_file(path: &str, pid: u32) -> Result<()> { + let pid_path = std::path::Path::new(path); + if let Some(parent) = pid_path.parent() { + std::fs::create_dir_all(parent).into_diagnostic()?; + } + std::fs::write(pid_path, format!("{pid}\n")).into_diagnostic()?; + info!( + path, + pid, "Published workload entrypoint PID for network sidecar" + ); + Ok(()) +} + +fn ssh_proxy_url_for_policy( + policy: &SandboxPolicy, + netns_proxy_host: Option, +) -> Option { + if !matches!(policy.network.mode, NetworkMode::Proxy) { + return None; + } + + let proxy = policy.network.proxy.as_ref()?; + if let Some(host) = netns_proxy_host { + let port = proxy.http_addr.map_or(3128, |addr| addr.port()); + return Some(format!("http://{host}:{port}")); + } + + proxy.http_addr.map(|addr| format!("http://{addr}")) +} + /// Eagerly fetch initial settings and install the agent-driven policy /// proposal skill if the flag is on at startup. /// @@ -451,3 +478,53 @@ async fn install_initial_agent_skill(sandbox_id: Option<&str>, openshell_endpoin ); } } + +#[cfg(test)] +mod tests { + use super::*; + use openshell_core::policy::{ + FilesystemPolicy, LandlockPolicy, NetworkMode, NetworkPolicy, ProcessPolicy, ProxyPolicy, + }; + + fn policy(mode: NetworkMode, http_addr: Option) -> SandboxPolicy { + SandboxPolicy { + version: 1, + filesystem: FilesystemPolicy::default(), + network: NetworkPolicy { + mode, + proxy: http_addr.map(|http_addr| ProxyPolicy { + http_addr: Some(http_addr), + }), + }, + landlock: LandlockPolicy::default(), + process: ProcessPolicy::default(), + } + } + + #[test] + fn ssh_proxy_url_uses_policy_addr_without_netns() { + let policy = policy(NetworkMode::Proxy, Some(([127, 0, 0, 1], 3128).into())); + + assert_eq!( + ssh_proxy_url_for_policy(&policy, None).as_deref(), + Some("http://127.0.0.1:3128") + ); + } + + #[test] + fn ssh_proxy_url_prefers_netns_host_with_policy_port() { + let policy = policy(NetworkMode::Proxy, Some(([127, 0, 0, 1], 8080).into())); + + assert_eq!( + ssh_proxy_url_for_policy(&policy, Some([10, 200, 0, 1].into())).as_deref(), + Some("http://10.200.0.1:8080") + ); + } + + #[test] + fn ssh_proxy_url_skips_non_proxy_mode() { + let policy = policy(NetworkMode::Allow, Some(([127, 0, 0, 1], 3128).into())); + + assert_eq!(ssh_proxy_url_for_policy(&policy, None), None); + } +} diff --git a/crates/openshell-supervisor-process/src/ssh.rs b/crates/openshell-supervisor-process/src/ssh.rs index 62d10f374d..c55a6d877e 100644 --- a/crates/openshell-supervisor-process/src/ssh.rs +++ b/crates/openshell-supervisor-process/src/ssh.rs @@ -6,7 +6,7 @@ use crate::child_env; #[cfg(target_os = "linux")] use crate::managed_children; -use crate::process::{drop_privileges, is_supervisor_only_env_var}; +use crate::process::{ProcessEnforcementMode, drop_privileges, is_supervisor_only_env_var}; use crate::sandbox; use miette::{IntoDiagnostic, Result}; use nix::pty::{Winsize, openpty}; @@ -42,6 +42,7 @@ type SshServerInit = ( fn ssh_server_init( listen_path: &Path, ca_file_paths: &Option<(PathBuf, PathBuf)>, + enforcement_mode: ProcessEnforcementMode, ) -> Result { let mut rng = OsRng; let host_key = PrivateKey::random(&mut rng, Algorithm::Ed25519).into_diagnostic()?; @@ -55,13 +56,16 @@ fn ssh_server_init( let config = Arc::new(config); let ca_paths = ca_file_paths.as_ref().map(|p| Arc::new(p.clone())); - // Ensure the parent directory exists and is root-owned with 0700 - // permissions. The sandbox entrypoint runs as an unprivileged user; it - // must not be able to enter this directory and connect to the socket. + // In full enforcement mode the supervisor starts as root and can isolate + // the SSH socket in a root-only directory before spawning unprivileged + // children. In network-only sidecar mode the process supervisor itself + // runs as the sandbox UID, so the driver points the socket at a writable + // sidecar state volume and accepts that Unix permissions no longer isolate + // same-UID child processes from the socket. if let Some(parent) = listen_path.parent() { std::fs::create_dir_all(parent).into_diagnostic()?; #[cfg(unix)] - { + if enforcement_mode.enforces_process_controls() { use std::os::unix::fs::PermissionsExt; let perms = std::fs::Permissions::from_mode(0o700); std::fs::set_permissions(parent, perms).into_diagnostic()?; @@ -108,21 +112,23 @@ pub async fn run_ssh_server( ca_file_paths: Option<(PathBuf, PathBuf)>, provider_credentials: ProviderCredentialState, user_environment: HashMap, + enforcement_mode: ProcessEnforcementMode, ) -> Result<()> { - let (listener, config, ca_paths) = match ssh_server_init(&listen_path, &ca_file_paths) { - Ok(v) => { - // Signal that the SSH server has bound the socket and is ready to - // accept connections. The parent task awaits this before spawning - // the entrypoint process, ensuring exec requests won't race - // against server startup. - let _ = ready_tx.send(Ok(())); - v - } - Err(err) => { - let _ = ready_tx.send(Err(err)); - return Ok(()); - } - }; + let (listener, config, ca_paths) = + match ssh_server_init(&listen_path, &ca_file_paths, enforcement_mode) { + Ok(v) => { + // Signal that the SSH server has bound the socket and is ready to + // accept connections. The parent task awaits this before spawning + // the entrypoint process, ensuring exec requests won't race + // against server startup. + let _ = ready_tx.send(Ok(())); + v + } + Err(err) => { + let _ = ready_tx.send(Err(err)); + return Ok(()); + } + }; loop { let (stream, _peer) = listener.accept().await.into_diagnostic()?; @@ -145,6 +151,7 @@ pub async fn run_ssh_server( ca_paths, provider_credentials, user_environment, + enforcement_mode, ) .await { @@ -172,6 +179,7 @@ async fn handle_connection( ca_file_paths: Option>, provider_credentials: ProviderCredentialState, user_environment: HashMap, + enforcement_mode: ProcessEnforcementMode, ) -> Result<()> { // Access is gated by the Unix-socket filesystem permissions (root-only), // not by an application-level preface. The supervisor bridges the @@ -195,6 +203,7 @@ async fn handle_connection( ca_file_paths, provider_credentials, user_environment, + enforcement_mode, ); russh::server::run_stream(config, stream, handler) .await @@ -223,6 +232,7 @@ struct SshHandler { ca_file_paths: Option>, provider_credentials: ProviderCredentialState, user_environment: HashMap, + enforcement_mode: ProcessEnforcementMode, channels: HashMap, } @@ -236,6 +246,7 @@ impl SshHandler { ca_file_paths: Option>, provider_credentials: ProviderCredentialState, user_environment: HashMap, + enforcement_mode: ProcessEnforcementMode, ) -> Self { Self { policy, @@ -245,6 +256,7 @@ impl SshHandler { ca_file_paths, provider_credentials, user_environment, + enforcement_mode, channels: HashMap::new(), } } @@ -468,6 +480,7 @@ impl russh::server::Handler for SshHandler { self.ca_file_paths.clone(), &self.provider_credentials.child_env_with_gcp_resolved(), &self.user_environment, + self.enforcement_mode, )?; let state = self.channels.get_mut(&channel).ok_or_else(|| { anyhow::anyhow!("subsystem_request on unknown channel {channel:?}") @@ -564,6 +577,7 @@ impl SshHandler { self.ca_file_paths.clone(), &provider_env, &self.user_environment, + self.enforcement_mode, )?; state.pty_master = Some(pty_master); state.input_sender = Some(input_sender); @@ -582,6 +596,7 @@ impl SshHandler { self.ca_file_paths.clone(), &provider_env, &self.user_environment, + self.enforcement_mode, )?; state.input_sender = Some(input_sender); } @@ -748,6 +763,7 @@ fn spawn_pty_shell( ca_file_paths: Option>, provider_env: &HashMap, user_environment: &HashMap, + enforcement_mode: ProcessEnforcementMode, ) -> anyhow::Result<(std::fs::File, mpsc::Sender>)> { let winsize = Winsize { ws_row: to_u16(pty.row_height.max(1)), @@ -806,12 +822,20 @@ fn spawn_pty_shell( // Probe Landlock availability from the parent process where tracing works. #[cfg(target_os = "linux")] - sandbox::linux::log_sandbox_readiness(policy, workdir.as_deref()); + if enforcement_mode.enforces_process_controls() { + sandbox::linux::log_sandbox_readiness(policy, workdir.as_deref()); + } // Phase 1 (as root): Prepare Landlock ruleset before drop_privileges. #[cfg(target_os = "linux")] - let prepared_sandbox = sandbox::linux::prepare(policy, workdir.as_deref()) - .map_err(|err| anyhow::anyhow!("Failed to prepare sandbox: {err}"))?; + let prepared_sandbox = if enforcement_mode.enforces_process_controls() { + Some( + sandbox::linux::prepare(policy, workdir.as_deref()) + .map_err(|err| anyhow::anyhow!("Failed to prepare sandbox: {err}"))?, + ) + } else { + None + }; #[cfg(unix)] { @@ -821,6 +845,7 @@ fn spawn_pty_shell( workdir.clone(), slave_fd, netns_fd, + enforcement_mode, #[cfg(target_os = "linux")] prepared_sandbox, )?; @@ -913,6 +938,7 @@ fn spawn_pipe_exec( ca_file_paths: Option>, provider_env: &HashMap, user_environment: &HashMap, + enforcement_mode: ProcessEnforcementMode, ) -> anyhow::Result>> { let mut cmd = command.map_or_else( || { @@ -955,12 +981,20 @@ fn spawn_pipe_exec( // Probe Landlock availability from the parent process where tracing works. #[cfg(target_os = "linux")] - sandbox::linux::log_sandbox_readiness(policy, workdir.as_deref()); + if enforcement_mode.enforces_process_controls() { + sandbox::linux::log_sandbox_readiness(policy, workdir.as_deref()); + } // Phase 1 (as root): Prepare Landlock ruleset before drop_privileges. #[cfg(target_os = "linux")] - let prepared_sandbox = sandbox::linux::prepare(policy, workdir.as_deref()) - .map_err(|err| anyhow::anyhow!("Failed to prepare sandbox: {err}"))?; + let prepared_sandbox = if enforcement_mode.enforces_process_controls() { + Some( + sandbox::linux::prepare(policy, workdir.as_deref()) + .map_err(|err| anyhow::anyhow!("Failed to prepare sandbox: {err}"))?, + ) + } else { + None + }; #[cfg(unix)] { @@ -969,6 +1003,7 @@ fn spawn_pipe_exec( policy.clone(), workdir.clone(), netns_fd, + enforcement_mode, #[cfg(target_os = "linux")] prepared_sandbox, )?; @@ -1068,7 +1103,9 @@ fn spawn_pipe_exec( mod unsafe_pty { #[cfg(not(target_os = "linux"))] use super::sandbox; - use super::{Command, RawFd, SandboxPolicy, Winsize, drop_privileges, setsid}; + use super::{ + Command, ProcessEnforcementMode, RawFd, SandboxPolicy, Winsize, drop_privileges, setsid, + }; #[cfg(unix)] use std::os::unix::process::CommandExt; @@ -1107,17 +1144,21 @@ mod unsafe_pty { _workdir: Option, slave_fd: RawFd, netns_fd: Option, - #[cfg(target_os = "linux")] prepared: crate::sandbox::linux::PreparedSandbox, + enforcement_mode: ProcessEnforcementMode, + #[cfg(target_os = "linux")] prepared: Option, ) -> anyhow::Result<()> { // Wrap in Option so we can .take() it out of the FnMut closure. // pre_exec is only called once (after fork, before exec). #[cfg(target_os = "linux")] - let mut prepared = Some(prepared); + let mut prepared = prepared; #[cfg(target_os = "linux")] - let supervisor_identity_mount = crate::process::supervisor_identity_mount_from_env() - .map_err(|err| { + let supervisor_identity_mount = if enforcement_mode.enforces_process_controls() { + crate::process::supervisor_identity_mount_from_env().map_err(|err| { anyhow::anyhow!("failed to prepare supervisor identity isolation: {err}") - })?; + })? + } else { + None + }; unsafe { cmd.pre_exec(move || { setsid().map_err(|err| std::io::Error::other(err.to_string()))?; @@ -1126,6 +1167,7 @@ mod unsafe_pty { enter_netns_and_sandbox( netns_fd, &policy, + enforcement_mode, #[cfg(target_os = "linux")] supervisor_identity_mount, #[cfg(target_os = "linux")] @@ -1152,20 +1194,25 @@ mod unsafe_pty { policy: SandboxPolicy, _workdir: Option, netns_fd: Option, - #[cfg(target_os = "linux")] prepared: crate::sandbox::linux::PreparedSandbox, + enforcement_mode: ProcessEnforcementMode, + #[cfg(target_os = "linux")] prepared: Option, ) -> anyhow::Result<()> { #[cfg(target_os = "linux")] - let mut prepared = Some(prepared); + let mut prepared = prepared; #[cfg(target_os = "linux")] - let supervisor_identity_mount = crate::process::supervisor_identity_mount_from_env() - .map_err(|err| { + let supervisor_identity_mount = if enforcement_mode.enforces_process_controls() { + crate::process::supervisor_identity_mount_from_env().map_err(|err| { anyhow::anyhow!("failed to prepare supervisor identity isolation: {err}") - })?; + })? + } else { + None + }; unsafe { cmd.pre_exec(move || { enter_netns_and_sandbox( netns_fd, &policy, + enforcement_mode, #[cfg(target_os = "linux")] supervisor_identity_mount, #[cfg(target_os = "linux")] @@ -1179,6 +1226,7 @@ mod unsafe_pty { fn enter_netns_and_sandbox( netns_fd: Option, policy: &SandboxPolicy, + enforcement_mode: ProcessEnforcementMode, #[cfg(target_os = "linux")] supervisor_identity_mount: Option< &crate::process::SupervisorIdentityMountNamespace, >, @@ -1207,7 +1255,9 @@ mod unsafe_pty { // Drop privileges. initgroups/setgid/setuid need /etc/group and // /etc/passwd which would be blocked if Landlock were already enforced. - drop_privileges(policy).map_err(|err| std::io::Error::other(err.to_string()))?; + if enforcement_mode.enforces_process_controls() { + drop_privileges(policy).map_err(|err| std::io::Error::other(err.to_string()))?; + } crate::process::harden_child_process() .map_err(|err| std::io::Error::other(err.to_string()))?; @@ -1220,7 +1270,9 @@ mod unsafe_pty { } #[cfg(not(target_os = "linux"))] - sandbox::apply(policy, None).map_err(|err| std::io::Error::other(err.to_string()))?; + if enforcement_mode.enforces_process_controls() { + sandbox::apply(policy, None).map_err(|err| std::io::Error::other(err.to_string()))?; + } Ok(()) } @@ -1681,21 +1733,24 @@ mod tests { policy, None, None, // no netns fd + ProcessEnforcementMode::Full, #[cfg(target_os = "linux")] - sandbox::linux::prepare( - &SandboxPolicy { - version: 0, - filesystem: FilesystemPolicy::default(), - network: NetworkPolicy::default(), - landlock: LandlockPolicy::default(), - process: ProcessPolicy { - run_as_user: None, - run_as_group: None, + Some( + sandbox::linux::prepare( + &SandboxPolicy { + version: 0, + filesystem: FilesystemPolicy::default(), + network: NetworkPolicy::default(), + landlock: LandlockPolicy::default(), + process: ProcessPolicy { + run_as_user: None, + run_as_group: None, + }, }, - }, - None, - ) - .expect("prepare should succeed in test environment"), + None, + ) + .expect("prepare should succeed in test environment"), + ), ) .expect("install pre_exec should succeed"); diff --git a/deploy/docker/Dockerfile.supervisor b/deploy/docker/Dockerfile.supervisor index c84cc70e9e..c760bbc890 100644 --- a/deploy/docker/Dockerfile.supervisor +++ b/deploy/docker/Dockerfile.supervisor @@ -5,10 +5,10 @@ # Supervisor image build. # -# The final image is `scratch`: it only carries the static `openshell-sandbox` -# binary used by Docker extraction, Podman image volumes, and the Kubernetes -# init container copy-self path. A static musl binary lets the image stay -# `scratch` while still being executable as an init container. +# The final image carries the static `openshell-sandbox` binary used by Docker +# extraction, Podman image volumes, and the Kubernetes init container copy-self +# path. It also includes nftables so the Kubernetes supervisor sidecar can +# install pod-namespace egress enforcement rules. # # The Rust binary is built natively before this image build runs and staged at: # deploy/docker/.build/prebuilt-binaries//openshell-sandbox @@ -19,17 +19,16 @@ # target) and uploads it as an artifact, which is downloaded into the same # staging directory before the image build job runs. -FROM scratch AS supervisor +FROM alpine:3.22 AS supervisor ARG TARGETARCH -# --chmod=0550 drops world-execute and survives the actions/upload-artifact -# + download-artifact roundtrip (which strips exec perms). Ownership is left -# at root (0:0) deliberately: the Podman driver mounts this image as a -# read-only image volume into the sandbox container and drops DAC_OVERRIDE, -# so the container's UID 0 must own the binary to read+exec it. Mode 0550 -# (r-xr-x---) is the security win; the chown to a non-root UID was breaking -# Podman without buying anything since the container is always UID 0. -COPY --chmod=0550 deploy/docker/.build/prebuilt-binaries/${TARGETARCH}/openshell-sandbox /openshell-sandbox +RUN apk add --no-cache nftables iptables iptables-legacy + +# --chmod=0555 restores execute bits after the actions/upload-artifact + +# download-artifact roundtrip strips them. Ownership stays root (0:0) for +# Podman image-volume mounts, while world-execute lets the Kubernetes +# network sidecar run this binary as the dedicated non-root proxy UID. +COPY --chmod=0555 deploy/docker/.build/prebuilt-binaries/${TARGETARCH}/openshell-sandbox /openshell-sandbox ENTRYPOINT ["/openshell-sandbox"] diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index f3a86f884f..0ed0959153 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -237,8 +237,10 @@ add `ci/values-spire.yaml` to the OpenShell release values files. | supervisor.image.pullPolicy | string | `""` | Supervisor image pull policy. Defaults to the gateway image pull policy when empty. | | supervisor.image.repository | string | `"ghcr.io/nvidia/openshell/supervisor"` | Supervisor image repository. | | supervisor.image.tag | string | `""` | Supervisor image tag. Defaults to the chart appVersion when empty. | +| supervisor.processEnforcement | string | `"network-only"` | Process/filesystem controls applied by the agent process supervisor in non-combined topologies. "network-only" keeps the low-permission agent shape; "full" grants combined-mode process/filesystem controls. | +| supervisor.proxyUid | int | `1337` | UID for the long-running network sidecar in sidecar topology. The network init container installs nftables rules that exempt this UID. | | supervisor.sideloadMethod | string | `""` | How the supervisor binary is delivered into sandbox pods. Empty (default) = auto-detect from cluster version: K8s >= v1.35 -> "image-volume" (ImageVolume enabled by default; GA in v1.36) K8s < v1.35 -> "init-container" (copies via init container + emptyDir) On K8s v1.33-v1.34 with the ImageVolume feature gate manually enabled, set this to "image-volume" explicitly. | -| supervisor.topology | string | `"combined"` | Supervisor pod topology for Kubernetes sandboxes. "combined" runs networking and process supervision in the agent container. | +| supervisor.topology | string | `"combined"` | Supervisor pod topology for Kubernetes sandboxes. "combined" runs the current single supervisor container in the agent pod. "sidecar" runs network enforcement in a dedicated sidecar and the process supervisor as a low-capability wrapper in the agent container. | | tolerations | list | `[]` | Tolerations for the gateway pod. | | workload.allowMultiReplicaStatefulSet | bool | `false` | Allow replicaCount > 1 while rendering a StatefulSet. Prefer workload.kind=deployment for external database-backed multi-replica gateways; this override exists for operators who explicitly require StatefulSet identity or storage semantics. | | workload.kind | string | `"statefulset"` | Gateway workload controller kind. Use `statefulset` for the default SQLite database, or `deployment` when server.externalDbSecret points at an external database. | diff --git a/deploy/helm/openshell/ci/values-sidecar.yaml b/deploy/helm/openshell/ci/values-sidecar.yaml new file mode 100644 index 0000000000..dac9e810f0 --- /dev/null +++ b/deploy/helm/openshell/ci/values-sidecar.yaml @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# CI/dev overlay for exercising the Kubernetes supervisor sidecar topology. +# +# Merge after values.yaml and ci/values-skaffold.yaml: +# helm install ... -f values.yaml -f ci/values-skaffold.yaml -f ci/values-sidecar.yaml +# +# Or set: +# OPENSHELL_E2E_KUBE_EXTRA_VALUES=deploy/helm/openshell/ci/values-sidecar.yaml +# before running `mise run e2e:kubernetes`. +supervisor: + topology: sidecar diff --git a/deploy/helm/openshell/skaffold.yaml b/deploy/helm/openshell/skaffold.yaml index d571c3bd9e..76a9e7a5b2 100644 --- a/deploy/helm/openshell/skaffold.yaml +++ b/deploy/helm/openshell/skaffold.yaml @@ -119,6 +119,8 @@ deploy: # To enable SPIFFE/SPIRE provider token grants (requires the # spire-crds and spire releases above): #- ci/values-spire.yaml + # To exercise the Kubernetes supervisor sidecar topology: + #- ci/values-sidecar.yaml # To test multi-replica external PostgreSQL behavior: #- ci/values-high-availability.yaml setValueTemplates: @@ -126,3 +128,9 @@ deploy: image.tag: '{{.IMAGE_TAG_openshell_gateway}}' supervisor.image.repository: '{{.IMAGE_REPO_openshell_supervisor}}' supervisor.image.tag: '{{.IMAGE_TAG_openshell_supervisor}}' +profiles: + - name: sidecar + patches: + - op: add + path: /deploy/helm/releases/0/valuesFiles/- + value: ci/values-sidecar.yaml diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index 0e75a311f0..9637c33284 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -114,6 +114,8 @@ data: service_account_name = {{ include "openshell.sandboxServiceAccountName" . | quote }} supervisor_sideload_method = {{ include "openshell.supervisorSideloadMethod" . | quote }} supervisor_topology = {{ .Values.supervisor.topology | default "combined" | quote }} + process_enforcement = {{ .Values.supervisor.processEnforcement | default "network-only" | quote }} + proxy_uid = {{ .Values.supervisor.proxyUid | default 1337 }} sa_token_ttl_secs = {{ .Values.server.sandboxJwt.k8sSaTokenTtlSecs | default 3600 }} {{- if .Values.server.providerTokenGrants.spiffe.enabled }} provider_spiffe_workload_api_socket_path = {{ .Values.server.providerTokenGrants.spiffe.workloadApiSocketPath | quote }} diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index 50051e0036..509eb4279b 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -83,21 +83,39 @@ tests: path: data["gateway.toml"] pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?service_account_name\s*=\s*"openshell-sandbox"' - - it: renders combined supervisor topology by default under [openshell.drivers.kubernetes] + - it: renders supervisor topology under [openshell.drivers.kubernetes] + template: templates/gateway-config.yaml + set: + supervisor.topology: sidecar + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?supervisor_topology\s*=\s*"sidecar"' + + - it: renders process enforcement under [openshell.drivers.kubernetes] + template: templates/gateway-config.yaml + set: + supervisor.processEnforcement: full + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?process_enforcement\s*=\s*"full"' + + - it: renders default process enforcement under [openshell.drivers.kubernetes] template: templates/gateway-config.yaml asserts: - matchRegex: path: data["gateway.toml"] - pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?supervisor_topology\s*=\s*"combined"' + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?process_enforcement\s*=\s*"network-only"' - - it: renders explicit combined supervisor topology under [openshell.drivers.kubernetes] + - it: renders proxy uid under [openshell.drivers.kubernetes] template: templates/gateway-config.yaml set: - supervisor.topology: combined + supervisor.proxyUid: 2200 asserts: - matchRegex: path: data["gateway.toml"] - pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?supervisor_topology\s*=\s*"combined"' + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?proxy_uid\s*=\s*2200' - it: renders sandbox image pull secrets under [openshell.drivers.kubernetes] template: templates/gateway-config.yaml diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index fd73dd0713..c670a97b8a 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -45,8 +45,17 @@ supervisor: # set this to "image-volume" explicitly. sideloadMethod: "" # -- Supervisor pod topology for Kubernetes sandboxes. - # "combined" runs networking and process supervision in the agent container. + # "combined" runs the current single supervisor container in the agent pod. + # "sidecar" runs network enforcement in a dedicated sidecar and the process + # supervisor as a low-capability wrapper in the agent container. topology: "combined" + # -- Process/filesystem controls applied by the agent process supervisor in + # non-combined topologies. "network-only" keeps the low-permission agent + # shape; "full" grants combined-mode process/filesystem controls. + processEnforcement: "network-only" + # -- UID for the long-running network sidecar in sidecar topology. The + # network init container installs nftables rules that exempt this UID. + proxyUid: 1337 # -- Image pull secrets attached to gateway and helper pods. imagePullSecrets: [] diff --git a/docs/kubernetes/access-control.mdx b/docs/kubernetes/access-control.mdx index 8824b6de1e..5409a4b11d 100644 --- a/docs/kubernetes/access-control.mdx +++ b/docs/kubernetes/access-control.mdx @@ -5,7 +5,7 @@ title: "Access Control" sidebar-title: "Access Control" description: "Configure OIDC user authentication or reverse-proxy auth termination for a Kubernetes-deployed OpenShell gateway." keywords: "Generative AI, Cybersecurity, Kubernetes, Authentication, mTLS, OIDC, Keycloak, Entra ID, Okta, Gateway Auth" -position: 4 +position: 5 --- The OpenShell gateway supports two access-control models for human callers on Kubernetes: diff --git a/docs/kubernetes/ingress.mdx b/docs/kubernetes/ingress.mdx index 844167246a..66178bc3a0 100644 --- a/docs/kubernetes/ingress.mdx +++ b/docs/kubernetes/ingress.mdx @@ -5,7 +5,7 @@ title: "Ingress" sidebar-title: "Ingress" description: "Expose the OpenShell gateway externally using the Kubernetes Gateway API and a GRPCRoute." keywords: "Generative AI, Cybersecurity, Kubernetes, Gateway API, Envoy Gateway, GRPCRoute, Ingress, External Access" -position: 3 +position: 4 --- By default, the OpenShell gateway is only reachable inside the cluster. To let CLI clients connect without a `kubectl port-forward`, expose the gateway through an ingress. diff --git a/docs/kubernetes/managing-certificates.mdx b/docs/kubernetes/managing-certificates.mdx index d929463504..b66419b505 100644 --- a/docs/kubernetes/managing-certificates.mdx +++ b/docs/kubernetes/managing-certificates.mdx @@ -5,7 +5,7 @@ title: "Managing Certificates" sidebar-title: "Managing Certificates" description: "Configure the OpenShell Helm chart to use cert-manager for mTLS certificate issuance and automatic renewal." keywords: "Generative AI, Cybersecurity, Kubernetes, cert-manager, PKI, TLS, mTLS, Certificates" -position: 2 +position: 3 --- The OpenShell gateway uses mTLS certificates for transport between the gateway and sandbox supervisors. These certificates are not Kubernetes user authentication; configure OIDC or a trusted access proxy for user access. The Helm chart supports two ways to provision and manage the certificate bundle: diff --git a/docs/kubernetes/openshift.mdx b/docs/kubernetes/openshift.mdx index b8313bdfe6..caf799b51f 100644 --- a/docs/kubernetes/openshift.mdx +++ b/docs/kubernetes/openshift.mdx @@ -5,7 +5,7 @@ title: "OpenShift" sidebar-title: "OpenShift" description: "Install the OpenShell Helm chart on OpenShift, including the SCC binding and chart overrides required by OpenShift's Security Context Constraints." keywords: "Generative AI, Cybersecurity, Kubernetes, OpenShift, SCC, Security Context Constraints, Helm, Gateway, Installation" -position: 5 +position: 6 --- diff --git a/docs/kubernetes/setup.mdx b/docs/kubernetes/setup.mdx index f6051b1235..cc886c1688 100644 --- a/docs/kubernetes/setup.mdx +++ b/docs/kubernetes/setup.mdx @@ -161,6 +161,7 @@ The most commonly changed values are: | `pkiInitJob.serverDnsNames` / `certManager.serverDnsNames` | Additional gateway server DNS SANs. Wildcard SANs also enable sandbox service URLs under that domain. | | `supervisor.sideloadMethod` | How the supervisor binary is delivered into sandbox pods. Leave empty to auto-detect based on cluster version: clusters running Kubernetes 1.35 or later use `image-volume` (ImageVolume GA in 1.36); older clusters use `init-container`. Set explicitly to `image-volume` on Kubernetes 1.33 or 1.34 with the ImageVolume feature gate enabled, or to `init-container` to force the legacy path on any version. | | `supervisor.topology` | Sandbox pod topology. Refer to [Topology](/kubernetes/topology). | +| `supervisor.proxyUid` | Non-root UID for the long-running network sidecar when `supervisor.topology=sidecar`. The UID must not match the sandbox UID. | Use a values file for repeatable deployments: @@ -244,7 +245,7 @@ The gateway exposes `/healthz` for process liveness and `/readyz` for dependency ## Next Steps -- To review Kubernetes sandbox topology, refer to [Topology](/kubernetes/topology). +- To choose between combined and sidecar sandbox pods, refer to [Topology](/kubernetes/topology). - To enable automatic certificate rotation with cert-manager, refer to [Managing Certificates](/kubernetes/managing-certificates). - To expose the gateway externally without port-forwarding, refer to [Ingress](/kubernetes/ingress). - To configure OIDC or reverse-proxy authentication, refer to [Access Control](/kubernetes/access-control). diff --git a/docs/kubernetes/topology.mdx b/docs/kubernetes/topology.mdx index f3f69cf6ec..5bf942e35b 100644 --- a/docs/kubernetes/topology.mdx +++ b/docs/kubernetes/topology.mdx @@ -3,37 +3,36 @@ # SPDX-License-Identifier: Apache-2.0 title: "Kubernetes Sandbox Topology" sidebar-title: "Topology" -description: "Review the default combined supervisor topology for Kubernetes sandbox pods." -keywords: "Generative AI, Cybersecurity, Kubernetes, Sandboxing, RuntimeClass" +description: "Choose between combined and sidecar supervisor topology for Kubernetes sandbox pods." +keywords: "Generative AI, Cybersecurity, Kubernetes, Sandboxing, Sidecar, Network Policy, RuntimeClass" position: 2 --- -Kubernetes sandbox pods run the OpenShell supervisor in `combined` topology by -default. Combined topology keeps network, filesystem, and process controls in -the agent pod so the supervisor can enforce the complete OpenShell sandbox -contract before launching the workload. +Kubernetes sandbox pods can run the OpenShell supervisor in `combined` or +`sidecar` topology. Choose the topology based on which controls you need inside +the pod and how much privilege your cluster allows on the agent container. ## Choose a Topology The default `combined` topology preserves the full OpenShell enforcement model. -Use it when you need OpenShell to apply all sandbox controls inside the workload -pod and your cluster policy permits the required Linux capabilities. +Use `sidecar` only when you accept network-focused enforcement in exchange for a +lower-privilege agent container. | Topology | Use when | Main tradeoff | |---|---|---| | `combined` | You need OpenShell network, filesystem, and process controls in the sandbox workload. | The agent container carries the Linux capabilities the supervisor needs. | - -Additional Kubernetes sandbox topologies are still being designed. Until they -are documented as supported configuration values, `combined` is the only -supported value for `supervisor.topology`. +| `sidecar` | You need the agent container to run as non-root without added Linux capabilities, and network policy is the primary control. | Defaults to network-only process supervision unless you opt in to `processEnforcement=full`. | ## Privilege Model -The long-running container permissions for `combined` topology are: +The long-running container permissions differ by topology: | Topology | Pod or container | UID/GID | Privilege escalation | Capabilities | Result | |---|---|---|---|---|---| | `combined` | Agent container, which also runs the supervisor | Not forced by topology | Not explicitly disabled by the driver | Adds `SYS_ADMIN`, `NET_ADMIN`, `SYS_PTRACE`, and `SYSLOG`; adds `SETUID`, `SETGID`, and `DAC_READ_SEARCH` when user namespaces are enabled | Full supervisor controls run in the agent container. | +| `sidecar` | Agent container, process-only supervisor (`network-only`) | `sandbox_uid:sandbox_gid` | `false` | Drops `ALL` | Agent and workload run without added Linux capabilities. | +| `sidecar` | Agent container, process-only supervisor (`full`) | Root supervisor | Not explicitly disabled by the driver | Adds combined-mode capabilities | Agent keeps combined-mode process/filesystem guards. | +| `sidecar` | Network supervisor sidecar | `proxyUid:sandbox_gid` | `false` | Drops `ALL` | Long-running proxy sidecar is also non-root without added capabilities. | Short-lived setup containers still have the permissions needed to prepare the pod: @@ -41,6 +40,7 @@ pod: | Topology | Setup container | UID/GID | Privilege escalation | Capabilities | Purpose | |---|---|---|---|---|---| | `combined` | Supervisor install init container | `0` | Not set | Not set | Copies the supervisor binary into the agent container volume. | +| `sidecar` | Network init container | `0` | `false` | Drops `ALL`; adds `NET_ADMIN`, `NET_RAW`, `CHOWN`, and `FOWNER` | Installs pod-local nftables rules and prepares shared sidecar state. | ## Combined Topology @@ -48,6 +48,26 @@ Combined topology is the original Kubernetes mode and remains the default. The agent container starts the OpenShell supervisor, and the supervisor launches the workload after applying sandbox setup. +```mermaid +flowchart TB + Sandbox["agents.x-k8s.io Sandbox"] + + subgraph Pod["Sandbox pod"] + subgraph Agent["agent container"] + Supervisor["OpenShell supervisor
network + process + filesystem"] + Workload["Agent workload"] + end + end + + Gateway["OpenShell Gateway"] + External["External services"] + + Sandbox --> Pod + Supervisor --> Workload + Supervisor -->|"gateway callback / SSH relay"| Gateway + Supervisor -->|"policy-enforced egress"| External +``` + Combined topology keeps these controls in one supervisor path: - Network endpoint and L7 policy enforcement. @@ -61,12 +81,90 @@ controls from the agent container, Kubernetes grants that container elevated Linux capabilities. Use this mode when you need the complete OpenShell sandbox contract and your cluster policy permits those capabilities. +## Sidecar Topology + +Sidecar topology splits the supervisor into a network sidecar and a +low-privilege process supervisor in the agent container. + +```mermaid +flowchart TB + Sandbox["agents.x-k8s.io Sandbox"] + + subgraph Pod["Sandbox pod"] + Init["network init container
root setup capabilities"] + State["shared state + TLS volumes"] + NetNS["pod network namespace"] + + subgraph Agent["agent container"] + ProcessSupervisor["process supervisor
network-only by default"] + Workload["Agent workload"] + end + + NetworkSidecar["network supervisor sidecar
proxyUid"] + end + + Gateway["OpenShell Gateway"] + External["External services"] + + Sandbox --> Pod + Init -->|"installs nftables rules"| NetNS + ProcessSupervisor --> Workload + Workload -->|"egress redirected on loopback"| NetworkSidecar + NetworkSidecar -->|"gateway forwarding"| Gateway + NetworkSidecar -->|"policy-enforced egress"| External + ProcessSupervisor --- State + NetworkSidecar --- State +``` + +The pod contains these OpenShell-managed pieces: + +| Component | Runs as | Purpose | +|---|---|---| +| Network init container | Root with setup capabilities | Installs pod-level nftables rules and prepares shared sidecar state. | +| Network sidecar | `supervisor.proxyUid` | Runs the proxy, enforces network policy, writes proxy TLS material, and forwards gateway traffic on loopback. | +| Agent container | Resolved sandbox UID/GID | Runs the process supervisor and launches the user workload. | + +In this topology, the agent container defaults to `runAsNonRoot: true`, +`allowPrivilegeEscalation: false`, and `capabilities.drop: ["ALL"]`. Set +`supervisor.processEnforcement=full` only when you want combined-mode +process/filesystem guards and accept the added agent-container permissions. The +long-running network sidecar always drops all Linux capabilities. The root init +container keeps the setup capabilities needed to configure pod networking. + +Sidecar mode preserves gateway session behavior, including SSH connectivity, +because the process supervisor still owns the session relay. The network sidecar +handles outbound enforcement and forwards the process supervisor's gateway +traffic to the real gateway endpoint. + + +Sidecar mode defaults the process supervisor to `network-only`. OpenShell still +enforces network endpoint and L7 policy through the sidecar, but the process +supervisor does not apply Landlock filesystem policy, process privilege +dropping, or process/binary identity checks unless you opt in to +`supervisor.processEnforcement=full`. + + +## Credential Exposure + +Sidecar topology uses pod `fsGroup` and group-readable projected credentials so +the non-root process supervisor can authenticate to the gateway. This includes +the projected ServiceAccount token used for sandbox token bootstrap and the +sandbox client TLS secret. + +Treat the agent container as trusted with respect to those in-pod gateway +credentials. Use `combined` topology when that credential exposure is not +acceptable for your deployment. + ## RuntimeClass Isolation -RuntimeClass isolation can add a stronger container boundary for the sandbox -workload when the cluster supports it. Runtime classes do not replace the -combined topology's supervisor controls; they add another isolation boundary -around the same supervised workload. +Sidecar topology pairs well with runtime classes such as gVisor or Kata +Containers when the cluster supports them. A sandboxed runtime strengthens the +container boundary while OpenShell focuses on network policy enforcement from +the sidecar. + +Runtime classes do not re-enable the OpenShell filesystem and process controls +that sidecar mode relaxes. Use them as an additional workload boundary, not as a +replacement for the combined topology's full supervisor controls. You can set a default runtime class in the Kubernetes driver configuration or override it per sandbox with driver config: @@ -77,24 +175,37 @@ openshell sandbox create \ -- claude ``` -## Configure Combined Mode +## Enable Sidecar Mode -For direct gateway TOML configuration, leave `supervisor_topology` unset, or -set it to `combined`, to use the default single-container supervisor path: +For direct gateway TOML configuration, set the Kubernetes driver fields: ```toml [openshell.drivers.kubernetes] -supervisor_topology = "combined" +supervisor_topology = "sidecar" +process_enforcement = "network-only" +proxy_uid = 1337 ``` -When the Helm chart renders `gateway.toml`, leave `supervisor.topology` unset, -or set it to `combined`, to produce the same driver configuration: +`proxy_uid` must be a non-root UID and must not match the sandbox UID. +The network init container exempts this UID from proxy redirection so the +sidecar can reach the gateway. +Set `process_enforcement="full"` only when you want the agent process supervisor +to keep combined-mode process/filesystem guards and accept the added +agent-container permissions. + +When the Helm chart renders `gateway.toml`, set the equivalent chart values: ```yaml supervisor: - topology: combined + topology: sidecar + processEnforcement: network-only + proxyUid: 1337 ``` +Leave `supervisor_topology` unset, or set it to `combined`, to keep the +original single-container supervisor path. For Helm installs, leave +`supervisor.topology` unset or set it to `combined`. + ## Next Steps - To install OpenShell on Kubernetes, refer to [Setup](/kubernetes/setup). diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 88b82870de..fd125231e0 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -177,9 +177,17 @@ supervisor_image_pull_policy = "IfNotPresent" # Use the image volume on Kubernetes >= 1.35 (GA in 1.36); switch to "init-container" # on older clusters or where the ImageVolume feature gate is off. supervisor_sideload_method = "image-volume" -# "combined" runs networking and process supervision in the sandbox agent -# container and preserves the existing Kubernetes sandbox behavior. +# "combined" runs the existing single supervisor container with full process, +# filesystem, and network enforcement in the agent container. "sidecar" moves +# pod-level network enforcement and gateway forwarding into a network sidecar. supervisor_topology = "combined" +# Process/filesystem controls for non-combined topologies. "network-only" +# keeps the low-permission agent shape; "full" grants combined-mode +# process/filesystem controls to the agent process supervisor. +process_enforcement = "network-only" +# UID used by the long-running network sidecar. In sidecar topology the +# network init container installs nftables rules that exempt this UID. +proxy_uid = 1337 grpc_endpoint = "https://openshell-gateway.agents.svc:8080" ssh_socket_path = "/run/openshell/ssh.sock" client_tls_secret_name = "openshell-client-tls" diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 714ddd6c33..3978e5b28d 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -306,10 +306,30 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA | `supervisor_image` | `supervisor.image.repository` / `supervisor.image.tag` | Set the supervisor image that provides the `openshell-sandbox` binary. | | `supervisor_image_pull_policy` | `supervisor.image.pullPolicy` | Set the Kubernetes image pull policy for the supervisor image. | | `supervisor_sideload_method` | `supervisor.sideloadMethod` | How the supervisor binary is delivered into sandbox pods. Leave empty to auto-detect from cluster version. Set to `image-volume` to mount the supervisor OCI image directly as a volume (requires Kubernetes 1.33+ with the ImageVolume feature gate; GA in 1.36), or `init-container` to copy it through an init container on older clusters. | +| `supervisor_topology` | `supervisor.topology` | Set `combined` for the default single supervisor path, or `sidecar` to move pod-level network enforcement and gateway forwarding into a dedicated sidecar. | +| `process_enforcement` | `supervisor.processEnforcement` | Process/filesystem controls for non-combined topologies. `network-only` keeps the low-permission agent shape. `full` grants combined-mode process/filesystem controls to the agent process supervisor. | +| `proxy_uid` | `supervisor.proxyUid` | UID used by the long-running network sidecar in `sidecar` topology. The network init container exempts this UID from proxy redirection. | | `app_armor_profile` | `server.appArmorProfile` | Set the sandbox agent container's AppArmor profile. Helm defaults this to `Unconfined` so AppArmor-enabled nodes do not block supervisor network namespace setup. Set the Helm value to an empty string to omit the field, or use `RuntimeDefault` or `Localhost/` for operator-managed profiles. | | `workspace_default_storage_size` | `server.workspaceDefaultStorageSize` | Set the default workspace PVC size for new sandboxes. | | `sa_token_ttl_secs` | `server.sandboxJwt.k8sSaTokenTtlSecs` | Set the projected ServiceAccount token TTL used for the bootstrap token exchange. | +In `combined` topology, the agent container carries the Linux capabilities +needed by the supervisor for network namespace setup, Landlock filesystem +policy, process privilege changes, and network policy enforcement. In `sidecar` +topology, the agent container runs as the resolved sandbox UID/GID with no added +Linux capabilities. A root init container performs the nftables setup, and the +long-running sidecar runs non-root with no added Linux capabilities. Sidecar +mode keeps gateway session and SSH behavior, but the process supervisor runs in +`network-only` mode by default: filesystem policy, process privilege dropping, +and process/binary identity checks are not applied by the process container. +Set `process_enforcement = "full"` only when you want those combined-mode +process/filesystem guards and accept the added agent-container permissions. + +Sidecar mode uses pod `fsGroup` so the non-root process supervisor can read the +projected ServiceAccount token and sandbox client TLS secret required for +gateway authentication. Treat the workload container as trusted with respect to +those in-pod gateway credentials. + The Kubernetes driver creates namespaced `agents.x-k8s.io` `Sandbox` resources from the Kubernetes SIG Apps [agent-sandbox](https://github.com/kubernetes-sigs/agent-sandbox) project. It detects the served Sandbox API at runtime, caches the selected API version for the gateway process, and uses `v1beta1` when available before falling back to `v1alpha1`, so supported Agent Sandbox installations work without version-specific operator configuration. The Agent Sandbox controller turns those resources into sandbox pods and related storage. If Agent Sandbox is upgraded in place, restart the OpenShell gateway after the controller and CRD rollout completes so the gateway can detect the served API versions again. diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index 47b8730dc1..8ce1989dae 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -393,6 +393,46 @@ require_cmd() { fi } +configure_fixture_container_engine() { + local selected_engine="" + + if [ -n "${CONTAINER_ENGINE:-}" ]; then + selected_engine="$(printf '%s' "${CONTAINER_ENGINE}" | tr '[:upper:]' '[:lower:]')" + case "${selected_engine}" in + docker|podman) + export CONTAINER_ENGINE="${selected_engine}" + return + ;; + *) + echo "ERROR: CONTAINER_ENGINE=${CONTAINER_ENGINE} is invalid; expected docker or podman" >&2 + exit 2 + ;; + esac + fi + + case "${KUBE_CONTEXT}" in + k3d-*) + selected_engine="docker" + ;; + kind-*) + case "$(printf '%s' "${KIND_EXPERIMENTAL_PROVIDER:-}" | tr '[:upper:]' '[:lower:]')" in + podman) + selected_engine="podman" + ;; + *) + selected_engine="docker" + ;; + esac + ;; + *) + return + ;; + esac + + export CONTAINER_ENGINE="${selected_engine}" + echo "Using ${CONTAINER_ENGINE} for Kubernetes e2e host-side fixture containers." +} + require_cmd helm require_cmd kubectl require_cmd curl @@ -423,6 +463,8 @@ else KUBE_CONTEXT="k3d-${CLUSTER_NAME}" fi +configure_fixture_container_engine + if [ -z "${OPENSHELL_E2E_KUBE_BUILD_IMAGES+x}" ]; then if [ "${CLUSTER_CREATED_BY_US}" = "1" ]; then OPENSHELL_E2E_KUBE_BUILD_IMAGES=1 diff --git a/tasks/helm.toml b/tasks/helm.toml index f25dadb09c..433f04f32a 100644 --- a/tasks/helm.toml +++ b/tasks/helm.toml @@ -55,16 +55,31 @@ description = "Run skaffold dev for deploy/helm/openshell (iterative deploy)" dir = "deploy/helm/openshell" run = "skaffold dev" +["helm:skaffold:dev:sidecar"] +description = "Run skaffold dev with the Kubernetes supervisor sidecar topology" +dir = "deploy/helm/openshell" +run = "skaffold dev -p sidecar" + ["helm:skaffold:run"] description = "Run skaffold run for deploy/helm/openshell (one-shot deploy)" dir = "deploy/helm/openshell" run = "skaffold run" +["helm:skaffold:run:sidecar"] +description = "Run skaffold run with the Kubernetes supervisor sidecar topology" +dir = "deploy/helm/openshell" +run = "skaffold run -p sidecar" + ["helm:skaffold:delete"] description = "Run skaffold delete for deploy/helm/openshell" dir = "deploy/helm/openshell" run = "skaffold delete" +["helm:skaffold:delete:sidecar"] +description = "Run skaffold delete for the Kubernetes supervisor sidecar topology" +dir = "deploy/helm/openshell" +run = "skaffold delete -p sidecar" + ["helm:skaffold:diagnose"] description = "Run skaffold diagnose for deploy/helm/openshell" dir = "deploy/helm/openshell" diff --git a/tasks/test.toml b/tasks/test.toml index db3878f756..c08fcc5a04 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -114,6 +114,11 @@ run = [ "AGENT_SANDBOX_VERSION=v0.4.6 e2e/rust/e2e-kubernetes.sh", ] +["e2e:kubernetes:sidecar"] +description = "Run Kubernetes e2e with the supervisor sidecar topology overlay" +env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-sidecar.yaml" } +run = "e2e/rust/e2e-kubernetes.sh" + ["e2e:kubernetes:db"] description = "Run Kubernetes e2e with all database backend scenarios (SQLite and external PostgreSQL with existingSecret)" env = { OPENSHELL_E2E_KUBE_DB_SCENARIOS = "1" } From 566cf14aaa48b33c492ebfc17069f53f797bcb89 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Thu, 2 Jul 2026 14:32:57 -0700 Subject: [PATCH 02/24] fix(supervisor): avoid similar process id names Signed-off-by: Taylor Mutch --- crates/openshell-supervisor-network/src/proxy.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index c38ecbd3a5..5debc2926c 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -1470,25 +1470,25 @@ fn resolve_owner_identity( } #[cfg(target_os = "linux")] -fn collect_ancestor_identities(pid: u32, stop_pid: u32) -> Vec<(u32, PathBuf)> { +fn collect_ancestor_identities(start_pid: u32, stop_pid: u32) -> Vec<(u32, PathBuf)> { const MAX_DEPTH: usize = 64; let mut ancestors = Vec::new(); - let mut current = pid; + let mut current = start_pid; for _ in 0..MAX_DEPTH { - let ppid = match crate::procfs::read_ppid(current) { - Some(p) if p > 0 && p != current => p, + let parent_pid = match crate::procfs::read_ppid(current) { + Some(parent) if parent > 0 && parent != current => parent, _ => break, }; - if let Ok(path) = crate::procfs::binary_path(ppid.cast_signed()) { - ancestors.push((ppid, path)); + if let Ok(path) = crate::procfs::binary_path(parent_pid.cast_signed()) { + ancestors.push((parent_pid, path)); } - if ppid == stop_pid || ppid == 1 { + if parent_pid == stop_pid || parent_pid == 1 { break; } - current = ppid; + current = parent_pid; } ancestors From 9a16c6bba7dbac3ab98c24fc448d0e617cb6ec83 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Thu, 2 Jul 2026 14:32:57 -0700 Subject: [PATCH 03/24] fix(supervisor): avoid similar process id names Signed-off-by: Taylor Mutch --- crates/openshell-sandbox/src/main.rs | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index 9a165c643c..d2fa754e9b 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -313,18 +313,18 @@ fn copy_sidecar_client_tls_if_present( #[cfg(target_os = "linux")] fn run_network_init( - proxy_uid: u32, - proxy_gid: u32, + network_proxy_uid: u32, + network_proxy_gid: u32, sidecar_state_dir: &str, sidecar_tls_dir: &str, ) -> Result<()> { - if proxy_uid < openshell_policy::MIN_SANDBOX_UID { + if network_proxy_uid < openshell_policy::MIN_SANDBOX_UID { return Err(miette::miette!( "--proxy-uid must be at least {}", openshell_policy::MIN_SANDBOX_UID )); } - if proxy_gid < openshell_policy::MIN_SANDBOX_UID { + if network_proxy_gid < openshell_policy::MIN_SANDBOX_UID { return Err(miette::miette!( "--proxy-gid must be at least {}", openshell_policy::MIN_SANDBOX_UID @@ -333,15 +333,20 @@ fn run_network_init( let sidecar_state_dir = Path::new(sidecar_state_dir); let sidecar_tls_dir = Path::new(sidecar_tls_dir); - prepare_sidecar_directory(sidecar_state_dir, proxy_uid, proxy_gid, 0o775)?; - prepare_sidecar_directory(sidecar_tls_dir, proxy_uid, proxy_gid, 0o755)?; + prepare_sidecar_directory( + sidecar_state_dir, + network_proxy_uid, + network_proxy_gid, + 0o775, + )?; + prepare_sidecar_directory(sidecar_tls_dir, network_proxy_uid, network_proxy_gid, 0o755)?; copy_sidecar_client_tls_if_present( Path::new(CLIENT_TLS_DIR), sidecar_tls_dir, - proxy_uid, - proxy_gid, + network_proxy_uid, + network_proxy_gid, )?; - openshell_supervisor_process::netns::install_sidecar_bypass_rules(proxy_uid) + openshell_supervisor_process::netns::install_sidecar_bypass_rules(network_proxy_uid) } #[cfg(not(target_os = "linux"))] From ae3d3d8f4ee9e36e5e3a1bc7d4cb1e6c621eef28 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Thu, 2 Jul 2026 18:42:04 -0700 Subject: [PATCH 04/24] fix(sandbox): avoid similar proxy id names Signed-off-by: Taylor Mutch --- crates/openshell-sandbox/src/main.rs | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index d2fa754e9b..b5481f6759 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -313,18 +313,18 @@ fn copy_sidecar_client_tls_if_present( #[cfg(target_os = "linux")] fn run_network_init( - network_proxy_uid: u32, - network_proxy_gid: u32, + proxy_user_id: u32, + proxy_primary_group_id: u32, sidecar_state_dir: &str, sidecar_tls_dir: &str, ) -> Result<()> { - if network_proxy_uid < openshell_policy::MIN_SANDBOX_UID { + if proxy_user_id < openshell_policy::MIN_SANDBOX_UID { return Err(miette::miette!( "--proxy-uid must be at least {}", openshell_policy::MIN_SANDBOX_UID )); } - if network_proxy_gid < openshell_policy::MIN_SANDBOX_UID { + if proxy_primary_group_id < openshell_policy::MIN_SANDBOX_UID { return Err(miette::miette!( "--proxy-gid must be at least {}", openshell_policy::MIN_SANDBOX_UID @@ -335,18 +335,23 @@ fn run_network_init( let sidecar_tls_dir = Path::new(sidecar_tls_dir); prepare_sidecar_directory( sidecar_state_dir, - network_proxy_uid, - network_proxy_gid, + proxy_user_id, + proxy_primary_group_id, 0o775, )?; - prepare_sidecar_directory(sidecar_tls_dir, network_proxy_uid, network_proxy_gid, 0o755)?; + prepare_sidecar_directory( + sidecar_tls_dir, + proxy_user_id, + proxy_primary_group_id, + 0o755, + )?; copy_sidecar_client_tls_if_present( Path::new(CLIENT_TLS_DIR), sidecar_tls_dir, - network_proxy_uid, - network_proxy_gid, + proxy_user_id, + proxy_primary_group_id, )?; - openshell_supervisor_process::netns::install_sidecar_bypass_rules(network_proxy_uid) + openshell_supervisor_process::netns::install_sidecar_bypass_rules(proxy_user_id) } #[cfg(not(target_os = "linux"))] From 53a5b904bd9c1f683059bc9bef6dbf3af379c056 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Mon, 6 Jul 2026 10:11:32 -0700 Subject: [PATCH 05/24] docs(kubernetes): clarify sidecar topology limits Signed-off-by: Taylor Mutch --- .agents/skills/debug-openshell-cluster/SKILL.md | 11 ++++++----- .../src/netns/nft_ruleset.rs | 4 +++- docs/kubernetes/topology.mdx | 4 +++- docs/reference/sandbox-compute-drivers.mdx | 2 ++ 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index 08e4230139..063e0da798 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -275,11 +275,12 @@ If `supervisor_topology = "sidecar"` is rendered, sandbox pods should have an container owns nftables setup and should be the only sidecar topology container with `NET_ADMIN`. It also needs `CHOWN`/`FOWNER` to hand shared emptyDir state to `sidecar_proxy_uid`. The long-running network sidecar runs as -`sidecar_proxy_uid` with primary GID `0` so it can read the root-owned, -group-readable projected service-account token. In sidecar topology the -`openshell-sa-token` projected volume should render `defaultMode: 288` (`0440`); -if the proxy logs `failed to read K8s SA token`, verify this token mode and the -network sidecar security context. The process container should also publish the +`sidecar_proxy_uid` with primary GID `sandbox_gid`; the pod `fsGroup` is also +set to `sandbox_gid` so the projected service-account token is group-readable. +In sidecar topology the `openshell-sa-token` projected volume should render +`defaultMode: 288` (`0440`); if the proxy logs `failed to read K8s SA token`, +verify this token mode plus the pod `fsGroup` and network sidecar +`runAsGroup`. The process container should also publish the workload entrypoint PID to `OPENSHELL_ENTRYPOINT_PID_FILE` (`/run/openshell-sidecar/entrypoint.pid` by default), and the network sidecar should read it for binary-scoped policy decisions; if allowed network rules are diff --git a/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs b/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs index d7ec5132e2..378f715c92 100644 --- a/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs +++ b/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs @@ -58,7 +58,9 @@ pub fn generate_bypass_ruleset(host_ip: &str, proxy_port: u16, log_prefix: Optio /// The network sidecar and the process supervisor share a pod network /// namespace. The sidecar runs as `proxy_uid` and owns external egress; /// sandbox traffic must use loopback services hosted by that sidecar -/// (gateway forward and HTTP CONNECT proxy). +/// (gateway forward and HTTP CONNECT proxy). The generated fence rejects +/// TCP/UDP bypass attempts from non-proxy UIDs; other L4 protocols are outside +/// the sidecar policy fence. pub fn generate_sidecar_bypass_ruleset(proxy_uid: u32, log_prefix: Option<&str>) -> String { let log_tcp = log_prefix .map(|p| { diff --git a/docs/kubernetes/topology.mdx b/docs/kubernetes/topology.mdx index 5bf942e35b..7f4b323777 100644 --- a/docs/kubernetes/topology.mdx +++ b/docs/kubernetes/topology.mdx @@ -141,7 +141,9 @@ Sidecar mode defaults the process supervisor to `network-only`. OpenShell still enforces network endpoint and L7 policy through the sidecar, but the process supervisor does not apply Landlock filesystem policy, process privilege dropping, or process/binary identity checks unless you opt in to -`supervisor.processEnforcement=full`. +`supervisor.processEnforcement=full`. Network policy still runs in the sidecar +and remains endpoint/L7 scoped; `full` does not restore binary-scoped network +identity in sidecar topology. ## Credential Exposure diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 3978e5b28d..4b524dd75d 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -324,6 +324,8 @@ mode keeps gateway session and SSH behavior, but the process supervisor runs in and process/binary identity checks are not applied by the process container. Set `process_enforcement = "full"` only when you want those combined-mode process/filesystem guards and accept the added agent-container permissions. +Network policy still runs in the sidecar and remains endpoint/L7 scoped in +sidecar topology; `full` does not restore binary-scoped network identity. Sidecar mode uses pod `fsGroup` so the non-root process supervisor can read the projected ServiceAccount token and sandbox client TLS secret required for From 4904e44424600a420012b31b0eaf6e80a78d7075 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Mon, 6 Jul 2026 11:12:41 -0700 Subject: [PATCH 06/24] fix(kubernetes): keep sidecar process leaf capless Signed-off-by: Taylor Mutch --- .../skills/debug-openshell-cluster/SKILL.md | 22 +- Cargo.lock | 1 + architecture/compute-runtimes.md | 14 +- .../src/provider_credentials.rs | 26 + crates/openshell-core/src/sandbox_env.rs | 12 +- .../openshell-driver-kubernetes/src/driver.rs | 146 ++++-- crates/openshell-sandbox/Cargo.toml | 1 + crates/openshell-sandbox/src/lib.rs | 488 ++++++++++-------- crates/openshell-sandbox/src/main.rs | 2 +- .../openshell-supervisor-process/src/run.rs | 2 + .../openshell-supervisor-process/src/ssh.rs | 105 +++- docs/kubernetes/topology.mdx | 37 +- docs/reference/gateway-config.mdx | 2 +- docs/reference/sandbox-compute-drivers.mdx | 19 +- 14 files changed, 553 insertions(+), 324 deletions(-) diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index 063e0da798..26f0169e3e 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -276,15 +276,25 @@ container owns nftables setup and should be the only sidecar topology container with `NET_ADMIN`. It also needs `CHOWN`/`FOWNER` to hand shared emptyDir state to `sidecar_proxy_uid`. The long-running network sidecar runs as `sidecar_proxy_uid` with primary GID `sandbox_gid`; the pod `fsGroup` is also -set to `sandbox_gid` so the projected service-account token is group-readable. -In sidecar topology the `openshell-sa-token` projected volume should render -`defaultMode: 288` (`0440`); if the proxy logs `failed to read K8s SA token`, -verify this token mode plus the pod `fsGroup` and network sidecar -`runAsGroup`. The process container should also publish the -workload entrypoint PID to `OPENSHELL_ENTRYPOINT_PID_FILE` +set to `sandbox_gid`. + +In sidecar topology only the network sidecar should mount the gateway bootstrap +credentials (`openshell-sa-token` and `openshell-client-tls`). The process +container should not receive `OPENSHELL_ENDPOINT`, gateway TLS env vars, the +sandbox token file, or those credential mounts. Instead, the network sidecar +writes `/run/openshell-sidecar/policy.pb` and +`/run/openshell-sidecar/provider-env.json`, then writes the readiness file. If +the process supervisor fails before launching the workload, inspect those +snapshot files and the network sidecar logs. + +The process container should also publish the workload entrypoint PID to +`OPENSHELL_ENTRYPOINT_PID_FILE` (`/run/openshell-sidecar/entrypoint.pid` by default), and the network sidecar should read it for binary-scoped policy decisions; if allowed network rules are all denied, inspect that file and the network sidecar logs. +The shared state directory should preserve `sandbox_gid` inheritance +(`02775`), and the SSH socket should be group-connectable (`0660`) so the +network sidecar can bridge gateway relay requests to the process supervisor. Inspect all three when sandbox registration or egress enforcement fails: ```bash diff --git a/Cargo.lock b/Cargo.lock index e94eb56f08..85d3e7a25a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3833,6 +3833,7 @@ dependencies = [ "openshell-policy", "openshell-supervisor-network", "openshell-supervisor-process", + "prost", "rustls", "serde_json", "temp-env", diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index ac239bfb8c..24b92a5fa3 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -92,16 +92,18 @@ paths, and command metadata. Kubernetes can run the supervisor in the default combined topology or in a sidecar topology. Combined mode keeps network and process supervision in the agent container. Sidecar mode runs network enforcement, the proxy, and gateway -loopback forwarding in a dedicated sidecar, while the agent container runs only -the process-supervision leaf and launches the user workload after the sidecar -signals readiness. In sidecar mode, an init container performs the privileged +session in a dedicated sidecar, while the agent container runs only the +process-supervision leaf and launches the user workload after the sidecar +signals readiness. The sidecar writes local policy and provider-environment +snapshots into shared state so the process leaf can start without gateway +credentials. In sidecar mode, an init container performs the privileged pod-network nftables setup with `NET_ADMIN` and hands shared state ownership to the configured proxy UID; the long-running network sidecar runs as that UID and does not keep `NET_ADMIN`. The agent container runs as the resolved sandbox UID/GID with no added Linux capabilities. Sidecar mode preserves gateway session -and SSH behavior, but treats the process leaf as network-only: Landlock -filesystem policy, process privilege dropping, and process/binary identity -checks are not applied there. +and SSH behavior, but treats the process leaf as network-only by default: +Landlock filesystem policy, process privilege dropping, and process/binary +identity checks are not applied there. ## Images diff --git a/crates/openshell-core/src/provider_credentials.rs b/crates/openshell-core/src/provider_credentials.rs index fd38453ccd..6b66c23ce6 100644 --- a/crates/openshell-core/src/provider_credentials.rs +++ b/crates/openshell-core/src/provider_credentials.rs @@ -63,6 +63,32 @@ impl ProviderCredentialState { } } + /// Build a static provider state from an already-prepared child + /// environment snapshot. + /// + /// Kubernetes sidecar topology uses this in the process-only supervisor: + /// the network sidecar owns provider credential resolvers and writes the + /// workload-facing env map into a shared local snapshot. The process leaf + /// must inject that map into child processes without re-placeholderizing it + /// or holding the gateway-side resolver material. + pub fn from_child_env_snapshot(revision: u64, child_env: HashMap) -> Self { + let snapshot = Arc::new(ProviderCredentialSnapshot { + revision, + child_env, + dynamic_credentials: HashMap::new(), + }); + + Self { + inner: Arc::new(RwLock::new(ProviderCredentialStateInner { + current: snapshot, + generations: VecDeque::new(), + current_resolver: None, + combined_resolver: None, + suppressed_keys: HashSet::new(), + })), + } + } + pub fn snapshot(&self) -> Arc { self.inner .read() diff --git a/crates/openshell-core/src/sandbox_env.rs b/crates/openshell-core/src/sandbox_env.rs index ae3a21787e..87e75b0848 100644 --- a/crates/openshell-core/src/sandbox_env.rs +++ b/crates/openshell-core/src/sandbox_env.rs @@ -59,11 +59,15 @@ pub const SUPERVISOR_READY_FILE: &str = "OPENSHELL_SUPERVISOR_READY_FILE"; /// read by the network sidecar for process/binary-bound network policy checks. pub const ENTRYPOINT_PID_FILE: &str = "OPENSHELL_ENTRYPOINT_PID_FILE"; -/// Loopback address where the network sidecar forwards gateway gRPC traffic. -pub const GATEWAY_FORWARD_ADDR: &str = "OPENSHELL_GATEWAY_FORWARD_ADDR"; +/// Local protobuf policy snapshot written by the network sidecar and read by +/// the process-only supervisor in Kubernetes sidecar topology. +pub const SIDECAR_POLICY_SNAPSHOT_FILE: &str = "OPENSHELL_SIDECAR_POLICY_SNAPSHOT_FILE"; -/// Optional TLS server name used when the process supervisor reaches the -/// gateway through a loopback TCP forward. +/// Local provider environment snapshot written by the network sidecar and read +/// by the process-only supervisor in Kubernetes sidecar topology. +pub const SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE: &str = "OPENSHELL_SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE"; + +/// Optional TLS server name override used when connecting to the gateway. pub const GATEWAY_TLS_SERVER_NAME: &str = "OPENSHELL_GATEWAY_TLS_SERVER_NAME"; /// Directory where the network supervisor writes the proxy CA files consumed diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index ae45b3f50e..6c4f67462d 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -1057,6 +1057,8 @@ const SIDECAR_STATE_MOUNT_PATH: &str = "/run/openshell-sidecar"; const SIDECAR_READY_FILE: &str = "/run/openshell-sidecar/supervisor.ready"; const SIDECAR_ENTRYPOINT_PID_FILE: &str = "/run/openshell-sidecar/entrypoint.pid"; const SIDECAR_SSH_SOCKET_FILE: &str = "/run/openshell-sidecar/ssh.sock"; +const SIDECAR_POLICY_SNAPSHOT_FILE: &str = "/run/openshell-sidecar/policy.pb"; +const SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE: &str = "/run/openshell-sidecar/provider-env.json"; /// Shared TLS work directory. The network sidecar writes the proxy CA bundle /// here, while the agent container consumes it after the readiness file exists. @@ -1064,11 +1066,6 @@ const SIDECAR_TLS_VOLUME_NAME: &str = "openshell-supervisor-tls"; const SIDECAR_TLS_MOUNT_PATH: &str = "/etc/openshell-tls/proxy"; const SIDECAR_CLIENT_TLS_MOUNT_PATH: &str = "/etc/openshell-tls/proxy/client"; -/// Loopback listener owned by the network sidecar. The process-only supervisor -/// connects here for gateway gRPC, and the sidecar forwards bytes to the real -/// gateway endpoint using its own network privileges. -const SIDECAR_GATEWAY_FORWARD_ADDR: &str = "127.0.0.1:18080"; - /// Build the emptyDir volume that holds the supervisor binary. /// /// The init container writes the binary here; the agent container reads it. @@ -1282,32 +1279,6 @@ fn sidecar_tls_volume_mount() -> serde_json::Value { }) } -fn sidecar_process_gateway_endpoint(grpc_endpoint: &str) -> String { - if grpc_endpoint.is_empty() { - String::new() - } else if grpc_endpoint.starts_with("https://") { - format!("https://{SIDECAR_GATEWAY_FORWARD_ADDR}") - } else { - format!("http://{SIDECAR_GATEWAY_FORWARD_ADDR}") - } -} - -fn gateway_tls_server_name(grpc_endpoint: &str) -> Option { - let rest = grpc_endpoint.strip_prefix("https://")?; - let authority = rest.split('/').next().unwrap_or(rest); - if authority.is_empty() { - return None; - } - if let Some(bracketed) = authority.strip_prefix('[') { - return bracketed.split(']').next().map(str::to_string); - } - authority - .split(':') - .next() - .filter(|host| !host.is_empty()) - .map(str::to_string) -} - fn copy_log_level_env( env: &mut Vec, template_environment: &std::collections::HashMap, @@ -1381,8 +1352,18 @@ fn supervisor_sidecar_env( ); upsert_env( &mut env, - openshell_core::sandbox_env::GATEWAY_FORWARD_ADDR, - SIDECAR_GATEWAY_FORWARD_ADDR, + openshell_core::sandbox_env::SSH_SOCKET_PATH, + SIDECAR_SSH_SOCKET_FILE, + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::SIDECAR_POLICY_SNAPSHOT_FILE, + SIDECAR_POLICY_SNAPSHOT_FILE, + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE, + SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE, ); upsert_env( &mut env, @@ -1604,6 +1585,9 @@ fn apply_supervisor_sidecar_topology( .or_insert_with(|| serde_json::json!([])) .as_array_mut(); if let Some(volume_mounts) = volume_mounts { + remove_volume_mount(volume_mounts, "openshell-sa-token"); + remove_volume_mount(volume_mounts, "openshell-client-tls"); + remove_volume_mount(volume_mounts, SPIFFE_WORKLOAD_API_VOLUME_NAME); volume_mounts.push(supervisor_volume_mount()); volume_mounts.push(sidecar_state_volume_mount()); volume_mounts.push(sidecar_tls_volume_mount()); @@ -1614,19 +1598,18 @@ fn apply_supervisor_sidecar_topology( .or_insert_with(|| serde_json::json!([])) .as_array_mut(); if let Some(env) = env { - let process_endpoint = sidecar_process_gateway_endpoint(params.grpc_endpoint); - upsert_env( + remove_env(env, openshell_core::sandbox_env::ENDPOINT); + remove_env(env, openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME); + remove_env(env, openshell_core::sandbox_env::TLS_CA); + remove_env(env, openshell_core::sandbox_env::TLS_CERT); + remove_env(env, openshell_core::sandbox_env::TLS_KEY); + remove_env(env, openshell_core::sandbox_env::SANDBOX_TOKEN); + remove_env(env, openshell_core::sandbox_env::SANDBOX_TOKEN_FILE); + remove_env(env, openshell_core::sandbox_env::K8S_SA_TOKEN_FILE); + remove_env( env, - openshell_core::sandbox_env::ENDPOINT, - &process_endpoint, + openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, ); - if let Some(server_name) = gateway_tls_server_name(params.grpc_endpoint) { - upsert_env( - env, - openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME, - &server_name, - ); - } upsert_env( env, openshell_core::sandbox_env::SUPERVISOR_TOPOLOGY, @@ -1657,6 +1640,16 @@ fn apply_supervisor_sidecar_topology( openshell_core::sandbox_env::ENTRYPOINT_PID_FILE, SIDECAR_ENTRYPOINT_PID_FILE, ); + upsert_env( + env, + openshell_core::sandbox_env::SIDECAR_POLICY_SNAPSHOT_FILE, + SIDECAR_POLICY_SNAPSHOT_FILE, + ); + upsert_env( + env, + openshell_core::sandbox_env::SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE, + SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE, + ); upsert_env( env, openshell_core::sandbox_env::PROXY_TLS_DIR, @@ -2599,6 +2592,14 @@ fn upsert_env(env: &mut Vec, name: &str, value: &str) { env.push(serde_json::json!({"name": name, "value": value})); } +fn remove_env(env: &mut Vec, name: &str) { + env.retain(|item| item.get("name").and_then(|value| value.as_str()) != Some(name)); +} + +fn remove_volume_mount(volume_mounts: &mut Vec, name: &str) { + volume_mounts.retain(|mount| mount.get("name").and_then(|value| value.as_str()) != Some(name)); +} + /// Extract a string value from the template's `platform_config` Struct. fn platform_config_string(template: &SandboxTemplate, key: &str) -> Option { let config = template.platform_config.as_ref()?; @@ -3176,11 +3177,19 @@ mod tests { ); assert_eq!( rendered_env(agent, openshell_core::sandbox_env::ENDPOINT), - Some("https://127.0.0.1:18080") + None ); assert_eq!( rendered_env(agent, openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME), - Some("openshell-gateway.openshell.svc") + None + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::TLS_CA), + None + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::K8S_SA_TOKEN_FILE), + None ); assert_eq!( rendered_env(agent, openshell_core::sandbox_env::PROCESS_ENFORCEMENT_MODE), @@ -3198,6 +3207,20 @@ mod tests { rendered_env(agent, openshell_core::sandbox_env::ENTRYPOINT_PID_FILE), Some(SIDECAR_ENTRYPOINT_PID_FILE) ); + assert_eq!( + rendered_env( + agent, + openshell_core::sandbox_env::SIDECAR_POLICY_SNAPSHOT_FILE + ), + Some(SIDECAR_POLICY_SNAPSHOT_FILE) + ); + assert_eq!( + rendered_env( + agent, + openshell_core::sandbox_env::SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE + ), + Some(SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE) + ); assert_eq!( rendered_env(agent, openshell_core::sandbox_env::PROXY_TLS_DIR), Some(SIDECAR_TLS_MOUNT_PATH) @@ -3231,8 +3254,22 @@ mod tests { Some("https://openshell-gateway.openshell.svc:8080") ); assert_eq!( - rendered_env(sidecar, openshell_core::sandbox_env::GATEWAY_FORWARD_ADDR), - Some(SIDECAR_GATEWAY_FORWARD_ADDR) + rendered_env(sidecar, openshell_core::sandbox_env::SSH_SOCKET_PATH), + Some(SIDECAR_SSH_SOCKET_FILE) + ); + assert_eq!( + rendered_env( + sidecar, + openshell_core::sandbox_env::SIDECAR_POLICY_SNAPSHOT_FILE + ), + Some(SIDECAR_POLICY_SNAPSHOT_FILE) + ); + assert_eq!( + rendered_env( + sidecar, + openshell_core::sandbox_env::SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE + ), + Some(SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE) ); assert_eq!( rendered_env( @@ -3260,6 +3297,19 @@ mod tests { .any(|mount| mount["name"] == "openshell-client-tls"), "runtime sidecar should use the init-copied TLS files, not the root-owned Secret mount" ); + let agent_mounts = agent["volumeMounts"].as_array().unwrap(); + assert!( + !agent_mounts + .iter() + .any(|mount| mount["name"] == "openshell-sa-token"), + "agent container must not mount gateway bootstrap token in sidecar topology" + ); + assert!( + !agent_mounts + .iter() + .any(|mount| mount["name"] == "openshell-client-tls"), + "agent container must not mount gateway client TLS secret in sidecar topology" + ); let volumes = pod_template["spec"]["volumes"].as_array().unwrap(); let sa_token = volumes .iter() diff --git a/crates/openshell-sandbox/Cargo.toml b/crates/openshell-sandbox/Cargo.toml index a5d3449106..94a86b5e8e 100644 --- a/crates/openshell-sandbox/Cargo.toml +++ b/crates/openshell-sandbox/Cargo.toml @@ -41,6 +41,7 @@ rustls = { workspace = true } # Serialization (serde_json::json! for OCSF unmapped fields) serde_json = { workspace = true } +prost = { workspace = true } # Logging tracing = { workspace = true } diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 3ff260c7c3..ef634bb34d 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -13,7 +13,8 @@ mod mechanistic_mapper; #[cfg_attr(not(target_os = "linux"), allow(dead_code))] mod metadata_server; -use miette::{IntoDiagnostic, Result}; +use miette::{IntoDiagnostic, Result, WrapErr}; +use prost::Message; use std::future::Future; use std::sync::Arc; use std::sync::atomic::{AtomicU32, Ordering}; @@ -67,8 +68,6 @@ use openshell_supervisor_network::opa::OpaEngine; use openshell_supervisor_process::process::ProcessEnforcementMode; pub use openshell_supervisor_process::process::{ProcessHandle, ProcessStatus}; use openshell_supervisor_process::skills; -use tokio::io::copy_bidirectional; -use tokio::net::{TcpListener, TcpStream}; use tokio::sync::mpsc::UnboundedSender; #[cfg(target_os = "linux")] use tokio::time::timeout; @@ -148,14 +147,26 @@ pub async fn run_sandbox( // Load policy and initialize OPA engine let openshell_endpoint_for_proxy = openshell_endpoint.clone(); let sandbox_name_for_agg = sandbox.clone(); - let (mut policy, opa_engine, retained_proto) = load_policy( - sandbox_id.clone(), - sandbox, - openshell_endpoint.clone(), - policy_rules, - policy_data, - ) - .await?; + let process_uses_sidecar_snapshots = + process_enabled && !network_enabled && sidecar_network_enforcement; + let (mut policy, opa_engine, retained_proto) = if process_uses_sidecar_snapshots { + let snapshot_path = sidecar_policy_snapshot_file().ok_or_else(|| { + miette::miette!( + "{} is required for process-only sidecar topology", + openshell_core::sandbox_env::SIDECAR_POLICY_SNAPSHOT_FILE + ) + })?; + load_policy_from_sidecar_snapshot(&snapshot_path)? + } else { + load_policy( + sandbox_id.clone(), + sandbox, + openshell_endpoint.clone(), + policy_rules, + policy_data, + ) + .await? + }; // Override the policy's process identity with the driver-resolved UID/GID // from the pod environment. The policy defaults to the name "sandbox" which @@ -190,71 +201,82 @@ pub async fn run_sandbox( policy.process.run_as_group = Some(gid); } - // Fetch provider environment variables from the server. - // This is done after loading the policy so the sandbox can still start - // even if provider env fetch fails (graceful degradation). - let ( - provider_env_revision, - provider_env, - provider_credential_expires_at_ms, - dynamic_credentials, - ) = if let (Some(id), Some(endpoint)) = (&sandbox_id, &openshell_endpoint) { - match openshell_core::grpc_client::fetch_provider_environment(endpoint, id).await { - Ok(result) => { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .message(format!( - "Fetched provider environment [env_count:{}]", - result.environment.len() - )) - .build() - ); - ( - result.provider_env_revision, - result.environment, - result.credential_expires_at_ms, - result.dynamic_credentials, - ) - } - Err(e) => { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Other, "degraded") - .message(format!( - "Failed to fetch provider environment, continuing without: {e}" - )) - .build() - ); - ( - 0, - std::collections::HashMap::new(), - std::collections::HashMap::new(), - std::collections::HashMap::new(), - ) - } - } + #[cfg_attr(not(target_os = "linux"), allow(unused_mut))] + let (provider_credentials, mut provider_env) = if process_uses_sidecar_snapshots { + let snapshot_path = sidecar_provider_env_snapshot_file().ok_or_else(|| { + miette::miette!( + "{} is required for process-only sidecar topology", + openshell_core::sandbox_env::SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE + ) + })?; + read_sidecar_provider_env_snapshot(&snapshot_path)? } else { - ( - 0, - std::collections::HashMap::new(), - std::collections::HashMap::new(), - std::collections::HashMap::new(), - ) - }; + // Fetch provider environment variables from the server. + // This is done after loading the policy so the sandbox can still start + // even if provider env fetch fails (graceful degradation). + let ( + provider_env_revision, + provider_env, + provider_credential_expires_at_ms, + dynamic_credentials, + ) = if let (Some(id), Some(endpoint)) = (&sandbox_id, &openshell_endpoint) { + match openshell_core::grpc_client::fetch_provider_environment(endpoint, id).await { + Ok(result) => { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .message(format!( + "Fetched provider environment [env_count:{}]", + result.environment.len() + )) + .build() + ); + ( + result.provider_env_revision, + result.environment, + result.credential_expires_at_ms, + result.dynamic_credentials, + ) + } + Err(e) => { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .state(StateId::Other, "degraded") + .message(format!( + "Failed to fetch provider environment, continuing without: {e}" + )) + .build() + ); + ( + 0, + std::collections::HashMap::new(), + std::collections::HashMap::new(), + std::collections::HashMap::new(), + ) + } + } + } else { + ( + 0, + std::collections::HashMap::new(), + std::collections::HashMap::new(), + std::collections::HashMap::new(), + ) + }; - let provider_credentials = ProviderCredentialState::from_environment( - provider_env_revision, - provider_env, - provider_credential_expires_at_ms, - dynamic_credentials, - ); - #[cfg_attr(not(target_os = "linux"), allow(unused_mut))] - let mut provider_env = provider_credentials.child_env_with_gcp_resolved(); + let provider_credentials = ProviderCredentialState::from_environment( + provider_env_revision, + provider_env, + provider_credential_expires_at_ms, + dynamic_credentials, + ); + let provider_env = provider_credentials.child_env_with_gcp_resolved(); + (provider_credentials, provider_env) + }; // Initialize the agent-proposals feature flag. Default false until the // initial settings fetch (or the poll loop) tells us otherwise. The flag @@ -354,14 +376,31 @@ pub async fn run_sandbox( None }; - let _gateway_forward = if network_enabled && sidecar_network_enforcement { - let endpoint = openshell_endpoint_for_proxy.as_deref().ok_or_else(|| { - miette::miette!("sidecar network enforcement requires an OpenShell gateway endpoint") + if network_enabled && sidecar_network_enforcement { + let policy_snapshot_path = sidecar_policy_snapshot_file().ok_or_else(|| { + miette::miette!( + "{} is required for sidecar process snapshots", + openshell_core::sandbox_env::SIDECAR_POLICY_SNAPSHOT_FILE + ) })?; - Some(start_gateway_forward_from_env(endpoint).await?) - } else { - None - }; + let provider_snapshot_path = sidecar_provider_env_snapshot_file().ok_or_else(|| { + miette::miette!( + "{} is required for sidecar provider snapshots", + openshell_core::sandbox_env::SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE + ) + })?; + let proto = retained_proto.as_ref().ok_or_else(|| { + miette::miette!( + "sidecar topology requires a gateway policy snapshot for the process supervisor" + ) + })?; + write_sidecar_policy_snapshot(&policy_snapshot_path, proto)?; + write_sidecar_provider_env_snapshot( + &provider_snapshot_path, + provider_credentials.snapshot().revision, + &provider_env, + )?; + } #[cfg(target_os = "linux")] if network_enabled && sidecar_network_enforcement { @@ -375,6 +414,24 @@ pub async fn run_sandbox( } } + if network_enabled + && sidecar_network_enforcement + && let (Some(endpoint), Some(id), Some(socket)) = ( + openshell_endpoint.as_deref(), + sandbox_id.as_deref(), + ssh_socket_path.as_ref(), + ) + { + wait_for_sidecar_ssh_socket(socket).await?; + openshell_supervisor_process::supervisor_session::spawn( + endpoint.to_string(), + id.to_string(), + std::path::PathBuf::from(socket), + None, + ); + info!("sidecar supervisor session task spawned"); + } + #[cfg(not(target_os = "linux"))] if network_enabled && sidecar_network_enforcement { return Err(miette::miette!( @@ -452,11 +509,13 @@ pub async fn run_sandbox( } // Spawn background policy poll task (gRPC mode only). - if let (Some(id), Some(endpoint), Some(engine)) = ( - sandbox_id.as_deref(), - openshell_endpoint.as_deref(), - opa_engine.as_ref(), - ) { + if !process_uses_sidecar_snapshots + && let (Some(id), Some(endpoint), Some(engine)) = ( + sandbox_id.as_deref(), + openshell_endpoint.as_deref(), + opa_engine.as_ref(), + ) + { let poll_id = id.to_string(); let poll_endpoint = endpoint.to_string(); let poll_engine = engine.clone(); @@ -553,6 +612,7 @@ pub async fn run_sandbox( sandbox_id.as_deref(), openshell_endpoint.as_deref(), ssh_socket_path, + sidecar_network_enforcement, &process_policy, process_enforcement_mode, entrypoint_pid, @@ -695,6 +755,140 @@ fn write_supervisor_ready(path: &str) -> Result<()> { Ok(()) } +async fn wait_for_sidecar_ssh_socket(path: &str) -> Result<()> { + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + if sidecar_ssh_socket_ready(path) { + info!(path, "Sidecar SSH socket is ready for gateway relay"); + return Ok(()); + } + if tokio::time::Instant::now() >= deadline { + return Err(miette::miette!( + "timed out waiting for process supervisor SSH socket {path}" + )); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + +#[cfg(unix)] +fn sidecar_ssh_socket_ready(path: &str) -> bool { + use std::os::unix::fs::{FileTypeExt, PermissionsExt}; + + let Ok(metadata) = std::fs::metadata(path) else { + return false; + }; + metadata.file_type().is_socket() && metadata.permissions().mode() & 0o060 == 0o060 +} + +#[cfg(not(unix))] +fn sidecar_ssh_socket_ready(path: &str) -> bool { + std::path::Path::new(path).exists() +} + +fn sidecar_policy_snapshot_file() -> Option { + std::env::var(openshell_core::sandbox_env::SIDECAR_POLICY_SNAPSHOT_FILE) + .ok() + .filter(|path| !path.is_empty()) +} + +fn sidecar_provider_env_snapshot_file() -> Option { + std::env::var(openshell_core::sandbox_env::SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE) + .ok() + .filter(|path| !path.is_empty()) +} + +fn load_policy_from_sidecar_snapshot( + path: &str, +) -> Result<( + SandboxPolicy, + Option>, + Option, +)> { + let bytes = std::fs::read(path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read sidecar policy snapshot from {path}"))?; + let proto = openshell_core::proto::SandboxPolicy::decode(bytes.as_slice()) + .into_diagnostic() + .wrap_err_with(|| format!("failed to decode sidecar policy snapshot from {path}"))?; + let opa_engine = Some(Arc::new(OpaEngine::from_proto(&proto)?)); + let policy = SandboxPolicy::try_from(proto.clone())?; + info!(path, "Loaded sidecar policy snapshot"); + Ok((policy, opa_engine, Some(proto))) +} + +fn write_sidecar_policy_snapshot( + path: &str, + proto: &openshell_core::proto::SandboxPolicy, +) -> Result<()> { + let snapshot_path = std::path::Path::new(path); + if let Some(parent) = snapshot_path.parent() { + std::fs::create_dir_all(parent).into_diagnostic()?; + } + std::fs::write(snapshot_path, proto.encode_to_vec()) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write sidecar policy snapshot to {path}"))?; + info!(path, "Wrote sidecar policy snapshot"); + Ok(()) +} + +fn read_sidecar_provider_env_snapshot( + path: &str, +) -> Result<( + ProviderCredentialState, + std::collections::HashMap, +)> { + let bytes = std::fs::read(path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read sidecar provider env snapshot from {path}"))?; + let value: serde_json::Value = serde_json::from_slice(&bytes) + .into_diagnostic() + .wrap_err_with(|| format!("failed to parse sidecar provider env snapshot from {path}"))?; + let revision = value + .get("revision") + .and_then(serde_json::Value::as_u64) + .unwrap_or_default(); + let child_env: std::collections::HashMap = + serde_json::from_value(value.get("child_env").cloned().unwrap_or_default()) + .into_diagnostic() + .wrap_err_with(|| { + format!("failed to decode child_env from sidecar provider env snapshot {path}") + })?; + let provider_credentials = + ProviderCredentialState::from_child_env_snapshot(revision, child_env.clone()); + info!( + path, + env_count = child_env.len(), + "Loaded sidecar provider env snapshot" + ); + Ok((provider_credentials, child_env)) +} + +fn write_sidecar_provider_env_snapshot( + path: &str, + revision: u64, + child_env: &std::collections::HashMap, +) -> Result<()> { + let snapshot_path = std::path::Path::new(path); + if let Some(parent) = snapshot_path.parent() { + std::fs::create_dir_all(parent).into_diagnostic()?; + } + let value = serde_json::json!({ + "revision": revision, + "child_env": child_env, + }); + let bytes = serde_json::to_vec(&value).into_diagnostic()?; + std::fs::write(snapshot_path, bytes) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write sidecar provider env snapshot to {path}"))?; + info!( + path, + env_count = child_env.len(), + "Wrote sidecar provider env snapshot" + ); + Ok(()) +} + fn sidecar_ca_file_paths() -> Option<(std::path::PathBuf, std::path::PathBuf)> { let tls_dir = std::env::var(openshell_core::sandbox_env::PROXY_TLS_DIR) .unwrap_or_else(|_| SIDECAR_TLS_DIR.to_string()); @@ -720,100 +914,6 @@ fn process_policy_for_topology( Ok(process_policy) } -struct GatewayForwardHandle { - task: tokio::task::JoinHandle<()>, -} - -impl Drop for GatewayForwardHandle { - fn drop(&mut self) { - self.task.abort(); - } -} - -async fn start_gateway_forward_from_env(endpoint: &str) -> Result { - let listen_addr = - std::env::var(openshell_core::sandbox_env::GATEWAY_FORWARD_ADDR).map_err(|_| { - miette::miette!( - "{} is required for sidecar gateway forwarding", - openshell_core::sandbox_env::GATEWAY_FORWARD_ADDR - ) - })?; - start_gateway_forward(&listen_addr, endpoint).await -} - -async fn start_gateway_forward(listen_addr: &str, endpoint: &str) -> Result { - let upstream = gateway_tcp_addr(endpoint)?; - let listener = TcpListener::bind(listen_addr).await.into_diagnostic()?; - info!( - listen_addr, - upstream, "Gateway loopback TCP forward started for sidecar topology" - ); - - let task = tokio::spawn(async move { - loop { - let (mut inbound, peer) = match listener.accept().await { - Ok(accepted) => accepted, - Err(e) => { - warn!(error = %e, "Gateway forward accept failed"); - continue; - } - }; - let upstream = upstream.clone(); - tokio::spawn(async move { - let mut outbound = match TcpStream::connect(&upstream).await { - Ok(stream) => stream, - Err(e) => { - warn!(peer = %peer, upstream, error = %e, "Gateway forward connect failed"); - return; - } - }; - if let Err(e) = copy_bidirectional(&mut inbound, &mut outbound).await { - debug!(peer = %peer, error = %e, "Gateway forward connection closed with error"); - } - }); - } - }); - - Ok(GatewayForwardHandle { task }) -} - -fn gateway_tcp_addr(endpoint: &str) -> Result { - let (scheme, rest) = endpoint - .split_once("://") - .ok_or_else(|| miette::miette!("gateway endpoint must include a URL scheme"))?; - let default_port = match scheme { - "http" => 80, - "https" => 443, - other => { - return Err(miette::miette!( - "unsupported gateway endpoint scheme '{other}' for sidecar forwarding" - )); - } - }; - let authority = rest.split('/').next().unwrap_or(rest); - if authority.is_empty() { - return Err(miette::miette!("gateway endpoint is missing a host")); - } - if authority.starts_with('[') { - let closing = authority - .find(']') - .ok_or_else(|| miette::miette!("invalid bracketed IPv6 gateway endpoint"))?; - let host = &authority[..=closing]; - let port = authority[closing + 1..] - .strip_prefix(':') - .and_then(|value| value.parse::().ok()) - .unwrap_or(default_port); - return Ok(format!("{host}:{port}")); - } - let (host, port) = match authority.rsplit_once(':') { - Some((host, port)) if !host.is_empty() => { - (host, port.parse::().unwrap_or(default_port)) - } - _ => (authority, default_port), - }; - Ok(format!("{host}:{port}")) -} - /// Flush aggregated denial summaries to the gateway via `SubmitPolicyAnalysis`. async fn flush_proposals_to_gateway( endpoint: &str, @@ -2291,34 +2391,6 @@ mod tests { ); } - #[test] - fn gateway_tcp_addr_uses_explicit_port() { - assert_eq!( - gateway_tcp_addr("https://openshell-gateway.openshell.svc:8080").unwrap(), - "openshell-gateway.openshell.svc:8080" - ); - } - - #[test] - fn gateway_tcp_addr_uses_scheme_default_port() { - assert_eq!( - gateway_tcp_addr("https://openshell-gateway.openshell.svc").unwrap(), - "openshell-gateway.openshell.svc:443" - ); - assert_eq!( - gateway_tcp_addr("http://openshell-gateway.openshell.svc").unwrap(), - "openshell-gateway.openshell.svc:80" - ); - } - - #[test] - fn gateway_tcp_addr_preserves_ipv6_brackets() { - assert_eq!( - gateway_tcp_addr("https://[fd00::1]:8443").unwrap(), - "[fd00::1]:8443" - ); - } - #[test] fn apply_ocsf_json_setting_enables_from_initial_settings_snapshot() { let enabled = AtomicBool::new(false); diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index b5481f6759..24d809c334 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -337,7 +337,7 @@ fn run_network_init( sidecar_state_dir, proxy_user_id, proxy_primary_group_id, - 0o775, + 0o2775, )?; prepare_sidecar_directory( sidecar_tls_dir, diff --git a/crates/openshell-supervisor-process/src/run.rs b/crates/openshell-supervisor-process/src/run.rs index bd3fea91f1..f7e25fd917 100644 --- a/crates/openshell-supervisor-process/src/run.rs +++ b/crates/openshell-supervisor-process/src/run.rs @@ -56,6 +56,7 @@ pub async fn run_process( sandbox_id: Option<&str>, openshell_endpoint: Option<&str>, ssh_socket_path: Option, + shared_ssh_socket: bool, policy: &SandboxPolicy, enforcement_mode: ProcessEnforcementMode, entrypoint_pid: Arc, @@ -245,6 +246,7 @@ pub async fn run_process( provider_credentials_clone, user_env_clone, enforcement_mode, + shared_ssh_socket, ) .await { diff --git a/crates/openshell-supervisor-process/src/ssh.rs b/crates/openshell-supervisor-process/src/ssh.rs index c55a6d877e..85466cb14f 100644 --- a/crates/openshell-supervisor-process/src/ssh.rs +++ b/crates/openshell-supervisor-process/src/ssh.rs @@ -43,6 +43,7 @@ fn ssh_server_init( listen_path: &Path, ca_file_paths: &Option<(PathBuf, PathBuf)>, enforcement_mode: ProcessEnforcementMode, + shared_socket: bool, ) -> Result { let mut rng = OsRng; let host_key = PrivateKey::random(&mut rng, Algorithm::Ed25519).into_diagnostic()?; @@ -56,16 +57,15 @@ fn ssh_server_init( let config = Arc::new(config); let ca_paths = ca_file_paths.as_ref().map(|p| Arc::new(p.clone())); - // In full enforcement mode the supervisor starts as root and can isolate - // the SSH socket in a root-only directory before spawning unprivileged - // children. In network-only sidecar mode the process supervisor itself - // runs as the sandbox UID, so the driver points the socket at a writable - // sidecar state volume and accepts that Unix permissions no longer isolate - // same-UID child processes from the socket. + // In full enforcement mode the supervisor normally starts as root and can + // isolate the SSH socket in a root-only directory before spawning + // unprivileged children. Sidecar topology is different: the gateway relay + // runs in the network sidecar as a different UID, so the shared sidecar + // state directory must stay group-accessible. if let Some(parent) = listen_path.parent() { std::fs::create_dir_all(parent).into_diagnostic()?; #[cfg(unix)] - if enforcement_mode.enforces_process_controls() { + if enforcement_mode.enforces_process_controls() && !shared_socket { use std::os::unix::fs::PermissionsExt; let perms = std::fs::Permissions::from_mode(0o700); std::fs::set_permissions(parent, perms).into_diagnostic()?; @@ -78,14 +78,14 @@ fn ssh_server_init( } let listener = UnixListener::bind(listen_path).into_diagnostic()?; - // Tighten permissions so only the supervisor (root) can connect. The - // sandbox entrypoint runs as an unprivileged user and must not be able to - // dial the SSH daemon directly — all access goes through the relay from - // the gateway. + // Tighten permissions so only the supervisor owner can connect in combined + // mode. In sidecar mode, allow the shared sandbox GID so the network + // sidecar's relay can connect to the process supervisor's SSH socket. #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; - let perms = std::fs::Permissions::from_mode(0o600); + let mode = if shared_socket { 0o660 } else { 0o600 }; + let perms = std::fs::Permissions::from_mode(mode); std::fs::set_permissions(listen_path, perms).into_diagnostic()?; } @@ -113,22 +113,27 @@ pub async fn run_ssh_server( provider_credentials: ProviderCredentialState, user_environment: HashMap, enforcement_mode: ProcessEnforcementMode, + shared_socket: bool, ) -> Result<()> { - let (listener, config, ca_paths) = - match ssh_server_init(&listen_path, &ca_file_paths, enforcement_mode) { - Ok(v) => { - // Signal that the SSH server has bound the socket and is ready to - // accept connections. The parent task awaits this before spawning - // the entrypoint process, ensuring exec requests won't race - // against server startup. - let _ = ready_tx.send(Ok(())); - v - } - Err(err) => { - let _ = ready_tx.send(Err(err)); - return Ok(()); - } - }; + let (listener, config, ca_paths) = match ssh_server_init( + &listen_path, + &ca_file_paths, + enforcement_mode, + shared_socket, + ) { + Ok(v) => { + // Signal that the SSH server has bound the socket and is ready to + // accept connections. The parent task awaits this before spawning + // the entrypoint process, ensuring exec requests won't race + // against server startup. + let _ = ready_tx.send(Ok(())); + v + } + Err(err) => { + let _ = ready_tx.send(Err(err)); + return Ok(()); + } + }; loop { let (stream, _peer) = listener.accept().await.into_diagnostic()?; @@ -1327,6 +1332,52 @@ mod tests { use super::*; use std::process::Stdio; + #[cfg(unix)] + fn file_mode(path: &Path) -> u32 { + use std::os::unix::fs::PermissionsExt; + std::fs::metadata(path).unwrap().permissions().mode() & 0o7777 + } + + #[cfg(unix)] + fn set_file_mode(path: &Path, mode: u32) { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(mode)).unwrap(); + } + + #[cfg(unix)] + #[tokio::test] + async fn ssh_server_init_full_enforcement_keeps_private_socket() { + let temp = tempfile::tempdir().unwrap(); + let parent = temp.path().join("ssh"); + std::fs::create_dir_all(&parent).unwrap(); + set_file_mode(&parent, 0o775); + let socket = parent.join("ssh.sock"); + + let (listener, _, _) = + ssh_server_init(&socket, &None, ProcessEnforcementMode::Full, false).unwrap(); + drop(listener); + + assert_eq!(file_mode(&parent), 0o700); + assert_eq!(file_mode(&socket), 0o600); + } + + #[cfg(unix)] + #[tokio::test] + async fn ssh_server_init_shared_socket_keeps_group_access() { + let temp = tempfile::tempdir().unwrap(); + let parent = temp.path().join("ssh"); + std::fs::create_dir_all(&parent).unwrap(); + set_file_mode(&parent, 0o775); + let socket = parent.join("ssh.sock"); + + let (listener, _, _) = + ssh_server_init(&socket, &None, ProcessEnforcementMode::Full, true).unwrap(); + drop(listener); + + assert_eq!(file_mode(&parent), 0o775); + assert_eq!(file_mode(&socket), 0o660); + } + /// Verify that dropping the input sender (the operation `channel_eof` /// performs) causes the stdin writer loop to exit and close the child's /// stdin pipe. Without this, commands like `cat | tar xf -` used by diff --git a/docs/kubernetes/topology.mdx b/docs/kubernetes/topology.mdx index 7f4b323777..bbf68edd63 100644 --- a/docs/kubernetes/topology.mdx +++ b/docs/kubernetes/topology.mdx @@ -110,9 +110,10 @@ flowchart TB Init -->|"installs nftables rules"| NetNS ProcessSupervisor --> Workload Workload -->|"egress redirected on loopback"| NetworkSidecar - NetworkSidecar -->|"gateway forwarding"| Gateway + NetworkSidecar -->|"gateway session + relays"| Gateway NetworkSidecar -->|"policy-enforced egress"| External - ProcessSupervisor --- State + NetworkSidecar -->|"policy/provider snapshots"| State + ProcessSupervisor -->|"reads snapshots"| State NetworkSidecar --- State ``` @@ -121,7 +122,7 @@ The pod contains these OpenShell-managed pieces: | Component | Runs as | Purpose | |---|---|---| | Network init container | Root with setup capabilities | Installs pod-level nftables rules and prepares shared sidecar state. | -| Network sidecar | `supervisor.proxyUid` | Runs the proxy, enforces network policy, writes proxy TLS material, and forwards gateway traffic on loopback. | +| Network sidecar | `supervisor.proxyUid` | Runs the proxy, enforces network policy, owns gateway authentication and the gateway session, and writes proxy TLS plus local policy/provider snapshots. | | Agent container | Resolved sandbox UID/GID | Runs the process supervisor and launches the user workload. | In this topology, the agent container defaults to `runAsNonRoot: true`, @@ -132,9 +133,10 @@ long-running network sidecar always drops all Linux capabilities. The root init container keeps the setup capabilities needed to configure pod networking. Sidecar mode preserves gateway session behavior, including SSH connectivity, -because the process supervisor still owns the session relay. The network sidecar -handles outbound enforcement and forwards the process supervisor's gateway -traffic to the real gateway endpoint. +because the network sidecar owns the gateway session and bridges relay requests +to the process supervisor's local SSH socket. The agent container does not get a +gateway endpoint, gateway TLS material, or the sandbox bootstrap token in the +default sidecar path. Sidecar mode defaults the process supervisor to `network-only`. OpenShell still @@ -148,14 +150,23 @@ identity in sidecar topology. ## Credential Exposure -Sidecar topology uses pod `fsGroup` and group-readable projected credentials so -the non-root process supervisor can authenticate to the gateway. This includes -the projected ServiceAccount token used for sandbox token bootstrap and the -sandbox client TLS secret. +Sidecar topology keeps gateway credentials in the network sidecar. The agent +container does not mount the projected ServiceAccount token used for sandbox +token bootstrap, does not mount the sandbox client TLS secret, and does not get +gateway callback environment variables. -Treat the agent container as trusted with respect to those in-pod gateway -credentials. Use `combined` topology when that credential exposure is not -acceptable for your deployment. +The network sidecar writes local snapshots that the process supervisor needs at +startup: + +- A protobuf policy snapshot. +- A workload-facing provider environment snapshot. + +The provider snapshot contains the environment variables OpenShell intentionally +injects into the workload. It does not give the agent container gateway +authentication material. Use `combined` topology when you need full +process/filesystem enforcement in the same supervisor path; use additional +runtime isolation when you need a stronger container boundary around +network-only sidecar workloads. ## RuntimeClass Isolation diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index fd125231e0..0e17a1b522 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -179,7 +179,7 @@ supervisor_image_pull_policy = "IfNotPresent" supervisor_sideload_method = "image-volume" # "combined" runs the existing single supervisor container with full process, # filesystem, and network enforcement in the agent container. "sidecar" moves -# pod-level network enforcement and gateway forwarding into a network sidecar. +# pod-level network enforcement and gateway session handling into a network sidecar. supervisor_topology = "combined" # Process/filesystem controls for non-combined topologies. "network-only" # keeps the low-permission agent shape; "full" grants combined-mode diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 4b524dd75d..7c52007fc5 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -306,7 +306,7 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA | `supervisor_image` | `supervisor.image.repository` / `supervisor.image.tag` | Set the supervisor image that provides the `openshell-sandbox` binary. | | `supervisor_image_pull_policy` | `supervisor.image.pullPolicy` | Set the Kubernetes image pull policy for the supervisor image. | | `supervisor_sideload_method` | `supervisor.sideloadMethod` | How the supervisor binary is delivered into sandbox pods. Leave empty to auto-detect from cluster version. Set to `image-volume` to mount the supervisor OCI image directly as a volume (requires Kubernetes 1.33+ with the ImageVolume feature gate; GA in 1.36), or `init-container` to copy it through an init container on older clusters. | -| `supervisor_topology` | `supervisor.topology` | Set `combined` for the default single supervisor path, or `sidecar` to move pod-level network enforcement and gateway forwarding into a dedicated sidecar. | +| `supervisor_topology` | `supervisor.topology` | Set `combined` for the default single supervisor path, or `sidecar` to move pod-level network enforcement and the gateway session into a dedicated sidecar. | | `process_enforcement` | `supervisor.processEnforcement` | Process/filesystem controls for non-combined topologies. `network-only` keeps the low-permission agent shape. `full` grants combined-mode process/filesystem controls to the agent process supervisor. | | `proxy_uid` | `supervisor.proxyUid` | UID used by the long-running network sidecar in `sidecar` topology. The network init container exempts this UID from proxy redirection. | | `app_armor_profile` | `server.appArmorProfile` | Set the sandbox agent container's AppArmor profile. Helm defaults this to `Unconfined` so AppArmor-enabled nodes do not block supervisor network namespace setup. Set the Helm value to an empty string to omit the field, or use `RuntimeDefault` or `Localhost/` for operator-managed profiles. | @@ -318,20 +318,19 @@ needed by the supervisor for network namespace setup, Landlock filesystem policy, process privilege changes, and network policy enforcement. In `sidecar` topology, the agent container runs as the resolved sandbox UID/GID with no added Linux capabilities. A root init container performs the nftables setup, and the -long-running sidecar runs non-root with no added Linux capabilities. Sidecar -mode keeps gateway session and SSH behavior, but the process supervisor runs in -`network-only` mode by default: filesystem policy, process privilege dropping, -and process/binary identity checks are not applied by the process container. +long-running sidecar runs non-root with no added Linux capabilities. The +network sidecar owns gateway authentication and writes local policy/provider +snapshots for the process supervisor, so the agent container does not mount the +sandbox bootstrap token or client TLS secret in the default sidecar path. +Sidecar mode keeps gateway session and SSH behavior, but the process supervisor +runs in `network-only` mode by default: filesystem policy, process privilege +dropping, and process/binary identity checks are not applied by the process +container. Set `process_enforcement = "full"` only when you want those combined-mode process/filesystem guards and accept the added agent-container permissions. Network policy still runs in the sidecar and remains endpoint/L7 scoped in sidecar topology; `full` does not restore binary-scoped network identity. -Sidecar mode uses pod `fsGroup` so the non-root process supervisor can read the -projected ServiceAccount token and sandbox client TLS secret required for -gateway authentication. Treat the workload container as trusted with respect to -those in-pod gateway credentials. - The Kubernetes driver creates namespaced `agents.x-k8s.io` `Sandbox` resources from the Kubernetes SIG Apps [agent-sandbox](https://github.com/kubernetes-sigs/agent-sandbox) project. It detects the served Sandbox API at runtime, caches the selected API version for the gateway process, and uses `v1beta1` when available before falling back to `v1alpha1`, so supported Agent Sandbox installations work without version-specific operator configuration. The Agent Sandbox controller turns those resources into sandbox pods and related storage. If Agent Sandbox is upgraded in place, restart the OpenShell gateway after the controller and CRD rollout completes so the gateway can detect the served API versions again. From e49234207f7d606cffdcb07ade8e92445d22d7c5 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Mon, 6 Jul 2026 11:35:26 -0700 Subject: [PATCH 07/24] fix(kubernetes): refresh sidecar provider env snapshots Signed-off-by: Taylor Mutch --- .../skills/debug-openshell-cluster/SKILL.md | 6 +- architecture/compute-runtimes.md | 19 +- .../src/provider_credentials.rs | 64 ++++++ .../openshell-driver-kubernetes/src/driver.rs | 1 + crates/openshell-sandbox/src/lib.rs | 191 ++++++++++++++++-- docs/kubernetes/topology.mdx | 16 +- docs/reference/sandbox-compute-drivers.mdx | 5 +- 7 files changed, 271 insertions(+), 31 deletions(-) diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index 26f0169e3e..d5fa6aa6fc 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -285,7 +285,11 @@ sandbox token file, or those credential mounts. Instead, the network sidecar writes `/run/openshell-sidecar/policy.pb` and `/run/openshell-sidecar/provider-env.json`, then writes the readiness file. If the process supervisor fails before launching the workload, inspect those -snapshot files and the network sidecar logs. +snapshot files and the network sidecar logs. If new SSH/exec sessions do not +pick up refreshed provider environment, inspect the provider-env snapshot +revision and network sidecar settings-poll logs; the process container should +consume newer provider-env snapshot revisions without receiving gateway +credentials. The process container should also publish the workload entrypoint PID to `OPENSHELL_ENTRYPOINT_PID_FILE` diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 24b92a5fa3..40bb3b84db 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -96,14 +96,17 @@ session in a dedicated sidecar, while the agent container runs only the process-supervision leaf and launches the user workload after the sidecar signals readiness. The sidecar writes local policy and provider-environment snapshots into shared state so the process leaf can start without gateway -credentials. In sidecar mode, an init container performs the privileged -pod-network nftables setup with `NET_ADMIN` and hands shared state ownership to -the configured proxy UID; the long-running network sidecar runs as that UID and -does not keep `NET_ADMIN`. The agent container runs as the resolved sandbox -UID/GID with no added Linux capabilities. Sidecar mode preserves gateway session -and SSH behavior, but treats the process leaf as network-only by default: -Landlock filesystem policy, process privilege dropping, and process/binary -identity checks are not applied there. +credentials. The network sidecar also refreshes the workload-facing provider +environment snapshot after settings polls so future process sessions see +updated provider env without giving the process leaf gateway access. In sidecar +mode, an init container performs the privileged pod-network nftables setup with +`NET_ADMIN` and hands shared state ownership to the configured proxy UID; the +long-running network sidecar runs as that UID and does not keep `NET_ADMIN`. +The agent container runs as the resolved sandbox UID/GID with no added Linux +capabilities. Sidecar mode preserves gateway session and SSH behavior, but +treats the process leaf as network-only by default: Landlock filesystem policy, +process privilege dropping, and process/binary identity checks are not applied +there. ## Images diff --git a/crates/openshell-core/src/provider_credentials.rs b/crates/openshell-core/src/provider_credentials.rs index 6b66c23ce6..e3e62e1b9a 100644 --- a/crates/openshell-core/src/provider_credentials.rs +++ b/crates/openshell-core/src/provider_credentials.rs @@ -89,6 +89,37 @@ impl ProviderCredentialState { } } + /// Install an already-prepared child environment snapshot. + /// + /// This is intentionally narrower than [`Self::install_environment`]: it + /// updates only the workload-facing env map and clears resolver state so a + /// process that does not own gateway/provider resolver material can still + /// pick up refreshed provider env for future child processes. + pub fn install_child_env_snapshot( + &self, + revision: u64, + mut child_env: HashMap, + ) -> usize { + let mut inner = self + .inner + .write() + .expect("provider credential state poisoned"); + + for key in &inner.suppressed_keys { + child_env.remove(key); + } + + inner.current = Arc::new(ProviderCredentialSnapshot { + revision, + child_env, + dynamic_credentials: HashMap::new(), + }); + inner.generations.clear(); + inner.current_resolver = None; + inner.combined_resolver = None; + inner.current.child_env.len() + } + pub fn snapshot(&self) -> Arc { self.inner .read() @@ -620,6 +651,39 @@ mod tests { ); } + #[test] + fn child_env_snapshot_install_updates_env_without_resolver_material() { + let state = ProviderCredentialState::from_child_env_snapshot( + 1, + HashMap::from([ + ("GITHUB_TOKEN".to_string(), "old".to_string()), + ("GCE_METADATA_HOST".to_string(), "marker".to_string()), + ]), + ); + state.remove_env_key("GCE_METADATA_HOST"); + + let env_count = state.install_child_env_snapshot( + 2, + HashMap::from([ + ("GITHUB_TOKEN".to_string(), "new".to_string()), + ("GCE_METADATA_HOST".to_string(), "marker".to_string()), + ]), + ); + + let snapshot = state.snapshot(); + assert_eq!(snapshot.revision, 2); + assert_eq!(env_count, 1); + assert_eq!( + snapshot.child_env.get("GITHUB_TOKEN").map(String::as_str), + Some("new") + ); + assert!(!snapshot.child_env.contains_key("GCE_METADATA_HOST")); + assert!( + state.resolver().is_none(), + "child-env snapshots must not install provider resolver material" + ); + } + #[test] fn stale_generation_falls_back_to_current_credential_after_retention_window() { let state = ProviderCredentialState::from_environment( diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 6c4f67462d..de568d040b 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -3169,6 +3169,7 @@ mod tests { assert_eq!(agent["securityContext"]["runAsUser"], 1500); assert_eq!(agent["securityContext"]["runAsGroup"], 1500); assert_eq!(agent["securityContext"]["runAsNonRoot"], true); + assert_eq!(agent["securityContext"]["allowPrivilegeEscalation"], false); assert_eq!( agent["securityContext"]["capabilities"], serde_json::json!({ diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index ef634bb34d..8b0a887981 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -201,6 +201,7 @@ pub async fn run_sandbox( policy.process.run_as_group = Some(gid); } + let mut process_sidecar_provider_snapshot_path = None; #[cfg_attr(not(target_os = "linux"), allow(unused_mut))] let (provider_credentials, mut provider_env) = if process_uses_sidecar_snapshots { let snapshot_path = sidecar_provider_env_snapshot_file().ok_or_else(|| { @@ -209,6 +210,7 @@ pub async fn run_sandbox( openshell_core::sandbox_env::SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE ) })?; + process_sidecar_provider_snapshot_path = Some(snapshot_path.clone()); read_sidecar_provider_env_snapshot(&snapshot_path)? } else { // Fetch provider environment variables from the server. @@ -277,6 +279,9 @@ pub async fn run_sandbox( let provider_env = provider_credentials.child_env_with_gcp_resolved(); (provider_credentials, provider_env) }; + if let Some(snapshot_path) = process_sidecar_provider_snapshot_path { + spawn_sidecar_provider_env_snapshot_watcher(snapshot_path, provider_credentials.clone()); + } // Initialize the agent-proposals feature flag. Default false until the // initial settings fetch (or the poll loop) tells us otherwise. The flag @@ -523,6 +528,10 @@ pub async fn run_sandbox( let poll_pid = entrypoint_pid.clone(); let poll_provider_credentials = provider_credentials.clone(); let poll_policy_local = networking.as_ref().map(|n| n.policy_local_ctx.clone()); + let poll_sidecar_provider_env_snapshot_path = (network_enabled + && sidecar_network_enforcement) + .then(sidecar_provider_env_snapshot_file) + .flatten(); let poll_interval_secs: u64 = std::env::var("OPENSHELL_POLICY_POLL_INTERVAL_SECS") .ok() .and_then(|v| v.parse().ok()) @@ -536,6 +545,7 @@ pub async fn run_sandbox( ocsf_enabled: poll_ocsf_enabled, provider_credentials: poll_provider_credentials, policy_local_ctx: poll_policy_local, + sidecar_provider_env_snapshot_path: poll_sidecar_provider_env_snapshot_path, }; tokio::spawn(async move { @@ -832,12 +842,31 @@ fn write_sidecar_policy_snapshot( Ok(()) } +struct SidecarProviderEnvSnapshot { + revision: u64, + child_env: std::collections::HashMap, +} + fn read_sidecar_provider_env_snapshot( path: &str, ) -> Result<( ProviderCredentialState, std::collections::HashMap, )> { + let snapshot = read_sidecar_provider_env_snapshot_data(path)?; + let provider_credentials = ProviderCredentialState::from_child_env_snapshot( + snapshot.revision, + snapshot.child_env.clone(), + ); + info!( + path, + env_count = snapshot.child_env.len(), + "Loaded sidecar provider env snapshot" + ); + Ok((provider_credentials, snapshot.child_env)) +} + +fn read_sidecar_provider_env_snapshot_data(path: &str) -> Result { let bytes = std::fs::read(path) .into_diagnostic() .wrap_err_with(|| format!("failed to read sidecar provider env snapshot from {path}"))?; @@ -854,14 +883,10 @@ fn read_sidecar_provider_env_snapshot( .wrap_err_with(|| { format!("failed to decode child_env from sidecar provider env snapshot {path}") })?; - let provider_credentials = - ProviderCredentialState::from_child_env_snapshot(revision, child_env.clone()); - info!( - path, - env_count = child_env.len(), - "Loaded sidecar provider env snapshot" - ); - Ok((provider_credentials, child_env)) + Ok(SidecarProviderEnvSnapshot { + revision, + child_env, + }) } fn write_sidecar_provider_env_snapshot( @@ -878,8 +903,7 @@ fn write_sidecar_provider_env_snapshot( "child_env": child_env, }); let bytes = serde_json::to_vec(&value).into_diagnostic()?; - std::fs::write(snapshot_path, bytes) - .into_diagnostic() + write_atomic(snapshot_path, &bytes) .wrap_err_with(|| format!("failed to write sidecar provider env snapshot to {path}"))?; info!( path, @@ -889,6 +913,70 @@ fn write_sidecar_provider_env_snapshot( Ok(()) } +fn write_atomic(path: &std::path::Path, bytes: &[u8]) -> Result<()> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).into_diagnostic()?; + } + let file_name = path + .file_name() + .and_then(std::ffi::OsStr::to_str) + .unwrap_or("snapshot"); + let tmp_path = path.with_file_name(format!(".{file_name}.{}.tmp", std::process::id())); + std::fs::write(&tmp_path, bytes).into_diagnostic()?; + std::fs::rename(&tmp_path, path).into_diagnostic()?; + Ok(()) +} + +fn refresh_sidecar_provider_env_snapshot( + path: &str, + provider_credentials: &ProviderCredentialState, +) -> Result> { + let snapshot = read_sidecar_provider_env_snapshot_data(path)?; + let current_revision = provider_credentials.snapshot().revision; + if snapshot.revision <= current_revision { + return Ok(None); + } + let env_count = + provider_credentials.install_child_env_snapshot(snapshot.revision, snapshot.child_env); + Ok(Some((snapshot.revision, env_count))) +} + +fn spawn_sidecar_provider_env_snapshot_watcher( + path: String, + provider_credentials: ProviderCredentialState, +) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + let mut interval = tokio::time::interval(Duration::from_secs(1)); + interval.tick().await; + loop { + interval.tick().await; + match refresh_sidecar_provider_env_snapshot(&path, &provider_credentials) { + Ok(Some((revision, env_count))) => { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped("provider_env_revision", serde_json::json!(revision)) + .message(format!( + "Sidecar provider environment snapshot refreshed [revision:{revision} env_count:{env_count}]" + )) + .build() + ); + } + Ok(None) => {} + Err(e) => { + warn!( + error = %e, + path, + "Failed to refresh sidecar provider environment snapshot" + ); + } + } + } + }) +} + fn sidecar_ca_file_paths() -> Option<(std::path::PathBuf, std::path::PathBuf)> { let tls_dir = std::env::var(openshell_core::sandbox_env::PROXY_TLS_DIR) .unwrap_or_else(|_| SIDECAR_TLS_DIR.to_string()); @@ -1978,6 +2066,7 @@ struct PolicyPollLoopContext { ocsf_enabled: Arc, provider_credentials: ProviderCredentialState, policy_local_ctx: Option>, + sidecar_provider_env_snapshot_path: Option, } async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { @@ -2055,13 +2144,38 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { .await { Ok(env_result) => { - let env_count = ctx.provider_credentials.install_environment( + ctx.provider_credentials.install_environment( env_result.provider_env_revision, env_result.environment, env_result.credential_expires_at_ms, env_result.dynamic_credentials, ); - current_provider_env_revision = env_result.provider_env_revision; + let child_env = ctx.provider_credentials.child_env_with_gcp_resolved(); + let env_count = child_env.len(); + let snapshot_write_ok = match ctx.sidecar_provider_env_snapshot_path.as_deref() + { + Some(snapshot_path) => { + match write_sidecar_provider_env_snapshot( + snapshot_path, + env_result.provider_env_revision, + &child_env, + ) { + Ok(()) => true, + Err(e) => { + warn!( + error = %e, + path = snapshot_path, + "Settings poll: failed to write sidecar provider environment snapshot" + ); + false + } + } + } + None => true, + }; + if snapshot_write_ok { + current_provider_env_revision = env_result.provider_env_revision; + } ocsf_emit!( ConfigStateChangeBuilder::new(ocsf_ctx()) .severity(SeverityId::Informational) @@ -2069,10 +2183,11 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { .state(StateId::Enabled, "loaded") .unmapped( "provider_env_revision", - serde_json::json!(current_provider_env_revision) + serde_json::json!(env_result.provider_env_revision) ) .message(format!( - "Provider environment refreshed [revision:{current_provider_env_revision} env_count:{env_count}]" + "Provider environment refreshed [revision:{} env_count:{env_count}]", + env_result.provider_env_revision )) .build() ); @@ -2391,6 +2506,54 @@ mod tests { ); } + #[test] + fn sidecar_provider_env_snapshot_refresh_installs_newer_revision() { + let dir = tempfile::tempdir().unwrap(); + let snapshot_path = dir.path().join("provider-env.json"); + let snapshot_path = snapshot_path.to_str().unwrap(); + let provider_credentials = ProviderCredentialState::from_child_env_snapshot( + 1, + std::collections::HashMap::from([("TOKEN".to_string(), "old".to_string())]), + ); + + write_sidecar_provider_env_snapshot( + snapshot_path, + 2, + &std::collections::HashMap::from([("TOKEN".to_string(), "new".to_string())]), + ) + .unwrap(); + + assert_eq!( + refresh_sidecar_provider_env_snapshot(snapshot_path, &provider_credentials).unwrap(), + Some((2, 1)) + ); + let snapshot = provider_credentials.snapshot(); + assert_eq!(snapshot.revision, 2); + assert_eq!( + snapshot.child_env.get("TOKEN").map(String::as_str), + Some("new") + ); + + write_sidecar_provider_env_snapshot( + snapshot_path, + 1, + &std::collections::HashMap::from([("TOKEN".to_string(), "stale".to_string())]), + ) + .unwrap(); + assert_eq!( + refresh_sidecar_provider_env_snapshot(snapshot_path, &provider_credentials).unwrap(), + None + ); + assert_eq!( + provider_credentials + .snapshot() + .child_env + .get("TOKEN") + .map(String::as_str), + Some("new") + ); + } + #[test] fn apply_ocsf_json_setting_enables_from_initial_settings_snapshot() { let enabled = AtomicBool::new(false); diff --git a/docs/kubernetes/topology.mdx b/docs/kubernetes/topology.mdx index bbf68edd63..ed5d6f6bec 100644 --- a/docs/kubernetes/topology.mdx +++ b/docs/kubernetes/topology.mdx @@ -155,18 +155,20 @@ container does not mount the projected ServiceAccount token used for sandbox token bootstrap, does not mount the sandbox client TLS secret, and does not get gateway callback environment variables. -The network sidecar writes local snapshots that the process supervisor needs at -startup: +The network sidecar writes local snapshots that the process supervisor needs: - A protobuf policy snapshot. - A workload-facing provider environment snapshot. The provider snapshot contains the environment variables OpenShell intentionally -injects into the workload. It does not give the agent container gateway -authentication material. Use `combined` topology when you need full -process/filesystem enforcement in the same supervisor path; use additional -runtime isolation when you need a stronger container boundary around -network-only sidecar workloads. +injects into the workload. The process supervisor loads it at startup and +watches for newer revisions that the network sidecar writes after settings +polls, so future child processes can see refreshed provider env without giving +the agent container gateway authentication material. This does not mutate the +environment of the already-running workload entrypoint. Use `combined` topology +when you need full process/filesystem enforcement in the same supervisor path; +use additional runtime isolation when you need a stronger container boundary +around network-only sidecar workloads. ## RuntimeClass Isolation diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 7c52007fc5..4def214955 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -321,7 +321,10 @@ Linux capabilities. A root init container performs the nftables setup, and the long-running sidecar runs non-root with no added Linux capabilities. The network sidecar owns gateway authentication and writes local policy/provider snapshots for the process supervisor, so the agent container does not mount the -sandbox bootstrap token or client TLS secret in the default sidecar path. +sandbox bootstrap token or client TLS secret in the default sidecar path. The +provider environment snapshot is refreshed by the network sidecar after +settings polls so future child processes can see updated provider env without +gateway access in the agent container. Sidecar mode keeps gateway session and SSH behavior, but the process supervisor runs in `network-only` mode by default: filesystem policy, process privilege dropping, and process/binary identity checks are not applied by the process From 8e6c504e68bc9c7051b503e89a9f3e2a648b4666 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Mon, 6 Jul 2026 12:51:56 -0700 Subject: [PATCH 08/24] test(supervisor): align hot-swap identity regression Signed-off-by: Taylor Mutch --- .../openshell-supervisor-network/src/proxy.rs | 100 ++++++++---------- 1 file changed, 44 insertions(+), 56 deletions(-) diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 5debc2926c..0a23689e58 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -1503,10 +1503,10 @@ fn collect_ancestor_identities(start_pid: u32, stop_pid: u32) -> Vec<(u32, PathB /// /// This is the identity-resolution block of [`evaluate_opa_tcp`] extracted /// into a standalone helper so it can be exercised by Linux-only regression -/// tests without a full OPA engine. The key invariant under test is that on -/// a hot-swap of the peer binary, the failure mode is -/// `"Binary integrity violation"` (from the identity cache) rather than -/// `"Failed to stat ... (deleted)"` (from the kernel-tainted path). +/// tests without a full OPA engine. The key hot-swap invariant under test is +/// that display paths are stripped for policy/logging, while integrity hashing +/// reads the live executable via `/proc//exe` instead of the replacement +/// file that now exists at the display path. #[cfg(target_os = "linux")] fn resolve_process_identity( entrypoint_pid: u32, @@ -7973,27 +7973,23 @@ network_policies: assert_eq!(resp_str[body_start..].len(), cl); } - /// End-to-end regression for the `docker cp` hot-swap hazard that - /// motivated `binary_path()` stripping the kernel's `" (deleted)"` - /// suffix (PR #844). + /// End-to-end regression for the `docker cp` hot-swap hazard around + /// unlinked process executables. /// - /// Before the strip, the identity-resolution chain inside - /// `evaluate_opa_tcp` failed with `"Failed to stat - /// /opt/openshell/bin/openshell-sandbox (deleted)"` because - /// `BinaryIdentityCache::verify_or_cache()` tried to `metadata()` the - /// tainted path. That masked the real security signal: a live process - /// was now bound to a *different* binary on disk than the one that was - /// TOFU-cached. After the strip, `binary_path()` returns a path that - /// stats fine, the cache rehashes the new bytes, and the hash mismatch - /// surfaces as a `Binary integrity violation` error — the contract this - /// PR is trying to establish. + /// `binary_path()` strips the kernel's `" (deleted)"` suffix so policy + /// identity and logs use a clean display path. Integrity verification must + /// not hash that display path after a hot-swap, because it may now point to + /// unrelated replacement bytes. It hashes `/proc//exe` instead, which + /// resolves to the live executable inode even after the original path was + /// unlinked. /// /// Test shape (from the review comment on the initial PR): /// 1. Start a `TcpListener` in the test process. /// 2. Copy `/bin/bash` to a temp path we control. /// 3. Prime `BinaryIdentityCache` with that temp binary's hash. /// 4. Spawn the temp bash as a child with a `/dev/tcp` one-liner that - /// opens a real TCP connection to the listener and holds it open. + /// opens a real TCP connection to the listener and holds it open + /// inside the bash process. /// 5. Accept the connection on the listener side and capture the peer's /// ephemeral port — that's what `resolve_process_identity` uses to /// walk `/proc/net/tcp` back to the child PID. @@ -8003,13 +7999,12 @@ network_policies: /// now readlink to `" (deleted)"` OR the overwritten file, depending /// on whether the filesystem reused the inode. /// 7. Call `resolve_process_identity` and assert: - /// - the error reason contains `"Binary integrity violation"` (the - /// cache detected the tampered on-disk bytes), and - /// - the error reason does NOT contain `"Failed to stat"` or - /// `"(deleted)"` (the old pre-strip failure mode). + /// - identity resolution succeeds using the live executable hash, and + /// - the returned display path does not contain the kernel-added + /// `"(deleted)"` suffix. #[cfg(target_os = "linux")] #[test] - fn resolve_process_identity_surfaces_binary_integrity_violation_on_hot_swap() { + fn resolve_process_identity_hashes_live_exe_after_hot_swap() { use crate::identity::BinaryIdentityCache; use std::io::Read; use std::net::TcpListener; @@ -8041,9 +8036,12 @@ network_policies: assert!(!v1_hash.is_empty()); // 4. Spawn the temp bash with a /dev/tcp one-liner that opens a real - // connection to the listener and sleeps to keep it open. The - // `read -t` blocks on stdin so the shell stays resident. - let script = format!("exec 3<>/dev/tcp/127.0.0.1/{listener_port}; sleep 30 <&3"); + // connection to the listener and blocks in bash's `read` builtin + // to keep it open. Do not use an external command like `sleep`: + // it inherits the socket fd and intentionally trips the shared + // socket ambiguity guard instead of exercising the hot-swap path. + let script = + format!("exec 3<>/dev/tcp/127.0.0.1/{listener_port}; read -r -t 30 _ <&3 || true"); let mut child = Command::new(&bash_v1) .arg("-c") .arg(&script) @@ -8087,10 +8085,11 @@ network_policies: std::fs::write(&bash_v1, tampered_bytes).expect("write replacement bytes"); // 7. Resolve identity through the real helper and assert the - // contract: we want "Binary integrity violation", not - // "Failed to stat ... (deleted)". + // contract: hash the live executable via /proc//exe while + // returning a clean display path for policy/logging. let test_pid = std::process::id(); let result = resolve_process_identity(test_pid, peer_port, &cache); + let child_pid = child.id(); // Always clean up the child before asserting so a failure doesn't // leak a sleeping process across test runs. @@ -8098,40 +8097,29 @@ network_policies: let _ = child.wait(); match result { - Ok(_) => panic!( - "resolve_process_identity unexpectedly succeeded after hot-swap; \ - the cache should have detected the tampered on-disk bytes" - ), - Err(err) => { - assert!( - err.reason.contains("Binary integrity violation"), - "expected 'Binary integrity violation' error, got: {}", - err.reason + Ok(identity) => { + assert_eq!( + identity.binary_pid, child_pid, + "expected the hot-swapped bash child to own the socket" ); - assert!( - !err.reason.contains("Failed to stat"), - "pre-PR-#844 failure mode leaked: {}", - err.reason + assert_eq!( + identity.bin_path, bash_v1, + "expected stripped display path to remain the original binary path" ); assert!( - !err.reason.contains("(deleted)"), - "resolved path still contains '(deleted)' suffix: {}", - err.reason + !identity.bin_path.to_string_lossy().contains("(deleted)"), + "resolved binary path still tainted: {}", + identity.bin_path.display() ); - // The binary field should be populated — we did resolve a - // path before failing. - assert!( - err.binary.is_some(), - "expected resolved binary path on integrity failure" + assert_eq!( + identity.bin_hash, v1_hash, + "expected integrity hash from the live executable, not replacement bytes" ); - if let Some(path) = &err.binary { - assert!( - !path.to_string_lossy().contains("(deleted)"), - "resolved binary path still tainted: {}", - path.display() - ); - } } + Err(err) => panic!( + "resolve_process_identity failed after hot-swap; expected live-exe identity: {}", + err.reason + ), } } From 254cdbe4bdde2aa9324c055e8ee92b31675db766 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Mon, 6 Jul 2026 19:52:32 -0700 Subject: [PATCH 09/24] fix(kubernetes): stage sidecar mtls files before proxy chown Signed-off-by: Taylor Mutch --- .agents/skills/helm-dev-environment/SKILL.md | 6 ++ crates/openshell-sandbox/src/main.rs | 75 +++++++++++++++++-- .../openshell/ci/values-sidecar-mtls.yaml | 17 +++++ deploy/helm/openshell/skaffold.yaml | 5 ++ tasks/helm.toml | 15 ++++ 5 files changed, 111 insertions(+), 7 deletions(-) create mode 100644 deploy/helm/openshell/ci/values-sidecar-mtls.yaml diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index 7d6ad7cd5b..620d6250d6 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -65,6 +65,11 @@ mise run helm:skaffold:run mise run helm:skaffold:run:sidecar ``` +**Supervisor sidecar topology with TLS/mTLS enabled** (build once and leave running): +```bash +mise run helm:skaffold:run:sidecar-mtls +``` + Both commands build the `gateway` and `supervisor` images and deploy the OpenShell Helm chart. The sidecar profile renders an `openshell-network-init` init container for nftables setup and a non-root `openshell-supervisor-network` runtime sidecar for @@ -265,6 +270,7 @@ for dependencies still declared in `Chart.yaml`. | `deploy/helm/openshell/ci/values-high-availability.yaml` | HA test overlay (`replicaCount: 2` with external PostgreSQL Secret) | | `deploy/helm/openshell/ci/values-keycloak.yaml` | Keycloak OIDC overlay | | `deploy/helm/openshell/ci/values-sidecar.yaml` | Supervisor sidecar topology overlay for Kubernetes e2e/dev | +| `deploy/helm/openshell/ci/values-sidecar-mtls.yaml` | Supervisor sidecar topology overlay with built-in TLS/mTLS re-enabled after Skaffold dev values | | `deploy/helm/openshell/ci/values-spire.yaml` | SPIFFE/SPIRE provider token grant overlay | | `deploy/helm/openshell/ci/values-spire-stack.yaml` | SPIRE hardened chart values for local dev | | `deploy/helm/openshell/ci/values-tls-disabled.yaml` | Lint-only: TLS + auth disabled (reverse-proxy edge termination) | diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index 24d809c334..d6f06c06ae 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -43,6 +43,16 @@ const CLIENT_TLS_DIR: &str = "/etc/openshell-tls/client"; const SIDECAR_CLIENT_TLS_SUBDIR: &str = "client"; #[cfg(target_os = "linux")] const CLIENT_TLS_FILES: [&str; 3] = ["ca.crt", "tls.crt", "tls.key"]; +#[cfg(target_os = "linux")] +const SIDECAR_STATE_DIR_MODE: u32 = 0o2775; +#[cfg(target_os = "linux")] +const SIDECAR_TLS_DIR_MODE: u32 = 0o755; +#[cfg(target_os = "linux")] +const SIDECAR_TLS_STAGING_DIR_MODE: u32 = 0o700; +#[cfg(target_os = "linux")] +const SIDECAR_CLIENT_TLS_DIR_MODE: u32 = 0o750; +#[cfg(target_os = "linux")] +const SIDECAR_CLIENT_TLS_FILE_MODE: u32 = 0o400; /// Which supervisor leaves are enabled in this process. /// @@ -256,6 +266,35 @@ fn prepare_sidecar_directory(path: &Path, uid: u32, gid: u32, mode: u32) -> Resu Ok(()) } +#[cfg(target_os = "linux")] +fn prepare_sidecar_directory_for_current_user(path: &Path, mode: u32) -> Result<()> { + use miette::Context as _; + use nix::unistd::{Gid, Uid, chown}; + use std::os::unix::fs::PermissionsExt; + + let uid = Uid::current(); + let gid = Gid::current(); + std::fs::create_dir_all(path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create sidecar directory {}", path.display()))?; + chown(path, Some(uid), Some(gid)) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to chown sidecar directory {} to {}:{}", + path.display(), + uid.as_raw(), + gid.as_raw() + ) + })?; + let mut perms = std::fs::metadata(path).into_diagnostic()?.permissions(); + perms.set_mode(mode); + std::fs::set_permissions(path, perms) + .into_diagnostic() + .wrap_err_with(|| format!("failed to chmod sidecar directory {}", path.display()))?; + Ok(()) +} + #[cfg(target_os = "linux")] fn copy_sidecar_client_tls_if_present( source_dir: &Path, @@ -272,7 +311,7 @@ fn copy_sidecar_client_tls_if_present( } let dest_dir = sidecar_tls_dir.join(SIDECAR_CLIENT_TLS_SUBDIR); - prepare_sidecar_directory(&dest_dir, uid, gid, 0o750)?; + prepare_sidecar_directory_for_current_user(&dest_dir, SIDECAR_TLS_STAGING_DIR_MODE)?; for file_name in CLIENT_TLS_FILES { let source = source_dir.join(file_name); if !source.exists() { @@ -282,6 +321,13 @@ fn copy_sidecar_client_tls_if_present( )); } let dest = dest_dir.join(file_name); + if dest.exists() { + std::fs::remove_file(&dest) + .into_diagnostic() + .wrap_err_with(|| { + format!("failed to remove stale client TLS file {}", dest.display()) + })?; + } std::fs::copy(&source, &dest) .into_diagnostic() .wrap_err_with(|| { @@ -292,7 +338,7 @@ fn copy_sidecar_client_tls_if_present( ) })?; let mut perms = std::fs::metadata(&dest).into_diagnostic()?.permissions(); - perms.set_mode(0o400); + perms.set_mode(SIDECAR_CLIENT_TLS_FILE_MODE); std::fs::set_permissions(&dest, perms) .into_diagnostic() .wrap_err_with(|| { @@ -308,6 +354,8 @@ fn copy_sidecar_client_tls_if_present( })?; } + prepare_sidecar_directory(&dest_dir, uid, gid, SIDECAR_CLIENT_TLS_DIR_MODE)?; + Ok(()) } @@ -337,19 +385,23 @@ fn run_network_init( sidecar_state_dir, proxy_user_id, proxy_primary_group_id, - 0o2775, + SIDECAR_STATE_DIR_MODE, )?; - prepare_sidecar_directory( + // The init container runs as uid 0 with CAP_DAC_OVERRIDE dropped. Keep the + // TLS work directory owned by the init user until the client cert copy is + // complete, then hand it to the long-running proxy UID. + prepare_sidecar_directory_for_current_user(sidecar_tls_dir, SIDECAR_TLS_DIR_MODE)?; + copy_sidecar_client_tls_if_present( + Path::new(CLIENT_TLS_DIR), sidecar_tls_dir, proxy_user_id, proxy_primary_group_id, - 0o755, )?; - copy_sidecar_client_tls_if_present( - Path::new(CLIENT_TLS_DIR), + prepare_sidecar_directory( sidecar_tls_dir, proxy_user_id, proxy_primary_group_id, + SIDECAR_TLS_DIR_MODE, )?; openshell_supervisor_process::netns::install_sidecar_bypass_rules(proxy_user_id) } @@ -623,4 +675,13 @@ mod tests { let err = "".parse::().unwrap_err(); assert!(err.contains("at least one")); } + + #[cfg(target_os = "linux")] + #[test] + fn sidecar_tls_modes_preserve_proxy_owned_parent_and_private_client_dir() { + assert_eq!(SIDECAR_TLS_DIR_MODE, 0o755); + assert_eq!(SIDECAR_TLS_STAGING_DIR_MODE, 0o700); + assert_eq!(SIDECAR_CLIENT_TLS_DIR_MODE, 0o750); + assert_eq!(SIDECAR_CLIENT_TLS_FILE_MODE, 0o400); + } } diff --git a/deploy/helm/openshell/ci/values-sidecar-mtls.yaml b/deploy/helm/openshell/ci/values-sidecar-mtls.yaml new file mode 100644 index 0000000000..ef5419a99d --- /dev/null +++ b/deploy/helm/openshell/ci/values-sidecar-mtls.yaml @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# CI/dev overlay for exercising the Kubernetes supervisor sidecar topology with +# the built-in PKI init job and gateway TLS/mTLS enabled. +# +# Merge after ci/values-skaffold.yaml so local image pull policy and developer +# auth settings remain active while this file restores TLS: +# helm install ... -f values.yaml -f ci/values-skaffold.yaml -f ci/values-sidecar-mtls.yaml +# +# Or use the Skaffold profile: +# mise run helm:skaffold:run:sidecar-mtls +server: + disableTls: false + +supervisor: + topology: sidecar diff --git a/deploy/helm/openshell/skaffold.yaml b/deploy/helm/openshell/skaffold.yaml index 76a9e7a5b2..1d370770fe 100644 --- a/deploy/helm/openshell/skaffold.yaml +++ b/deploy/helm/openshell/skaffold.yaml @@ -134,3 +134,8 @@ profiles: - op: add path: /deploy/helm/releases/0/valuesFiles/- value: ci/values-sidecar.yaml + - name: sidecar-mtls + patches: + - op: add + path: /deploy/helm/releases/0/valuesFiles/- + value: ci/values-sidecar-mtls.yaml diff --git a/tasks/helm.toml b/tasks/helm.toml index 433f04f32a..24b6667b1d 100644 --- a/tasks/helm.toml +++ b/tasks/helm.toml @@ -60,6 +60,11 @@ description = "Run skaffold dev with the Kubernetes supervisor sidecar topology" dir = "deploy/helm/openshell" run = "skaffold dev -p sidecar" +["helm:skaffold:dev:sidecar-mtls"] +description = "Run skaffold dev with the Kubernetes supervisor sidecar topology and TLS/mTLS enabled" +dir = "deploy/helm/openshell" +run = "skaffold dev -p sidecar-mtls" + ["helm:skaffold:run"] description = "Run skaffold run for deploy/helm/openshell (one-shot deploy)" dir = "deploy/helm/openshell" @@ -70,6 +75,11 @@ description = "Run skaffold run with the Kubernetes supervisor sidecar topology" dir = "deploy/helm/openshell" run = "skaffold run -p sidecar" +["helm:skaffold:run:sidecar-mtls"] +description = "Run skaffold run with the Kubernetes supervisor sidecar topology and TLS/mTLS enabled" +dir = "deploy/helm/openshell" +run = "skaffold run -p sidecar-mtls" + ["helm:skaffold:delete"] description = "Run skaffold delete for deploy/helm/openshell" dir = "deploy/helm/openshell" @@ -80,6 +90,11 @@ description = "Run skaffold delete for the Kubernetes supervisor sidecar topolog dir = "deploy/helm/openshell" run = "skaffold delete -p sidecar" +["helm:skaffold:delete:sidecar-mtls"] +description = "Run skaffold delete for the Kubernetes supervisor sidecar topology with TLS/mTLS enabled" +dir = "deploy/helm/openshell" +run = "skaffold delete -p sidecar-mtls" + ["helm:skaffold:diagnose"] description = "Run skaffold diagnose for deploy/helm/openshell" dir = "deploy/helm/openshell" From aa87aad5314d5ceaf31e57d240b7c3ddec64ea18 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Tue, 7 Jul 2026 08:39:26 -0700 Subject: [PATCH 10/24] fix(kubernetes): simplify sidecar supervisor topology Signed-off-by: Taylor Mutch --- architecture/compute-runtimes.md | 5 +- crates/openshell-core/src/sandbox_env.rs | 9 -- crates/openshell-driver-kubernetes/README.md | 4 +- .../openshell-driver-kubernetes/src/config.rs | 64 +-------- .../openshell-driver-kubernetes/src/driver.rs | 124 +++--------------- crates/openshell-driver-kubernetes/src/lib.rs | 4 +- .../openshell-driver-kubernetes/src/main.rs | 11 +- .../openshell-driver-podman/src/container.rs | 1 + crates/openshell-sandbox/src/lib.rs | 8 +- .../src/netns/mod.rs | 61 ++++++++- deploy/helm/openshell/README.md | 1 - .../openshell/templates/gateway-config.yaml | 1 - .../openshell/tests/gateway_config_test.yaml | 16 --- deploy/helm/openshell/values.yaml | 4 - docs/kubernetes/topology.mdx | 22 +--- docs/reference/gateway-config.mdx | 4 - docs/reference/sandbox-compute-drivers.mdx | 11 +- e2e/with-kube-gateway.sh | 39 +----- 18 files changed, 109 insertions(+), 280 deletions(-) diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 40bb3b84db..8884b7d341 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -104,9 +104,8 @@ mode, an init container performs the privileged pod-network nftables setup with long-running network sidecar runs as that UID and does not keep `NET_ADMIN`. The agent container runs as the resolved sandbox UID/GID with no added Linux capabilities. Sidecar mode preserves gateway session and SSH behavior, but -treats the process leaf as network-only by default: Landlock filesystem policy, -process privilege dropping, and process/binary identity checks are not applied -there. +treats the process leaf as network-only: Landlock filesystem policy, process +privilege dropping, and process/binary identity checks are not applied there. ## Images diff --git a/crates/openshell-core/src/sandbox_env.rs b/crates/openshell-core/src/sandbox_env.rs index 87e75b0848..0174a37fd6 100644 --- a/crates/openshell-core/src/sandbox_env.rs +++ b/crates/openshell-core/src/sandbox_env.rs @@ -36,15 +36,6 @@ pub const SUPERVISOR_TOPOLOGY: &str = "OPENSHELL_SUPERVISOR_TOPOLOGY"; /// Network enforcement backend selected by the compute driver. pub const NETWORK_ENFORCEMENT_MODE: &str = "OPENSHELL_NETWORK_ENFORCEMENT_MODE"; -/// Process enforcement mode selected by the compute driver. -/// -/// The default when unset is `"full"`, where the process supervisor enforces -/// filesystem/process policy before spawning workloads. Kubernetes sidecar -/// topology sets this to `"network-only"` so the process wrapper can run as -/// the sandbox UID without Linux capabilities while preserving SSH/session -/// behavior. -pub const PROCESS_ENFORCEMENT_MODE: &str = "OPENSHELL_PROCESS_ENFORCEMENT_MODE"; - /// Whether network policy evaluation must bind requests to the peer binary. /// /// The default when unset is `"required"`. Kubernetes sidecar experiments may diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index 9e8c26c9d3..8736e62a46 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -62,11 +62,9 @@ container and runs the long-lived network sidecar as a non-root UID with no added Linux capabilities. The agent container also runs as the resolved sandbox UID/GID with `allowPrivilegeEscalation: false` and `capabilities.drop: ["ALL"]`. In this mode OpenShell preserves gateway session and SSH behavior, but the -process supervisor defaults to network-only mode and does not apply Landlock +process supervisor runs in network-only mode and does not apply Landlock filesystem policy, process privilege dropping, or process/binary identity checks. Network endpoint and L7 policy remain enforced by the network sidecar. -Set `process_enforcement = "full"` only when you want combined-mode -process/filesystem guards and accept the added agent-container permissions. Sidecar mode uses the pod `fsGroup` to make the projected service-account token and sandbox client TLS secret group-readable so the non-root process supervisor diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index a0d3920cd4..e33f156044 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -88,43 +88,6 @@ impl FromStr for SupervisorTopology { } } -/// Process/filesystem controls applied by the process supervisor in split -/// Kubernetes topologies. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub enum ProcessEnforcementMode { - /// Preserve process launch and session relay behavior, but leave - /// filesystem/process guards to the network supervisor topology. - #[default] - NetworkOnly, - /// Run the process supervisor with the same process/filesystem controls as - /// combined topology. - Full, -} - -impl std::fmt::Display for ProcessEnforcementMode { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::NetworkOnly => f.write_str("network-only"), - Self::Full => f.write_str("full"), - } - } -} - -impl FromStr for ProcessEnforcementMode { - type Err = String; - - fn from_str(s: &str) -> Result { - match s { - "network-only" => Ok(Self::NetworkOnly), - "full" => Ok(Self::Full), - other => Err(format!( - "unknown process enforcement mode '{other}'; expected 'network-only' or 'full'" - )), - } - } -} - /// Kubernetes `AppArmor` profile requested for the sandbox agent container. #[derive(Debug, Clone, PartialEq, Eq)] pub enum AppArmorProfile { @@ -251,10 +214,6 @@ pub struct KubernetesComputeConfig { pub supervisor_sideload_method: SupervisorSideloadMethod, /// How the supervisor is arranged for Kubernetes sandbox pods. pub supervisor_topology: SupervisorTopology, - /// Process/filesystem enforcement mode used by the agent container in - /// non-combined topologies. `network-only` keeps the low-permission agent - /// shape; `full` grants the agent supervisor combined-mode controls. - pub process_enforcement: ProcessEnforcementMode, /// UID used by the long-running network sidecar in `sidecar` topology. /// The network init container installs nftables rules that exempt this /// UID, so it must not match the sandbox workload UID. @@ -345,7 +304,6 @@ impl Default for KubernetesComputeConfig { supervisor_image_pull_policy: String::new(), supervisor_sideload_method: SupervisorSideloadMethod::default(), supervisor_topology: SupervisorTopology::default(), - process_enforcement: ProcessEnforcementMode::default(), proxy_uid: DEFAULT_PROXY_UID, grpc_endpoint: String::new(), ssh_socket_path: "/run/openshell/ssh.sock".to_string(), @@ -540,13 +498,6 @@ mod tests { assert_eq!(cfg.proxy_uid, DEFAULT_PROXY_UID); } - #[test] - fn default_process_enforcement_is_network_only() { - let cfg = KubernetesComputeConfig::default(); - assert_eq!(cfg.process_enforcement, ProcessEnforcementMode::NetworkOnly); - assert_eq!(cfg.process_enforcement.to_string(), "network-only"); - } - #[test] fn serde_override_supervisor_topology_sidecar() { let json = serde_json::json!({ @@ -565,15 +516,6 @@ mod tests { assert_eq!(cfg.supervisor_topology, SupervisorTopology::Combined); } - #[test] - fn serde_override_process_enforcement_full() { - let json = serde_json::json!({ - "process_enforcement": "full" - }); - let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); - assert_eq!(cfg.process_enforcement, ProcessEnforcementMode::Full); - } - #[test] fn serde_override_proxy_uid() { let json = serde_json::json!({ @@ -604,12 +546,12 @@ mod tests { } #[test] - fn serde_rejects_invalid_process_enforcement() { + fn serde_rejects_removed_process_enforcement_field() { let json = serde_json::json!({ - "process_enforcement": "privileged" + "process_enforcement": "network-only" }); let err = serde_json::from_value::(json).unwrap_err(); - assert!(err.to_string().contains("unknown variant")); + assert!(err.to_string().contains("unknown field")); } #[test] diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index de568d040b..8069927f2b 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -6,8 +6,8 @@ use super::AppArmorProfile; use crate::config::{ DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_SANDBOX_UID, - DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, ProcessEnforcementMode, - SupervisorSideloadMethod, SupervisorTopology, + DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, SupervisorSideloadMethod, + SupervisorTopology, }; use futures::{Stream, StreamExt, TryStreamExt}; use k8s_openapi::api::core::v1::{Event as KubeEventObj, Namespace, Node}; @@ -564,7 +564,6 @@ impl KubernetesComputeDriver { supervisor_image_pull_policy: &self.config.supervisor_image_pull_policy, supervisor_sideload_method: self.config.supervisor_sideload_method, supervisor_topology: self.config.supervisor_topology, - process_enforcement: self.config.process_enforcement, proxy_uid: self.config.proxy_uid, service_account_name: &self.config.service_account_name, sandbox_id: &sandbox.id, @@ -1544,40 +1543,25 @@ fn apply_supervisor_sidecar_topology( .entry("securityContext") .or_insert_with(|| serde_json::json!({})); if let Some(sc) = security_context.as_object_mut() { - match params.process_enforcement { - ProcessEnforcementMode::NetworkOnly => { - sc.insert( - "runAsUser".to_string(), - serde_json::json!(params.sandbox_uid), - ); - sc.insert( - "runAsGroup".to_string(), - serde_json::json!(params.sandbox_gid), - ); - sc.insert("runAsNonRoot".to_string(), serde_json::json!(true)); - sc.insert( - "allowPrivilegeEscalation".to_string(), - serde_json::json!(false), - ); - sc.insert( - "capabilities".to_string(), - serde_json::json!({ - "drop": ["ALL"] - }), - ); - } - ProcessEnforcementMode::Full => { - sc.insert("runAsUser".to_string(), serde_json::json!(0)); - sc.remove("runAsGroup"); - sc.remove("runAsNonRoot"); - sc.remove("allowPrivilegeEscalation"); - sc.entry("capabilities".to_string()).or_insert_with(|| { - serde_json::json!({ - "add": ["SYS_ADMIN", "NET_ADMIN", "SYS_PTRACE", "SYSLOG"] - }) - }); - } - } + sc.insert( + "runAsUser".to_string(), + serde_json::json!(params.sandbox_uid), + ); + sc.insert( + "runAsGroup".to_string(), + serde_json::json!(params.sandbox_gid), + ); + sc.insert("runAsNonRoot".to_string(), serde_json::json!(true)); + sc.insert( + "allowPrivilegeEscalation".to_string(), + serde_json::json!(false), + ); + sc.insert( + "capabilities".to_string(), + serde_json::json!({ + "drop": ["ALL"] + }), + ); } let volume_mounts = container @@ -1620,11 +1604,6 @@ fn apply_supervisor_sidecar_topology( openshell_core::sandbox_env::NETWORK_ENFORCEMENT_MODE, "sidecar-nftables", ); - upsert_env( - env, - openshell_core::sandbox_env::PROCESS_ENFORCEMENT_MODE, - ¶ms.process_enforcement.to_string(), - ); upsert_env( env, openshell_core::sandbox_env::SSH_SOCKET_PATH, @@ -1825,7 +1804,6 @@ struct SandboxPodParams<'a> { supervisor_image_pull_policy: &'a str, supervisor_sideload_method: SupervisorSideloadMethod, supervisor_topology: SupervisorTopology, - process_enforcement: ProcessEnforcementMode, proxy_uid: u32, service_account_name: &'a str, sandbox_id: &'a str, @@ -1859,7 +1837,6 @@ impl Default for SandboxPodParams<'_> { supervisor_image_pull_policy: "", supervisor_sideload_method: SupervisorSideloadMethod::default(), supervisor_topology: SupervisorTopology::default(), - process_enforcement: ProcessEnforcementMode::default(), proxy_uid: DEFAULT_PROXY_UID, service_account_name: DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, sandbox_id: "", @@ -3192,10 +3169,6 @@ mod tests { rendered_env(agent, openshell_core::sandbox_env::K8S_SA_TOKEN_FILE), None ); - assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::PROCESS_ENFORCEMENT_MODE), - Some("network-only") - ); assert_eq!( rendered_env(agent, openshell_core::sandbox_env::SSH_SOCKET_PATH), Some(SIDECAR_SSH_SOCKET_FILE) @@ -3409,61 +3382,6 @@ mod tests { } } - #[test] - fn sidecar_topology_full_process_enforcement_keeps_combined_agent_permissions() { - let params = SandboxPodParams { - supervisor_topology: SupervisorTopology::Sidecar, - process_enforcement: ProcessEnforcementMode::Full, - supervisor_sideload_method: SupervisorSideloadMethod::InitContainer, - supervisor_image: "supervisor-image:latest", - grpc_endpoint: "https://openshell-gateway.openshell.svc:8080", - sandbox_uid: 1500, - sandbox_gid: 1500, - proxy_uid: 2200, - ..SandboxPodParams::default() - }; - let pod_template = sandbox_template_to_k8s( - &SandboxTemplate::default(), - false, - &std::collections::HashMap::new(), - false, - ¶ms, - ); - - let containers = pod_template["spec"]["containers"].as_array().unwrap(); - let agent = containers - .iter() - .find(|container| container["name"] == "agent") - .unwrap(); - let sc = &agent["securityContext"]; - assert_eq!(sc["runAsUser"], 0); - assert!(sc.get("runAsGroup").is_none()); - assert!(sc.get("runAsNonRoot").is_none()); - assert!(sc.get("allowPrivilegeEscalation").is_none()); - assert_eq!( - sc["capabilities"], - serde_json::json!({ - "add": ["SYS_ADMIN", "NET_ADMIN", "SYS_PTRACE", "SYSLOG"] - }) - ); - assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::PROCESS_ENFORCEMENT_MODE), - Some("full") - ); - - let sidecar = containers - .iter() - .find(|container| container["name"] == SUPERVISOR_NETWORK_SIDECAR_NAME) - .unwrap(); - assert_eq!(sidecar["securityContext"]["runAsUser"], 2200); - assert_eq!( - sidecar["securityContext"]["capabilities"], - serde_json::json!({ - "drop": ["ALL"] - }) - ); - } - #[test] fn sidecar_topology_rejects_proxy_uid_matching_sandbox_uid() { let params = SandboxPodParams { diff --git a/crates/openshell-driver-kubernetes/src/lib.rs b/crates/openshell-driver-kubernetes/src/lib.rs index 114055bcc5..8c326f6af8 100644 --- a/crates/openshell-driver-kubernetes/src/lib.rs +++ b/crates/openshell-driver-kubernetes/src/lib.rs @@ -7,8 +7,8 @@ pub mod grpc; pub use config::{ AppArmorProfile, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, - DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, ProcessEnforcementMode, - SupervisorSideloadMethod, SupervisorTopology, + DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, SupervisorSideloadMethod, + SupervisorTopology, }; pub use driver::{KubernetesComputeDriver, KubernetesDriverError}; pub use grpc::ComputeDriverService; diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 34ec1b55d7..3271e56a47 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -11,8 +11,7 @@ use openshell_core::VERSION; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; use openshell_driver_kubernetes::{ AppArmorProfile, ComputeDriverService, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, - KubernetesComputeConfig, KubernetesComputeDriver, ProcessEnforcementMode, - SupervisorSideloadMethod, SupervisorTopology, + KubernetesComputeConfig, KubernetesComputeDriver, SupervisorSideloadMethod, SupervisorTopology, }; #[derive(Parser, Debug)] @@ -88,13 +87,6 @@ struct Args { )] supervisor_topology: SupervisorTopology, - #[arg( - long, - env = "OPENSHELL_PROCESS_ENFORCEMENT", - default_value = "network-only" - )] - process_enforcement: ProcessEnforcementMode, - #[arg(long, env = "OPENSHELL_PROXY_UID", default_value_t = DEFAULT_PROXY_UID)] proxy_uid: u32, @@ -142,7 +134,6 @@ async fn main() -> Result<()> { supervisor_image_pull_policy: args.supervisor_image_pull_policy.unwrap_or_default(), supervisor_sideload_method: args.supervisor_sideload_method, supervisor_topology: args.supervisor_topology, - process_enforcement: args.process_enforcement, proxy_uid: args.proxy_uid, grpc_endpoint: args.grpc_endpoint.unwrap_or_default(), ssh_socket_path: args.sandbox_ssh_socket_path, diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index b4f7c176be..9be46df80d 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -14,6 +14,7 @@ use openshell_core::{driver_mounts, proto_struct}; use serde::Serialize; use serde_json::Value; use std::collections::{BTreeMap, HashSet}; +#[cfg(target_os = "linux")] use std::path::Path; /// Returns `true` when `SELinux` is enabled (enforcing or permissive). diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 8b0a887981..694efe25c9 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -691,11 +691,11 @@ fn sidecar_network_enforcement_enabled() -> bool { } fn process_enforcement_mode() -> ProcessEnforcementMode { - match std::env::var(openshell_core::sandbox_env::PROCESS_ENFORCEMENT_MODE) - .unwrap_or_else(|_| "full".to_string()) - .as_str() + match std::env::var(openshell_core::sandbox_env::SUPERVISOR_TOPOLOGY) + .ok() + .as_deref() { - "network-only" => ProcessEnforcementMode::NetworkOnly, + Some("sidecar") => ProcessEnforcementMode::NetworkOnly, _ => ProcessEnforcementMode::Full, } } diff --git a/crates/openshell-supervisor-process/src/netns/mod.rs b/crates/openshell-supervisor-process/src/netns/mod.rs index 86a5406ade..472cacd03a 100644 --- a/crates/openshell-supervisor-process/src/netns/mod.rs +++ b/crates/openshell-supervisor-process/src/netns/mod.rs @@ -513,9 +513,43 @@ fn install_sidecar_iptables_legacy_bypass_rules(proxy_uid: u32) -> Result<()> { "trusted iptables-legacy helper not found; sidecar network enforcement fallback unavailable" ) })?; + let ip6tables_cmd = find_ip6tables_legacy().ok_or_else(|| { + miette::miette!( + "trusted ip6tables-legacy helper not found; sidecar network enforcement fallback cannot fence IPv6" + ) + })?; cleanup_sidecar_iptables_legacy_rules(&iptables_cmd); + cleanup_sidecar_iptables_legacy_rules(&ip6tables_cmd); + + if let Err(e) = install_sidecar_iptables_legacy_family_rules( + &iptables_cmd, + proxy_uid, + "icmp-port-unreachable", + ) { + cleanup_sidecar_iptables_legacy_rules(&iptables_cmd); + cleanup_sidecar_iptables_legacy_rules(&ip6tables_cmd); + return Err(e); + } + + if let Err(e) = install_sidecar_iptables_legacy_family_rules( + &ip6tables_cmd, + proxy_uid, + "icmp6-port-unreachable", + ) { + cleanup_sidecar_iptables_legacy_rules(&iptables_cmd); + cleanup_sidecar_iptables_legacy_rules(&ip6tables_cmd); + return Err(e); + } + Ok(()) +} + +fn install_sidecar_iptables_legacy_family_rules( + cmd: &str, + proxy_uid: u32, + udp_reject_with: &str, +) -> Result<()> { let proxy_uid_arg = proxy_uid.to_string(); let commands: Vec> = vec![ vec!["-N", SIDECAR_IPTABLES_CHAIN], @@ -558,14 +592,14 @@ fn install_sidecar_iptables_legacy_bypass_rules(proxy_uid: u32) -> Result<()> { "-j", "REJECT", "--reject-with", - "icmp-port-unreachable", + udp_reject_with, ], vec!["-A", "OUTPUT", "-j", SIDECAR_IPTABLES_CHAIN], ]; for args in commands { - if let Err(e) = run_iptables_legacy_current_namespace(&iptables_cmd, &args) { - cleanup_sidecar_iptables_legacy_rules(&iptables_cmd); + if let Err(e) = run_iptables_legacy_current_namespace(cmd, &args) { + cleanup_sidecar_iptables_legacy_rules(cmd); return Err(e); } } @@ -783,6 +817,11 @@ const IPTABLES_LEGACY_SEARCH_PATHS: &[&str] = &[ "/sbin/iptables-legacy", "/usr/bin/iptables-legacy", ]; +const IP6TABLES_LEGACY_SEARCH_PATHS: &[&str] = &[ + "/usr/sbin/ip6tables-legacy", + "/sbin/ip6tables-legacy", + "/usr/bin/ip6tables-legacy", +]; fn find_trusted_binary<'a>(name: &str, paths: &'a [&str]) -> Result<&'a str> { paths @@ -813,6 +852,12 @@ fn find_iptables_legacy() -> Option { .map(String::from) } +fn find_ip6tables_legacy() -> Option { + find_trusted_binary("ip6tables-legacy", IP6TABLES_LEGACY_SEARCH_PATHS) + .ok() + .map(String::from) +} + #[cfg(test)] mod tests { use super::*; @@ -862,6 +907,16 @@ mod tests { } } + #[test] + fn ip6tables_legacy_search_paths_are_absolute() { + for path in IP6TABLES_LEGACY_SEARCH_PATHS { + assert!( + path.starts_with('/'), + "IP6TABLES_LEGACY_SEARCH_PATHS entry must be absolute: {path}" + ); + } + } + #[test] #[ignore = "requires root privileges"] fn test_create_and_drop_namespace() { diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 0ed0959153..4c71165398 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -237,7 +237,6 @@ add `ci/values-spire.yaml` to the OpenShell release values files. | supervisor.image.pullPolicy | string | `""` | Supervisor image pull policy. Defaults to the gateway image pull policy when empty. | | supervisor.image.repository | string | `"ghcr.io/nvidia/openshell/supervisor"` | Supervisor image repository. | | supervisor.image.tag | string | `""` | Supervisor image tag. Defaults to the chart appVersion when empty. | -| supervisor.processEnforcement | string | `"network-only"` | Process/filesystem controls applied by the agent process supervisor in non-combined topologies. "network-only" keeps the low-permission agent shape; "full" grants combined-mode process/filesystem controls. | | supervisor.proxyUid | int | `1337` | UID for the long-running network sidecar in sidecar topology. The network init container installs nftables rules that exempt this UID. | | supervisor.sideloadMethod | string | `""` | How the supervisor binary is delivered into sandbox pods. Empty (default) = auto-detect from cluster version: K8s >= v1.35 -> "image-volume" (ImageVolume enabled by default; GA in v1.36) K8s < v1.35 -> "init-container" (copies via init container + emptyDir) On K8s v1.33-v1.34 with the ImageVolume feature gate manually enabled, set this to "image-volume" explicitly. | | supervisor.topology | string | `"combined"` | Supervisor pod topology for Kubernetes sandboxes. "combined" runs the current single supervisor container in the agent pod. "sidecar" runs network enforcement in a dedicated sidecar and the process supervisor as a low-capability wrapper in the agent container. | diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index 9637c33284..122ee39732 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -114,7 +114,6 @@ data: service_account_name = {{ include "openshell.sandboxServiceAccountName" . | quote }} supervisor_sideload_method = {{ include "openshell.supervisorSideloadMethod" . | quote }} supervisor_topology = {{ .Values.supervisor.topology | default "combined" | quote }} - process_enforcement = {{ .Values.supervisor.processEnforcement | default "network-only" | quote }} proxy_uid = {{ .Values.supervisor.proxyUid | default 1337 }} sa_token_ttl_secs = {{ .Values.server.sandboxJwt.k8sSaTokenTtlSecs | default 3600 }} {{- if .Values.server.providerTokenGrants.spiffe.enabled }} diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index 509eb4279b..26453ed49a 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -92,22 +92,6 @@ tests: path: data["gateway.toml"] pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?supervisor_topology\s*=\s*"sidecar"' - - it: renders process enforcement under [openshell.drivers.kubernetes] - template: templates/gateway-config.yaml - set: - supervisor.processEnforcement: full - asserts: - - matchRegex: - path: data["gateway.toml"] - pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?process_enforcement\s*=\s*"full"' - - - it: renders default process enforcement under [openshell.drivers.kubernetes] - template: templates/gateway-config.yaml - asserts: - - matchRegex: - path: data["gateway.toml"] - pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?process_enforcement\s*=\s*"network-only"' - - it: renders proxy uid under [openshell.drivers.kubernetes] template: templates/gateway-config.yaml set: diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index c670a97b8a..d10204e089 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -49,10 +49,6 @@ supervisor: # "sidecar" runs network enforcement in a dedicated sidecar and the process # supervisor as a low-capability wrapper in the agent container. topology: "combined" - # -- Process/filesystem controls applied by the agent process supervisor in - # non-combined topologies. "network-only" keeps the low-permission agent - # shape; "full" grants combined-mode process/filesystem controls. - processEnforcement: "network-only" # -- UID for the long-running network sidecar in sidecar topology. The # network init container installs nftables rules that exempt this UID. proxyUid: 1337 diff --git a/docs/kubernetes/topology.mdx b/docs/kubernetes/topology.mdx index ed5d6f6bec..c2e13c5c64 100644 --- a/docs/kubernetes/topology.mdx +++ b/docs/kubernetes/topology.mdx @@ -21,7 +21,7 @@ lower-privilege agent container. | Topology | Use when | Main tradeoff | |---|---|---| | `combined` | You need OpenShell network, filesystem, and process controls in the sandbox workload. | The agent container carries the Linux capabilities the supervisor needs. | -| `sidecar` | You need the agent container to run as non-root without added Linux capabilities, and network policy is the primary control. | Defaults to network-only process supervision unless you opt in to `processEnforcement=full`. | +| `sidecar` | You need the agent container to run as non-root without added Linux capabilities, and network policy is the primary control. | The process supervisor is network-only, so filesystem/process controls do not run in the agent container. | ## Privilege Model @@ -31,7 +31,6 @@ The long-running container permissions differ by topology: |---|---|---|---|---|---| | `combined` | Agent container, which also runs the supervisor | Not forced by topology | Not explicitly disabled by the driver | Adds `SYS_ADMIN`, `NET_ADMIN`, `SYS_PTRACE`, and `SYSLOG`; adds `SETUID`, `SETGID`, and `DAC_READ_SEARCH` when user namespaces are enabled | Full supervisor controls run in the agent container. | | `sidecar` | Agent container, process-only supervisor (`network-only`) | `sandbox_uid:sandbox_gid` | `false` | Drops `ALL` | Agent and workload run without added Linux capabilities. | -| `sidecar` | Agent container, process-only supervisor (`full`) | Root supervisor | Not explicitly disabled by the driver | Adds combined-mode capabilities | Agent keeps combined-mode process/filesystem guards. | | `sidecar` | Network supervisor sidecar | `proxyUid:sandbox_gid` | `false` | Drops `ALL` | Long-running proxy sidecar is also non-root without added capabilities. | Short-lived setup containers still have the permissions needed to prepare the @@ -96,7 +95,7 @@ flowchart TB NetNS["pod network namespace"] subgraph Agent["agent container"] - ProcessSupervisor["process supervisor
network-only by default"] + ProcessSupervisor["process supervisor
network-only"] Workload["Agent workload"] end @@ -126,9 +125,7 @@ The pod contains these OpenShell-managed pieces: | Agent container | Resolved sandbox UID/GID | Runs the process supervisor and launches the user workload. | In this topology, the agent container defaults to `runAsNonRoot: true`, -`allowPrivilegeEscalation: false`, and `capabilities.drop: ["ALL"]`. Set -`supervisor.processEnforcement=full` only when you want combined-mode -process/filesystem guards and accept the added agent-container permissions. The +`allowPrivilegeEscalation: false`, and `capabilities.drop: ["ALL"]`. The long-running network sidecar always drops all Linux capabilities. The root init container keeps the setup capabilities needed to configure pod networking. @@ -139,13 +136,11 @@ gateway endpoint, gateway TLS material, or the sandbox bootstrap token in the default sidecar path. -Sidecar mode defaults the process supervisor to `network-only`. OpenShell still +Sidecar mode runs the process supervisor in `network-only` mode. OpenShell still enforces network endpoint and L7 policy through the sidecar, but the process supervisor does not apply Landlock filesystem policy, process privilege -dropping, or process/binary identity checks unless you opt in to -`supervisor.processEnforcement=full`. Network policy still runs in the sidecar -and remains endpoint/L7 scoped; `full` does not restore binary-scoped network -identity in sidecar topology. +dropping, or process/binary identity checks. Use `combined` topology when you +need those controls in the same supervisor path. ## Credential Exposure @@ -197,23 +192,18 @@ For direct gateway TOML configuration, set the Kubernetes driver fields: ```toml [openshell.drivers.kubernetes] supervisor_topology = "sidecar" -process_enforcement = "network-only" proxy_uid = 1337 ``` `proxy_uid` must be a non-root UID and must not match the sandbox UID. The network init container exempts this UID from proxy redirection so the sidecar can reach the gateway. -Set `process_enforcement="full"` only when you want the agent process supervisor -to keep combined-mode process/filesystem guards and accept the added -agent-container permissions. When the Helm chart renders `gateway.toml`, set the equivalent chart values: ```yaml supervisor: topology: sidecar - processEnforcement: network-only proxyUid: 1337 ``` diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 0e17a1b522..eb3a23566d 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -181,10 +181,6 @@ supervisor_sideload_method = "image-volume" # filesystem, and network enforcement in the agent container. "sidecar" moves # pod-level network enforcement and gateway session handling into a network sidecar. supervisor_topology = "combined" -# Process/filesystem controls for non-combined topologies. "network-only" -# keeps the low-permission agent shape; "full" grants combined-mode -# process/filesystem controls to the agent process supervisor. -process_enforcement = "network-only" # UID used by the long-running network sidecar. In sidecar topology the # network init container installs nftables rules that exempt this UID. proxy_uid = 1337 diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 4def214955..b2cee81543 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -307,7 +307,6 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA | `supervisor_image_pull_policy` | `supervisor.image.pullPolicy` | Set the Kubernetes image pull policy for the supervisor image. | | `supervisor_sideload_method` | `supervisor.sideloadMethod` | How the supervisor binary is delivered into sandbox pods. Leave empty to auto-detect from cluster version. Set to `image-volume` to mount the supervisor OCI image directly as a volume (requires Kubernetes 1.33+ with the ImageVolume feature gate; GA in 1.36), or `init-container` to copy it through an init container on older clusters. | | `supervisor_topology` | `supervisor.topology` | Set `combined` for the default single supervisor path, or `sidecar` to move pod-level network enforcement and the gateway session into a dedicated sidecar. | -| `process_enforcement` | `supervisor.processEnforcement` | Process/filesystem controls for non-combined topologies. `network-only` keeps the low-permission agent shape. `full` grants combined-mode process/filesystem controls to the agent process supervisor. | | `proxy_uid` | `supervisor.proxyUid` | UID used by the long-running network sidecar in `sidecar` topology. The network init container exempts this UID from proxy redirection. | | `app_armor_profile` | `server.appArmorProfile` | Set the sandbox agent container's AppArmor profile. Helm defaults this to `Unconfined` so AppArmor-enabled nodes do not block supervisor network namespace setup. Set the Helm value to an empty string to omit the field, or use `RuntimeDefault` or `Localhost/` for operator-managed profiles. | | `workspace_default_storage_size` | `server.workspaceDefaultStorageSize` | Set the default workspace PVC size for new sandboxes. | @@ -326,13 +325,9 @@ provider environment snapshot is refreshed by the network sidecar after settings polls so future child processes can see updated provider env without gateway access in the agent container. Sidecar mode keeps gateway session and SSH behavior, but the process supervisor -runs in `network-only` mode by default: filesystem policy, process privilege -dropping, and process/binary identity checks are not applied by the process -container. -Set `process_enforcement = "full"` only when you want those combined-mode -process/filesystem guards and accept the added agent-container permissions. -Network policy still runs in the sidecar and remains endpoint/L7 scoped in -sidecar topology; `full` does not restore binary-scoped network identity. +runs in `network-only` mode: filesystem policy, process privilege dropping, and +process/binary identity checks are not applied by the process container. Network +policy still runs in the sidecar and remains endpoint/L7 scoped. The Kubernetes driver creates namespaced `agents.x-k8s.io` `Sandbox` resources from the Kubernetes SIG Apps [agent-sandbox](https://github.com/kubernetes-sigs/agent-sandbox) project. It detects the served Sandbox API at runtime, caches the selected API version for the gateway process, and uses `v1beta1` when available before falling back to `v1alpha1`, so supported Agent Sandbox installations work without version-specific operator configuration. The Agent Sandbox controller turns those resources into sandbox pods and related storage. diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index 8ce1989dae..b4b52e7204 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -394,43 +394,18 @@ require_cmd() { } configure_fixture_container_engine() { - local selected_engine="" - - if [ -n "${CONTAINER_ENGINE:-}" ]; then - selected_engine="$(printf '%s' "${CONTAINER_ENGINE}" | tr '[:upper:]' '[:lower:]')" - case "${selected_engine}" in - docker|podman) - export CONTAINER_ENGINE="${selected_engine}" - return - ;; - *) - echo "ERROR: CONTAINER_ENGINE=${CONTAINER_ENGINE} is invalid; expected docker or podman" >&2 - exit 2 - ;; - esac - fi - - case "${KUBE_CONTEXT}" in - k3d-*) - selected_engine="docker" - ;; - kind-*) - case "$(printf '%s' "${KIND_EXPERIMENTAL_PROVIDER:-}" | tr '[:upper:]' '[:lower:]')" in - podman) - selected_engine="podman" - ;; - *) - selected_engine="docker" - ;; - esac + [ -n "${CONTAINER_ENGINE:-}" ] || return + local selected_engine + selected_engine="$(printf '%s' "${CONTAINER_ENGINE}" | tr '[:upper:]' '[:lower:]')" + case "${selected_engine}" in + docker|podman) ;; *) - return + echo "ERROR: CONTAINER_ENGINE=${CONTAINER_ENGINE} is invalid; expected docker or podman" >&2 + exit 2 ;; esac - export CONTAINER_ENGINE="${selected_engine}" - echo "Using ${CONTAINER_ENGINE} for Kubernetes e2e host-side fixture containers." } require_cmd helm From 6e0e7816db75ec4f65a3f35206a82ed0355ab387 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Tue, 7 Jul 2026 08:58:47 -0700 Subject: [PATCH 11/24] chore(helm): reuse sidecar skaffold values Signed-off-by: Taylor Mutch --- .agents/skills/helm-dev-environment/SKILL.md | 11 ++++++----- .../helm/openshell/ci/values-sidecar-mtls.yaml | 17 ----------------- deploy/helm/openshell/skaffold.yaml | 6 +++++- 3 files changed, 11 insertions(+), 23 deletions(-) delete mode 100644 deploy/helm/openshell/ci/values-sidecar-mtls.yaml diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index 620d6250d6..0283644aa4 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -73,9 +73,10 @@ mise run helm:skaffold:run:sidecar-mtls Both commands build the `gateway` and `supervisor` images and deploy the OpenShell Helm chart. The sidecar profile renders an `openshell-network-init` init container for nftables setup and a non-root `openshell-supervisor-network` runtime sidecar for -proxying. The `pkiInitJob` hook (a pre-install Job that runs `openshell-gateway -generate-certs`) generates mTLS secrets on first install. Envoy Gateway opt-in; -see the Optional Add-ons section below. +proxying. The sidecar-mTLS profile reuses `ci/values-sidecar.yaml` and restores +`server.disableTls=false` inline for Skaffold. The `pkiInitJob` hook (a pre-install +Job that runs `openshell-gateway generate-certs`) generates mTLS secrets on first +install. Envoy Gateway opt-in; see the Optional Add-ons section below. The gateway Service uses ClusterIP. Access is via Envoy Gateway (port `8080`) or `kubectl port-forward`. @@ -87,7 +88,8 @@ create the Secret named `openshell-ha-pg` with a `uri` key, then run ### TLS behaviour `ci/values-skaffold.yaml` sets `server.disableTls: true`, so Skaffold-based deploys run -plaintext by default. To test with TLS enabled, comment out that line and redeploy. +plaintext by default. To test sidecar topology with TLS enabled, use +`mise run helm:skaffold:run:sidecar-mtls`. | Mode | `server.disableTls` | Gateway scheme | |------|---------------------|----------------| @@ -270,7 +272,6 @@ for dependencies still declared in `Chart.yaml`. | `deploy/helm/openshell/ci/values-high-availability.yaml` | HA test overlay (`replicaCount: 2` with external PostgreSQL Secret) | | `deploy/helm/openshell/ci/values-keycloak.yaml` | Keycloak OIDC overlay | | `deploy/helm/openshell/ci/values-sidecar.yaml` | Supervisor sidecar topology overlay for Kubernetes e2e/dev | -| `deploy/helm/openshell/ci/values-sidecar-mtls.yaml` | Supervisor sidecar topology overlay with built-in TLS/mTLS re-enabled after Skaffold dev values | | `deploy/helm/openshell/ci/values-spire.yaml` | SPIFFE/SPIRE provider token grant overlay | | `deploy/helm/openshell/ci/values-spire-stack.yaml` | SPIRE hardened chart values for local dev | | `deploy/helm/openshell/ci/values-tls-disabled.yaml` | Lint-only: TLS + auth disabled (reverse-proxy edge termination) | diff --git a/deploy/helm/openshell/ci/values-sidecar-mtls.yaml b/deploy/helm/openshell/ci/values-sidecar-mtls.yaml deleted file mode 100644 index ef5419a99d..0000000000 --- a/deploy/helm/openshell/ci/values-sidecar-mtls.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# CI/dev overlay for exercising the Kubernetes supervisor sidecar topology with -# the built-in PKI init job and gateway TLS/mTLS enabled. -# -# Merge after ci/values-skaffold.yaml so local image pull policy and developer -# auth settings remain active while this file restores TLS: -# helm install ... -f values.yaml -f ci/values-skaffold.yaml -f ci/values-sidecar-mtls.yaml -# -# Or use the Skaffold profile: -# mise run helm:skaffold:run:sidecar-mtls -server: - disableTls: false - -supervisor: - topology: sidecar diff --git a/deploy/helm/openshell/skaffold.yaml b/deploy/helm/openshell/skaffold.yaml index 1d370770fe..119adf086b 100644 --- a/deploy/helm/openshell/skaffold.yaml +++ b/deploy/helm/openshell/skaffold.yaml @@ -138,4 +138,8 @@ profiles: patches: - op: add path: /deploy/helm/releases/0/valuesFiles/- - value: ci/values-sidecar-mtls.yaml + value: ci/values-sidecar.yaml + - op: add + path: /deploy/helm/releases/0/setValues + value: + server.disableTls: "false" From b21a98dc186ed4c58d91eddbcfa95e3a0c799595 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Tue, 7 Jul 2026 09:07:01 -0700 Subject: [PATCH 12/24] fix(supervisor): avoid similar iptables helper names Signed-off-by: Taylor Mutch --- .../src/netns/mod.rs | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/crates/openshell-supervisor-process/src/netns/mod.rs b/crates/openshell-supervisor-process/src/netns/mod.rs index 472cacd03a..837d7030bb 100644 --- a/crates/openshell-supervisor-process/src/netns/mod.rs +++ b/crates/openshell-supervisor-process/src/netns/mod.rs @@ -508,37 +508,37 @@ fn install_sidecar_nft_bypass_rules(proxy_uid: u32) -> Result<()> { const SIDECAR_IPTABLES_CHAIN: &str = "OPENSHELL_SIDECAR_BYPASS"; fn install_sidecar_iptables_legacy_bypass_rules(proxy_uid: u32) -> Result<()> { - let iptables_cmd = find_iptables_legacy().ok_or_else(|| { + let ipv4_filter_tool = find_iptables_legacy().ok_or_else(|| { miette::miette!( "trusted iptables-legacy helper not found; sidecar network enforcement fallback unavailable" ) })?; - let ip6tables_cmd = find_ip6tables_legacy().ok_or_else(|| { + let ipv6_fence_tool = find_ip6tables_legacy().ok_or_else(|| { miette::miette!( "trusted ip6tables-legacy helper not found; sidecar network enforcement fallback cannot fence IPv6" ) })?; - cleanup_sidecar_iptables_legacy_rules(&iptables_cmd); - cleanup_sidecar_iptables_legacy_rules(&ip6tables_cmd); + cleanup_sidecar_iptables_legacy_rules(&ipv4_filter_tool); + cleanup_sidecar_iptables_legacy_rules(&ipv6_fence_tool); if let Err(e) = install_sidecar_iptables_legacy_family_rules( - &iptables_cmd, + &ipv4_filter_tool, proxy_uid, "icmp-port-unreachable", ) { - cleanup_sidecar_iptables_legacy_rules(&iptables_cmd); - cleanup_sidecar_iptables_legacy_rules(&ip6tables_cmd); + cleanup_sidecar_iptables_legacy_rules(&ipv4_filter_tool); + cleanup_sidecar_iptables_legacy_rules(&ipv6_fence_tool); return Err(e); } if let Err(e) = install_sidecar_iptables_legacy_family_rules( - &ip6tables_cmd, + &ipv6_fence_tool, proxy_uid, "icmp6-port-unreachable", ) { - cleanup_sidecar_iptables_legacy_rules(&iptables_cmd); - cleanup_sidecar_iptables_legacy_rules(&ip6tables_cmd); + cleanup_sidecar_iptables_legacy_rules(&ipv4_filter_tool); + cleanup_sidecar_iptables_legacy_rules(&ipv6_fence_tool); return Err(e); } From 969e97bff14ee78c21f502342d32ac4db54c13ae Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Tue, 7 Jul 2026 09:44:26 -0700 Subject: [PATCH 13/24] fix(e2e): harden kube gateway wrapper setup Signed-off-by: Taylor Mutch --- e2e/with-kube-gateway.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index b4b52e7204..e4f47008c9 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -394,7 +394,7 @@ require_cmd() { } configure_fixture_container_engine() { - [ -n "${CONTAINER_ENGINE:-}" ] || return + [ -n "${CONTAINER_ENGINE:-}" ] || return 0 local selected_engine selected_engine="$(printf '%s' "${CONTAINER_ENGINE}" | tr '[:upper:]' '[:lower:]')" case "${selected_engine}" in @@ -518,7 +518,7 @@ if [ -z "${HOST_GATEWAY_IP}" ] \ # is unreachable for the typical test-host listener (0.0.0.0 bind). detected="$(docker network inspect "${net}" \ -f '{{range .IPAM.Config}}{{.Gateway}}{{"\n"}}{{end}}' 2>/dev/null \ - | awk '/^[0-9.]+$/ { print; exit }')" + | awk '/^[0-9.]+$/ { print; exit }' || true)" if [ -n "${detected}" ]; then HOST_GATEWAY_IP="${detected}" echo "Detected host gateway IP ${HOST_GATEWAY_IP} from docker network '${net}'." From 568118787170bf3158fd5c6573f0b711e12eee2e Mon Sep 17 00:00:00 2001 From: Seth Jennings Date: Tue, 7 Jul 2026 17:58:53 -0700 Subject: [PATCH 14/24] fix(supervisor): avoid nft batch rollback on OCP Run nftables setup as individual commands so optional conntrack and log expressions can fail without rolling back required table, chain, and reject rules. Signed-off-by: Seth Jennings Signed-off-by: Taylor Mutch --- architecture/compute-runtimes.md | 3 + .../src/netns/mod.rs | 173 +++--- .../src/netns/nft_ruleset.rs | 554 ++++++++++++++---- docs/kubernetes/openshift.mdx | 5 + 4 files changed, 525 insertions(+), 210 deletions(-) diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 8884b7d341..a2d5e21ee3 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -102,6 +102,9 @@ updated provider env without giving the process leaf gateway access. In sidecar mode, an init container performs the privileged pod-network nftables setup with `NET_ADMIN` and hands shared state ownership to the configured proxy UID; the long-running network sidecar runs as that UID and does not keep `NET_ADMIN`. +The init path applies nftables as individual commands so optional conntrack and +log expressions can fail without rolling back the required table, chain, and +reject rules. The agent container runs as the resolved sandbox UID/GID with no added Linux capabilities. Sidecar mode preserves gateway session and SSH behavior, but treats the process leaf as network-only: Landlock filesystem policy, process diff --git a/crates/openshell-supervisor-process/src/netns/mod.rs b/crates/openshell-supervisor-process/src/netns/mod.rs index 837d7030bb..c9e1a46eb9 100644 --- a/crates/openshell-supervisor-process/src/netns/mod.rs +++ b/crates/openshell-supervisor-process/src/netns/mod.rs @@ -285,43 +285,22 @@ impl NetworkNamespace { // monitor can see log entries from the sandbox namespace. enable_nf_log_all_netns(); - // Try combined ruleset with log rules first. Log rules must appear - // before reject rules in the chain so packets are logged before being - // rejected. If the kernel lacks nft_log support, fall back to the - // reject-only ruleset. - let ruleset_with_log = - nft_ruleset::generate_bypass_ruleset(&host_ip_str, proxy_port, Some(&log_prefix)); - - if let Err(e) = run_nft_netns(&self.name, &nft_path, &ruleset_with_log) { + let commands = + nft_ruleset::generate_bypass_commands(&host_ip_str, proxy_port, Some(&log_prefix)); + + if let Err(e) = run_nft_commands_netns(&self.name, &nft_path, &commands) { openshell_ocsf::ocsf_emit!( openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(openshell_ocsf::SeverityId::Low) + .severity(openshell_ocsf::SeverityId::Medium) .status(openshell_ocsf::StatusId::Failure) - .state(openshell_ocsf::StateId::Other, "degraded") + .state(openshell_ocsf::StateId::Disabled, "failed") .message(format!( - "Failed to install bypass log rules (non-fatal), falling back to reject-only [ns:{}]: {e}", + "Failed to install bypass detection rules [ns:{}]: {e}", self.name )) .build() ); - - let ruleset_no_log = - nft_ruleset::generate_bypass_ruleset(&host_ip_str, proxy_port, None); - - if let Err(e) = run_nft_netns(&self.name, &nft_path, &ruleset_no_log) { - openshell_ocsf::ocsf_emit!( - openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(openshell_ocsf::SeverityId::Medium) - .status(openshell_ocsf::StatusId::Failure) - .state(openshell_ocsf::StateId::Disabled, "failed") - .message(format!( - "Failed to install bypass detection rules [ns:{}]: {e}", - self.name - )) - .build() - ); - return Err(e); - } + return Err(e); } openshell_ocsf::ocsf_emit!( @@ -501,8 +480,8 @@ fn install_sidecar_nft_bypass_rules(proxy_uid: u32) -> Result<()> { ) })?; let log_prefix = Some("openshell:sidecar-bypass:"); - let ruleset = nft_ruleset::generate_sidecar_bypass_ruleset(proxy_uid, log_prefix); - run_nft_current_namespace(&nft_cmd, &ruleset) + let commands = nft_ruleset::generate_sidecar_bypass_commands(proxy_uid, log_prefix); + run_nft_commands_current_namespace(&nft_cmd, &commands) } const SIDECAR_IPTABLES_CHAIN: &str = "OPENSHELL_SIDECAR_BYPASS"; @@ -664,36 +643,42 @@ fn run_iptables_legacy_current_namespace(iptables_cmd: &str, args: &[&str]) -> R Ok(()) } -fn run_nft_current_namespace(nft_cmd: &str, ruleset: &str) -> Result<()> { - use std::io::Write; - let mut tmp = tempfile::Builder::new() - .prefix("openshell-sidecar-nft-") - .suffix(".conf") - .tempfile() - .into_diagnostic()?; - tmp.write_all(ruleset.as_bytes()).into_diagnostic()?; - let ruleset_path = tmp.path().to_string_lossy().to_string(); - - debug!( - command = %format!("{nft_cmd} -f {ruleset_path}"), - "Loading nftables sidecar ruleset" - ); - - let output = Command::new(nft_cmd) - .args(["-f", &ruleset_path]) - .output() - .into_diagnostic()?; - - drop(tmp); - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(miette::miette!( - "sidecar nft ruleset load failed: {}", - stderr.trim() - )); +/// Run a sequence of nft commands in the current network namespace. +/// +/// Each command is executed as a separate `nft` invocation to avoid atomic +/// batch rollback (where one unsupported expression like `ct state` or `log` +/// causes the entire transaction, including table creation, to fail). +/// +/// Commands marked as non-required are allowed to fail with a warning. +/// Required commands that fail abort the sequence immediately. +fn run_nft_commands_current_namespace( + nft_cmd: &str, + commands: &[nft_ruleset::NftCommand], +) -> Result<()> { + for cmd in commands { + let args_str = cmd.args.join(" "); + debug!(command = %format!("{nft_cmd} {args_str}"), "Running nft command"); + + let output = Command::new(nft_cmd) + .args(&cmd.args) + .output() + .into_diagnostic()?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + if cmd.required { + return Err(miette::miette!( + "{nft_cmd} {args_str} failed: {}", + stderr.trim() + )); + } + warn!( + command = %args_str, + error = %stderr.trim(), + "non-required nft command failed (continuing)" + ); + } } - Ok(()) } @@ -739,47 +724,51 @@ fn run_ip_netns(netns: &str, args: &[&str]) -> Result<()> { Ok(()) } -/// Load an nftables ruleset inside a network namespace via `nsenter --net=`. +/// Run a sequence of nft commands inside a network namespace via `nsenter --net=`. /// -/// Writes the ruleset to a temp file and loads it with `nft -f `. -/// A temp file is used instead of piping to stdin (`nft -f -`) because -/// `nft` resolves `-` to `/dev/stdin`, which may not exist in minimal -/// VM guest environments (e.g. virtiofs rootfs without /proc mounted -/// at nft invocation time). -fn run_nft_netns(netns: &str, nft_cmd: &str, ruleset: &str) -> Result<()> { - use std::io::Write; - let mut tmp = tempfile::Builder::new() - .prefix("openshell-nft-") - .suffix(".conf") - .tempfile() - .into_diagnostic()?; - tmp.write_all(ruleset.as_bytes()).into_diagnostic()?; - let ruleset_path = tmp.path().to_string_lossy().to_string(); - +/// Each command is executed as a separate invocation to avoid atomic batch +/// rollback. See [`run_nft_commands_current_namespace`] for rationale. +fn run_nft_commands_netns( + netns: &str, + nft_cmd: &str, + commands: &[nft_ruleset::NftCommand], +) -> Result<()> { let nsenter_path = find_trusted_binary("nsenter", NSENTER_SEARCH_PATHS)?; let ns_path = format!("/var/run/netns/{netns}"); let net_flag = format!("--net={ns_path}"); - debug!( - command = %format!("{nsenter_path} {net_flag} -- {nft_cmd} -f {ruleset_path}"), - "Loading nftables ruleset in namespace" - ); + for cmd in commands { + let args_str = cmd.args.join(" "); + debug!( + command = %format!("{nsenter_path} {net_flag} -- {nft_cmd} {args_str}"), + "Running nft command in namespace" + ); - let output = Command::new(nsenter_path) - .args([net_flag.as_str(), "--", nft_cmd, "-f", &ruleset_path]) - .output() - .into_diagnostic()?; + let mut full_args = vec![net_flag.as_str(), "--", nft_cmd]; + let arg_refs: Vec<&str> = cmd.args.iter().map(String::as_str).collect(); + full_args.extend(&arg_refs); - drop(tmp); + let output = Command::new(nsenter_path) + .args(&full_args) + .output() + .into_diagnostic()?; - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(miette::miette!( - "nft ruleset load failed in netns {netns}: {}", - stderr.trim() - )); + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + if cmd.required { + return Err(miette::miette!( + "nft {args_str} failed in netns {netns}: {}", + stderr.trim() + )); + } + warn!( + command = %args_str, + error = %stderr.trim(), + netns = %netns, + "non-required nft command failed in namespace (continuing)" + ); + } } - Ok(()) } diff --git a/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs b/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs index 378f715c92..25b4549ab5 100644 --- a/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs +++ b/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs @@ -6,54 +6,202 @@ //! This module provides pure functions to generate nftables rulesets that enforce //! the sandbox network policy: all traffic must go through the proxy, with bypass //! attempts logged and rejected. +//! +//! Rulesets are returned as a sequence of individual nft commands rather than a +//! monolithic file. Running each command as a separate `nft` invocation avoids +//! `nft -f` atomic batch semantics, where a single unsupported expression (e.g. +//! `ct state` without `nf_conntrack`, `log` without `nf_log`) rolls back the +//! entire transaction including table/chain creation. + +/// A single nft command with metadata about whether it is required. +pub struct NftCommand { + /// The nft command arguments (e.g. `["add", "table", "inet", "openshell_bypass"]`). + pub args: Vec, + /// When false, failure of this command is non-fatal; the caller should + /// log a warning and continue with the remaining commands. + pub required: bool, +} -/// Generate a complete nftables ruleset for sandbox network bypass enforcement. +/// Generate nft commands for sandbox network bypass enforcement. /// /// Creates an `inet` family table (handles both IPv4 and IPv6) with rules that: /// 1. Accept traffic to the proxy (IPv4 only) /// 2. Accept loopback traffic -/// 3. Accept established/related connections +/// 3. Accept established/related connections (optional; requires `nf_conntrack`) /// 4. Reject TCP and UDP bypass attempts (both IPv4 and IPv6) /// /// If `log_prefix` is provided, log rules are inserted before each reject rule /// so that bypass attempts are recorded in the kernel ring buffer before being -/// rejected. The `log` expression requires kernel `nft_log` module support; -/// pass `None` for `log_prefix` as a fallback when that module is unavailable. -pub fn generate_bypass_ruleset(host_ip: &str, proxy_port: u16, log_prefix: Option<&str>) -> String { - let log_tcp = log_prefix - .map(|p| { - format!( - "\n tcp flags syn limit rate 5/second burst 10 packets log prefix \"{p}\" flags skuid" - ) - }) - .unwrap_or_default(); - let log_udp = log_prefix - .map(|p| { - format!( - "\n meta l4proto udp limit rate 5/second burst 10 packets log prefix \"{p}\" flags skuid" - ) - }) - .unwrap_or_default(); - - format!( - r#"table inet openshell_bypass {{ - chain output {{ - type filter hook output priority 0; policy accept; - - ip daddr {host_ip} tcp dport {proxy_port} accept - oifname "lo" accept - ct state established,related accept{log_tcp} - meta nfproto ipv4 meta l4proto tcp reject with icmp type port-unreachable - meta nfproto ipv6 meta l4proto tcp reject with icmpv6 type port-unreachable{log_udp} - meta nfproto ipv4 meta l4proto udp reject with icmp type port-unreachable - meta nfproto ipv6 meta l4proto udp reject with icmpv6 type port-unreachable - }} -}} -"# - ) +/// rejected. Log rules are always non-required since they need `nf_log` support. +pub fn generate_bypass_commands( + host_ip: &str, + proxy_port: u16, + log_prefix: Option<&str>, +) -> Vec { + let table = "openshell_bypass"; + let mut cmds = vec![ + nft_cmd(true, &["add", "table", "inet", table]), + nft_cmd(true, &["flush", "table", "inet", table]), + nft_cmd( + true, + &[ + "add", + "chain", + "inet", + table, + "output", + "{ type filter hook output priority 0; policy accept; }", + ], + ), + nft_cmd( + true, + &[ + "add", + "rule", + "inet", + table, + "output", + "ip", + "daddr", + host_ip, + "tcp", + "dport", + &proxy_port.to_string(), + "accept", + ], + ), + nft_cmd( + true, + &[ + "add", "rule", "inet", table, "output", "oifname", "lo", "accept", + ], + ), + nft_cmd( + false, + &[ + "add", + "rule", + "inet", + table, + "output", + "ct", + "state", + "established,related", + "accept", + ], + ), + ]; + + if let Some(prefix) = log_prefix { + cmds.push(nft_cmd( + false, + &[ + "add", "rule", "inet", table, "output", "tcp", "flags", "syn", "limit", "rate", + "5/second", "burst", "10", "packets", "log", "prefix", prefix, "flags", "skuid", + ], + )); + } + + cmds.push(nft_cmd( + true, + &[ + "add", + "rule", + "inet", + table, + "output", + "meta", + "nfproto", + "ipv4", + "meta", + "l4proto", + "tcp", + "reject", + "with", + "icmp", + "type", + "port-unreachable", + ], + )); + cmds.push(nft_cmd( + true, + &[ + "add", + "rule", + "inet", + table, + "output", + "meta", + "nfproto", + "ipv6", + "meta", + "l4proto", + "tcp", + "reject", + "with", + "icmpv6", + "type", + "port-unreachable", + ], + )); + + if let Some(prefix) = log_prefix { + cmds.push(nft_cmd( + false, + &[ + "add", "rule", "inet", table, "output", "meta", "l4proto", "udp", "limit", "rate", + "5/second", "burst", "10", "packets", "log", "prefix", prefix, "flags", "skuid", + ], + )); + } + + cmds.push(nft_cmd( + true, + &[ + "add", + "rule", + "inet", + table, + "output", + "meta", + "nfproto", + "ipv4", + "meta", + "l4proto", + "udp", + "reject", + "with", + "icmp", + "type", + "port-unreachable", + ], + )); + cmds.push(nft_cmd( + true, + &[ + "add", + "rule", + "inet", + table, + "output", + "meta", + "nfproto", + "ipv6", + "meta", + "l4proto", + "udp", + "reject", + "with", + "icmpv6", + "type", + "port-unreachable", + ], + )); + + cmds } -/// Generate a pod-network ruleset for Kubernetes sidecar enforcement. +/// Generate nft commands for Kubernetes sidecar enforcement. /// /// The network sidecar and the process supervisor share a pod network /// namespace. The sidecar runs as `proxy_uid` and owns external egress; @@ -61,72 +209,214 @@ pub fn generate_bypass_ruleset(host_ip: &str, proxy_port: u16, log_prefix: Optio /// (gateway forward and HTTP CONNECT proxy). The generated fence rejects /// TCP/UDP bypass attempts from non-proxy UIDs; other L4 protocols are outside /// the sidecar policy fence. -pub fn generate_sidecar_bypass_ruleset(proxy_uid: u32, log_prefix: Option<&str>) -> String { - let log_tcp = log_prefix - .map(|p| { - format!( - "\n tcp flags syn limit rate 5/second burst 10 packets log prefix \"{p}\" flags skuid" - ) - }) - .unwrap_or_default(); - let log_udp = log_prefix - .map(|p| { - format!( - "\n meta l4proto udp limit rate 5/second burst 10 packets log prefix \"{p}\" flags skuid" - ) - }) - .unwrap_or_default(); - - format!( - r#"table inet openshell_sidecar_bypass {{ - chain output {{ - type filter hook output priority 0; policy accept; - - oifname "lo" accept - ct state established,related accept - meta skuid {proxy_uid} accept{log_tcp} - meta nfproto ipv4 meta l4proto tcp reject with icmp type port-unreachable - meta nfproto ipv6 meta l4proto tcp reject with icmpv6 type port-unreachable{log_udp} - meta nfproto ipv4 meta l4proto udp reject with icmp type port-unreachable - meta nfproto ipv6 meta l4proto udp reject with icmpv6 type port-unreachable - }} -}} -"# - ) +pub fn generate_sidecar_bypass_commands( + proxy_uid: u32, + log_prefix: Option<&str>, +) -> Vec { + let table = "openshell_sidecar_bypass"; + let uid_str = proxy_uid.to_string(); + let mut cmds = vec![ + nft_cmd(true, &["add", "table", "inet", table]), + nft_cmd(true, &["flush", "table", "inet", table]), + nft_cmd( + true, + &[ + "add", + "chain", + "inet", + table, + "output", + "{ type filter hook output priority 0; policy accept; }", + ], + ), + nft_cmd( + true, + &[ + "add", "rule", "inet", table, "output", "oifname", "lo", "accept", + ], + ), + nft_cmd( + false, + &[ + "add", + "rule", + "inet", + table, + "output", + "ct", + "state", + "established,related", + "accept", + ], + ), + nft_cmd( + true, + &[ + "add", "rule", "inet", table, "output", "meta", "skuid", &uid_str, "accept", + ], + ), + ]; + + if let Some(prefix) = log_prefix { + cmds.push(nft_cmd( + false, + &[ + "add", "rule", "inet", table, "output", "tcp", "flags", "syn", "limit", "rate", + "5/second", "burst", "10", "packets", "log", "prefix", prefix, "flags", "skuid", + ], + )); + } + + cmds.push(nft_cmd( + true, + &[ + "add", + "rule", + "inet", + table, + "output", + "meta", + "nfproto", + "ipv4", + "meta", + "l4proto", + "tcp", + "reject", + "with", + "icmp", + "type", + "port-unreachable", + ], + )); + cmds.push(nft_cmd( + true, + &[ + "add", + "rule", + "inet", + table, + "output", + "meta", + "nfproto", + "ipv6", + "meta", + "l4proto", + "tcp", + "reject", + "with", + "icmpv6", + "type", + "port-unreachable", + ], + )); + + if let Some(prefix) = log_prefix { + cmds.push(nft_cmd( + false, + &[ + "add", "rule", "inet", table, "output", "meta", "l4proto", "udp", "limit", "rate", + "5/second", "burst", "10", "packets", "log", "prefix", prefix, "flags", "skuid", + ], + )); + } + + cmds.push(nft_cmd( + true, + &[ + "add", + "rule", + "inet", + table, + "output", + "meta", + "nfproto", + "ipv4", + "meta", + "l4proto", + "udp", + "reject", + "with", + "icmp", + "type", + "port-unreachable", + ], + )); + cmds.push(nft_cmd( + true, + &[ + "add", + "rule", + "inet", + table, + "output", + "meta", + "nfproto", + "ipv6", + "meta", + "l4proto", + "udp", + "reject", + "with", + "icmpv6", + "type", + "port-unreachable", + ], + )); + + cmds +} + +fn nft_cmd(required: bool, args: &[&str]) -> NftCommand { + NftCommand { + args: args.iter().map(|s| (*s).to_string()).collect(), + required, + } } #[cfg(test)] mod tests { use super::*; + fn cmd_str(cmd: &NftCommand) -> String { + cmd.args.join(" ") + } + + fn all_strs(cmds: &[NftCommand]) -> String { + cmds.iter().map(cmd_str).collect::>().join("\n") + } + #[test] - fn generates_bypass_ruleset_with_proxy_rule() { - let ruleset = generate_bypass_ruleset("10.0.2.2", 8080, None); - assert!(ruleset.contains("table inet openshell_bypass")); - assert!(ruleset.contains("chain output")); - assert!(ruleset.contains("ip daddr 10.0.2.2 tcp dport 8080 accept")); + fn generates_bypass_commands_with_proxy_rule() { + let cmds = generate_bypass_commands("10.0.2.2", 8080, None); + let text = all_strs(&cmds); + assert!(text.contains("add table inet openshell_bypass")); + assert!(text.contains("add chain inet openshell_bypass output")); + assert!(text.contains("ip daddr 10.0.2.2 tcp dport 8080 accept")); } #[test] - fn ruleset_has_inet_family_table_and_output_chain() { - let ruleset = generate_bypass_ruleset("192.168.1.1", 3128, None); - assert!(ruleset.contains("table inet openshell_bypass")); - assert!(ruleset.contains("type filter hook output priority 0; policy accept;")); + fn bypass_commands_have_table_and_chain() { + let cmds = generate_bypass_commands("192.168.1.1", 3128, None); + let text = all_strs(&cmds); + assert!(text.contains("add table inet openshell_bypass")); + assert!(text.contains("type filter hook output priority 0; policy accept;")); } #[test] fn proxy_accept_rule_uses_provided_ip_and_port() { - let ruleset = generate_bypass_ruleset("172.16.0.1", 9999, None); - assert!(ruleset.contains("ip daddr 172.16.0.1 tcp dport 9999 accept")); + let cmds = generate_bypass_commands("172.16.0.1", 9999, None); + let text = all_strs(&cmds); + assert!(text.contains("ip daddr 172.16.0.1 tcp dport 9999 accept")); } #[test] fn rules_are_ordered_accept_then_reject() { - let ruleset = generate_bypass_ruleset("10.0.2.2", 8080, None); - let proxy_pos = ruleset.find("ip daddr").unwrap(); - let lo_pos = ruleset.find("oifname \"lo\"").unwrap(); - let ct_pos = ruleset.find("ct state established,related").unwrap(); - let reject_pos = ruleset.find("reject with icmp type").unwrap(); + let cmds = generate_bypass_commands("10.0.2.2", 8080, None); + let text = all_strs(&cmds); + let proxy_pos = text.find("ip daddr").unwrap(); + let lo_pos = text.find("oifname lo").unwrap(); + let ct_pos = text.find("ct state established,related").unwrap(); + let reject_pos = text.find("reject with icmp type").unwrap(); assert!(proxy_pos < lo_pos); assert!(lo_pos < ct_pos); @@ -135,11 +425,12 @@ mod tests { #[test] fn both_ipv4_and_ipv6_reject_types_are_present() { - let ruleset = generate_bypass_ruleset("10.0.2.2", 8080, None); - let icmp_count = ruleset + let cmds = generate_bypass_commands("10.0.2.2", 8080, None); + let text = all_strs(&cmds); + let icmp_count = text .matches("reject with icmp type port-unreachable") .count(); - let icmpv6_count = ruleset + let icmpv6_count = text .matches("reject with icmpv6 type port-unreachable") .count(); assert_eq!(icmp_count, 2, "need IPv4 ICMP rejects for TCP + UDP"); @@ -147,34 +438,35 @@ mod tests { } #[test] - fn no_log_ruleset_omits_log_rules() { - let ruleset = generate_bypass_ruleset("10.0.2.2", 8080, None); + fn no_log_commands_omit_log_rules() { + let cmds = generate_bypass_commands("10.0.2.2", 8080, None); + let text = all_strs(&cmds); assert!( - !ruleset.contains("log prefix"), - "no-log ruleset must not contain log rules" + !text.contains("log prefix"), + "no-log commands must not contain log rules" ); } #[test] - fn log_ruleset_contains_prefix_for_tcp_and_udp() { - let ruleset = generate_bypass_ruleset("10.0.2.2", 8080, Some("openshell:bypass:test:")); - let count = ruleset - .matches("log prefix \"openshell:bypass:test:\"") - .count(); + fn log_commands_contain_prefix_for_tcp_and_udp() { + let cmds = generate_bypass_commands("10.0.2.2", 8080, Some("openshell:bypass:test:")); + let text = all_strs(&cmds); + let count = text.matches("log prefix openshell:bypass:test:").count(); assert_eq!(count, 2, "need log rules for both TCP and UDP"); - assert!(ruleset.contains("tcp flags syn limit rate 5/second burst 10 packets")); - assert!(ruleset.contains("meta l4proto udp limit rate 5/second burst 10 packets")); + assert!(text.contains("tcp flags syn limit rate 5/second burst 10 packets")); + assert!(text.contains("meta l4proto udp limit rate 5/second burst 10 packets")); } #[test] fn log_rules_appear_before_reject_rules() { - let ruleset = generate_bypass_ruleset("10.0.2.2", 8080, Some("openshell:bypass:test:")); - let tcp_log_pos = ruleset.find("tcp flags syn").unwrap(); - let tcp_reject_pos = ruleset + let cmds = generate_bypass_commands("10.0.2.2", 8080, Some("openshell:bypass:test:")); + let text = all_strs(&cmds); + let tcp_log_pos = text.find("tcp flags syn").unwrap(); + let tcp_reject_pos = text .find("meta nfproto ipv4 meta l4proto tcp reject") .unwrap(); - let udp_log_pos = ruleset.find("meta l4proto udp limit rate").unwrap(); - let udp_reject_pos = ruleset + let udp_log_pos = text.find("meta l4proto udp limit rate").unwrap(); + let udp_reject_pos = text .find("meta nfproto ipv4 meta l4proto udp reject") .unwrap(); @@ -189,24 +481,50 @@ mod tests { } #[test] - fn sidecar_ruleset_allows_supervisor_uid_and_loopback() { - let ruleset = generate_sidecar_bypass_ruleset(1337, None); - assert!(ruleset.contains("table inet openshell_sidecar_bypass")); - assert!(ruleset.contains("oifname \"lo\" accept")); - assert!(ruleset.contains("meta skuid 1337 accept")); + fn ct_state_rule_is_not_required() { + let cmds = generate_bypass_commands("10.0.2.2", 8080, None); + let ct_cmd = cmds + .iter() + .find(|c| cmd_str(c).contains("ct state")) + .unwrap(); + assert!( + !ct_cmd.required, + "ct state rule should be non-required (needs nf_conntrack)" + ); + } + + #[test] + fn log_rules_are_not_required() { + let cmds = generate_bypass_commands("10.0.2.2", 8080, Some("openshell:bypass:test:")); + for cmd in &cmds { + if cmd_str(cmd).contains("log prefix") { + assert!( + !cmd.required, + "log rules should be non-required (needs nf_log)" + ); + } + } + } + + #[test] + fn sidecar_commands_allow_supervisor_uid_and_loopback() { + let cmds = generate_sidecar_bypass_commands(1337, None); + let text = all_strs(&cmds); + assert!(text.contains("add table inet openshell_sidecar_bypass")); + assert!(text.contains("oifname lo accept")); + assert!(text.contains("meta skuid 1337 accept")); } #[test] - fn sidecar_ruleset_rejects_tcp_and_udp_egress() { - let ruleset = generate_sidecar_bypass_ruleset(0, Some("openshell:sidecar:test:")); - assert!(ruleset.contains("meta nfproto ipv4 meta l4proto tcp reject")); - assert!(ruleset.contains("meta nfproto ipv6 meta l4proto tcp reject")); - assert!(ruleset.contains("meta nfproto ipv4 meta l4proto udp reject")); - assert!(ruleset.contains("meta nfproto ipv6 meta l4proto udp reject")); + fn sidecar_commands_reject_tcp_and_udp_egress() { + let cmds = generate_sidecar_bypass_commands(0, Some("openshell:sidecar:test:")); + let text = all_strs(&cmds); + assert!(text.contains("meta nfproto ipv4 meta l4proto tcp reject")); + assert!(text.contains("meta nfproto ipv6 meta l4proto tcp reject")); + assert!(text.contains("meta nfproto ipv4 meta l4proto udp reject")); + assert!(text.contains("meta nfproto ipv6 meta l4proto udp reject")); assert_eq!( - ruleset - .matches("log prefix \"openshell:sidecar:test:\"") - .count(), + text.matches("log prefix openshell:sidecar:test:").count(), 2 ); } diff --git a/docs/kubernetes/openshift.mdx b/docs/kubernetes/openshift.mdx index caf799b51f..2d4c2fc30a 100644 --- a/docs/kubernetes/openshift.mdx +++ b/docs/kubernetes/openshift.mdx @@ -14,6 +14,11 @@ The OpenShift install path is experimental. It currently requires running sandbo OpenShift's [Security Context Constraints](https://docs.openshift.com/container-platform/latest/authentication/managing-security-context-constraints.html) reject the chart's default pod security settings. Installing on OpenShift requires precreating the namespace, granting the `privileged` SCC to the sandbox service account, and overriding a few chart values so the cluster admission controller can assign UIDs and FS groups itself. +OpenShell installs sandbox nftables rules as individual commands. On OpenShift +nodes where optional conntrack or packet log expressions are unavailable, those +optional rules can fail without rolling back the required proxy bypass reject +rules. + ## Prerequisites - OpenShift 4.x cluster with `oc` configured From cd0477fb23b26ca0352db3f86b1d096467da7c70 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Wed, 8 Jul 2026 12:12:35 -0700 Subject: [PATCH 15/24] fix(kubernetes): preserve process identity in sidecar topology Render sidecar pods with a shared process namespace, keep binary-aware network policy enabled, and move Kubernetes sidecar settings under the nested sidecar config table. Also apply unprivileged Landlock/seccomp setup in NetworkOnly supervisor mode so sidecar topology keeps sandbox child hardening without privileged process setup. Signed-off-by: Taylor Mutch --- architecture/compute-runtimes.md | 8 +- crates/openshell-driver-kubernetes/README.md | 20 +-- .../openshell-driver-kubernetes/src/config.rs | 114 +++++++++++++----- .../openshell-driver-kubernetes/src/driver.rs | 18 +-- crates/openshell-driver-kubernetes/src/lib.rs | 4 +- .../openshell-driver-kubernetes/src/main.rs | 23 ++-- .../src/process.rs | 65 +++++++--- .../openshell-supervisor-process/src/run.rs | 6 +- .../src/sandbox/linux/landlock.rs | 80 +++++++++++- .../src/sandbox/linux/mod.rs | 17 ++- .../openshell-supervisor-process/src/ssh.rs | 40 +++--- deploy/helm/openshell/README.md | 2 +- .../openshell/templates/gateway-config.yaml | 6 +- .../openshell/tests/gateway_config_test.yaml | 13 +- deploy/helm/openshell/values.yaml | 7 +- docs/kubernetes/setup.mdx | 2 +- docs/kubernetes/topology.mdx | 40 +++--- docs/reference/gateway-config.mdx | 10 +- docs/reference/sandbox-compute-drivers.mdx | 14 ++- 19 files changed, 338 insertions(+), 151 deletions(-) diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index a2d5e21ee3..71a31328e0 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -107,8 +107,12 @@ log expressions can fail without rolling back the required table, chain, and reject rules. The agent container runs as the resolved sandbox UID/GID with no added Linux capabilities. Sidecar mode preserves gateway session and SSH behavior, but -treats the process leaf as network-only: Landlock filesystem policy, process -privilege dropping, and process/binary identity checks are not applied there. +treats the process leaf as network-only: Landlock filesystem policy and child +seccomp still apply where supported, while process privilege dropping and +supervisor identity mount isolation do not run because the agent container is +already unprivileged. Sidecar pods use a shared process namespace so the +network sidecar can resolve workload process and binary identity through +`/proc/`. ## Images diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index 8736e62a46..e9efc2867b 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -62,15 +62,17 @@ container and runs the long-lived network sidecar as a non-root UID with no added Linux capabilities. The agent container also runs as the resolved sandbox UID/GID with `allowPrivilegeEscalation: false` and `capabilities.drop: ["ALL"]`. In this mode OpenShell preserves gateway session and SSH behavior, but the -process supervisor runs in network-only mode and does not apply Landlock -filesystem policy, process privilege dropping, or process/binary identity -checks. Network endpoint and L7 policy remain enforced by the network sidecar. - -Sidecar mode uses the pod `fsGroup` to make the projected service-account token -and sandbox client TLS secret group-readable so the non-root process supervisor -can authenticate to the gateway. Treat the agent container as trusted with -respect to those in-pod gateway credentials until a narrower credential handoff -exists. +process supervisor does not perform root-to-sandbox privilege dropping or +supervisor identity mount isolation. It still applies Landlock filesystem policy +and child seccomp filters where the kernel/runtime supports them. Network +endpoint and L7 policy remain enforced by the network sidecar, and +sidecar pods use a shared process namespace so the network sidecar can resolve +process/binary identity through `/proc/`. + +Sidecar mode keeps gateway credentials in the network sidecar. The agent +container does not mount the projected service-account token used for sandbox +token bootstrap, does not mount the sandbox client TLS secret, and does not get +gateway callback environment variables. The driver can request a Kubernetes AppArmor profile through `app_armor_profile`. diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index e33f156044..f07e4fb9a9 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -88,6 +88,35 @@ impl FromStr for SupervisorTopology { } } +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct KubernetesSidecarConfig { + /// UID used by the long-running network sidecar in `sidecar` topology. + /// The network init container installs nftables rules that exempt this + /// UID, so it must not match the sandbox workload UID. + pub proxy_uid: u32, +} + +impl Default for KubernetesSidecarConfig { + fn default() -> Self { + Self { + proxy_uid: DEFAULT_PROXY_UID, + } + } +} + +impl KubernetesSidecarConfig { + pub fn validate_proxy_uid(&self) -> Result<(), String> { + if self.proxy_uid < openshell_policy::MIN_SANDBOX_UID { + return Err(format!( + "sidecar.proxy_uid must be at least {}", + openshell_policy::MIN_SANDBOX_UID + )); + } + Ok(()) + } +} + /// Kubernetes `AppArmor` profile requested for the sandbox agent container. #[derive(Debug, Clone, PartialEq, Eq)] pub enum AppArmorProfile { @@ -213,11 +242,9 @@ pub struct KubernetesComputeConfig { /// How the supervisor binary is delivered into sandbox pods. pub supervisor_sideload_method: SupervisorSideloadMethod, /// How the supervisor is arranged for Kubernetes sandbox pods. - pub supervisor_topology: SupervisorTopology, - /// UID used by the long-running network sidecar in `sidecar` topology. - /// The network init container installs nftables rules that exempt this - /// UID, so it must not match the sandbox workload UID. - pub proxy_uid: u32, + pub topology: SupervisorTopology, + /// Sidecar-only settings used when `topology = "sidecar"`. + pub sidecar: KubernetesSidecarConfig, pub grpc_endpoint: String, pub ssh_socket_path: String, pub client_tls_secret_name: String, @@ -303,8 +330,8 @@ impl Default for KubernetesComputeConfig { supervisor_image: DEFAULT_SUPERVISOR_IMAGE.to_string(), supervisor_image_pull_policy: String::new(), supervisor_sideload_method: SupervisorSideloadMethod::default(), - supervisor_topology: SupervisorTopology::default(), - proxy_uid: DEFAULT_PROXY_UID, + topology: SupervisorTopology::default(), + sidecar: KubernetesSidecarConfig::default(), grpc_endpoint: String::new(), ssh_socket_path: "/run/openshell/ssh.sock".to_string(), client_tls_secret_name: String::new(), @@ -350,13 +377,7 @@ impl KubernetesComputeConfig { } pub fn validate_proxy_uid(&self) -> Result<(), String> { - if self.proxy_uid < openshell_policy::MIN_SANDBOX_UID { - return Err(format!( - "proxy_uid must be at least {}", - openshell_policy::MIN_SANDBOX_UID - )); - } - Ok(()) + self.sidecar.validate_proxy_uid() } /// Resolve the sandbox UID/GID pair. @@ -486,50 +507,63 @@ mod tests { } #[test] - fn default_supervisor_topology_is_combined() { + fn default_topology_is_combined() { let cfg = KubernetesComputeConfig::default(); - assert_eq!(cfg.supervisor_topology, SupervisorTopology::Combined); - assert_eq!(cfg.supervisor_topology.to_string(), "combined"); + assert_eq!(cfg.topology, SupervisorTopology::Combined); + assert_eq!(cfg.topology.to_string(), "combined"); } #[test] fn default_proxy_uid_is_dedicated_non_root_uid() { let cfg = KubernetesComputeConfig::default(); - assert_eq!(cfg.proxy_uid, DEFAULT_PROXY_UID); + assert_eq!(cfg.sidecar.proxy_uid, DEFAULT_PROXY_UID); } #[test] - fn serde_override_supervisor_topology_sidecar() { + fn serde_override_topology_sidecar() { let json = serde_json::json!({ - "supervisor_topology": "sidecar" + "topology": "sidecar" }); let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); - assert_eq!(cfg.supervisor_topology, SupervisorTopology::Sidecar); + assert_eq!(cfg.topology, SupervisorTopology::Sidecar); } #[test] - fn serde_override_supervisor_topology_combined() { + fn serde_override_topology_combined() { let json = serde_json::json!({ - "supervisor_topology": "combined" + "topology": "combined" }); let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); - assert_eq!(cfg.supervisor_topology, SupervisorTopology::Combined); + assert_eq!(cfg.topology, SupervisorTopology::Combined); + } + + #[test] + fn serde_rejects_sidecar_binary_identity_field() { + let json = serde_json::json!({ + "sidecar": { + "binary_identity": "shared-pid" + } + }); + let err = serde_json::from_value::(json).unwrap_err(); + assert!(err.to_string().contains("unknown field")); } #[test] - fn serde_override_proxy_uid() { + fn serde_override_sidecar_proxy_uid_nested() { let json = serde_json::json!({ - "proxy_uid": 2000 + "sidecar": { + "proxy_uid": 2000 + } }); let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); - assert_eq!(cfg.proxy_uid, 2000); + assert_eq!(cfg.sidecar.proxy_uid, 2000); cfg.validate_proxy_uid().unwrap(); } #[test] fn validate_proxy_uid_rejects_privileged_uid() { let cfg = KubernetesComputeConfig { - proxy_uid: 999, + sidecar: KubernetesSidecarConfig { proxy_uid: 999 }, ..KubernetesComputeConfig::default() }; let err = cfg.validate_proxy_uid().unwrap_err(); @@ -537,14 +571,34 @@ mod tests { } #[test] - fn serde_rejects_invalid_supervisor_topology() { + fn serde_rejects_invalid_topology() { let json = serde_json::json!({ - "supervisor_topology": "unsupported" + "topology": "unsupported" }); let err = serde_json::from_value::(json).unwrap_err(); assert!(err.to_string().contains("unknown variant")); } + #[test] + fn serde_rejects_removed_supervisor_topology_field() { + let json = serde_json::json!({ + "supervisor_topology": "sidecar" + }); + let err = serde_json::from_value::(json).unwrap_err(); + assert!(err.to_string().contains("unknown field")); + } + + #[test] + fn serde_rejects_removed_flat_sidecar_fields() { + for json in [ + serde_json::json!({ "sidecar_binary_identity": "shared-pid" }), + serde_json::json!({ "proxy_uid": 2000 }), + ] { + let err = serde_json::from_value::(json).unwrap_err(); + assert!(err.to_string().contains("unknown field")); + } + } + #[test] fn serde_rejects_removed_process_enforcement_field() { let json = serde_json::json!({ diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 8069927f2b..568901255d 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -563,8 +563,8 @@ impl KubernetesComputeDriver { supervisor_image: &self.config.supervisor_image, supervisor_image_pull_policy: &self.config.supervisor_image_pull_policy, supervisor_sideload_method: self.config.supervisor_sideload_method, - supervisor_topology: self.config.supervisor_topology, - proxy_uid: self.config.proxy_uid, + supervisor_topology: self.config.topology, + proxy_uid: self.config.sidecar.proxy_uid, service_account_name: &self.config.service_account_name, sandbox_id: &sandbox.id, sandbox_name: &sandbox.name, @@ -1334,11 +1334,6 @@ fn supervisor_sidecar_env( openshell_core::sandbox_env::NETWORK_ENFORCEMENT_MODE, "sidecar-nftables", ); - upsert_env( - &mut env, - openshell_core::sandbox_env::NETWORK_BINARY_IDENTITY, - "relaxed", - ); upsert_env( &mut env, openshell_core::sandbox_env::SUPERVISOR_READY_FILE, @@ -1488,6 +1483,8 @@ fn apply_supervisor_sidecar_topology( sc.insert("fsGroup".to_string(), serde_json::json!(params.sandbox_gid)); } + spec.insert("shareProcessNamespace".to_string(), serde_json::json!(true)); + apply_supervisor_binary_source( spec, params.supervisor_image, @@ -3124,10 +3121,7 @@ mod tests { ¶ms, ); - assert!( - pod_template["spec"]["shareProcessNamespace"].is_null(), - "sidecar mode no longer needs a shared process namespace when binary identity is relaxed" - ); + assert_eq!(pod_template["spec"]["shareProcessNamespace"], true); assert_eq!(pod_template["spec"]["securityContext"]["fsGroup"], 1500); let containers = pod_template["spec"]["containers"].as_array().unwrap(); assert_eq!(containers.len(), 2); @@ -3250,7 +3244,7 @@ mod tests { sidecar, openshell_core::sandbox_env::NETWORK_BINARY_IDENTITY ), - Some("relaxed") + None ); assert_eq!( rendered_env(sidecar, openshell_core::sandbox_env::ENTRYPOINT_PID_FILE), diff --git a/crates/openshell-driver-kubernetes/src/lib.rs b/crates/openshell-driver-kubernetes/src/lib.rs index 8c326f6af8..7c56c8de5b 100644 --- a/crates/openshell-driver-kubernetes/src/lib.rs +++ b/crates/openshell-driver-kubernetes/src/lib.rs @@ -7,8 +7,8 @@ pub mod grpc; pub use config::{ AppArmorProfile, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, - DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, SupervisorSideloadMethod, - SupervisorTopology, + DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, KubernetesSidecarConfig, + SupervisorSideloadMethod, SupervisorTopology, }; pub use driver::{KubernetesComputeDriver, KubernetesDriverError}; pub use grpc::ComputeDriverService; diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 3271e56a47..8f62087663 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -11,7 +11,8 @@ use openshell_core::VERSION; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; use openshell_driver_kubernetes::{ AppArmorProfile, ComputeDriverService, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, - KubernetesComputeConfig, KubernetesComputeDriver, SupervisorSideloadMethod, SupervisorTopology, + KubernetesComputeConfig, KubernetesComputeDriver, KubernetesSidecarConfig, + SupervisorSideloadMethod, SupervisorTopology, }; #[derive(Parser, Debug)] @@ -82,13 +83,19 @@ struct Args { #[arg( long, - env = "OPENSHELL_SUPERVISOR_TOPOLOGY", + alias = "supervisor-topology", + env = "OPENSHELL_K8S_TOPOLOGY", default_value = "combined" )] - supervisor_topology: SupervisorTopology, + topology: SupervisorTopology, - #[arg(long, env = "OPENSHELL_PROXY_UID", default_value_t = DEFAULT_PROXY_UID)] - proxy_uid: u32, + #[arg( + long = "sidecar-proxy-uid", + alias = "proxy-uid", + env = "OPENSHELL_K8S_SIDECAR_PROXY_UID", + default_value_t = DEFAULT_PROXY_UID + )] + sidecar_proxy_uid: u32, #[arg(long, env = "OPENSHELL_ENABLE_USER_NAMESPACES")] enable_user_namespaces: bool, @@ -133,8 +140,10 @@ async fn main() -> Result<()> { .unwrap_or_else(|| openshell_core::config::DEFAULT_SUPERVISOR_IMAGE.to_string()), supervisor_image_pull_policy: args.supervisor_image_pull_policy.unwrap_or_default(), supervisor_sideload_method: args.supervisor_sideload_method, - supervisor_topology: args.supervisor_topology, - proxy_uid: args.proxy_uid, + topology: args.topology, + sidecar: KubernetesSidecarConfig { + proxy_uid: args.sidecar_proxy_uid, + }, grpc_endpoint: args.grpc_endpoint.unwrap_or_default(), ssh_socket_path: args.sandbox_ssh_socket_path, client_tls_secret_name: args.client_tls_secret_name.unwrap_or_default(), diff --git a/crates/openshell-supervisor-process/src/process.rs b/crates/openshell-supervisor-process/src/process.rs index bd8be04c86..72effa98f8 100644 --- a/crates/openshell-supervisor-process/src/process.rs +++ b/crates/openshell-supervisor-process/src/process.rs @@ -42,9 +42,32 @@ pub enum ProcessEnforcementMode { impl ProcessEnforcementMode { #[must_use] - pub const fn enforces_process_controls(self) -> bool { + pub const fn uses_privileged_process_setup(self) -> bool { matches!(self, Self::Full) } + + #[must_use] + pub const fn enforces_child_sandbox(self) -> bool { + matches!(self, Self::Full | Self::NetworkOnly) + } +} + +#[cfg(target_os = "linux")] +pub(crate) fn prepare_child_sandbox( + policy: &SandboxPolicy, + workdir: Option<&str>, + enforcement_mode: ProcessEnforcementMode, +) -> Result> { + if !enforcement_mode.enforces_child_sandbox() { + return Ok(None); + } + + let prepared = if enforcement_mode.uses_privileged_process_setup() { + sandbox::linux::prepare(policy, workdir) + } else { + sandbox::linux::prepare_current_user(policy, workdir) + }?; + Ok(Some(prepared)) } const SUPERVISOR_ONLY_ENV_VARS: &[&str] = &[ @@ -580,24 +603,20 @@ impl ProcessHandle { // process where the tracing subscriber is functional. The child's // pre_exec context cannot reliably emit structured logs. #[cfg(target_os = "linux")] - if enforcement_mode.enforces_process_controls() { + if enforcement_mode.enforces_child_sandbox() { sandbox::linux::log_sandbox_readiness(policy, workdir); } - // Phase 1 (as root): Prepare Landlock ruleset by opening PathFds. - // This MUST happen before drop_privileges() so that root-only paths - // (e.g. mode 700 directories) can be opened. See issue #803. + // Phase 1: Prepare Landlock ruleset by opening PathFds. + // In full mode this runs before drop_privileges() so root-only paths + // can be opened. In sidecar network-only mode the container already + // runs as the sandbox UID, so inaccessible paths are unavailable to + // the workload and best-effort compatibility skips them. #[cfg(target_os = "linux")] - let prepared_sandbox = if enforcement_mode.enforces_process_controls() { - Some( - sandbox::linux::prepare(policy, workdir) - .map_err(|err| miette::miette!("Failed to prepare sandbox: {err}"))?, - ) - } else { - None - }; + let prepared_sandbox = prepare_child_sandbox(policy, workdir, enforcement_mode) + .map_err(|err| miette::miette!("Failed to prepare sandbox: {err}"))?; #[cfg(target_os = "linux")] - let supervisor_identity_mount = if enforcement_mode.enforces_process_controls() { + let supervisor_identity_mount = if enforcement_mode.uses_privileged_process_setup() { supervisor_identity_mount_from_env().map_err(|err| { miette::miette!("Failed to prepare supervisor identity isolation: {err}") })? @@ -640,7 +659,7 @@ impl ProcessHandle { // Drop privileges. initgroups/setgid/setuid need access to // /etc/group and /etc/passwd which would be blocked if // Landlock were already enforced. - if enforcement_mode.enforces_process_controls() { + if enforcement_mode.uses_privileged_process_setup() { drop_privileges(&policy) .map_err(|err| std::io::Error::other(err.to_string()))?; } @@ -741,14 +760,14 @@ impl ProcessHandle { // Drop privileges before applying sandbox restrictions. // initgroups/setgid/setuid need access to /etc/group and /etc/passwd // which may be blocked by Landlock. - if enforcement_mode.enforces_process_controls() { + if enforcement_mode.uses_privileged_process_setup() { drop_privileges(&policy) .map_err(|err| std::io::Error::other(err.to_string()))?; } harden_child_process().map_err(|err| std::io::Error::other(err.to_string()))?; - if enforcement_mode.enforces_process_controls() { + if enforcement_mode.enforces_child_sandbox() { sandbox::apply(&policy, workdir.as_deref()) .map_err(|err| std::io::Error::other(err.to_string()))?; } @@ -1504,6 +1523,18 @@ mod tests { ); } + #[test] + fn full_enforcement_uses_privileged_setup_and_child_sandbox() { + assert!(ProcessEnforcementMode::Full.uses_privileged_process_setup()); + assert!(ProcessEnforcementMode::Full.enforces_child_sandbox()); + } + + #[test] + fn network_only_enforcement_keeps_child_sandbox_without_privileged_setup() { + assert!(!ProcessEnforcementMode::NetworkOnly.uses_privileged_process_setup()); + assert!(ProcessEnforcementMode::NetworkOnly.enforces_child_sandbox()); + } + #[cfg(target_os = "linux")] fn capability_bounding_set_clear_available() -> bool { capctl::caps::CapState::get_current() diff --git a/crates/openshell-supervisor-process/src/run.rs b/crates/openshell-supervisor-process/src/run.rs index f7e25fd917..37f6ea77c2 100644 --- a/crates/openshell-supervisor-process/src/run.rs +++ b/crates/openshell-supervisor-process/src/run.rs @@ -73,7 +73,7 @@ pub async fn run_process( // /etc/group so the "sandbox" entry matches. Must run before // validate_sandbox_user so passwd lookups see the correct identity. #[cfg(unix)] - if enforcement_mode.enforces_process_controls() { + if enforcement_mode.uses_privileged_process_setup() { crate::process::update_sandbox_passwd_entries()?; } @@ -81,7 +81,7 @@ pub async fn run_process( // must include a "sandbox" user for privilege dropping; failing fast here // beats silently running children as root. #[cfg(unix)] - if enforcement_mode.enforces_process_controls() { + if enforcement_mode.uses_privileged_process_setup() { crate::process::validate_sandbox_user(policy)?; crate::process::validate_sandbox_group(policy)?; } @@ -90,7 +90,7 @@ pub async fn run_process( // sandbox user/group. Runs as the supervisor (root) before the child // is forked so the workload sees writable paths it owns. #[cfg(unix)] - if enforcement_mode.enforces_process_controls() { + if enforcement_mode.uses_privileged_process_setup() { crate::process::prepare_filesystem(policy)?; } diff --git a/crates/openshell-supervisor-process/src/sandbox/linux/landlock.rs b/crates/openshell-supervisor-process/src/sandbox/linux/landlock.rs index 8808a1a87a..222377b5d2 100644 --- a/crates/openshell-supervisor-process/src/sandbox/linux/landlock.rs +++ b/crates/openshell-supervisor-process/src/sandbox/linux/landlock.rs @@ -5,7 +5,7 @@ use landlock::{ ABI, Access, AccessFs, CompatLevel, Compatible, PathBeneath, PathFd, PathFdError, Ruleset, - RulesetAttr, RulesetCreatedAttr, + RulesetAttr, }; use miette::{IntoDiagnostic, Result}; use openshell_core::policy::{LandlockCompatibility, SandboxPolicy}; @@ -95,6 +95,12 @@ pub struct PreparedRuleset { compatibility: LandlockCompatibility, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum PathOpenMode { + Privileged, + CurrentUser, +} + /// Phase 1: Open `PathFds` and build the Landlock ruleset **as root**. /// /// This must run before `drop_privileges()` so that `PathFd::new()` can open @@ -103,6 +109,27 @@ pub struct PreparedRuleset { /// Returns `None` if there are no filesystem paths to restrict (no-op). /// Returns `Some(PreparedRuleset)` on success, or an error. pub fn prepare(policy: &SandboxPolicy, workdir: Option<&str>) -> Result> { + prepare_with_path_open_mode(policy, workdir, PathOpenMode::Privileged) +} + +/// Phase 1 for already-unprivileged workloads. +/// +/// Kubernetes sidecar mode starts the process supervisor as the sandbox UID, so +/// Landlock path FDs are opened as the same UID that will run the workload. +/// Paths this UID cannot open are already unavailable to the workload; omit +/// them from the allowlist and let the resulting ruleset deny everything else. +pub fn prepare_current_user( + policy: &SandboxPolicy, + workdir: Option<&str>, +) -> Result> { + prepare_with_path_open_mode(policy, workdir, PathOpenMode::CurrentUser) +} + +fn prepare_with_path_open_mode( + policy: &SandboxPolicy, + workdir: Option<&str>, + path_open_mode: PathOpenMode, +) -> Result> { let read_only = policy.filesystem.read_only.clone(); let mut read_write = policy.filesystem.read_write.clone(); @@ -188,7 +215,7 @@ pub fn prepare(policy: &SandboxPolicy, workdir: Option<&str>) -> Result) -> Result) -> Result<()> { /// /// In `HardRequirement` mode, any failure is fatal — the caller propagates the /// error, which ultimately aborts sandbox startup. -fn try_open_path(path: &Path, compatibility: &LandlockCompatibility) -> Result> { +fn try_open_path( + path: &Path, + compatibility: &LandlockCompatibility, + path_open_mode: PathOpenMode, +) -> Result> { match PathFd::new(path) { Ok(fd) => Ok(Some(fd)), Err(err) => { @@ -335,6 +366,28 @@ fn try_open_path(path: &Path, compatibility: &LandlockCompatibility) -> Result { // NotFound is expected for stale baseline paths (e.g. @@ -421,6 +474,7 @@ mod tests { let result = try_open_path( &PathBuf::from("/nonexistent/openshell/test/path"), &LandlockCompatibility::BestEffort, + PathOpenMode::Privileged, ); assert!(result.is_ok()); assert!(result.unwrap().is_none()); @@ -431,6 +485,7 @@ mod tests { let result = try_open_path( &PathBuf::from("/nonexistent/openshell/test/path"), &LandlockCompatibility::HardRequirement, + PathOpenMode::Privileged, ); assert!(result.is_err()); let err_msg = result.unwrap_err().to_string(); @@ -447,11 +502,26 @@ mod tests { #[test] fn try_open_path_succeeds_for_existing_path() { let dir = tempfile::tempdir().unwrap(); - let result = try_open_path(dir.path(), &LandlockCompatibility::BestEffort); + let result = try_open_path( + dir.path(), + &LandlockCompatibility::BestEffort, + PathOpenMode::Privileged, + ); assert!(result.is_ok()); assert!(result.unwrap().is_some()); } + #[test] + fn try_open_path_current_user_skips_missing_path_in_hard_requirement() { + let result = try_open_path( + &PathBuf::from("/nonexistent/openshell/test/path"), + &LandlockCompatibility::HardRequirement, + PathOpenMode::CurrentUser, + ); + assert!(result.is_ok()); + assert!(result.unwrap().is_none()); + } + #[test] fn classify_not_found() { let err = std::io::Error::from_raw_os_error(libc::ENOENT); diff --git a/crates/openshell-supervisor-process/src/sandbox/linux/mod.rs b/crates/openshell-supervisor-process/src/sandbox/linux/mod.rs index b5397ef079..107a50e370 100644 --- a/crates/openshell-supervisor-process/src/sandbox/linux/mod.rs +++ b/crates/openshell-supervisor-process/src/sandbox/linux/mod.rs @@ -12,7 +12,7 @@ use std::path::PathBuf; use std::sync::Once; /// Opaque handle to a prepared-but-not-yet-enforced sandbox. -/// Holds the Landlock ruleset with `PathFds` opened as root. +/// Holds the Landlock ruleset with `PathFds` opened before child exec. pub struct PreparedSandbox { landlock: Option, policy: SandboxPolicy, @@ -30,6 +30,21 @@ pub fn prepare(policy: &SandboxPolicy, workdir: Option<&str>) -> Result, +) -> Result { + let landlock = landlock::prepare_current_user(policy, workdir)?; + Ok(PreparedSandbox { + landlock, + policy: policy.clone(), + }) +} + /// Phase 2: Enforce prepared sandbox restrictions (after `drop_privileges`). /// /// Calls `restrict_self()` for Landlock and applies seccomp filters. diff --git a/crates/openshell-supervisor-process/src/ssh.rs b/crates/openshell-supervisor-process/src/ssh.rs index 85466cb14f..82118b8416 100644 --- a/crates/openshell-supervisor-process/src/ssh.rs +++ b/crates/openshell-supervisor-process/src/ssh.rs @@ -65,7 +65,7 @@ fn ssh_server_init( if let Some(parent) = listen_path.parent() { std::fs::create_dir_all(parent).into_diagnostic()?; #[cfg(unix)] - if enforcement_mode.enforces_process_controls() && !shared_socket { + if enforcement_mode.uses_privileged_process_setup() && !shared_socket { use std::os::unix::fs::PermissionsExt; let perms = std::fs::Permissions::from_mode(0o700); std::fs::set_permissions(parent, perms).into_diagnostic()?; @@ -827,20 +827,15 @@ fn spawn_pty_shell( // Probe Landlock availability from the parent process where tracing works. #[cfg(target_os = "linux")] - if enforcement_mode.enforces_process_controls() { + if enforcement_mode.enforces_child_sandbox() { sandbox::linux::log_sandbox_readiness(policy, workdir.as_deref()); } - // Phase 1 (as root): Prepare Landlock ruleset before drop_privileges. + // Phase 1: Prepare Landlock ruleset before the child applies it. #[cfg(target_os = "linux")] - let prepared_sandbox = if enforcement_mode.enforces_process_controls() { - Some( - sandbox::linux::prepare(policy, workdir.as_deref()) - .map_err(|err| anyhow::anyhow!("Failed to prepare sandbox: {err}"))?, - ) - } else { - None - }; + let prepared_sandbox = + crate::process::prepare_child_sandbox(policy, workdir.as_deref(), enforcement_mode) + .map_err(|err| anyhow::anyhow!("Failed to prepare sandbox: {err}"))?; #[cfg(unix)] { @@ -986,20 +981,15 @@ fn spawn_pipe_exec( // Probe Landlock availability from the parent process where tracing works. #[cfg(target_os = "linux")] - if enforcement_mode.enforces_process_controls() { + if enforcement_mode.enforces_child_sandbox() { sandbox::linux::log_sandbox_readiness(policy, workdir.as_deref()); } - // Phase 1 (as root): Prepare Landlock ruleset before drop_privileges. + // Phase 1: Prepare Landlock ruleset before the child applies it. #[cfg(target_os = "linux")] - let prepared_sandbox = if enforcement_mode.enforces_process_controls() { - Some( - sandbox::linux::prepare(policy, workdir.as_deref()) - .map_err(|err| anyhow::anyhow!("Failed to prepare sandbox: {err}"))?, - ) - } else { - None - }; + let prepared_sandbox = + crate::process::prepare_child_sandbox(policy, workdir.as_deref(), enforcement_mode) + .map_err(|err| anyhow::anyhow!("Failed to prepare sandbox: {err}"))?; #[cfg(unix)] { @@ -1157,7 +1147,7 @@ mod unsafe_pty { #[cfg(target_os = "linux")] let mut prepared = prepared; #[cfg(target_os = "linux")] - let supervisor_identity_mount = if enforcement_mode.enforces_process_controls() { + let supervisor_identity_mount = if enforcement_mode.uses_privileged_process_setup() { crate::process::supervisor_identity_mount_from_env().map_err(|err| { anyhow::anyhow!("failed to prepare supervisor identity isolation: {err}") })? @@ -1205,7 +1195,7 @@ mod unsafe_pty { #[cfg(target_os = "linux")] let mut prepared = prepared; #[cfg(target_os = "linux")] - let supervisor_identity_mount = if enforcement_mode.enforces_process_controls() { + let supervisor_identity_mount = if enforcement_mode.uses_privileged_process_setup() { crate::process::supervisor_identity_mount_from_env().map_err(|err| { anyhow::anyhow!("failed to prepare supervisor identity isolation: {err}") })? @@ -1260,7 +1250,7 @@ mod unsafe_pty { // Drop privileges. initgroups/setgid/setuid need /etc/group and // /etc/passwd which would be blocked if Landlock were already enforced. - if enforcement_mode.enforces_process_controls() { + if enforcement_mode.uses_privileged_process_setup() { drop_privileges(policy).map_err(|err| std::io::Error::other(err.to_string()))?; } crate::process::harden_child_process() @@ -1275,7 +1265,7 @@ mod unsafe_pty { } #[cfg(not(target_os = "linux"))] - if enforcement_mode.enforces_process_controls() { + if enforcement_mode.enforces_child_sandbox() { sandbox::apply(policy, None).map_err(|err| std::io::Error::other(err.to_string()))?; } diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 4c71165398..6093edf638 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -237,7 +237,7 @@ add `ci/values-spire.yaml` to the OpenShell release values files. | supervisor.image.pullPolicy | string | `""` | Supervisor image pull policy. Defaults to the gateway image pull policy when empty. | | supervisor.image.repository | string | `"ghcr.io/nvidia/openshell/supervisor"` | Supervisor image repository. | | supervisor.image.tag | string | `""` | Supervisor image tag. Defaults to the chart appVersion when empty. | -| supervisor.proxyUid | int | `1337` | UID for the long-running network sidecar in sidecar topology. The network init container installs nftables rules that exempt this UID. | +| supervisor.sidecar.proxyUid | int | `1337` | UID for the long-running network sidecar in sidecar topology. The network init container installs nftables rules that exempt this UID. | | supervisor.sideloadMethod | string | `""` | How the supervisor binary is delivered into sandbox pods. Empty (default) = auto-detect from cluster version: K8s >= v1.35 -> "image-volume" (ImageVolume enabled by default; GA in v1.36) K8s < v1.35 -> "init-container" (copies via init container + emptyDir) On K8s v1.33-v1.34 with the ImageVolume feature gate manually enabled, set this to "image-volume" explicitly. | | supervisor.topology | string | `"combined"` | Supervisor pod topology for Kubernetes sandboxes. "combined" runs the current single supervisor container in the agent pod. "sidecar" runs network enforcement in a dedicated sidecar and the process supervisor as a low-capability wrapper in the agent container. | | tolerations | list | `[]` | Tolerations for the gateway pod. | diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index 122ee39732..7ed16701da 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -113,8 +113,7 @@ data: grpc_endpoint = {{ include "openshell.grpcEndpoint" . | quote }} service_account_name = {{ include "openshell.sandboxServiceAccountName" . | quote }} supervisor_sideload_method = {{ include "openshell.supervisorSideloadMethod" . | quote }} - supervisor_topology = {{ .Values.supervisor.topology | default "combined" | quote }} - proxy_uid = {{ .Values.supervisor.proxyUid | default 1337 }} + topology = {{ .Values.supervisor.topology | default "combined" | quote }} sa_token_ttl_secs = {{ .Values.server.sandboxJwt.k8sSaTokenTtlSecs | default 3600 }} {{- if .Values.server.providerTokenGrants.spiffe.enabled }} provider_spiffe_workload_api_socket_path = {{ .Values.server.providerTokenGrants.spiffe.workloadApiSocketPath | quote }} @@ -143,3 +142,6 @@ data: {{- if .Values.supervisor.image.pullPolicy }} supervisor_image_pull_policy = {{ .Values.supervisor.image.pullPolicy | quote }} {{- end }} + + [openshell.drivers.kubernetes.sidecar] + proxy_uid = {{ .Values.supervisor.sidecar.proxyUid | default 1337 }} diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index 26453ed49a..9e3dfd93fb 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -83,23 +83,26 @@ tests: path: data["gateway.toml"] pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?service_account_name\s*=\s*"openshell-sandbox"' - - it: renders supervisor topology under [openshell.drivers.kubernetes] + - it: renders topology under [openshell.drivers.kubernetes] template: templates/gateway-config.yaml set: supervisor.topology: sidecar asserts: - matchRegex: path: data["gateway.toml"] - pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?supervisor_topology\s*=\s*"sidecar"' + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?topology\s*=\s*"sidecar"' + - notMatchRegex: + path: data["gateway.toml"] + pattern: 'supervisor_topology\s*=' - - it: renders proxy uid under [openshell.drivers.kubernetes] + - it: renders proxy uid under [openshell.drivers.kubernetes.sidecar] template: templates/gateway-config.yaml set: - supervisor.proxyUid: 2200 + supervisor.sidecar.proxyUid: 2200 asserts: - matchRegex: path: data["gateway.toml"] - pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?proxy_uid\s*=\s*2200' + pattern: '(?ms)\[openshell\.drivers\.kubernetes\.sidecar\].*?proxy_uid\s*=\s*2200' - it: renders sandbox image pull secrets under [openshell.drivers.kubernetes] template: templates/gateway-config.yaml diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index d10204e089..b756142d8b 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -49,9 +49,10 @@ supervisor: # "sidecar" runs network enforcement in a dedicated sidecar and the process # supervisor as a low-capability wrapper in the agent container. topology: "combined" - # -- UID for the long-running network sidecar in sidecar topology. The - # network init container installs nftables rules that exempt this UID. - proxyUid: 1337 + sidecar: + # -- UID for the long-running network sidecar in sidecar topology. The + # network init container installs nftables rules that exempt this UID. + proxyUid: 1337 # -- Image pull secrets attached to gateway and helper pods. imagePullSecrets: [] diff --git a/docs/kubernetes/setup.mdx b/docs/kubernetes/setup.mdx index cc886c1688..d97200b19e 100644 --- a/docs/kubernetes/setup.mdx +++ b/docs/kubernetes/setup.mdx @@ -161,7 +161,7 @@ The most commonly changed values are: | `pkiInitJob.serverDnsNames` / `certManager.serverDnsNames` | Additional gateway server DNS SANs. Wildcard SANs also enable sandbox service URLs under that domain. | | `supervisor.sideloadMethod` | How the supervisor binary is delivered into sandbox pods. Leave empty to auto-detect based on cluster version: clusters running Kubernetes 1.35 or later use `image-volume` (ImageVolume GA in 1.36); older clusters use `init-container`. Set explicitly to `image-volume` on Kubernetes 1.33 or 1.34 with the ImageVolume feature gate enabled, or to `init-container` to force the legacy path on any version. | | `supervisor.topology` | Sandbox pod topology. Refer to [Topology](/kubernetes/topology). | -| `supervisor.proxyUid` | Non-root UID for the long-running network sidecar when `supervisor.topology=sidecar`. The UID must not match the sandbox UID. | +| `supervisor.sidecar.proxyUid` | Non-root UID for the long-running network sidecar when `supervisor.topology=sidecar`. The UID must not match the sandbox UID. | Use a values file for repeatable deployments: diff --git a/docs/kubernetes/topology.mdx b/docs/kubernetes/topology.mdx index c2e13c5c64..7161415961 100644 --- a/docs/kubernetes/topology.mdx +++ b/docs/kubernetes/topology.mdx @@ -21,7 +21,7 @@ lower-privilege agent container. | Topology | Use when | Main tradeoff | |---|---|---| | `combined` | You need OpenShell network, filesystem, and process controls in the sandbox workload. | The agent container carries the Linux capabilities the supervisor needs. | -| `sidecar` | You need the agent container to run as non-root without added Linux capabilities, and network policy is the primary control. | The process supervisor is network-only, so filesystem/process controls do not run in the agent container. | +| `sidecar` | You need the agent container to run as non-root without added Linux capabilities, and network policy is the primary control. | Privilege-dropping and supervisor mount isolation do not run in the agent container. | ## Privilege Model @@ -121,7 +121,7 @@ The pod contains these OpenShell-managed pieces: | Component | Runs as | Purpose | |---|---|---| | Network init container | Root with setup capabilities | Installs pod-level nftables rules and prepares shared sidecar state. | -| Network sidecar | `supervisor.proxyUid` | Runs the proxy, enforces network policy, owns gateway authentication and the gateway session, and writes proxy TLS plus local policy/provider snapshots. | +| Network sidecar | `supervisor.sidecar.proxyUid` | Runs the proxy, enforces network policy, owns gateway authentication and the gateway session, and writes proxy TLS plus local policy/provider snapshots. | | Agent container | Resolved sandbox UID/GID | Runs the process supervisor and launches the user workload. | In this topology, the agent container defaults to `runAsNonRoot: true`, @@ -137,10 +137,14 @@ default sidecar path. Sidecar mode runs the process supervisor in `network-only` mode. OpenShell still -enforces network endpoint and L7 policy through the sidecar, but the process -supervisor does not apply Landlock filesystem policy, process privilege -dropping, or process/binary identity checks. Use `combined` topology when you -need those controls in the same supervisor path. +enforces network endpoint and L7 policy through the sidecar, and the process +supervisor applies Landlock filesystem policy and child seccomp filters where +the kernel/runtime supports them. The process supervisor does not perform +root-to-sandbox privilege dropping because Kubernetes starts the container as +the sandbox UID/GID, and it does not perform supervisor identity mount +isolation because gateway credentials are not mounted into the agent container. +Sidecar pods use `shareProcessNamespace: true` so the network sidecar can +resolve workload process and binary identity through `/proc/`. ## Credential Exposure @@ -161,9 +165,9 @@ watches for newer revisions that the network sidecar writes after settings polls, so future child processes can see refreshed provider env without giving the agent container gateway authentication material. This does not mutate the environment of the already-running workload entrypoint. Use `combined` topology -when you need full process/filesystem enforcement in the same supervisor path; -use additional runtime isolation when you need a stronger container boundary -around network-only sidecar workloads. +when you need the full single-supervisor enforcement path; use additional +runtime isolation when you need a stronger container boundary around sidecar +workloads. ## RuntimeClass Isolation @@ -172,9 +176,10 @@ Containers when the cluster supports them. A sandboxed runtime strengthens the container boundary while OpenShell focuses on network policy enforcement from the sidecar. -Runtime classes do not re-enable the OpenShell filesystem and process controls -that sidecar mode relaxes. Use them as an additional workload boundary, not as a -replacement for the combined topology's full supervisor controls. +Runtime classes do not re-enable the OpenShell privilege-drop or supervisor +mount-isolation controls that sidecar mode relaxes. Use them as an additional +workload boundary, not as a replacement for the combined topology's full +supervisor controls. You can set a default runtime class in the Kubernetes driver configuration or override it per sandbox with driver config: @@ -191,7 +196,9 @@ For direct gateway TOML configuration, set the Kubernetes driver fields: ```toml [openshell.drivers.kubernetes] -supervisor_topology = "sidecar" +topology = "sidecar" + +[openshell.drivers.kubernetes.sidecar] proxy_uid = 1337 ``` @@ -204,11 +211,12 @@ When the Helm chart renders `gateway.toml`, set the equivalent chart values: ```yaml supervisor: topology: sidecar - proxyUid: 1337 + sidecar: + proxyUid: 1337 ``` -Leave `supervisor_topology` unset, or set it to `combined`, to keep the -original single-container supervisor path. For Helm installs, leave +Leave `topology` unset, or set it to `combined`, to keep the original +single-container supervisor path. For Helm installs, leave `supervisor.topology` unset or set it to `combined`. ## Next Steps diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index eb3a23566d..0e190b3426 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -180,10 +180,7 @@ supervisor_sideload_method = "image-volume" # "combined" runs the existing single supervisor container with full process, # filesystem, and network enforcement in the agent container. "sidecar" moves # pod-level network enforcement and gateway session handling into a network sidecar. -supervisor_topology = "combined" -# UID used by the long-running network sidecar. In sidecar topology the -# network init container installs nftables rules that exempt this UID. -proxy_uid = 1337 +topology = "combined" grpc_endpoint = "https://openshell-gateway.agents.svc:8080" ssh_socket_path = "/run/openshell/ssh.sock" client_tls_secret_name = "openshell-client-tls" @@ -207,6 +204,11 @@ provider_spiffe_workload_api_socket_path = "/spiffe-workload-api/spire-agent.soc # back to 1000 on non-OpenShift clusters. # sandbox_uid = 1500 # sandbox_gid = 1500 + +[openshell.drivers.kubernetes.sidecar] +# UID used by the long-running network sidecar. In sidecar topology the +# network init container installs nftables rules that exempt this UID. +proxy_uid = 1337 ``` ### Docker diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index b2cee81543..71e76900d0 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -306,8 +306,8 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA | `supervisor_image` | `supervisor.image.repository` / `supervisor.image.tag` | Set the supervisor image that provides the `openshell-sandbox` binary. | | `supervisor_image_pull_policy` | `supervisor.image.pullPolicy` | Set the Kubernetes image pull policy for the supervisor image. | | `supervisor_sideload_method` | `supervisor.sideloadMethod` | How the supervisor binary is delivered into sandbox pods. Leave empty to auto-detect from cluster version. Set to `image-volume` to mount the supervisor OCI image directly as a volume (requires Kubernetes 1.33+ with the ImageVolume feature gate; GA in 1.36), or `init-container` to copy it through an init container on older clusters. | -| `supervisor_topology` | `supervisor.topology` | Set `combined` for the default single supervisor path, or `sidecar` to move pod-level network enforcement and the gateway session into a dedicated sidecar. | -| `proxy_uid` | `supervisor.proxyUid` | UID used by the long-running network sidecar in `sidecar` topology. The network init container exempts this UID from proxy redirection. | +| `topology` | `supervisor.topology` | Set `combined` for the default single supervisor path, or `sidecar` to move pod-level network enforcement and the gateway session into a dedicated sidecar. | +| `sidecar.proxy_uid` | `supervisor.sidecar.proxyUid` | UID used by the long-running network sidecar in `sidecar` topology. The network init container exempts this UID from proxy redirection. | | `app_armor_profile` | `server.appArmorProfile` | Set the sandbox agent container's AppArmor profile. Helm defaults this to `Unconfined` so AppArmor-enabled nodes do not block supervisor network namespace setup. Set the Helm value to an empty string to omit the field, or use `RuntimeDefault` or `Localhost/` for operator-managed profiles. | | `workspace_default_storage_size` | `server.workspaceDefaultStorageSize` | Set the default workspace PVC size for new sandboxes. | | `sa_token_ttl_secs` | `server.sandboxJwt.k8sSaTokenTtlSecs` | Set the projected ServiceAccount token TTL used for the bootstrap token exchange. | @@ -324,10 +324,12 @@ sandbox bootstrap token or client TLS secret in the default sidecar path. The provider environment snapshot is refreshed by the network sidecar after settings polls so future child processes can see updated provider env without gateway access in the agent container. -Sidecar mode keeps gateway session and SSH behavior, but the process supervisor -runs in `network-only` mode: filesystem policy, process privilege dropping, and -process/binary identity checks are not applied by the process container. Network -policy still runs in the sidecar and remains endpoint/L7 scoped. +Sidecar mode keeps gateway session and SSH behavior. The process supervisor +applies Landlock filesystem policy and child seccomp filters where supported, +but it does not perform root-to-sandbox privilege dropping or supervisor +identity mount isolation. Network policy still runs in the sidecar, and sidecar +pods set `shareProcessNamespace: true` so the network sidecar can resolve +process/binary identity through `/proc/`. The Kubernetes driver creates namespaced `agents.x-k8s.io` `Sandbox` resources from the Kubernetes SIG Apps [agent-sandbox](https://github.com/kubernetes-sigs/agent-sandbox) project. It detects the served Sandbox API at runtime, caches the selected API version for the gateway process, and uses `v1beta1` when available before falling back to `v1alpha1`, so supported Agent Sandbox installations work without version-specific operator configuration. The Agent Sandbox controller turns those resources into sandbox pods and related storage. From 43cce1dcd221c1a0f996278e479c0f33606961c7 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Wed, 8 Jul 2026 12:53:18 -0700 Subject: [PATCH 16/24] refactor(kubernetes): replace sidecar snapshots with control socket Coordinate sidecar policy and provider bootstrap over a local Unix socket so the process leaf no longer reads policy/provider snapshot files. Report entrypoint startup through the control channel and keep gateway credentials confined to the network sidecar. Signed-off-by: Taylor Mutch --- Cargo.lock | 1 + architecture/compute-runtimes.md | 8 +- .../src/provider_credentials.rs | 4 +- crates/openshell-core/src/sandbox_env.rs | 20 +- crates/openshell-driver-kubernetes/README.md | 4 +- .../openshell-driver-kubernetes/src/driver.rs | 96 +-- crates/openshell-sandbox/Cargo.toml | 1 + crates/openshell-sandbox/src/lib.rs | 773 ++++++++---------- .../openshell-sandbox/src/sidecar_control.rs | 570 +++++++++++++ .../openshell-supervisor-process/src/run.rs | 24 +- docs/kubernetes/topology.mdx | 29 +- docs/reference/sandbox-compute-drivers.mdx | 11 +- 12 files changed, 954 insertions(+), 587 deletions(-) create mode 100644 crates/openshell-sandbox/src/sidecar_control.rs diff --git a/Cargo.lock b/Cargo.lock index 85d3e7a25a..e7a8bfe627 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3835,6 +3835,7 @@ dependencies = [ "openshell-supervisor-process", "prost", "rustls", + "serde", "serde_json", "temp-env", "tempfile", diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 71a31328e0..c48a039788 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -94,10 +94,10 @@ sidecar topology. Combined mode keeps network and process supervision in the agent container. Sidecar mode runs network enforcement, the proxy, and gateway session in a dedicated sidecar, while the agent container runs only the process-supervision leaf and launches the user workload after the sidecar -signals readiness. The sidecar writes local policy and provider-environment -snapshots into shared state so the process leaf can start without gateway -credentials. The network sidecar also refreshes the workload-facing provider -environment snapshot after settings polls so future process sessions see +serves bootstrap state over a local control socket. The network sidecar owns +gateway credentials and sends policy plus workload-facing provider environment +state to the process leaf over that socket. It also streams provider +environment updates after settings polls so future process sessions see updated provider env without giving the process leaf gateway access. In sidecar mode, an init container performs the privileged pod-network nftables setup with `NET_ADMIN` and hands shared state ownership to the configured proxy UID; the diff --git a/crates/openshell-core/src/provider_credentials.rs b/crates/openshell-core/src/provider_credentials.rs index e3e62e1b9a..65310b3a43 100644 --- a/crates/openshell-core/src/provider_credentials.rs +++ b/crates/openshell-core/src/provider_credentials.rs @@ -67,8 +67,8 @@ impl ProviderCredentialState { /// environment snapshot. /// /// Kubernetes sidecar topology uses this in the process-only supervisor: - /// the network sidecar owns provider credential resolvers and writes the - /// workload-facing env map into a shared local snapshot. The process leaf + /// the network sidecar owns provider credential resolvers and sends the + /// workload-facing env map over a local control channel. The process leaf /// must inject that map into child processes without re-placeholderizing it /// or holding the gateway-side resolver material. pub fn from_child_env_snapshot(revision: u64, child_env: HashMap) -> Self { diff --git a/crates/openshell-core/src/sandbox_env.rs b/crates/openshell-core/src/sandbox_env.rs index 0174a37fd6..f066580636 100644 --- a/crates/openshell-core/src/sandbox_env.rs +++ b/crates/openshell-core/src/sandbox_env.rs @@ -43,20 +43,12 @@ pub const NETWORK_ENFORCEMENT_MODE: &str = "OPENSHELL_NETWORK_ENFORCEMENT_MODE"; /// `/proc` identity binding. pub const NETWORK_BINARY_IDENTITY: &str = "OPENSHELL_NETWORK_BINARY_IDENTITY"; -/// File written by the network supervisor when sidecar networking is ready. -pub const SUPERVISOR_READY_FILE: &str = "OPENSHELL_SUPERVISOR_READY_FILE"; - -/// File written by the process supervisor with the workload entrypoint PID and -/// read by the network sidecar for process/binary-bound network policy checks. -pub const ENTRYPOINT_PID_FILE: &str = "OPENSHELL_ENTRYPOINT_PID_FILE"; - -/// Local protobuf policy snapshot written by the network sidecar and read by -/// the process-only supervisor in Kubernetes sidecar topology. -pub const SIDECAR_POLICY_SNAPSHOT_FILE: &str = "OPENSHELL_SIDECAR_POLICY_SNAPSHOT_FILE"; - -/// Local provider environment snapshot written by the network sidecar and read -/// by the process-only supervisor in Kubernetes sidecar topology. -pub const SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE: &str = "OPENSHELL_SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE"; +/// Unix socket used by Kubernetes sidecar topology for local coordination. +/// +/// The network sidecar owns gateway credentials and serves policy/provider +/// state over this socket instead of exposing gateway credentials to the agent +/// container. +pub const SIDECAR_CONTROL_SOCKET: &str = "OPENSHELL_SIDECAR_CONTROL_SOCKET"; /// Optional TLS server name override used when connecting to the gateway. pub const GATEWAY_TLS_SERVER_NAME: &str = "OPENSHELL_GATEWAY_TLS_SERVER_NAME"; diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index e9efc2867b..d8f2de33f6 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -72,7 +72,9 @@ process/binary identity through `/proc/`. Sidecar mode keeps gateway credentials in the network sidecar. The agent container does not mount the projected service-account token used for sandbox token bootstrap, does not mount the sandbox client TLS secret, and does not get -gateway callback environment variables. +gateway callback environment variables. The process supervisor receives policy +and provider environment state from the sidecar over a local control socket in +the shared sidecar state volume. The driver can request a Kubernetes AppArmor profile through `app_armor_profile`. diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 568901255d..78804890c4 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -1049,18 +1049,15 @@ const SUPERVISOR_NETWORK_INIT_CONTAINER_NAME: &str = "openshell-network-init"; /// Container name for the network-only supervisor sidecar. const SUPERVISOR_NETWORK_SIDECAR_NAME: &str = "openshell-supervisor-network"; -/// Shared volume used by the network sidecar to signal readiness to the -/// process-only supervisor in the agent container. +/// Shared volume used by the network sidecar and process-only supervisor for +/// local coordination in sidecar topology. const SIDECAR_STATE_VOLUME_NAME: &str = "openshell-sidecar-state"; const SIDECAR_STATE_MOUNT_PATH: &str = "/run/openshell-sidecar"; -const SIDECAR_READY_FILE: &str = "/run/openshell-sidecar/supervisor.ready"; -const SIDECAR_ENTRYPOINT_PID_FILE: &str = "/run/openshell-sidecar/entrypoint.pid"; +const SIDECAR_CONTROL_SOCKET: &str = "/run/openshell-sidecar/control.sock"; const SIDECAR_SSH_SOCKET_FILE: &str = "/run/openshell-sidecar/ssh.sock"; -const SIDECAR_POLICY_SNAPSHOT_FILE: &str = "/run/openshell-sidecar/policy.pb"; -const SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE: &str = "/run/openshell-sidecar/provider-env.json"; /// Shared TLS work directory. The network sidecar writes the proxy CA bundle -/// here, while the agent container consumes it after the readiness file exists. +/// here, while the agent container consumes it after sidecar bootstrap. const SIDECAR_TLS_VOLUME_NAME: &str = "openshell-supervisor-tls"; const SIDECAR_TLS_MOUNT_PATH: &str = "/etc/openshell-tls/proxy"; const SIDECAR_CLIENT_TLS_MOUNT_PATH: &str = "/etc/openshell-tls/proxy/client"; @@ -1336,29 +1333,14 @@ fn supervisor_sidecar_env( ); upsert_env( &mut env, - openshell_core::sandbox_env::SUPERVISOR_READY_FILE, - SIDECAR_READY_FILE, - ); - upsert_env( - &mut env, - openshell_core::sandbox_env::ENTRYPOINT_PID_FILE, - SIDECAR_ENTRYPOINT_PID_FILE, + openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET, + SIDECAR_CONTROL_SOCKET, ); upsert_env( &mut env, openshell_core::sandbox_env::SSH_SOCKET_PATH, SIDECAR_SSH_SOCKET_FILE, ); - upsert_env( - &mut env, - openshell_core::sandbox_env::SIDECAR_POLICY_SNAPSHOT_FILE, - SIDECAR_POLICY_SNAPSHOT_FILE, - ); - upsert_env( - &mut env, - openshell_core::sandbox_env::SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE, - SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE, - ); upsert_env( &mut env, openshell_core::sandbox_env::PROXY_TLS_DIR, @@ -1608,23 +1590,8 @@ fn apply_supervisor_sidecar_topology( ); upsert_env( env, - openshell_core::sandbox_env::SUPERVISOR_READY_FILE, - SIDECAR_READY_FILE, - ); - upsert_env( - env, - openshell_core::sandbox_env::ENTRYPOINT_PID_FILE, - SIDECAR_ENTRYPOINT_PID_FILE, - ); - upsert_env( - env, - openshell_core::sandbox_env::SIDECAR_POLICY_SNAPSHOT_FILE, - SIDECAR_POLICY_SNAPSHOT_FILE, - ); - upsert_env( - env, - openshell_core::sandbox_env::SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE, - SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE, + openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET, + SIDECAR_CONTROL_SOCKET, ); upsert_env( env, @@ -3168,26 +3135,18 @@ mod tests { Some(SIDECAR_SSH_SOCKET_FILE) ); assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::SUPERVISOR_READY_FILE), - Some(SIDECAR_READY_FILE) - ); - assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::ENTRYPOINT_PID_FILE), - Some(SIDECAR_ENTRYPOINT_PID_FILE) + rendered_env(agent, openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET), + Some(SIDECAR_CONTROL_SOCKET) ); + assert_eq!(rendered_env(agent, "OPENSHELL_SUPERVISOR_READY_FILE"), None); + assert_eq!(rendered_env(agent, "OPENSHELL_ENTRYPOINT_PID_FILE"), None); assert_eq!( - rendered_env( - agent, - openshell_core::sandbox_env::SIDECAR_POLICY_SNAPSHOT_FILE - ), - Some(SIDECAR_POLICY_SNAPSHOT_FILE) + rendered_env(agent, "OPENSHELL_SIDECAR_POLICY_SNAPSHOT_FILE"), + None ); assert_eq!( - rendered_env( - agent, - openshell_core::sandbox_env::SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE - ), - Some(SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE) + rendered_env(agent, "OPENSHELL_SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE"), + None ); assert_eq!( rendered_env(agent, openshell_core::sandbox_env::PROXY_TLS_DIR), @@ -3226,18 +3185,16 @@ mod tests { Some(SIDECAR_SSH_SOCKET_FILE) ); assert_eq!( - rendered_env( - sidecar, - openshell_core::sandbox_env::SIDECAR_POLICY_SNAPSHOT_FILE - ), - Some(SIDECAR_POLICY_SNAPSHOT_FILE) + rendered_env(sidecar, openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET), + Some(SIDECAR_CONTROL_SOCKET) ); assert_eq!( - rendered_env( - sidecar, - openshell_core::sandbox_env::SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE - ), - Some(SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE) + rendered_env(sidecar, "OPENSHELL_SIDECAR_POLICY_SNAPSHOT_FILE"), + None + ); + assert_eq!( + rendered_env(sidecar, "OPENSHELL_SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE"), + None ); assert_eq!( rendered_env( @@ -3246,10 +3203,7 @@ mod tests { ), None ); - assert_eq!( - rendered_env(sidecar, openshell_core::sandbox_env::ENTRYPOINT_PID_FILE), - Some(SIDECAR_ENTRYPOINT_PID_FILE) - ); + assert_eq!(rendered_env(sidecar, "OPENSHELL_ENTRYPOINT_PID_FILE"), None); assert_eq!( rendered_env(sidecar, openshell_core::sandbox_env::PROXY_TLS_DIR), Some(SIDECAR_TLS_MOUNT_PATH) diff --git a/crates/openshell-sandbox/Cargo.toml b/crates/openshell-sandbox/Cargo.toml index 94a86b5e8e..0d7ff33925 100644 --- a/crates/openshell-sandbox/Cargo.toml +++ b/crates/openshell-sandbox/Cargo.toml @@ -40,6 +40,7 @@ nix = { workspace = true } rustls = { workspace = true } # Serialization (serde_json::json! for OCSF unmapped fields) +serde = { workspace = true } serde_json = { workspace = true } prost = { workspace = true } diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 694efe25c9..f6988cf426 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -12,12 +12,12 @@ mod google_cloud_metadata; mod mechanistic_mapper; #[cfg_attr(not(target_os = "linux"), allow(dead_code))] mod metadata_server; +mod sidecar_control; -use miette::{IntoDiagnostic, Result, WrapErr}; -use prost::Message; +use miette::{IntoDiagnostic, Result}; use std::future::Future; use std::sync::Arc; -use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::atomic::AtomicU32; use std::time::Duration; use tracing::{debug, info, warn}; @@ -136,38 +136,44 @@ pub async fn run_sandbox( let sidecar_network_enforcement = sidecar_network_enforcement_enabled(); let process_enforcement_mode = process_enforcement_mode(); - let sidecar_ready_file = supervisor_ready_file(); - if process_enabled - && !network_enabled - && let Some(path) = sidecar_ready_file.as_deref() - { - wait_for_supervisor_ready(path).await?; - } - - // Load policy and initialize OPA engine - let openshell_endpoint_for_proxy = openshell_endpoint.clone(); - let sandbox_name_for_agg = sandbox.clone(); - let process_uses_sidecar_snapshots = + let process_uses_sidecar_control = process_enabled && !network_enabled && sidecar_network_enforcement; - let (mut policy, opa_engine, retained_proto) = if process_uses_sidecar_snapshots { - let snapshot_path = sidecar_policy_snapshot_file().ok_or_else(|| { + let mut process_control_connection = None; + let sidecar_bootstrap = if process_uses_sidecar_control { + let socket = sidecar_control_socket().ok_or_else(|| { miette::miette!( "{} is required for process-only sidecar topology", - openshell_core::sandbox_env::SIDECAR_POLICY_SNAPSHOT_FILE + openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET ) })?; - load_policy_from_sidecar_snapshot(&snapshot_path)? - } else { - load_policy( - sandbox_id.clone(), - sandbox, - openshell_endpoint.clone(), - policy_rules, - policy_data, + let (bootstrap, connection) = sidecar_control::connect_process_client( + &socket, + Duration::from_secs(SIDECAR_READY_TIMEOUT_SECS), ) - .await? + .await?; + process_control_connection = Some(connection); + Some(bootstrap) + } else { + None }; + // Load policy and initialize OPA engine + let openshell_endpoint_for_proxy = openshell_endpoint.clone(); + let sandbox_name_for_agg = sandbox.clone(); + let (mut policy, opa_engine, retained_proto) = + if let Some(bootstrap) = sidecar_bootstrap.as_ref() { + load_policy_from_sidecar_bootstrap(bootstrap)? + } else { + load_policy( + sandbox_id.clone(), + sandbox, + openshell_endpoint.clone(), + policy_rules, + policy_data, + ) + .await? + }; + // Override the policy's process identity with the driver-resolved UID/GID // from the pod environment. The policy defaults to the name "sandbox" which // resolves via /etc/passwd, but the driver may have chosen a different @@ -201,86 +207,86 @@ pub async fn run_sandbox( policy.process.run_as_group = Some(gid); } - let mut process_sidecar_provider_snapshot_path = None; #[cfg_attr(not(target_os = "linux"), allow(unused_mut))] - let (provider_credentials, mut provider_env) = if process_uses_sidecar_snapshots { - let snapshot_path = sidecar_provider_env_snapshot_file().ok_or_else(|| { - miette::miette!( - "{} is required for process-only sidecar topology", - openshell_core::sandbox_env::SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE - ) - })?; - process_sidecar_provider_snapshot_path = Some(snapshot_path.clone()); - read_sidecar_provider_env_snapshot(&snapshot_path)? - } else { - // Fetch provider environment variables from the server. - // This is done after loading the policy so the sandbox can still start - // even if provider env fetch fails (graceful degradation). - let ( - provider_env_revision, - provider_env, - provider_credential_expires_at_ms, - dynamic_credentials, - ) = if let (Some(id), Some(endpoint)) = (&sandbox_id, &openshell_endpoint) { - match openshell_core::grpc_client::fetch_provider_environment(endpoint, id).await { - Ok(result) => { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .message(format!( - "Fetched provider environment [env_count:{}]", - result.environment.len() - )) - .build() - ); - ( - result.provider_env_revision, - result.environment, - result.credential_expires_at_ms, - result.dynamic_credentials, - ) - } - Err(e) => { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Other, "degraded") - .message(format!( - "Failed to fetch provider environment, continuing without: {e}" - )) - .build() - ); - ( - 0, - std::collections::HashMap::new(), - std::collections::HashMap::new(), - std::collections::HashMap::new(), - ) - } - } + let (provider_credentials, mut provider_env) = + if let Some(bootstrap) = sidecar_bootstrap.as_ref() { + let provider_credentials = ProviderCredentialState::from_child_env_snapshot( + bootstrap.provider_env_revision, + bootstrap.provider_child_env.clone(), + ); + (provider_credentials, bootstrap.provider_child_env.clone()) } else { - ( - 0, - std::collections::HashMap::new(), - std::collections::HashMap::new(), - std::collections::HashMap::new(), - ) - }; + // Fetch provider environment variables from the server. + // This is done after loading the policy so the sandbox can still start + // even if provider env fetch fails (graceful degradation). + let ( + provider_env_revision, + provider_env, + provider_credential_expires_at_ms, + dynamic_credentials, + ) = if let (Some(id), Some(endpoint)) = (&sandbox_id, &openshell_endpoint) { + match openshell_core::grpc_client::fetch_provider_environment(endpoint, id).await { + Ok(result) => { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .message(format!( + "Fetched provider environment [env_count:{}]", + result.environment.len() + )) + .build() + ); + ( + result.provider_env_revision, + result.environment, + result.credential_expires_at_ms, + result.dynamic_credentials, + ) + } + Err(e) => { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .state(StateId::Other, "degraded") + .message(format!( + "Failed to fetch provider environment, continuing without: {e}" + )) + .build() + ); + ( + 0, + std::collections::HashMap::new(), + std::collections::HashMap::new(), + std::collections::HashMap::new(), + ) + } + } + } else { + ( + 0, + std::collections::HashMap::new(), + std::collections::HashMap::new(), + std::collections::HashMap::new(), + ) + }; - let provider_credentials = ProviderCredentialState::from_environment( - provider_env_revision, - provider_env, - provider_credential_expires_at_ms, - dynamic_credentials, - ); - let provider_env = provider_credentials.child_env_with_gcp_resolved(); - (provider_credentials, provider_env) - }; - if let Some(snapshot_path) = process_sidecar_provider_snapshot_path { - spawn_sidecar_provider_env_snapshot_watcher(snapshot_path, provider_credentials.clone()); + let provider_credentials = ProviderCredentialState::from_environment( + provider_env_revision, + provider_env, + provider_credential_expires_at_ms, + dynamic_credentials, + ); + let provider_env = provider_credentials.child_env_with_gcp_resolved(); + (provider_credentials, provider_env) + }; + let process_control_writer = process_control_connection + .as_ref() + .map(|connection| connection.writer.clone()); + if let Some(connection) = process_control_connection { + spawn_sidecar_control_update_watcher(connection.updates, provider_credentials.clone()); } // Initialize the agent-proposals feature flag. Default false until the @@ -298,12 +304,6 @@ pub async fn run_sandbox( // Shared PID: set after process spawn so the proxy can look up // the entrypoint process's /proc/net/tcp for identity binding. let entrypoint_pid = Arc::new(AtomicU32::new(0)); - if network_enabled - && !process_enabled - && let Some(path) = entrypoint_pid_file() - { - spawn_entrypoint_pid_file_watcher(path, entrypoint_pid.clone()); - } // Create the workload's network namespace. It is shared infrastructure: // the proxy binds to its host-side veth IP, the bypass monitor reads @@ -381,62 +381,59 @@ pub async fn run_sandbox( None }; - if network_enabled && sidecar_network_enforcement { - let policy_snapshot_path = sidecar_policy_snapshot_file().ok_or_else(|| { - miette::miette!( - "{} is required for sidecar process snapshots", - openshell_core::sandbox_env::SIDECAR_POLICY_SNAPSHOT_FILE - ) - })?; - let provider_snapshot_path = sidecar_provider_env_snapshot_file().ok_or_else(|| { + #[cfg(target_os = "linux")] + let sidecar_control_server = if network_enabled && sidecar_network_enforcement { + if !matches!(policy.network.mode, NetworkMode::Proxy) { + return Err(miette::miette!( + "sidecar network enforcement requires proxy network mode" + )); + } + let socket = sidecar_control_socket().ok_or_else(|| { miette::miette!( - "{} is required for sidecar provider snapshots", - openshell_core::sandbox_env::SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE + "{} is required for sidecar topology", + openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET ) })?; let proto = retained_proto.as_ref().ok_or_else(|| { miette::miette!( - "sidecar topology requires a gateway policy snapshot for the process supervisor" + "sidecar topology requires gateway policy data for the process supervisor" ) })?; - write_sidecar_policy_snapshot(&policy_snapshot_path, proto)?; - write_sidecar_provider_env_snapshot( - &provider_snapshot_path, - provider_credentials.snapshot().revision, - &provider_env, - )?; - } + let ca_paths = networking.as_ref().and_then(|n| n.ca_file_paths.clone()); + Some(sidecar_control::spawn_server( + &socket, + sidecar_control::BootstrapData { + policy_proto: proto.clone(), + provider_env_revision: provider_credentials.snapshot().revision, + provider_child_env: provider_env.clone(), + proxy_ca_cert_path: ca_paths.as_ref().map(|paths| paths.0.clone()), + proxy_ca_bundle_path: ca_paths.as_ref().map(|paths| paths.1.clone()), + }, + )?) + } else { + None + }; + #[cfg(not(target_os = "linux"))] + let sidecar_control_server: Option = None; + + let sidecar_control_publisher = sidecar_control_server + .as_ref() + .map(sidecar_control::ServerHandle::publisher); #[cfg(target_os = "linux")] if network_enabled && sidecar_network_enforcement { - if !matches!(policy.network.mode, NetworkMode::Proxy) { - return Err(miette::miette!( - "sidecar network enforcement requires proxy network mode" - )); - } - if let Some(path) = sidecar_ready_file.as_deref() { - write_supervisor_ready(path)?; + if let Some(server) = sidecar_control_server { + spawn_sidecar_entrypoint_handler( + server.into_entrypoint_receiver(), + entrypoint_pid.clone(), + opa_engine.clone(), + retained_proto.clone(), + openshell_endpoint.clone(), + sandbox_id.clone(), + ); } } - if network_enabled - && sidecar_network_enforcement - && let (Some(endpoint), Some(id), Some(socket)) = ( - openshell_endpoint.as_deref(), - sandbox_id.as_deref(), - ssh_socket_path.as_ref(), - ) - { - wait_for_sidecar_ssh_socket(socket).await?; - openshell_supervisor_process::supervisor_session::spawn( - endpoint.to_string(), - id.to_string(), - std::path::PathBuf::from(socket), - None, - ); - info!("sidecar supervisor session task spawned"); - } - #[cfg(not(target_os = "linux"))] if network_enabled && sidecar_network_enforcement { return Err(miette::miette!( @@ -514,7 +511,7 @@ pub async fn run_sandbox( } // Spawn background policy poll task (gRPC mode only). - if !process_uses_sidecar_snapshots + if !process_uses_sidecar_control && let (Some(id), Some(endpoint), Some(engine)) = ( sandbox_id.as_deref(), openshell_endpoint.as_deref(), @@ -528,10 +525,6 @@ pub async fn run_sandbox( let poll_pid = entrypoint_pid.clone(); let poll_provider_credentials = provider_credentials.clone(); let poll_policy_local = networking.as_ref().map(|n| n.policy_local_ctx.clone()); - let poll_sidecar_provider_env_snapshot_path = (network_enabled - && sidecar_network_enforcement) - .then(sidecar_provider_env_snapshot_file) - .flatten(); let poll_interval_secs: u64 = std::env::var("OPENSHELL_POLICY_POLL_INTERVAL_SECS") .ok() .and_then(|v| v.parse().ok()) @@ -545,7 +538,7 @@ pub async fn run_sandbox( ocsf_enabled: poll_ocsf_enabled, provider_credentials: poll_provider_credentials, policy_local_ctx: poll_policy_local, - sidecar_provider_env_snapshot_path: poll_sidecar_provider_env_snapshot_path, + sidecar_control_publisher: sidecar_control_publisher.clone(), }; tokio::spawn(async move { @@ -602,17 +595,59 @@ pub async fn run_sandbox( } let process_policy = process_policy_for_topology(&policy, sidecar_network_enforcement)?; + let sidecar_bootstrap_ca_file_paths = sidecar_bootstrap.as_ref().and_then(|bootstrap| { + bootstrap + .proxy_ca_cert_path + .clone() + .zip(bootstrap.proxy_ca_bundle_path.clone()) + }); let exit_code = if process_enabled { let ca_file_paths = networking .as_ref() .and_then(|n| n.ca_file_paths.clone()) .or_else(|| { - sidecar_network_enforcement - .then(sidecar_ca_file_paths) - .flatten() + if sidecar_network_enforcement { + sidecar_bootstrap_ca_file_paths + .clone() + .or_else(sidecar_ca_file_paths) + } else { + None + } }); + let entrypoint_started_tx = + if process_uses_sidecar_control && let Some(writer) = process_control_writer.clone() { + let ssh_socket = ssh_socket_path.clone().ok_or_else(|| { + miette::miette!( + "{} is required for process-only sidecar topology", + openshell_core::sandbox_env::SSH_SOCKET_PATH + ) + })?; + let (tx, rx) = tokio::sync::oneshot::channel(); + tokio::spawn(async move { + match rx.await { + Ok(pid) => { + if let Err(err) = sidecar_control::send_entrypoint_started( + &writer, + pid, + std::path::Path::new(&ssh_socket), + ) + .await + { + warn!(error = %err, "Failed to send sidecar entrypoint event"); + } + } + Err(_closed) => { + debug!("Entrypoint exited before sidecar entrypoint event was sent"); + } + } + }); + Some(tx) + } else { + None + }; + openshell_supervisor_process::run::run_process( program, args, @@ -626,6 +661,7 @@ pub async fn run_sandbox( &process_policy, process_enforcement_mode, entrypoint_pid, + entrypoint_started_tx, provider_credentials, provider_env, ca_file_paths, @@ -700,258 +736,43 @@ fn process_enforcement_mode() -> ProcessEnforcementMode { } } -fn supervisor_ready_file() -> Option { - std::env::var(openshell_core::sandbox_env::SUPERVISOR_READY_FILE) - .ok() - .filter(|value| !value.is_empty()) -} - -fn entrypoint_pid_file() -> Option { - std::env::var(openshell_core::sandbox_env::ENTRYPOINT_PID_FILE) - .ok() - .filter(|value| !value.is_empty()) -} - -fn spawn_entrypoint_pid_file_watcher(path: String, entrypoint_pid: Arc) { - tokio::spawn(async move { - let pid_path = std::path::PathBuf::from(&path); - loop { - match std::fs::read_to_string(&pid_path) { - Ok(contents) => match contents.trim().parse::() { - Ok(pid) if pid > 0 => { - entrypoint_pid.store(pid, Ordering::Release); - info!(path, pid, "Loaded sidecar workload entrypoint PID"); - return; - } - Ok(_) | Err(_) => { - debug!(path, contents = %contents.trim(), "Ignoring invalid entrypoint PID file contents"); - } - }, - Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} - Err(err) => { - debug!(path, error = %err, "Failed to read entrypoint PID file"); - } - } - tokio::time::sleep(Duration::from_millis(100)).await; - } - }); -} - -async fn wait_for_supervisor_ready(path: &str) -> Result<()> { - let ready_path = std::path::Path::new(path); - let deadline = tokio::time::Instant::now() + Duration::from_secs(SIDECAR_READY_TIMEOUT_SECS); - loop { - if ready_path.exists() { - info!(path, "Network supervisor sidecar is ready"); - return Ok(()); - } - if tokio::time::Instant::now() >= deadline { - return Err(miette::miette!( - "timed out waiting for network supervisor sidecar readiness file {path}" - )); - } - tokio::time::sleep(Duration::from_millis(250)).await; - } -} - -#[cfg(target_os = "linux")] -fn write_supervisor_ready(path: &str) -> Result<()> { - let ready_path = std::path::Path::new(path); - if let Some(parent) = ready_path.parent() { - std::fs::create_dir_all(parent).into_diagnostic()?; - } - std::fs::write(ready_path, b"ready\n").into_diagnostic()?; - info!(path, "Network supervisor sidecar readiness file written"); - Ok(()) -} - -async fn wait_for_sidecar_ssh_socket(path: &str) -> Result<()> { - let deadline = tokio::time::Instant::now() + Duration::from_secs(30); - loop { - if sidecar_ssh_socket_ready(path) { - info!(path, "Sidecar SSH socket is ready for gateway relay"); - return Ok(()); - } - if tokio::time::Instant::now() >= deadline { - return Err(miette::miette!( - "timed out waiting for process supervisor SSH socket {path}" - )); - } - tokio::time::sleep(Duration::from_millis(100)).await; - } -} - -#[cfg(unix)] -fn sidecar_ssh_socket_ready(path: &str) -> bool { - use std::os::unix::fs::{FileTypeExt, PermissionsExt}; - - let Ok(metadata) = std::fs::metadata(path) else { - return false; - }; - metadata.file_type().is_socket() && metadata.permissions().mode() & 0o060 == 0o060 -} - -#[cfg(not(unix))] -fn sidecar_ssh_socket_ready(path: &str) -> bool { - std::path::Path::new(path).exists() -} - -fn sidecar_policy_snapshot_file() -> Option { - std::env::var(openshell_core::sandbox_env::SIDECAR_POLICY_SNAPSHOT_FILE) - .ok() - .filter(|path| !path.is_empty()) -} - -fn sidecar_provider_env_snapshot_file() -> Option { - std::env::var(openshell_core::sandbox_env::SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE) +fn sidecar_control_socket() -> Option { + std::env::var(openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET) .ok() .filter(|path| !path.is_empty()) + .map(std::path::PathBuf::from) } -fn load_policy_from_sidecar_snapshot( - path: &str, +fn load_policy_from_sidecar_bootstrap( + bootstrap: &sidecar_control::BootstrapData, ) -> Result<( SandboxPolicy, Option>, Option, )> { - let bytes = std::fs::read(path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to read sidecar policy snapshot from {path}"))?; - let proto = openshell_core::proto::SandboxPolicy::decode(bytes.as_slice()) - .into_diagnostic() - .wrap_err_with(|| format!("failed to decode sidecar policy snapshot from {path}"))?; + let proto = bootstrap.policy_proto.clone(); let opa_engine = Some(Arc::new(OpaEngine::from_proto(&proto)?)); let policy = SandboxPolicy::try_from(proto.clone())?; - info!(path, "Loaded sidecar policy snapshot"); + info!("Loaded sidecar policy from control socket bootstrap"); Ok((policy, opa_engine, Some(proto))) } -fn write_sidecar_policy_snapshot( - path: &str, - proto: &openshell_core::proto::SandboxPolicy, -) -> Result<()> { - let snapshot_path = std::path::Path::new(path); - if let Some(parent) = snapshot_path.parent() { - std::fs::create_dir_all(parent).into_diagnostic()?; - } - std::fs::write(snapshot_path, proto.encode_to_vec()) - .into_diagnostic() - .wrap_err_with(|| format!("failed to write sidecar policy snapshot to {path}"))?; - info!(path, "Wrote sidecar policy snapshot"); - Ok(()) -} - -struct SidecarProviderEnvSnapshot { - revision: u64, - child_env: std::collections::HashMap, -} - -fn read_sidecar_provider_env_snapshot( - path: &str, -) -> Result<( - ProviderCredentialState, - std::collections::HashMap, -)> { - let snapshot = read_sidecar_provider_env_snapshot_data(path)?; - let provider_credentials = ProviderCredentialState::from_child_env_snapshot( - snapshot.revision, - snapshot.child_env.clone(), - ); - info!( - path, - env_count = snapshot.child_env.len(), - "Loaded sidecar provider env snapshot" - ); - Ok((provider_credentials, snapshot.child_env)) -} - -fn read_sidecar_provider_env_snapshot_data(path: &str) -> Result { - let bytes = std::fs::read(path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to read sidecar provider env snapshot from {path}"))?; - let value: serde_json::Value = serde_json::from_slice(&bytes) - .into_diagnostic() - .wrap_err_with(|| format!("failed to parse sidecar provider env snapshot from {path}"))?; - let revision = value - .get("revision") - .and_then(serde_json::Value::as_u64) - .unwrap_or_default(); - let child_env: std::collections::HashMap = - serde_json::from_value(value.get("child_env").cloned().unwrap_or_default()) - .into_diagnostic() - .wrap_err_with(|| { - format!("failed to decode child_env from sidecar provider env snapshot {path}") - })?; - Ok(SidecarProviderEnvSnapshot { - revision, - child_env, - }) -} - -fn write_sidecar_provider_env_snapshot( - path: &str, - revision: u64, - child_env: &std::collections::HashMap, -) -> Result<()> { - let snapshot_path = std::path::Path::new(path); - if let Some(parent) = snapshot_path.parent() { - std::fs::create_dir_all(parent).into_diagnostic()?; - } - let value = serde_json::json!({ - "revision": revision, - "child_env": child_env, - }); - let bytes = serde_json::to_vec(&value).into_diagnostic()?; - write_atomic(snapshot_path, &bytes) - .wrap_err_with(|| format!("failed to write sidecar provider env snapshot to {path}"))?; - info!( - path, - env_count = child_env.len(), - "Wrote sidecar provider env snapshot" - ); - Ok(()) -} - -fn write_atomic(path: &std::path::Path, bytes: &[u8]) -> Result<()> { - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent).into_diagnostic()?; - } - let file_name = path - .file_name() - .and_then(std::ffi::OsStr::to_str) - .unwrap_or("snapshot"); - let tmp_path = path.with_file_name(format!(".{file_name}.{}.tmp", std::process::id())); - std::fs::write(&tmp_path, bytes).into_diagnostic()?; - std::fs::rename(&tmp_path, path).into_diagnostic()?; - Ok(()) -} - -fn refresh_sidecar_provider_env_snapshot( - path: &str, - provider_credentials: &ProviderCredentialState, -) -> Result> { - let snapshot = read_sidecar_provider_env_snapshot_data(path)?; - let current_revision = provider_credentials.snapshot().revision; - if snapshot.revision <= current_revision { - return Ok(None); - } - let env_count = - provider_credentials.install_child_env_snapshot(snapshot.revision, snapshot.child_env); - Ok(Some((snapshot.revision, env_count))) -} - -fn spawn_sidecar_provider_env_snapshot_watcher( - path: String, +fn spawn_sidecar_control_update_watcher( + mut updates: tokio::sync::mpsc::UnboundedReceiver, provider_credentials: ProviderCredentialState, ) -> tokio::task::JoinHandle<()> { tokio::spawn(async move { - let mut interval = tokio::time::interval(Duration::from_secs(1)); - interval.tick().await; - loop { - interval.tick().await; - match refresh_sidecar_provider_env_snapshot(&path, &provider_credentials) { - Ok(Some((revision, env_count))) => { + while let Some(update) = updates.recv().await { + match update { + sidecar_control::ControlUpdate::ProviderEnvUpdated { + revision, + provider_child_env, + } => { + if revision <= provider_credentials.snapshot().revision { + continue; + } + let env_count = provider_credentials + .install_child_env_snapshot(revision, provider_child_env); ocsf_emit!( ConfigStateChangeBuilder::new(ocsf_ctx()) .severity(SeverityId::Informational) @@ -959,17 +780,21 @@ fn spawn_sidecar_provider_env_snapshot_watcher( .state(StateId::Enabled, "loaded") .unmapped("provider_env_revision", serde_json::json!(revision)) .message(format!( - "Sidecar provider environment snapshot refreshed [revision:{revision} env_count:{env_count}]" + "Sidecar provider environment refreshed [revision:{revision} env_count:{env_count}]" )) .build() ); } - Ok(None) => {} - Err(e) => { - warn!( - error = %e, - path, - "Failed to refresh sidecar provider environment snapshot" + sidecar_control::ControlUpdate::PolicyUpdated { + policy_proto, + policy_hash, + config_revision, + } => { + debug!( + version = policy_proto.version, + policy_hash, + config_revision, + "Received sidecar policy update for process supervisor" ); } } @@ -977,6 +802,53 @@ fn spawn_sidecar_provider_env_snapshot_watcher( }) } +#[cfg(target_os = "linux")] +fn spawn_sidecar_entrypoint_handler( + mut entrypoint_rx: tokio::sync::mpsc::UnboundedReceiver, + entrypoint_pid: Arc, + opa_engine: Option>, + retained_proto: Option, + openshell_endpoint: Option, + sandbox_id: Option, +) { + tokio::spawn(async move { + let Some(started) = entrypoint_rx.recv().await else { + return; + }; + + entrypoint_pid.store(started.pid, std::sync::atomic::Ordering::Release); + info!( + pid = started.pid, + ssh_socket = %started.ssh_socket_path.display(), + "Sidecar process supervisor reported entrypoint start" + ); + + if let (Some(engine), Some(proto)) = (opa_engine.as_ref(), retained_proto.as_ref()) { + match engine.reload_from_proto_with_pid(proto, started.pid) { + Ok(()) => info!( + pid = started.pid, + "Policy binary symlink resolution complete for sidecar entrypoint" + ), + Err(err) => warn!( + error = %err, + pid = started.pid, + "Failed to rebuild OPA engine with sidecar entrypoint PID" + ), + } + } + + if let (Some(endpoint), Some(id)) = (openshell_endpoint, sandbox_id) { + openshell_supervisor_process::supervisor_session::spawn( + endpoint, + id, + started.ssh_socket_path, + None, + ); + info!("sidecar supervisor session task spawned"); + } + }); +} + fn sidecar_ca_file_paths() -> Option<(std::path::PathBuf, std::path::PathBuf)> { let tls_dir = std::env::var(openshell_core::sandbox_env::PROXY_TLS_DIR) .unwrap_or_else(|_| SIDECAR_TLS_DIR.to_string()); @@ -2066,7 +1938,7 @@ struct PolicyPollLoopContext { ocsf_enabled: Arc, provider_credentials: ProviderCredentialState, policy_local_ctx: Option>, - sidecar_provider_env_snapshot_path: Option, + sidecar_control_publisher: Option, } async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { @@ -2152,30 +2024,13 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { ); let child_env = ctx.provider_credentials.child_env_with_gcp_resolved(); let env_count = child_env.len(); - let snapshot_write_ok = match ctx.sidecar_provider_env_snapshot_path.as_deref() - { - Some(snapshot_path) => { - match write_sidecar_provider_env_snapshot( - snapshot_path, - env_result.provider_env_revision, - &child_env, - ) { - Ok(()) => true, - Err(e) => { - warn!( - error = %e, - path = snapshot_path, - "Settings poll: failed to write sidecar provider environment snapshot" - ); - false - } - } - } - None => true, - }; - if snapshot_write_ok { - current_provider_env_revision = env_result.provider_env_revision; + if let Some(publisher) = ctx.sidecar_control_publisher.as_ref() { + publisher.publish_provider_env( + env_result.provider_env_revision, + child_env.clone(), + ); } + current_provider_env_revision = env_result.provider_env_revision; ocsf_emit!( ConfigStateChangeBuilder::new(ocsf_ctx()) .severity(SeverityId::Informational) @@ -2223,6 +2078,13 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { if let Some(policy_local_ctx) = ctx.policy_local_ctx.as_ref() { policy_local_ctx.set_current_policy(policy.clone()).await; } + if let Some(publisher) = ctx.sidecar_control_publisher.as_ref() { + publisher.publish_policy( + policy.clone(), + result.policy_hash.clone(), + result.config_revision, + ); + } if result.global_policy_version > 0 { ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) .severity(SeverityId::Informational) @@ -2506,27 +2368,34 @@ mod tests { ); } - #[test] - fn sidecar_provider_env_snapshot_refresh_installs_newer_revision() { - let dir = tempfile::tempdir().unwrap(); - let snapshot_path = dir.path().join("provider-env.json"); - let snapshot_path = snapshot_path.to_str().unwrap(); + #[tokio::test] + async fn sidecar_control_provider_env_update_installs_newer_revision() { + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); let provider_credentials = ProviderCredentialState::from_child_env_snapshot( 1, std::collections::HashMap::from([("TOKEN".to_string(), "old".to_string())]), ); - - write_sidecar_provider_env_snapshot( - snapshot_path, - 2, - &std::collections::HashMap::from([("TOKEN".to_string(), "new".to_string())]), - ) + let handle = spawn_sidecar_control_update_watcher(rx, provider_credentials.clone()); + + tx.send(sidecar_control::ControlUpdate::ProviderEnvUpdated { + revision: 2, + provider_child_env: std::collections::HashMap::from([( + "TOKEN".to_string(), + "new".to_string(), + )]), + }) .unwrap(); - assert_eq!( - refresh_sidecar_provider_env_snapshot(snapshot_path, &provider_credentials).unwrap(), - Some((2, 1)) - ); + tokio::time::timeout(Duration::from_secs(1), async { + loop { + if provider_credentials.snapshot().revision == 2 { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); let snapshot = provider_credentials.snapshot(); assert_eq!(snapshot.revision, 2); assert_eq!( @@ -2534,16 +2403,15 @@ mod tests { Some("new") ); - write_sidecar_provider_env_snapshot( - snapshot_path, - 1, - &std::collections::HashMap::from([("TOKEN".to_string(), "stale".to_string())]), - ) + tx.send(sidecar_control::ControlUpdate::ProviderEnvUpdated { + revision: 1, + provider_child_env: std::collections::HashMap::from([( + "TOKEN".to_string(), + "stale".to_string(), + )]), + }) .unwrap(); - assert_eq!( - refresh_sidecar_provider_env_snapshot(snapshot_path, &provider_credentials).unwrap(), - None - ); + tokio::time::sleep(Duration::from_millis(20)).await; assert_eq!( provider_credentials .snapshot() @@ -2552,6 +2420,7 @@ mod tests { .map(String::as_str), Some("new") ); + handle.abort(); } #[test] diff --git a/crates/openshell-sandbox/src/sidecar_control.rs b/crates/openshell-sandbox/src/sidecar_control.rs new file mode 100644 index 0000000000..48bcf9bda2 --- /dev/null +++ b/crates/openshell-sandbox/src/sidecar_control.rs @@ -0,0 +1,570 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Local control channel for Kubernetes sidecar topology. +//! +//! The network sidecar owns gateway credentials. The process supervisor in the +//! agent container connects over this Unix socket to receive policy/provider +//! state without mounting gateway credentials into the agent container. + +use miette::{IntoDiagnostic, Result, WrapErr}; +use prost::Message; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, RwLock}; +use std::time::Duration; +use tokio::io::{AsyncBufReadExt, AsyncWrite, AsyncWriteExt, BufReader}; +use tokio::net::UnixListener; +use tokio::net::unix::OwnedWriteHalf; +use tokio::sync::{Mutex, broadcast, mpsc}; +use tracing::{debug, info, warn}; + +#[derive(Debug, Clone)] +pub struct BootstrapData { + pub policy_proto: openshell_core::proto::SandboxPolicy, + pub provider_env_revision: u64, + pub provider_child_env: HashMap, + pub proxy_ca_cert_path: Option, + pub proxy_ca_bundle_path: Option, +} + +#[derive(Debug, Clone)] +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] +pub struct EntrypointStarted { + pub pid: u32, + pub ssh_socket_path: PathBuf, +} + +#[derive(Debug, Clone)] +pub enum ControlUpdate { + ProviderEnvUpdated { + revision: u64, + provider_child_env: HashMap, + }, + PolicyUpdated { + policy_proto: openshell_core::proto::SandboxPolicy, + policy_hash: String, + config_revision: u64, + }, +} + +#[derive(Clone)] +pub struct Publisher { + state: Arc>, + updates: broadcast::Sender, +} + +impl Publisher { + pub fn publish_provider_env(&self, revision: u64, provider_child_env: HashMap) { + { + let mut state = self.state.write().expect("sidecar control state poisoned"); + if revision <= state.provider_env_revision { + return; + } + state.provider_env_revision = revision; + state.provider_child_env.clone_from(&provider_child_env); + } + + let _ = self.updates.send(WireServerMessage::ProviderEnvUpdated { + revision, + provider_child_env, + }); + } + + pub fn publish_policy( + &self, + policy_proto: openshell_core::proto::SandboxPolicy, + policy_hash: String, + config_revision: u64, + ) { + { + let mut state = self.state.write().expect("sidecar control state poisoned"); + state.policy_proto = policy_proto.clone(); + } + + let _ = self.updates.send(WireServerMessage::PolicyUpdated { + policy_proto: policy_proto.encode_to_vec(), + policy_hash, + config_revision, + }); + } +} + +pub struct ServerHandle { + publisher: Publisher, + #[cfg_attr(not(target_os = "linux"), allow(dead_code))] + entrypoint_rx: mpsc::UnboundedReceiver, +} + +impl ServerHandle { + pub fn publisher(&self) -> Publisher { + self.publisher.clone() + } + + #[cfg_attr(not(target_os = "linux"), allow(dead_code))] + pub fn into_entrypoint_receiver(self) -> mpsc::UnboundedReceiver { + self.entrypoint_rx + } +} + +pub struct ProcessConnection { + pub writer: Arc>, + pub updates: mpsc::UnboundedReceiver, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +enum WireClientMessage { + BootstrapRequest, + EntrypointStarted { pid: u32, ssh_socket_path: String }, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +enum WireServerMessage { + BootstrapResponse { + policy_proto: Vec, + provider_env_revision: u64, + provider_child_env: HashMap, + proxy_ca_cert_path: Option, + proxy_ca_bundle_path: Option, + }, + ProviderEnvUpdated { + revision: u64, + provider_child_env: HashMap, + }, + PolicyUpdated { + policy_proto: Vec, + policy_hash: String, + config_revision: u64, + }, +} + +impl BootstrapData { + #[cfg_attr(not(target_os = "linux"), allow(dead_code))] + fn to_wire(&self) -> WireServerMessage { + WireServerMessage::BootstrapResponse { + policy_proto: self.policy_proto.encode_to_vec(), + provider_env_revision: self.provider_env_revision, + provider_child_env: self.provider_child_env.clone(), + proxy_ca_cert_path: self + .proxy_ca_cert_path + .as_ref() + .map(|path| path.display().to_string()), + proxy_ca_bundle_path: self + .proxy_ca_bundle_path + .as_ref() + .map(|path| path.display().to_string()), + } + } +} + +impl TryFrom for BootstrapData { + type Error = miette::Report; + + fn try_from(message: WireServerMessage) -> Result { + let WireServerMessage::BootstrapResponse { + policy_proto, + provider_env_revision, + provider_child_env, + proxy_ca_cert_path, + proxy_ca_bundle_path, + } = message + else { + return Err(miette::miette!( + "expected sidecar bootstrap response, received update message" + )); + }; + + let policy_proto = openshell_core::proto::SandboxPolicy::decode(policy_proto.as_slice()) + .into_diagnostic() + .wrap_err("failed to decode sidecar bootstrap policy")?; + + Ok(Self { + policy_proto, + provider_env_revision, + provider_child_env, + proxy_ca_cert_path: proxy_ca_cert_path.map(PathBuf::from), + proxy_ca_bundle_path: proxy_ca_bundle_path.map(PathBuf::from), + }) + } +} + +impl TryFrom for ControlUpdate { + type Error = miette::Report; + + fn try_from(message: WireServerMessage) -> Result { + match message { + WireServerMessage::ProviderEnvUpdated { + revision, + provider_child_env, + } => Ok(Self::ProviderEnvUpdated { + revision, + provider_child_env, + }), + WireServerMessage::PolicyUpdated { + policy_proto, + policy_hash, + config_revision, + } => { + let policy_proto = + openshell_core::proto::SandboxPolicy::decode(policy_proto.as_slice()) + .into_diagnostic() + .wrap_err("failed to decode sidecar policy update")?; + Ok(Self::PolicyUpdated { + policy_proto, + policy_hash, + config_revision, + }) + } + WireServerMessage::BootstrapResponse { .. } => Err(miette::miette!( + "unexpected sidecar bootstrap response after initial handshake" + )), + } + } +} + +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] +pub fn spawn_server(path: &Path, bootstrap: BootstrapData) -> Result { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to create sidecar control socket dir {}", + parent.display() + ) + })?; + } + match std::fs::remove_file(path) { + Ok(()) => {} + Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} + Err(err) => { + return Err(err).into_diagnostic().wrap_err_with(|| { + format!( + "failed to remove stale sidecar control socket {}", + path.display() + ) + }); + } + } + + let listener = UnixListener::bind(path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to bind sidecar control socket {}", path.display()))?; + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o660)) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to set permissions on sidecar control socket {}", + path.display() + ) + })?; + } + + let state = Arc::new(RwLock::new(bootstrap)); + let (updates, _) = broadcast::channel(32); + let (entrypoint_tx, entrypoint_rx) = mpsc::unbounded_channel(); + let publisher = Publisher { + state: state.clone(), + updates: updates.clone(), + }; + + tokio::spawn(accept_loop(listener, state, updates, entrypoint_tx)); + info!(path = %path.display(), "Sidecar control socket listening"); + + Ok(ServerHandle { + publisher, + entrypoint_rx, + }) +} + +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] +async fn accept_loop( + listener: UnixListener, + state: Arc>, + updates: broadcast::Sender, + entrypoint_tx: mpsc::UnboundedSender, +) { + loop { + match listener.accept().await { + Ok((stream, _addr)) => { + let state = state.clone(); + let updates = updates.clone(); + let entrypoint_tx = entrypoint_tx.clone(); + tokio::spawn(async move { + if let Err(err) = handle_connection(stream, state, updates, entrypoint_tx).await + { + debug!(error = %err, "Sidecar control connection closed"); + } + }); + } + Err(err) => { + warn!(error = %err, "Failed to accept sidecar control connection"); + } + } + } +} + +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] +async fn handle_connection( + stream: tokio::net::UnixStream, + state: Arc>, + updates: broadcast::Sender, + entrypoint_tx: mpsc::UnboundedSender, +) -> Result<()> { + let (reader, mut writer) = stream.into_split(); + let mut lines = BufReader::new(reader).lines(); + + let first_line = + lines.next_line().await.into_diagnostic()?.ok_or_else(|| { + miette::miette!("sidecar control client disconnected before bootstrap") + })?; + match decode_client_message(&first_line)? { + WireClientMessage::BootstrapRequest => {} + WireClientMessage::EntrypointStarted { .. } => { + return Err(miette::miette!( + "sidecar control client sent entrypoint event before bootstrap" + )); + } + } + + let bootstrap = { + let state = state.read().expect("sidecar control state poisoned"); + state.to_wire() + }; + write_json_line(&mut writer, &bootstrap).await?; + + let mut update_rx = updates.subscribe(); + loop { + tokio::select! { + line = lines.next_line() => { + let Some(line) = line.into_diagnostic()? else { + return Ok(()); + }; + match decode_client_message(&line)? { + WireClientMessage::BootstrapRequest => { + debug!("Ignoring duplicate sidecar bootstrap request"); + } + WireClientMessage::EntrypointStarted { pid, ssh_socket_path } => { + if pid == 0 { + warn!("Ignoring sidecar entrypoint event with pid=0"); + continue; + } + let _ = entrypoint_tx.send(EntrypointStarted { + pid, + ssh_socket_path: PathBuf::from(ssh_socket_path), + }); + } + } + } + update = update_rx.recv() => { + match update { + Ok(message) => write_json_line(&mut writer, &message).await?, + Err(broadcast::error::RecvError::Lagged(skipped)) => { + warn!(skipped, "Sidecar control client lagged behind updates"); + } + Err(broadcast::error::RecvError::Closed) => return Ok(()), + } + } + } + } +} + +pub async fn connect_process_client( + path: &Path, + timeout: Duration, +) -> Result<(BootstrapData, ProcessConnection)> { + let stream = connect_with_retry(path, timeout).await?; + let (reader, mut writer) = stream.into_split(); + write_json_line(&mut writer, &WireClientMessage::BootstrapRequest).await?; + + let mut lines = BufReader::new(reader).lines(); + let first_line = lines + .next_line() + .await + .into_diagnostic()? + .ok_or_else(|| miette::miette!("sidecar control closed before bootstrap response"))?; + let bootstrap = BootstrapData::try_from(decode_server_message(&first_line)?)?; + + let (update_tx, updates) = mpsc::unbounded_channel(); + tokio::spawn(async move { + while let Ok(Some(line)) = lines.next_line().await { + match decode_server_message(&line).and_then(ControlUpdate::try_from) { + Ok(update) => { + if update_tx.send(update).is_err() { + return; + } + } + Err(err) => { + warn!(error = %err, "Ignoring invalid sidecar control update"); + } + } + } + }); + + Ok(( + bootstrap, + ProcessConnection { + writer: Arc::new(Mutex::new(writer)), + updates, + }, + )) +} + +async fn connect_with_retry(path: &Path, timeout: Duration) -> Result { + let deadline = tokio::time::Instant::now() + timeout; + loop { + match tokio::net::UnixStream::connect(path).await { + Ok(stream) => return Ok(stream), + Err(err) if tokio::time::Instant::now() < deadline => { + debug!( + path = %path.display(), + error = %err, + "Waiting for sidecar control socket" + ); + tokio::time::sleep(Duration::from_millis(100)).await; + } + Err(err) => { + return Err(err).into_diagnostic().wrap_err_with(|| { + format!( + "timed out waiting for sidecar control socket {}", + path.display() + ) + }); + } + } + } +} + +pub async fn send_entrypoint_started( + writer: &Arc>, + pid: u32, + ssh_socket_path: &Path, +) -> Result<()> { + let message = WireClientMessage::EntrypointStarted { + pid, + ssh_socket_path: ssh_socket_path.display().to_string(), + }; + let mut writer = writer.lock().await; + write_json_line(&mut *writer, &message).await +} + +async fn write_json_line(writer: &mut W, value: &T) -> Result<()> +where + W: AsyncWrite + Unpin + Send, + T: Serialize + Sync, +{ + let bytes = serde_json::to_vec(value).into_diagnostic()?; + writer.write_all(&bytes).await.into_diagnostic()?; + writer.write_all(b"\n").await.into_diagnostic()?; + writer.flush().await.into_diagnostic()?; + Ok(()) +} + +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] +fn decode_client_message(line: &str) -> Result { + serde_json::from_str(line) + .into_diagnostic() + .wrap_err("failed to decode sidecar client message") +} + +fn decode_server_message(line: &str) -> Result { + serde_json::from_str(line) + .into_diagnostic() + .wrap_err("failed to decode sidecar server message") +} + +#[cfg(test)] +mod tests { + use super::*; + use openshell_core::proto::SandboxPolicy; + + #[tokio::test] + async fn bootstrap_round_trips_policy_and_provider_env() { + let dir = tempfile::tempdir().unwrap(); + let socket = dir.path().join("control.sock"); + let mut env = HashMap::new(); + env.insert("GITHUB_TOKEN".to_string(), "secret".to_string()); + let bootstrap = BootstrapData { + policy_proto: SandboxPolicy { + version: 7, + ..SandboxPolicy::default() + }, + provider_env_revision: 3, + provider_child_env: env.clone(), + proxy_ca_cert_path: Some(PathBuf::from("/tmp/ca.pem")), + proxy_ca_bundle_path: Some(PathBuf::from("/tmp/bundle.pem")), + }; + + let _server = spawn_server(&socket, bootstrap).unwrap(); + let (received, _connection) = connect_process_client(&socket, Duration::from_secs(1)) + .await + .unwrap(); + + assert_eq!(received.policy_proto.version, 7); + assert_eq!(received.provider_env_revision, 3); + assert_eq!(received.provider_child_env, env); + assert_eq!( + received.proxy_ca_cert_path, + Some(PathBuf::from("/tmp/ca.pem")) + ); + assert_eq!( + received.proxy_ca_bundle_path, + Some(PathBuf::from("/tmp/bundle.pem")) + ); + } + + #[tokio::test] + async fn entrypoint_started_is_delivered_to_server() { + let dir = tempfile::tempdir().unwrap(); + let socket = dir.path().join("control.sock"); + let server = spawn_server( + &socket, + BootstrapData { + policy_proto: SandboxPolicy::default(), + provider_env_revision: 0, + provider_child_env: HashMap::new(), + proxy_ca_cert_path: None, + proxy_ca_bundle_path: None, + }, + ) + .unwrap(); + let mut entrypoint_rx = server.into_entrypoint_receiver(); + let (_bootstrap, connection) = connect_process_client(&socket, Duration::from_secs(1)) + .await + .unwrap(); + + send_entrypoint_started( + &connection.writer, + 4242, + Path::new("/run/openshell-sidecar/ssh.sock"), + ) + .await + .unwrap(); + + let started = tokio::time::timeout(Duration::from_secs(1), entrypoint_rx.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!(started.pid, 4242); + assert_eq!( + started.ssh_socket_path, + PathBuf::from("/run/openshell-sidecar/ssh.sock") + ); + } + + #[test] + fn malformed_client_message_is_rejected() { + let err = decode_client_message("not-json").unwrap_err(); + assert!( + err.to_string() + .contains("failed to decode sidecar client message") + ); + } +} diff --git a/crates/openshell-supervisor-process/src/run.rs b/crates/openshell-supervisor-process/src/run.rs index 37f6ea77c2..5276eb300f 100644 --- a/crates/openshell-supervisor-process/src/run.rs +++ b/crates/openshell-supervisor-process/src/run.rs @@ -60,6 +60,7 @@ pub async fn run_process( policy: &SandboxPolicy, enforcement_mode: ProcessEnforcementMode, entrypoint_pid: Arc, + entrypoint_started_tx: Option>, provider_credentials: ProviderCredentialState, provider_env: std::collections::HashMap, ca_file_paths: Option<(std::path::PathBuf, std::path::PathBuf)>, @@ -333,8 +334,8 @@ pub async fn run_process( // Store the entrypoint PID so the proxy can resolve TCP peer identity entrypoint_pid.store(handle.pid(), Ordering::Release); - if let Some(path) = entrypoint_pid_file() { - write_entrypoint_pid_file(&path, handle.pid())?; + if let Some(tx) = entrypoint_started_tx { + let _ = tx.send(handle.pid()); } ocsf_emit!( ProcessActivityBuilder::new(ocsf_ctx()) @@ -388,25 +389,6 @@ pub async fn run_process( Ok(status.code()) } -fn entrypoint_pid_file() -> Option { - std::env::var(openshell_core::sandbox_env::ENTRYPOINT_PID_FILE) - .ok() - .filter(|value| !value.is_empty()) -} - -fn write_entrypoint_pid_file(path: &str, pid: u32) -> Result<()> { - let pid_path = std::path::Path::new(path); - if let Some(parent) = pid_path.parent() { - std::fs::create_dir_all(parent).into_diagnostic()?; - } - std::fs::write(pid_path, format!("{pid}\n")).into_diagnostic()?; - info!( - path, - pid, "Published workload entrypoint PID for network sidecar" - ); - Ok(()) -} - fn ssh_proxy_url_for_policy( policy: &SandboxPolicy, netns_proxy_host: Option, diff --git a/docs/kubernetes/topology.mdx b/docs/kubernetes/topology.mdx index 7161415961..8e0bb9e934 100644 --- a/docs/kubernetes/topology.mdx +++ b/docs/kubernetes/topology.mdx @@ -111,8 +111,8 @@ flowchart TB Workload -->|"egress redirected on loopback"| NetworkSidecar NetworkSidecar -->|"gateway session + relays"| Gateway NetworkSidecar -->|"policy-enforced egress"| External - NetworkSidecar -->|"policy/provider snapshots"| State - ProcessSupervisor -->|"reads snapshots"| State + NetworkSidecar -->|"control socket + proxy TLS"| State + ProcessSupervisor -->|"bootstrap + updates"| State NetworkSidecar --- State ``` @@ -121,7 +121,7 @@ The pod contains these OpenShell-managed pieces: | Component | Runs as | Purpose | |---|---|---| | Network init container | Root with setup capabilities | Installs pod-level nftables rules and prepares shared sidecar state. | -| Network sidecar | `supervisor.sidecar.proxyUid` | Runs the proxy, enforces network policy, owns gateway authentication and the gateway session, and writes proxy TLS plus local policy/provider snapshots. | +| Network sidecar | `supervisor.sidecar.proxyUid` | Runs the proxy, enforces network policy, owns gateway authentication and the gateway session, and serves local policy/provider state over the sidecar control socket. | | Agent container | Resolved sandbox UID/GID | Runs the process supervisor and launches the user workload. | In this topology, the agent container defaults to `runAsNonRoot: true`, @@ -154,20 +154,15 @@ container does not mount the projected ServiceAccount token used for sandbox token bootstrap, does not mount the sandbox client TLS secret, and does not get gateway callback environment variables. -The network sidecar writes local snapshots that the process supervisor needs: - -- A protobuf policy snapshot. -- A workload-facing provider environment snapshot. - -The provider snapshot contains the environment variables OpenShell intentionally -injects into the workload. The process supervisor loads it at startup and -watches for newer revisions that the network sidecar writes after settings -polls, so future child processes can see refreshed provider env without giving -the agent container gateway authentication material. This does not mutate the -environment of the already-running workload entrypoint. Use `combined` topology -when you need the full single-supervisor enforcement path; use additional -runtime isolation when you need a stronger container boundary around sidecar -workloads. +The network sidecar serves the policy and workload-facing provider environment +over a Unix control socket in the shared sidecar state volume. The process +supervisor connects to that socket at startup, receives bootstrap state, and +then listens for provider-environment updates after settings polls. Future child +processes can see refreshed provider env without giving the agent container +gateway authentication material. This does not mutate the environment of the +already-running workload entrypoint. Use `combined` topology when you need the +full single-supervisor enforcement path; use additional runtime isolation when +you need a stronger container boundary around sidecar workloads. ## RuntimeClass Isolation diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 71e76900d0..864a1ea242 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -319,11 +319,12 @@ topology, the agent container runs as the resolved sandbox UID/GID with no added Linux capabilities. A root init container performs the nftables setup, and the long-running sidecar runs non-root with no added Linux capabilities. The network sidecar owns gateway authentication and writes local policy/provider -snapshots for the process supervisor, so the agent container does not mount the -sandbox bootstrap token or client TLS secret in the default sidecar path. The -provider environment snapshot is refreshed by the network sidecar after -settings polls so future child processes can see updated provider env without -gateway access in the agent container. +state to the process supervisor over a local control socket, so the agent +container does not mount the sandbox bootstrap token or client TLS secret in +the default sidecar path. The provider environment is refreshed by the network +sidecar after settings polls and streamed to the process supervisor so future +child processes can see updated provider env without gateway access in the +agent container. Sidecar mode keeps gateway session and SSH behavior. The process supervisor applies Landlock filesystem policy and child seccomp filters where supported, but it does not perform root-to-sandbox privilege dropping or supervisor From 0458baa5efe63c6cb11cdef1ed20b473c590cdca Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Wed, 8 Jul 2026 14:05:33 -0700 Subject: [PATCH 17/24] feat(kubernetes): support relaxed sidecar network identity Signed-off-by: Taylor Mutch --- architecture/compute-runtimes.md | 6 +- crates/openshell-driver-kubernetes/README.md | 10 ++- .../openshell-driver-kubernetes/src/config.rs | 28 ++++++- .../openshell-driver-kubernetes/src/driver.rs | 73 ++++++++++++++++++- .../openshell-driver-kubernetes/src/main.rs | 11 ++- crates/openshell-sandbox/src/lib.rs | 69 ++++++++++-------- .../openshell-sandbox/src/sidecar_control.rs | 32 +++++++- .../src/sandbox/linux/landlock.rs | 2 +- deploy/helm/openshell/README.md | 1 + deploy/helm/openshell/ci/values-sidecar.yaml | 5 ++ .../openshell/templates/gateway-config.yaml | 1 + .../openshell/tests/gateway_config_test.yaml | 9 +++ deploy/helm/openshell/values.yaml | 5 ++ docs/kubernetes/topology.mdx | 16 +++- docs/reference/gateway-config.mdx | 4 + docs/reference/sandbox-compute-drivers.mdx | 6 +- 16 files changed, 230 insertions(+), 48 deletions(-) diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index c48a039788..a1a2bf8b2b 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -101,7 +101,11 @@ environment updates after settings polls so future process sessions see updated provider env without giving the process leaf gateway access. In sidecar mode, an init container performs the privileged pod-network nftables setup with `NET_ADMIN` and hands shared state ownership to the configured proxy UID; the -long-running network sidecar runs as that UID and does not keep `NET_ADMIN`. +long-running network sidecar runs as that UID, does not keep `NET_ADMIN`, and +adds only `SYS_PTRACE` so it can resolve workload process/binary identity +through shared `/proc`. Operators can set the sidecar +`process_binary_aware_network_policy` flag false to omit `SYS_PTRACE`; that +downgrades network policy to endpoint/L7 matching without `policy.binaries`. The init path applies nftables as individual commands so optional conntrack and log expressions can fail without rolling back the required table, chain, and reject rules. diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index d8f2de33f6..67caae699b 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -58,9 +58,13 @@ the Linux capabilities the supervisor needs for namespace setup and process, filesystem, and network policy enforcement. The `sidecar` supervisor topology moves pod-level network setup into a root init -container and runs the long-lived network sidecar as a non-root UID with no -added Linux capabilities. The agent container also runs as the resolved sandbox -UID/GID with `allowPrivilegeEscalation: false` and `capabilities.drop: ["ALL"]`. +container and runs the long-lived network sidecar as a non-root UID that drops +default Linux capabilities and adds only `SYS_PTRACE` for workload `/proc` +inspection. The agent container also runs as the resolved sandbox UID/GID with +`allowPrivilegeEscalation: false` and `capabilities.drop: ["ALL"]`. +Set `sidecar.process_binary_aware_network_policy = false` to omit `SYS_PTRACE` +from the sidecar and enforce endpoint/L7 network policy without matching +`policy.binaries`. In this mode OpenShell preserves gateway session and SSH behavior, but the process supervisor does not perform root-to-sandbox privilege dropping or supervisor identity mount isolation. It still applies Landlock filesystem policy diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index f07e4fb9a9..2ab5179edc 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -95,12 +95,18 @@ pub struct KubernetesSidecarConfig { /// The network init container installs nftables rules that exempt this /// UID, so it must not match the sandbox workload UID. pub proxy_uid: u32, + /// Require process/binary-aware network policy enforcement in sidecar + /// topology. When disabled, the network sidecar drops the extra `/proc` + /// inspection permission and evaluates endpoint/L7 policy without + /// matching `policy.binaries`. + pub process_binary_aware_network_policy: bool, } impl Default for KubernetesSidecarConfig { fn default() -> Self { Self { proxy_uid: DEFAULT_PROXY_UID, + process_binary_aware_network_policy: true, } } } @@ -519,6 +525,12 @@ mod tests { assert_eq!(cfg.sidecar.proxy_uid, DEFAULT_PROXY_UID); } + #[test] + fn default_sidecar_requires_process_binary_aware_network_policy() { + let cfg = KubernetesComputeConfig::default(); + assert!(cfg.sidecar.process_binary_aware_network_policy); + } + #[test] fn serde_override_topology_sidecar() { let json = serde_json::json!({ @@ -548,6 +560,17 @@ mod tests { assert!(err.to_string().contains("unknown field")); } + #[test] + fn serde_override_sidecar_process_binary_aware_network_policy_nested() { + let json = serde_json::json!({ + "sidecar": { + "process_binary_aware_network_policy": false + } + }); + let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); + assert!(!cfg.sidecar.process_binary_aware_network_policy); + } + #[test] fn serde_override_sidecar_proxy_uid_nested() { let json = serde_json::json!({ @@ -563,7 +586,10 @@ mod tests { #[test] fn validate_proxy_uid_rejects_privileged_uid() { let cfg = KubernetesComputeConfig { - sidecar: KubernetesSidecarConfig { proxy_uid: 999 }, + sidecar: KubernetesSidecarConfig { + proxy_uid: 999, + ..KubernetesSidecarConfig::default() + }, ..KubernetesComputeConfig::default() }; let err = cfg.validate_proxy_uid().unwrap_err(); diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 78804890c4..f74274939a 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -565,6 +565,10 @@ impl KubernetesComputeDriver { supervisor_sideload_method: self.config.supervisor_sideload_method, supervisor_topology: self.config.topology, proxy_uid: self.config.sidecar.proxy_uid, + process_binary_aware_network_policy: self + .config + .sidecar + .process_binary_aware_network_policy, service_account_name: &self.config.service_account_name, sandbox_id: &sandbox.id, sandbox_name: &sandbox.name, @@ -1346,6 +1350,13 @@ fn supervisor_sidecar_env( openshell_core::sandbox_env::PROXY_TLS_DIR, SIDECAR_TLS_MOUNT_PATH, ); + if !params.process_binary_aware_network_policy { + upsert_env( + &mut env, + openshell_core::sandbox_env::NETWORK_BINARY_IDENTITY, + "relaxed", + ); + } env } @@ -1354,6 +1365,16 @@ fn supervisor_sidecar_container( spec_environment: &std::collections::HashMap, params: &SandboxPodParams<'_>, ) -> serde_json::Value { + let capabilities = if params.process_binary_aware_network_policy { + serde_json::json!({ + "drop": ["ALL"], + "add": ["SYS_PTRACE"] + }) + } else { + serde_json::json!({ + "drop": ["ALL"] + }) + }; let mut container = serde_json::json!({ "name": SUPERVISOR_NETWORK_SIDECAR_NAME, "image": params.supervisor_image, @@ -1367,9 +1388,7 @@ fn supervisor_sidecar_container( "runAsGroup": params.sandbox_gid, "runAsNonRoot": true, "allowPrivilegeEscalation": false, - "capabilities": { - "drop": ["ALL"] - } + "capabilities": capabilities }, "volumeMounts": [ sidecar_state_volume_mount(), @@ -1769,6 +1788,7 @@ struct SandboxPodParams<'a> { supervisor_sideload_method: SupervisorSideloadMethod, supervisor_topology: SupervisorTopology, proxy_uid: u32, + process_binary_aware_network_policy: bool, service_account_name: &'a str, sandbox_id: &'a str, sandbox_name: &'a str, @@ -1802,6 +1822,7 @@ impl Default for SandboxPodParams<'_> { supervisor_sideload_method: SupervisorSideloadMethod::default(), supervisor_topology: SupervisorTopology::default(), proxy_uid: DEFAULT_PROXY_UID, + process_binary_aware_network_policy: true, service_account_name: DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, sandbox_id: "", sandbox_name: "", @@ -3173,7 +3194,8 @@ mod tests { assert_eq!( sidecar["securityContext"]["capabilities"], serde_json::json!({ - "drop": ["ALL"] + "drop": ["ALL"], + "add": ["SYS_PTRACE"] }) ); assert_eq!( @@ -3280,6 +3302,49 @@ mod tests { })); } + #[test] + fn sidecar_topology_can_relax_process_binary_aware_network_policy() { + let params = SandboxPodParams { + supervisor_topology: SupervisorTopology::Sidecar, + supervisor_sideload_method: SupervisorSideloadMethod::InitContainer, + supervisor_image: "supervisor-image:latest", + proxy_uid: 2200, + sandbox_uid: 1500, + sandbox_gid: 1500, + process_binary_aware_network_policy: false, + ..SandboxPodParams::default() + }; + let pod_template = sandbox_template_to_k8s( + &SandboxTemplate { + image: "agent-image:latest".to_string(), + ..SandboxTemplate::default() + }, + false, + &std::collections::HashMap::new(), + false, + ¶ms, + ); + + let containers = pod_template["spec"]["containers"].as_array().unwrap(); + let sidecar = containers + .iter() + .find(|container| container["name"] == SUPERVISOR_NETWORK_SIDECAR_NAME) + .unwrap(); + assert_eq!( + sidecar["securityContext"]["capabilities"], + serde_json::json!({ + "drop": ["ALL"] + }) + ); + assert_eq!( + rendered_env( + sidecar, + openshell_core::sandbox_env::NETWORK_BINARY_IDENTITY + ), + Some("relaxed") + ); + } + #[test] fn sidecar_topology_adds_shared_state_and_tls_volumes() { let params = SandboxPodParams { diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 8f62087663..81ed57d818 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use clap::Parser; +use clap::{ArgAction, Parser}; use miette::{IntoDiagnostic, Result}; use std::net::SocketAddr; use tracing::info; @@ -97,6 +97,14 @@ struct Args { )] sidecar_proxy_uid: u32, + #[arg( + long = "sidecar-process-binary-aware-network-policy", + env = "OPENSHELL_K8S_SIDECAR_PROCESS_BINARY_AWARE_NETWORK_POLICY", + default_value_t = true, + action = ArgAction::Set + )] + sidecar_process_binary_aware_network_policy: bool, + #[arg(long, env = "OPENSHELL_ENABLE_USER_NAMESPACES")] enable_user_namespaces: bool, @@ -143,6 +151,7 @@ async fn main() -> Result<()> { topology: args.topology, sidecar: KubernetesSidecarConfig { proxy_uid: args.sidecar_proxy_uid, + process_binary_aware_network_policy: args.sidecar_process_binary_aware_network_policy, }, grpc_endpoint: args.grpc_endpoint.unwrap_or_default(), ssh_socket_path: args.sandbox_ssh_socket_path, diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index f6988cf426..2c119ed3f3 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -812,39 +812,50 @@ fn spawn_sidecar_entrypoint_handler( sandbox_id: Option, ) { tokio::spawn(async move { - let Some(started) = entrypoint_rx.recv().await else { - return; - }; - - entrypoint_pid.store(started.pid, std::sync::atomic::Ordering::Release); - info!( - pid = started.pid, - ssh_socket = %started.ssh_socket_path.display(), - "Sidecar process supervisor reported entrypoint start" - ); - - if let (Some(engine), Some(proto)) = (opa_engine.as_ref(), retained_proto.as_ref()) { - match engine.reload_from_proto_with_pid(proto, started.pid) { - Ok(()) => info!( + let mut session_started = false; + while let Some(started) = entrypoint_rx.recv().await { + entrypoint_pid.store(started.pid, std::sync::atomic::Ordering::Release); + if started.start_session { + info!( pid = started.pid, - "Policy binary symlink resolution complete for sidecar entrypoint" - ), - Err(err) => warn!( - error = %err, + ssh_socket = %started.ssh_socket_path.display(), + "Sidecar process supervisor reported entrypoint start" + ); + } else { + info!( pid = started.pid, - "Failed to rebuild OPA engine with sidecar entrypoint PID" - ), + "Sidecar process supervisor reported initial process anchor" + ); } - } - if let (Some(endpoint), Some(id)) = (openshell_endpoint, sandbox_id) { - openshell_supervisor_process::supervisor_session::spawn( - endpoint, - id, - started.ssh_socket_path, - None, - ); - info!("sidecar supervisor session task spawned"); + if let (Some(engine), Some(proto)) = (opa_engine.as_ref(), retained_proto.as_ref()) { + match engine.reload_from_proto_with_pid(proto, started.pid) { + Ok(()) => info!( + pid = started.pid, + "Policy binary symlink resolution complete for sidecar process anchor" + ), + Err(err) => warn!( + error = %err, + pid = started.pid, + "Failed to rebuild OPA engine with sidecar process anchor PID" + ), + } + } + + if started.start_session + && !session_started + && let (Some(endpoint), Some(id)) = + (openshell_endpoint.as_ref(), sandbox_id.as_ref()) + { + openshell_supervisor_process::supervisor_session::spawn( + endpoint.clone(), + id.clone(), + started.ssh_socket_path, + None, + ); + session_started = true; + info!("sidecar supervisor session task spawned"); + } } }); } diff --git a/crates/openshell-sandbox/src/sidecar_control.rs b/crates/openshell-sandbox/src/sidecar_control.rs index 48bcf9bda2..8176ffe742 100644 --- a/crates/openshell-sandbox/src/sidecar_control.rs +++ b/crates/openshell-sandbox/src/sidecar_control.rs @@ -34,6 +34,7 @@ pub struct BootstrapData { pub struct EntrypointStarted { pub pid: u32, pub ssh_socket_path: PathBuf, + pub start_session: bool, } #[derive(Debug, Clone)] @@ -116,7 +117,7 @@ pub struct ProcessConnection { #[derive(Debug, Serialize, Deserialize)] #[serde(tag = "type", rename_all = "snake_case")] enum WireClientMessage { - BootstrapRequest, + BootstrapRequest { supervisor_pid: Option }, EntrypointStarted { pid: u32, ssh_socket_path: String }, } @@ -326,7 +327,15 @@ async fn handle_connection( miette::miette!("sidecar control client disconnected before bootstrap") })?; match decode_client_message(&first_line)? { - WireClientMessage::BootstrapRequest => {} + WireClientMessage::BootstrapRequest { supervisor_pid } => { + if let Some(pid) = supervisor_pid.filter(|pid| *pid != 0) { + let _ = entrypoint_tx.send(EntrypointStarted { + pid, + ssh_socket_path: PathBuf::new(), + start_session: false, + }); + } + } WireClientMessage::EntrypointStarted { .. } => { return Err(miette::miette!( "sidecar control client sent entrypoint event before bootstrap" @@ -348,7 +357,7 @@ async fn handle_connection( return Ok(()); }; match decode_client_message(&line)? { - WireClientMessage::BootstrapRequest => { + WireClientMessage::BootstrapRequest { .. } => { debug!("Ignoring duplicate sidecar bootstrap request"); } WireClientMessage::EntrypointStarted { pid, ssh_socket_path } => { @@ -359,6 +368,7 @@ async fn handle_connection( let _ = entrypoint_tx.send(EntrypointStarted { pid, ssh_socket_path: PathBuf::from(ssh_socket_path), + start_session: true, }); } } @@ -382,7 +392,13 @@ pub async fn connect_process_client( ) -> Result<(BootstrapData, ProcessConnection)> { let stream = connect_with_retry(path, timeout).await?; let (reader, mut writer) = stream.into_split(); - write_json_line(&mut writer, &WireClientMessage::BootstrapRequest).await?; + write_json_line( + &mut writer, + &WireClientMessage::BootstrapRequest { + supervisor_pid: Some(std::process::id()), + }, + ) + .await?; let mut lines = BufReader::new(reader).lines(); let first_line = lines @@ -540,6 +556,13 @@ mod tests { .await .unwrap(); + let anchor = tokio::time::timeout(Duration::from_secs(1), entrypoint_rx.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!(anchor.pid, std::process::id()); + assert!(!anchor.start_session); + send_entrypoint_started( &connection.writer, 4242, @@ -557,6 +580,7 @@ mod tests { started.ssh_socket_path, PathBuf::from("/run/openshell-sidecar/ssh.sock") ); + assert!(started.start_session); } #[test] diff --git a/crates/openshell-supervisor-process/src/sandbox/linux/landlock.rs b/crates/openshell-supervisor-process/src/sandbox/linux/landlock.rs index 222377b5d2..fb82cf02fd 100644 --- a/crates/openshell-supervisor-process/src/sandbox/linux/landlock.rs +++ b/crates/openshell-supervisor-process/src/sandbox/linux/landlock.rs @@ -5,7 +5,7 @@ use landlock::{ ABI, Access, AccessFs, CompatLevel, Compatible, PathBeneath, PathFd, PathFdError, Ruleset, - RulesetAttr, + RulesetAttr, RulesetCreatedAttr, }; use miette::{IntoDiagnostic, Result}; use openshell_core::policy::{LandlockCompatibility, SandboxPolicy}; diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 6093edf638..8426060807 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -237,6 +237,7 @@ add `ci/values-spire.yaml` to the OpenShell release values files. | supervisor.image.pullPolicy | string | `""` | Supervisor image pull policy. Defaults to the gateway image pull policy when empty. | | supervisor.image.repository | string | `"ghcr.io/nvidia/openshell/supervisor"` | Supervisor image repository. | | supervisor.image.tag | string | `""` | Supervisor image tag. Defaults to the chart appVersion when empty. | +| supervisor.sidecar.processBinaryAwareNetworkPolicy | bool | `true` | Keep process/binary-aware network policy enabled in sidecar topology. When false, the network sidecar drops the SYS_PTRACE capability used for cross-UID /proc inspection and enforces endpoint/L7 policy without matching policy.binaries. | | supervisor.sidecar.proxyUid | int | `1337` | UID for the long-running network sidecar in sidecar topology. The network init container installs nftables rules that exempt this UID. | | supervisor.sideloadMethod | string | `""` | How the supervisor binary is delivered into sandbox pods. Empty (default) = auto-detect from cluster version: K8s >= v1.35 -> "image-volume" (ImageVolume enabled by default; GA in v1.36) K8s < v1.35 -> "init-container" (copies via init container + emptyDir) On K8s v1.33-v1.34 with the ImageVolume feature gate manually enabled, set this to "image-volume" explicitly. | | supervisor.topology | string | `"combined"` | Supervisor pod topology for Kubernetes sandboxes. "combined" runs the current single supervisor container in the agent pod. "sidecar" runs network enforcement in a dedicated sidecar and the process supervisor as a low-capability wrapper in the agent container. | diff --git a/deploy/helm/openshell/ci/values-sidecar.yaml b/deploy/helm/openshell/ci/values-sidecar.yaml index dac9e810f0..ba8dc50ef1 100644 --- a/deploy/helm/openshell/ci/values-sidecar.yaml +++ b/deploy/helm/openshell/ci/values-sidecar.yaml @@ -11,3 +11,8 @@ # before running `mise run e2e:kubernetes`. supervisor: topology: sidecar + sidecar: + # The strict sidecar default requires cross-container /proc identity access. + # CI/dev e2e uses the explicit downgraded mode so endpoint and L7 policy + # coverage remains runnable on local k3d while that path is hardened. + processBinaryAwareNetworkPolicy: false diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index 7ed16701da..0b4d0c9d9f 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -145,3 +145,4 @@ data: [openshell.drivers.kubernetes.sidecar] proxy_uid = {{ .Values.supervisor.sidecar.proxyUid | default 1337 }} + process_binary_aware_network_policy = {{ .Values.supervisor.sidecar.processBinaryAwareNetworkPolicy }} diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index 9e3dfd93fb..ee7d8fdc32 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -104,6 +104,15 @@ tests: path: data["gateway.toml"] pattern: '(?ms)\[openshell\.drivers\.kubernetes\.sidecar\].*?proxy_uid\s*=\s*2200' + - it: renders process binary aware network policy under [openshell.drivers.kubernetes.sidecar] + template: templates/gateway-config.yaml + set: + supervisor.sidecar.processBinaryAwareNetworkPolicy: false + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\.sidecar\].*?process_binary_aware_network_policy\s*=\s*false' + - it: renders sandbox image pull secrets under [openshell.drivers.kubernetes] template: templates/gateway-config.yaml set: diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index b756142d8b..e18e3b3586 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -53,6 +53,11 @@ supervisor: # -- UID for the long-running network sidecar in sidecar topology. The # network init container installs nftables rules that exempt this UID. proxyUid: 1337 + # -- Keep process/binary-aware network policy enabled in sidecar topology. + # When false, the network sidecar drops the SYS_PTRACE capability used for + # cross-UID /proc inspection and enforces endpoint/L7 policy without + # matching policy.binaries. + processBinaryAwareNetworkPolicy: true # -- Image pull secrets attached to gateway and helper pods. imagePullSecrets: [] diff --git a/docs/kubernetes/topology.mdx b/docs/kubernetes/topology.mdx index 8e0bb9e934..579e7634d4 100644 --- a/docs/kubernetes/topology.mdx +++ b/docs/kubernetes/topology.mdx @@ -31,7 +31,7 @@ The long-running container permissions differ by topology: |---|---|---|---|---|---| | `combined` | Agent container, which also runs the supervisor | Not forced by topology | Not explicitly disabled by the driver | Adds `SYS_ADMIN`, `NET_ADMIN`, `SYS_PTRACE`, and `SYSLOG`; adds `SETUID`, `SETGID`, and `DAC_READ_SEARCH` when user namespaces are enabled | Full supervisor controls run in the agent container. | | `sidecar` | Agent container, process-only supervisor (`network-only`) | `sandbox_uid:sandbox_gid` | `false` | Drops `ALL` | Agent and workload run without added Linux capabilities. | -| `sidecar` | Network supervisor sidecar | `proxyUid:sandbox_gid` | `false` | Drops `ALL` | Long-running proxy sidecar is also non-root without added capabilities. | +| `sidecar` | Network supervisor sidecar | `proxyUid:sandbox_gid` | `false` | Drops `ALL`; adds `SYS_PTRACE` by default | Long-running proxy sidecar is non-root and can inspect workload `/proc` entries for process/binary-aware network policy. | Short-lived setup containers still have the permissions needed to prepare the pod: @@ -126,8 +126,12 @@ The pod contains these OpenShell-managed pieces: In this topology, the agent container defaults to `runAsNonRoot: true`, `allowPrivilegeEscalation: false`, and `capabilities.drop: ["ALL"]`. The -long-running network sidecar always drops all Linux capabilities. The root init -container keeps the setup capabilities needed to configure pod networking. +long-running network sidecar drops default Linux capabilities and adds only +`SYS_PTRACE` for workload process identity resolution by default. Setting +`supervisor.sidecar.processBinaryAwareNetworkPolicy=false` omits that capability +and downgrades network policy to endpoint/L7 enforcement without binary +matching. The root init container keeps the setup capabilities needed to +configure pod networking. Sidecar mode preserves gateway session behavior, including SSH connectivity, because the network sidecar owns the gateway session and bridges relay requests @@ -208,12 +212,18 @@ supervisor: topology: sidecar sidecar: proxyUid: 1337 + processBinaryAwareNetworkPolicy: true ``` Leave `topology` unset, or set it to `combined`, to keep the original single-container supervisor path. For Helm installs, leave `supervisor.topology` unset or set it to `combined`. +Set `supervisor.sidecar.processBinaryAwareNetworkPolicy=false` only when you +accept downgrading sidecar network policy to endpoint/L7 enforcement without +matching `policy.binaries`. This removes the sidecar's `SYS_PTRACE` capability, +which is used for cross-UID `/proc` inspection. + ## Next Steps - To install OpenShell on Kubernetes, refer to [Setup](/kubernetes/setup). diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 0e190b3426..451a5ead5f 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -209,6 +209,10 @@ provider_spiffe_workload_api_socket_path = "/spiffe-workload-api/spire-agent.soc # UID used by the long-running network sidecar. In sidecar topology the # network init container installs nftables rules that exempt this UID. proxy_uid = 1337 +# Keep process/binary-aware network policy enabled in sidecar topology. Set +# false to drop the sidecar's SYS_PTRACE capability used for cross-UID /proc +# inspection and enforce endpoint/L7 policy without matching policy.binaries. +process_binary_aware_network_policy = true ``` ### Docker diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 864a1ea242..a1cac67924 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -308,6 +308,7 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA | `supervisor_sideload_method` | `supervisor.sideloadMethod` | How the supervisor binary is delivered into sandbox pods. Leave empty to auto-detect from cluster version. Set to `image-volume` to mount the supervisor OCI image directly as a volume (requires Kubernetes 1.33+ with the ImageVolume feature gate; GA in 1.36), or `init-container` to copy it through an init container on older clusters. | | `topology` | `supervisor.topology` | Set `combined` for the default single supervisor path, or `sidecar` to move pod-level network enforcement and the gateway session into a dedicated sidecar. | | `sidecar.proxy_uid` | `supervisor.sidecar.proxyUid` | UID used by the long-running network sidecar in `sidecar` topology. The network init container exempts this UID from proxy redirection. | +| `sidecar.process_binary_aware_network_policy` | `supervisor.sidecar.processBinaryAwareNetworkPolicy` | Keep process/binary-aware network policy enabled in `sidecar` topology. Set false to drop the sidecar's `SYS_PTRACE` capability used for cross-UID `/proc` inspection and enforce endpoint/L7 policy without matching `policy.binaries`. | | `app_armor_profile` | `server.appArmorProfile` | Set the sandbox agent container's AppArmor profile. Helm defaults this to `Unconfined` so AppArmor-enabled nodes do not block supervisor network namespace setup. Set the Helm value to an empty string to omit the field, or use `RuntimeDefault` or `Localhost/` for operator-managed profiles. | | `workspace_default_storage_size` | `server.workspaceDefaultStorageSize` | Set the default workspace PVC size for new sandboxes. | | `sa_token_ttl_secs` | `server.sandboxJwt.k8sSaTokenTtlSecs` | Set the projected ServiceAccount token TTL used for the bootstrap token exchange. | @@ -317,7 +318,10 @@ needed by the supervisor for network namespace setup, Landlock filesystem policy, process privilege changes, and network policy enforcement. In `sidecar` topology, the agent container runs as the resolved sandbox UID/GID with no added Linux capabilities. A root init container performs the nftables setup, and the -long-running sidecar runs non-root with no added Linux capabilities. The +long-running sidecar runs non-root, drops default capabilities, and adds only +`SYS_PTRACE` for workload process identity resolution through shared `/proc`. The +`sidecar.process_binary_aware_network_policy = false` setting removes that +sidecar capability and relaxes network policy to endpoint/L7 matching only. The network sidecar owns gateway authentication and writes local policy/provider state to the process supervisor over a local control socket, so the agent container does not mount the sandbox bootstrap token or client TLS secret in From d46cd483909e87fd0e30a79520d6a5eb4c2c2c74 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Wed, 8 Jul 2026 14:23:04 -0700 Subject: [PATCH 18/24] fix(sandbox): satisfy sidecar clippy lint Signed-off-by: Taylor Mutch --- crates/openshell-sandbox/src/lib.rs | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 2c119ed3f3..81b2a9a005 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -421,17 +421,18 @@ pub async fn run_sandbox( .map(sidecar_control::ServerHandle::publisher); #[cfg(target_os = "linux")] - if network_enabled && sidecar_network_enforcement { - if let Some(server) = sidecar_control_server { - spawn_sidecar_entrypoint_handler( - server.into_entrypoint_receiver(), - entrypoint_pid.clone(), - opa_engine.clone(), - retained_proto.clone(), - openshell_endpoint.clone(), - sandbox_id.clone(), - ); - } + if network_enabled + && sidecar_network_enforcement + && let Some(server) = sidecar_control_server + { + spawn_sidecar_entrypoint_handler( + server.into_entrypoint_receiver(), + entrypoint_pid.clone(), + opa_engine.clone(), + retained_proto.clone(), + openshell_endpoint.clone(), + sandbox_id.clone(), + ); } #[cfg(not(target_os = "linux"))] From 5d43818934613cf0da9651c7611b63d33e7185aa Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Wed, 8 Jul 2026 14:28:39 -0700 Subject: [PATCH 19/24] refactor(kubernetes): standardize topology naming Signed-off-by: Taylor Mutch --- .../openshell-driver-kubernetes/src/config.rs | 2 +- .../openshell-driver-kubernetes/src/driver.rs | 24 +++++++++---------- .../openshell-driver-kubernetes/src/main.rs | 7 +----- 3 files changed, 13 insertions(+), 20 deletions(-) diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 2ab5179edc..a9fa730c5e 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -83,7 +83,7 @@ impl FromStr for SupervisorTopology { match s { "combined" => Ok(Self::Combined), "sidecar" => Ok(Self::Sidecar), - other => Err(format!("unknown supervisor topology '{other}'")), + other => Err(format!("unknown topology '{other}'")), } } } diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index f74274939a..6957a69a43 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -563,7 +563,7 @@ impl KubernetesComputeDriver { supervisor_image: &self.config.supervisor_image, supervisor_image_pull_policy: &self.config.supervisor_image_pull_policy, supervisor_sideload_method: self.config.supervisor_sideload_method, - supervisor_topology: self.config.topology, + topology: self.config.topology, proxy_uid: self.config.sidecar.proxy_uid, process_binary_aware_network_policy: self .config @@ -1786,7 +1786,7 @@ struct SandboxPodParams<'a> { supervisor_image: &'a str, supervisor_image_pull_policy: &'a str, supervisor_sideload_method: SupervisorSideloadMethod, - supervisor_topology: SupervisorTopology, + topology: SupervisorTopology, proxy_uid: u32, process_binary_aware_network_policy: bool, service_account_name: &'a str, @@ -1820,7 +1820,7 @@ impl Default for SandboxPodParams<'_> { supervisor_image: "", supervisor_image_pull_policy: "", supervisor_sideload_method: SupervisorSideloadMethod::default(), - supervisor_topology: SupervisorTopology::default(), + topology: SupervisorTopology::default(), proxy_uid: DEFAULT_PROXY_UID, process_binary_aware_network_policy: true, service_account_name: DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, @@ -1846,9 +1846,7 @@ impl Default for SandboxPodParams<'_> { fn validate_sidecar_proxy_identity( params: &SandboxPodParams<'_>, ) -> Result<(), KubernetesDriverError> { - if params.supervisor_topology == SupervisorTopology::Sidecar - && params.proxy_uid == params.sandbox_uid - { + if params.topology == SupervisorTopology::Sidecar && params.proxy_uid == params.sandbox_uid { return Err(KubernetesDriverError::Precondition(format!( "proxy_uid ({}) must not match sandbox_uid ({}) in sidecar topology", params.proxy_uid, params.sandbox_uid @@ -2153,7 +2151,7 @@ fn sandbox_template_to_k8s_with_gpu_requirements( // pod fsGroup + 0440 to preserve gateway session and SSH control behavior. let mut volumes: Vec = Vec::new(); if !params.client_tls_secret_name.is_empty() { - let client_tls_default_mode = match params.supervisor_topology { + let client_tls_default_mode = match params.topology { SupervisorTopology::Combined => 0o400, SupervisorTopology::Sidecar => 0o440, }; @@ -2179,7 +2177,7 @@ fn sandbox_template_to_k8s_with_gpu_requirements( // it automatically. The supervisor exchanges this for a gateway-minted // JWT via `IssueSandboxToken` once at startup. In sidecar topology both // supervisor containers run with the sandbox GID and need group-read access. - let sa_token_default_mode = match params.supervisor_topology { + let sa_token_default_mode = match params.topology { SupervisorTopology::Combined => 0o400, SupervisorTopology::Sidecar => 0o440, }; @@ -2217,7 +2215,7 @@ fn sandbox_template_to_k8s_with_gpu_requirements( let mut result = serde_json::Value::Object(template_value); - match params.supervisor_topology { + match params.topology { SupervisorTopology::Combined => { apply_supervisor_sideload( &mut result, @@ -3087,7 +3085,7 @@ mod tests { #[test] fn sidecar_topology_renders_process_agent_and_network_sidecar() { let params = SandboxPodParams { - supervisor_topology: SupervisorTopology::Sidecar, + topology: SupervisorTopology::Sidecar, supervisor_sideload_method: SupervisorSideloadMethod::InitContainer, supervisor_image: "supervisor-image:latest", supervisor_image_pull_policy: "IfNotPresent", @@ -3305,7 +3303,7 @@ mod tests { #[test] fn sidecar_topology_can_relax_process_binary_aware_network_policy() { let params = SandboxPodParams { - supervisor_topology: SupervisorTopology::Sidecar, + topology: SupervisorTopology::Sidecar, supervisor_sideload_method: SupervisorSideloadMethod::InitContainer, supervisor_image: "supervisor-image:latest", proxy_uid: 2200, @@ -3348,7 +3346,7 @@ mod tests { #[test] fn sidecar_topology_adds_shared_state_and_tls_volumes() { let params = SandboxPodParams { - supervisor_topology: SupervisorTopology::Sidecar, + topology: SupervisorTopology::Sidecar, supervisor_sideload_method: SupervisorSideloadMethod::ImageVolume, supervisor_image: "supervisor-image:latest", grpc_endpoint: "http://openshell-gateway.openshell.svc:8080", @@ -3398,7 +3396,7 @@ mod tests { #[test] fn sidecar_topology_rejects_proxy_uid_matching_sandbox_uid() { let params = SandboxPodParams { - supervisor_topology: SupervisorTopology::Sidecar, + topology: SupervisorTopology::Sidecar, proxy_uid: 1500, sandbox_uid: 1500, ..SandboxPodParams::default() diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 81ed57d818..d43202e6b2 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -81,12 +81,7 @@ struct Args { )] supervisor_sideload_method: SupervisorSideloadMethod, - #[arg( - long, - alias = "supervisor-topology", - env = "OPENSHELL_K8S_TOPOLOGY", - default_value = "combined" - )] + #[arg(long, env = "OPENSHELL_K8S_TOPOLOGY", default_value = "combined")] topology: SupervisorTopology, #[arg( From ac4211c42e1058d791b2e3e00f69c823cec3957e Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Wed, 8 Jul 2026 15:39:21 -0700 Subject: [PATCH 20/24] fix(sandbox): satisfy linux clippy timeout import Signed-off-by: Taylor Mutch --- crates/openshell-sandbox/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 81b2a9a005..7635a0f791 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -69,7 +69,7 @@ use openshell_supervisor_process::process::ProcessEnforcementMode; pub use openshell_supervisor_process::process::{ProcessHandle, ProcessStatus}; use openshell_supervisor_process::skills; use tokio::sync::mpsc::UnboundedSender; -#[cfg(target_os = "linux")] +#[cfg(any(test, target_os = "linux"))] use tokio::time::timeout; const SIDECAR_NETWORK_ENFORCEMENT_MODE: &str = "sidecar-nftables"; @@ -2398,7 +2398,7 @@ mod tests { }) .unwrap(); - tokio::time::timeout(Duration::from_secs(1), async { + timeout(Duration::from_secs(1), async { loop { if provider_credentials.snapshot().revision == 2 { break; From 49192b8d92b79a8fab4ed3c7ee934d88246fbfdc Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Wed, 8 Jul 2026 17:32:02 -0700 Subject: [PATCH 21/24] fix(kubernetes): support kata sidecar on ipv4 pods Signed-off-by: Taylor Mutch --- .../src/netns/mod.rs | 96 +++++++++++++++---- .../openshell/ci/values-sidecar-kata.yaml | 24 +++++ 2 files changed, 103 insertions(+), 17 deletions(-) create mode 100644 deploy/helm/openshell/ci/values-sidecar-kata.yaml diff --git a/crates/openshell-supervisor-process/src/netns/mod.rs b/crates/openshell-supervisor-process/src/netns/mod.rs index c9e1a46eb9..55b87f131d 100644 --- a/crates/openshell-supervisor-process/src/netns/mod.rs +++ b/crates/openshell-supervisor-process/src/netns/mod.rs @@ -485,6 +485,7 @@ fn install_sidecar_nft_bypass_rules(proxy_uid: u32) -> Result<()> { } const SIDECAR_IPTABLES_CHAIN: &str = "OPENSHELL_SIDECAR_BYPASS"; +const PROC_NET_IF_INET6_PATH: &str = "/proc/net/if_inet6"; fn install_sidecar_iptables_legacy_bypass_rules(proxy_uid: u32) -> Result<()> { let ipv4_filter_tool = find_iptables_legacy().ok_or_else(|| { @@ -492,38 +493,69 @@ fn install_sidecar_iptables_legacy_bypass_rules(proxy_uid: u32) -> Result<()> { "trusted iptables-legacy helper not found; sidecar network enforcement fallback unavailable" ) })?; - let ipv6_fence_tool = find_ip6tables_legacy().ok_or_else(|| { - miette::miette!( - "trusted ip6tables-legacy helper not found; sidecar network enforcement fallback cannot fence IPv6" - ) - })?; - cleanup_sidecar_iptables_legacy_rules(&ipv4_filter_tool); - cleanup_sidecar_iptables_legacy_rules(&ipv6_fence_tool); + let ipv6_fence_tool = if current_namespace_has_non_loopback_ipv6()? { + Some(find_ip6tables_legacy().ok_or_else(|| { + miette::miette!( + "trusted ip6tables-legacy helper not found; sidecar network enforcement fallback cannot fence IPv6" + ) + })?) + } else { + warn!( + "Skipping IPv6 sidecar iptables-legacy fallback because the current namespace has no non-loopback IPv6 interface" + ); + None + }; + + cleanup_sidecar_iptables_legacy_rule_families(&ipv4_filter_tool, ipv6_fence_tool.as_deref()); if let Err(e) = install_sidecar_iptables_legacy_family_rules( &ipv4_filter_tool, proxy_uid, "icmp-port-unreachable", ) { - cleanup_sidecar_iptables_legacy_rules(&ipv4_filter_tool); - cleanup_sidecar_iptables_legacy_rules(&ipv6_fence_tool); + cleanup_sidecar_iptables_legacy_rule_families( + &ipv4_filter_tool, + ipv6_fence_tool.as_deref(), + ); return Err(e); } - if let Err(e) = install_sidecar_iptables_legacy_family_rules( - &ipv6_fence_tool, - proxy_uid, - "icmp6-port-unreachable", - ) { - cleanup_sidecar_iptables_legacy_rules(&ipv4_filter_tool); - cleanup_sidecar_iptables_legacy_rules(&ipv6_fence_tool); - return Err(e); + if let Some(ipv6_fence_tool) = ipv6_fence_tool { + if let Err(e) = install_sidecar_iptables_legacy_family_rules( + &ipv6_fence_tool, + proxy_uid, + "icmp6-port-unreachable", + ) { + cleanup_sidecar_iptables_legacy_rule_families( + &ipv4_filter_tool, + Some(&ipv6_fence_tool), + ); + return Err(e); + } } Ok(()) } +fn current_namespace_has_non_loopback_ipv6() -> Result { + match std::fs::read_to_string(PROC_NET_IF_INET6_PATH) { + Ok(content) => Ok(has_non_loopback_ipv6_interface(&content)), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(e) => Err(miette::miette!( + "failed to inspect {PROC_NET_IF_INET6_PATH} before installing sidecar IPv6 fence: {e}" + )), + } +} + +fn has_non_loopback_ipv6_interface(content: &str) -> bool { + content.lines().any(|line| { + line.split_whitespace() + .nth(5) + .is_some_and(|iface| iface != "lo") + }) +} + fn install_sidecar_iptables_legacy_family_rules( cmd: &str, proxy_uid: u32, @@ -597,6 +629,13 @@ fn cleanup_sidecar_iptables_legacy_rules(iptables_cmd: &str) { let _ = run_iptables_legacy_current_namespace(iptables_cmd, &["-X", SIDECAR_IPTABLES_CHAIN]); } +fn cleanup_sidecar_iptables_legacy_rule_families(ipv4_cmd: &str, ipv6_cmd: Option<&str>) { + cleanup_sidecar_iptables_legacy_rules(ipv4_cmd); + if let Some(ipv6_cmd) = ipv6_cmd { + cleanup_sidecar_iptables_legacy_rules(ipv6_cmd); + } +} + /// Run an `ip` command on the host. fn run_ip(args: &[&str]) -> Result<()> { let ip_path = find_trusted_binary("ip", IP_SEARCH_PATHS)?; @@ -906,6 +945,29 @@ mod tests { } } + #[test] + fn non_loopback_ipv6_detector_ignores_empty_input() { + assert!(!has_non_loopback_ipv6_interface("")); + assert!(!has_non_loopback_ipv6_interface("\n\n")); + } + + #[test] + fn non_loopback_ipv6_detector_ignores_loopback() { + let content = "00000000000000000000000000000001 01 80 10 80 lo\n"; + + assert!(!has_non_loopback_ipv6_interface(content)); + } + + #[test] + fn non_loopback_ipv6_detector_detects_pod_interface() { + let content = "\ +00000000000000000000000000000001 01 80 10 80 lo +fe800000000000000000000000000001 02 40 20 80 eth0 +"; + + assert!(has_non_loopback_ipv6_interface(content)); + } + #[test] #[ignore = "requires root privileges"] fn test_create_and_drop_namespace() { diff --git a/deploy/helm/openshell/ci/values-sidecar-kata.yaml b/deploy/helm/openshell/ci/values-sidecar-kata.yaml new file mode 100644 index 0000000000..2e23a1009e --- /dev/null +++ b/deploy/helm/openshell/ci/values-sidecar-kata.yaml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# CI/dev overlay for exercising the Kubernetes supervisor sidecar topology under +# a Kata RuntimeClass. +# +# Use with e2e/with-kube-gateway.sh by setting: +# OPENSHELL_E2E_KUBE_EXTRA_VALUES=deploy/helm/openshell/ci/values-sidecar-kata.yaml +# The e2e wrapper supplies the image repository and tag through OPENSHELL_REGISTRY +# and IMAGE_TAG for existing-cluster runs. + +supervisor: + # Use the sidecar topology under Kata so network enforcement runs in the + # sidecar and the sandbox agent container stays low-privilege. + topology: sidecar + sidecar: + # Keep strict process/binary-aware network policy enabled for the Kata + # validation path. Set this false only when intentionally validating the + # documented endpoint/L7-only downgrade mode. + processBinaryAwareNetworkPolicy: true + +# Kata validation clusters normally install this RuntimeClass. +server: + defaultRuntimeClassName: kata-qemu From 5ab5a52292edc227f84e1a3661a1972c32a2dcbc Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Wed, 8 Jul 2026 18:13:41 -0700 Subject: [PATCH 22/24] fix(kubernetes): satisfy linux clippy for sidecar fallback Signed-off-by: Taylor Mutch --- .../openshell-supervisor-process/src/netns/mod.rs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/crates/openshell-supervisor-process/src/netns/mod.rs b/crates/openshell-supervisor-process/src/netns/mod.rs index 55b87f131d..44e9470931 100644 --- a/crates/openshell-supervisor-process/src/netns/mod.rs +++ b/crates/openshell-supervisor-process/src/netns/mod.rs @@ -521,18 +521,15 @@ fn install_sidecar_iptables_legacy_bypass_rules(proxy_uid: u32) -> Result<()> { return Err(e); } - if let Some(ipv6_fence_tool) = ipv6_fence_tool { - if let Err(e) = install_sidecar_iptables_legacy_family_rules( + if let Some(ipv6_fence_tool) = ipv6_fence_tool + && let Err(e) = install_sidecar_iptables_legacy_family_rules( &ipv6_fence_tool, proxy_uid, "icmp6-port-unreachable", - ) { - cleanup_sidecar_iptables_legacy_rule_families( - &ipv4_filter_tool, - Some(&ipv6_fence_tool), - ); - return Err(e); - } + ) + { + cleanup_sidecar_iptables_legacy_rule_families(&ipv4_filter_tool, Some(&ipv6_fence_tool)); + return Err(e); } Ok(()) From 552605ca6e0b8d1397255fdea7d13efe240444f1 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Wed, 8 Jul 2026 22:27:15 -0700 Subject: [PATCH 23/24] chore(kubernetes): remove stale supervisor topology references Signed-off-by: Taylor Mutch --- .../skills/debug-openshell-cluster/SKILL.md | 48 +++++++++---------- .../openshell-driver-kubernetes/src/config.rs | 14 ++++-- .../openshell/tests/gateway_config_test.yaml | 2 +- 3 files changed, 34 insertions(+), 30 deletions(-) diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index d5fa6aa6fc..788bac3116 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -268,41 +268,41 @@ kubectl -n openshell get configmap openshell-config -o jsonpath='{.data.gateway\ kubectl -n get sandbox -o jsonpath='{.spec.template.spec.serviceAccountName}{"\n"}' ``` -If `supervisor_topology = "sidecar"` is rendered, sandbox pods should have an -`openshell-network-init` init container running `--mode=network-init`, an -`agent` container running `openshell-sandbox --mode=process`, and an -`openshell-supervisor-network` container running `--mode=network`. The init -container owns nftables setup and should be the only sidecar topology container -with `NET_ADMIN`. It also needs `CHOWN`/`FOWNER` to hand shared emptyDir state -to `sidecar_proxy_uid`. The long-running network sidecar runs as -`sidecar_proxy_uid` with primary GID `sandbox_gid`; the pod `fsGroup` is also -set to `sandbox_gid`. +If `topology = "sidecar"` is rendered under `[openshell.drivers.kubernetes]`, +sandbox pods should have an `openshell-network-init` init container running +`--mode=network-init`, an `agent` container running +`openshell-sandbox --mode=process`, and an `openshell-supervisor-network` +container running `--mode=network`. The init container owns nftables setup and +should be the only sidecar topology container with `NET_ADMIN`. It also needs +`CHOWN`/`FOWNER` to hand shared emptyDir state to `proxy_uid`. The long-running +network sidecar runs as `proxy_uid` with primary GID `sandbox_gid`; the pod +`fsGroup` is also set to `sandbox_gid`. In sidecar topology only the network sidecar should mount the gateway bootstrap credentials (`openshell-sa-token` and `openshell-client-tls`). The process container should not receive `OPENSHELL_ENDPOINT`, gateway TLS env vars, the sandbox token file, or those credential mounts. Instead, the network sidecar -writes `/run/openshell-sidecar/policy.pb` and -`/run/openshell-sidecar/provider-env.json`, then writes the readiness file. If -the process supervisor fails before launching the workload, inspect those -snapshot files and the network sidecar logs. If new SSH/exec sessions do not -pick up refreshed provider environment, inspect the provider-env snapshot -revision and network sidecar settings-poll logs; the process container should -consume newer provider-env snapshot revisions without receiving gateway -credentials. - -The process container should also publish the workload entrypoint PID to -`OPENSHELL_ENTRYPOINT_PID_FILE` -(`/run/openshell-sidecar/entrypoint.pid` by default), and the network sidecar -should read it for binary-scoped policy decisions; if allowed network rules are -all denied, inspect that file and the network sidecar logs. +serves policy and provider environment state over the Unix control socket from +`OPENSHELL_SIDECAR_CONTROL_SOCKET` (`/run/openshell-sidecar/control.sock` by +default). If the process supervisor fails before launching the workload, +inspect both containers for control-socket bind, connect, bootstrap, or update +errors. If new SSH/exec sessions do not pick up refreshed provider environment, +inspect the network sidecar settings-poll logs and the process container logs +for provider environment update handling; the process container should consume +newer provider-env revisions without receiving gateway credentials. + +The process container reports the workload entrypoint PID over the same control +socket, and the network sidecar uses that PID for binary-scoped policy +decisions through `/proc`. If rules with `policy.binaries` are unexpectedly +denied, inspect the sidecar control logs and confirm the pod has +`shareProcessNamespace: true`. The shared state directory should preserve `sandbox_gid` inheritance (`02775`), and the SSH socket should be group-connectable (`0660`) so the network sidecar can bridge gateway relay requests to the process supervisor. Inspect all three when sandbox registration or egress enforcement fails: ```bash -kubectl -n openshell get configmap openshell-config -o jsonpath='{.data.gateway\.toml}' | grep supervisor_topology +kubectl -n openshell get configmap openshell-config -o jsonpath='{.data.gateway\.toml}' | grep -E '^\[openshell\.drivers\.kubernetes\]|^topology\s*=' kubectl -n get pod -o jsonpath='{range .spec.initContainers[*]}{.name}{" "}{.command}{"\n"}{end}' kubectl -n get pod -o jsonpath='{range .spec.containers[*]}{.name}{" "}{.command}{"\n"}{end}' kubectl -n logs -c openshell-network-init --tail=200 diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index a9fa730c5e..cc834b6de5 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -606,11 +606,15 @@ mod tests { } #[test] - fn serde_rejects_removed_supervisor_topology_field() { - let json = serde_json::json!({ - "supervisor_topology": "sidecar" - }); - let err = serde_json::from_value::(json).unwrap_err(); + fn serde_rejects_removed_topology_alias_field() { + let mut json = serde_json::Map::new(); + json.insert( + ["supervisor", "topology"].join("_"), + serde_json::json!("sidecar"), + ); + let err = + serde_json::from_value::(serde_json::Value::Object(json)) + .unwrap_err(); assert!(err.to_string().contains("unknown field")); } diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index ee7d8fdc32..b47225ebd2 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -93,7 +93,7 @@ tests: pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?topology\s*=\s*"sidecar"' - notMatchRegex: path: data["gateway.toml"] - pattern: 'supervisor_topology\s*=' + pattern: 'supervisor[_]topology\s*=' - it: renders proxy uid under [openshell.drivers.kubernetes.sidecar] template: templates/gateway-config.yaml From 497f3e3f0104a87052e8f58ad203c31ef8d68f9d Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Thu, 9 Jul 2026 13:16:32 -0700 Subject: [PATCH 24/24] fix(kubernetes): enable sidecar binary policy inspection Signed-off-by: Taylor Mutch --- crates/openshell-driver-kubernetes/README.md | 17 +++-- .../openshell-driver-kubernetes/src/config.rs | 16 +++-- .../openshell-driver-kubernetes/src/driver.rs | 72 ++++++++++++++++--- crates/openshell-sandbox/src/main.rs | 47 ++++++++---- deploy/helm/openshell/README.md | 4 +- deploy/helm/openshell/values.yaml | 13 ++-- docs/reference/gateway-config.mdx | 11 +-- 7 files changed, 135 insertions(+), 45 deletions(-) diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index 67caae699b..f6c0e01330 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -58,13 +58,16 @@ the Linux capabilities the supervisor needs for namespace setup and process, filesystem, and network policy enforcement. The `sidecar` supervisor topology moves pod-level network setup into a root init -container and runs the long-lived network sidecar as a non-root UID that drops -default Linux capabilities and adds only `SYS_PTRACE` for workload `/proc` -inspection. The agent container also runs as the resolved sandbox UID/GID with -`allowPrivilegeEscalation: false` and `capabilities.drop: ["ALL"]`. -Set `sidecar.process_binary_aware_network_policy = false` to omit `SYS_PTRACE` -from the sidecar and enforce endpoint/L7 network policy without matching -`policy.binaries`. +container. In the default process/binary-aware mode, the long-lived network +sidecar runs as UID 0 with `allowPrivilegeEscalation: false`, drops default +Linux capabilities, and adds only `SYS_PTRACE` plus `DAC_READ_SEARCH` for +cross-UID workload `/proc` inspection. The agent container also runs as the +resolved sandbox UID/GID with `allowPrivilegeEscalation: false` and +`capabilities.drop: ["ALL"]`. +Set `sidecar.process_binary_aware_network_policy = false` to run the network +sidecar as the configured non-root `sidecar.proxy_uid`, omit the extra `/proc` +inspection capabilities, and enforce endpoint/L7 network policy without +matching `policy.binaries`. In this mode OpenShell preserves gateway session and SSH behavior, but the process supervisor does not perform root-to-sandbox privilege dropping or supervisor identity mount isolation. It still applies Landlock filesystem policy diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index cc834b6de5..d2f841147c 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -15,7 +15,7 @@ pub const DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME: &str = "default"; /// Default storage size for the workspace PVC. pub const DEFAULT_WORKSPACE_STORAGE_SIZE: &str = "2Gi"; -/// Default UID for the long-running Kubernetes network supervisor sidecar. +/// Default non-root UID for relaxed Kubernetes network supervisor sidecars. pub const DEFAULT_PROXY_UID: u32 = 1337; /// How the supervisor binary is delivered into sandbox pods. @@ -91,14 +91,16 @@ impl FromStr for SupervisorTopology { #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct KubernetesSidecarConfig { - /// UID used by the long-running network sidecar in `sidecar` topology. - /// The network init container installs nftables rules that exempt this - /// UID, so it must not match the sandbox workload UID. + /// UID used by relaxed long-running network sidecars in `sidecar` + /// topology. The network init container installs nftables rules that + /// exempt this UID, so it must not match the sandbox workload UID. + /// Strict process/binary-aware sidecars run as UID 0 so Kubernetes grants + /// the requested `/proc` inspection capabilities into the effective set. pub proxy_uid: u32, /// Require process/binary-aware network policy enforcement in sidecar - /// topology. When disabled, the network sidecar drops the extra `/proc` - /// inspection permission and evaluates endpoint/L7 policy without - /// matching `policy.binaries`. + /// topology. When disabled, the network sidecar runs as `proxy_uid`, + /// drops the extra `/proc` inspection permissions, and evaluates + /// endpoint/L7 policy without matching `policy.binaries`. pub process_binary_aware_network_policy: bool, } diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 6957a69a43..de02c9ce2c 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -1053,6 +1053,10 @@ const SUPERVISOR_NETWORK_INIT_CONTAINER_NAME: &str = "openshell-network-init"; /// Container name for the network-only supervisor sidecar. const SUPERVISOR_NETWORK_SIDECAR_NAME: &str = "openshell-supervisor-network"; +/// UID used by strict process/binary-aware sidecars so Kubernetes grants the +/// requested capability set into the effective set without privilege escalation. +const BINARY_AWARE_SIDECAR_PROXY_UID: u32 = 0; + /// Shared volume used by the network sidecar and process-only supervisor for /// local coordination in sidecar topology. const SIDECAR_STATE_VOLUME_NAME: &str = "openshell-sidecar-state"; @@ -1365,10 +1369,11 @@ fn supervisor_sidecar_container( spec_environment: &std::collections::HashMap, params: &SandboxPodParams<'_>, ) -> serde_json::Value { + let proxy_uid = effective_sidecar_proxy_uid(params); let capabilities = if params.process_binary_aware_network_policy { serde_json::json!({ "drop": ["ALL"], - "add": ["SYS_PTRACE"] + "add": ["SYS_PTRACE", "DAC_READ_SEARCH"] }) } else { serde_json::json!({ @@ -1384,9 +1389,9 @@ fn supervisor_sidecar_container( ], "env": supervisor_sidecar_env(template_environment, spec_environment, params), "securityContext": { - "runAsUser": params.proxy_uid, + "runAsUser": proxy_uid, "runAsGroup": params.sandbox_gid, - "runAsNonRoot": true, + "runAsNonRoot": proxy_uid != 0, "allowPrivilegeEscalation": false, "capabilities": capabilities }, @@ -1419,7 +1424,16 @@ fn supervisor_sidecar_container( container } +fn effective_sidecar_proxy_uid(params: &SandboxPodParams<'_>) -> u32 { + if params.process_binary_aware_network_policy { + BINARY_AWARE_SIDECAR_PROXY_UID + } else { + params.proxy_uid + } +} + fn supervisor_network_init_container(params: &SandboxPodParams<'_>) -> serde_json::Value { + let proxy_uid = effective_sidecar_proxy_uid(params); let mut container = serde_json::json!({ "name": SUPERVISOR_NETWORK_INIT_CONTAINER_NAME, "image": params.supervisor_image, @@ -1427,7 +1441,7 @@ fn supervisor_network_init_container(params: &SandboxPodParams<'_>) -> serde_jso SUPERVISOR_IMAGE_BINARY_PATH, "--mode=network-init", "--proxy-uid", - params.proxy_uid.to_string(), + proxy_uid.to_string(), "--proxy-gid", params.sandbox_gid.to_string(), "--sidecar-state-dir", @@ -3186,14 +3200,18 @@ mod tests { sidecar["command"], serde_json::json!([SUPERVISOR_IMAGE_BINARY_PATH, "--mode=network"]) ); - assert_eq!(sidecar["securityContext"]["runAsUser"], 2200); + assert_eq!(sidecar["securityContext"]["runAsUser"], 0); assert_eq!(sidecar["securityContext"]["runAsGroup"], 1500); - assert_eq!(sidecar["securityContext"]["runAsNonRoot"], true); + assert_eq!(sidecar["securityContext"]["runAsNonRoot"], false); + assert_eq!( + sidecar["securityContext"]["allowPrivilegeEscalation"], + false + ); assert_eq!( sidecar["securityContext"]["capabilities"], serde_json::json!({ "drop": ["ALL"], - "add": ["SYS_PTRACE"] + "add": ["SYS_PTRACE", "DAC_READ_SEARCH"] }) ); assert_eq!( @@ -3277,7 +3295,7 @@ mod tests { SUPERVISOR_IMAGE_BINARY_PATH, "--mode=network-init", "--proxy-uid", - "2200", + "0", "--proxy-gid", "1500", "--sidecar-state-dir", @@ -3328,6 +3346,13 @@ mod tests { .iter() .find(|container| container["name"] == SUPERVISOR_NETWORK_SIDECAR_NAME) .unwrap(); + assert_eq!(sidecar["securityContext"]["runAsUser"], 2200); + assert_eq!(sidecar["securityContext"]["runAsGroup"], 1500); + assert_eq!(sidecar["securityContext"]["runAsNonRoot"], true); + assert_eq!( + sidecar["securityContext"]["allowPrivilegeEscalation"], + false + ); assert_eq!( sidecar["securityContext"]["capabilities"], serde_json::json!({ @@ -3341,6 +3366,12 @@ mod tests { ), Some("relaxed") ); + let init_containers = pod_template["spec"]["initContainers"].as_array().unwrap(); + let network_init = init_containers + .iter() + .find(|container| container["name"] == SUPERVISOR_NETWORK_INIT_CONTAINER_NAME) + .unwrap(); + assert_eq!(network_init["command"][3], "2200"); } #[test] @@ -3376,6 +3407,25 @@ mod tests { })); let containers = pod_template["spec"]["containers"].as_array().unwrap(); + let sidecar = containers + .iter() + .find(|container| container["name"] == SUPERVISOR_NETWORK_SIDECAR_NAME) + .unwrap(); + assert_eq!( + sidecar["securityContext"]["capabilities"], + serde_json::json!({ + "drop": ["ALL"], + "add": ["SYS_PTRACE", "DAC_READ_SEARCH"] + }) + ); + assert_eq!(sidecar["securityContext"]["runAsUser"], 0); + assert_eq!(sidecar["securityContext"]["runAsGroup"], 1000); + assert_eq!(sidecar["securityContext"]["runAsNonRoot"], false); + assert_eq!( + sidecar["securityContext"]["allowPrivilegeEscalation"], + false + ); + for container_name in ["agent", SUPERVISOR_NETWORK_SIDECAR_NAME] { let container = containers .iter() @@ -3391,6 +3441,12 @@ mod tests { && mount["mountPath"] == SIDECAR_TLS_MOUNT_PATH })); } + let init_containers = pod_template["spec"]["initContainers"].as_array().unwrap(); + let network_init = init_containers + .iter() + .find(|container| container["name"] == SUPERVISOR_NETWORK_INIT_CONTAINER_NAME) + .unwrap(); + assert_eq!(network_init["command"][3], "0"); } #[test] diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index d6f06c06ae..43e513a135 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -366,18 +366,7 @@ fn run_network_init( sidecar_state_dir: &str, sidecar_tls_dir: &str, ) -> Result<()> { - if proxy_user_id < openshell_policy::MIN_SANDBOX_UID { - return Err(miette::miette!( - "--proxy-uid must be at least {}", - openshell_policy::MIN_SANDBOX_UID - )); - } - if proxy_primary_group_id < openshell_policy::MIN_SANDBOX_UID { - return Err(miette::miette!( - "--proxy-gid must be at least {}", - openshell_policy::MIN_SANDBOX_UID - )); - } + validate_network_init_ids(proxy_user_id, proxy_primary_group_id)?; let sidecar_state_dir = Path::new(sidecar_state_dir); let sidecar_tls_dir = Path::new(sidecar_tls_dir); @@ -406,6 +395,23 @@ fn run_network_init( openshell_supervisor_process::netns::install_sidecar_bypass_rules(proxy_user_id) } +#[cfg(target_os = "linux")] +fn validate_network_init_ids(proxy_user_id: u32, proxy_primary_group_id: u32) -> Result<()> { + if proxy_user_id != 0 && proxy_user_id < openshell_policy::MIN_SANDBOX_UID { + return Err(miette::miette!( + "--proxy-uid must be 0 or at least {}", + openshell_policy::MIN_SANDBOX_UID + )); + } + if proxy_primary_group_id < openshell_policy::MIN_SANDBOX_UID { + return Err(miette::miette!( + "--proxy-gid must be at least {}", + openshell_policy::MIN_SANDBOX_UID + )); + } + Ok(()) +} + #[cfg(not(target_os = "linux"))] fn run_network_init( _proxy_uid: u32, @@ -684,4 +690,21 @@ mod tests { assert_eq!(SIDECAR_CLIENT_TLS_DIR_MODE, 0o750); assert_eq!(SIDECAR_CLIENT_TLS_FILE_MODE, 0o400); } + + #[cfg(target_os = "linux")] + #[test] + fn network_init_accepts_root_proxy_uid_for_binary_aware_sidecar() { + validate_network_init_ids(0, openshell_policy::MIN_SANDBOX_UID).unwrap(); + } + + #[cfg(target_os = "linux")] + #[test] + fn network_init_still_rejects_low_non_root_proxy_ids() { + let uid_err = + validate_network_init_ids(999, openshell_policy::MIN_SANDBOX_UID).unwrap_err(); + assert!(uid_err.to_string().contains("--proxy-uid")); + + let gid_err = validate_network_init_ids(0, 999).unwrap_err(); + assert!(gid_err.to_string().contains("--proxy-gid")); + } } diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 8426060807..881b7f24fa 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -237,8 +237,8 @@ add `ci/values-spire.yaml` to the OpenShell release values files. | supervisor.image.pullPolicy | string | `""` | Supervisor image pull policy. Defaults to the gateway image pull policy when empty. | | supervisor.image.repository | string | `"ghcr.io/nvidia/openshell/supervisor"` | Supervisor image repository. | | supervisor.image.tag | string | `""` | Supervisor image tag. Defaults to the chart appVersion when empty. | -| supervisor.sidecar.processBinaryAwareNetworkPolicy | bool | `true` | Keep process/binary-aware network policy enabled in sidecar topology. When false, the network sidecar drops the SYS_PTRACE capability used for cross-UID /proc inspection and enforces endpoint/L7 policy without matching policy.binaries. | -| supervisor.sidecar.proxyUid | int | `1337` | UID for the long-running network sidecar in sidecar topology. The network init container installs nftables rules that exempt this UID. | +| supervisor.sidecar.processBinaryAwareNetworkPolicy | bool | `true` | Keep process/binary-aware network policy enabled in sidecar topology. When false, the network sidecar runs as proxyUid, drops the extra /proc inspection capabilities, and enforces endpoint/L7 policy without matching policy.binaries. | +| supervisor.sidecar.proxyUid | int | `1337` | UID for relaxed long-running network sidecars in sidecar topology. Strict process/binary-aware sidecars run as UID 0 so Kubernetes grants the required /proc inspection capabilities into the effective set. The network init container installs nftables rules that exempt the effective sidecar UID. | | supervisor.sideloadMethod | string | `""` | How the supervisor binary is delivered into sandbox pods. Empty (default) = auto-detect from cluster version: K8s >= v1.35 -> "image-volume" (ImageVolume enabled by default; GA in v1.36) K8s < v1.35 -> "init-container" (copies via init container + emptyDir) On K8s v1.33-v1.34 with the ImageVolume feature gate manually enabled, set this to "image-volume" explicitly. | | supervisor.topology | string | `"combined"` | Supervisor pod topology for Kubernetes sandboxes. "combined" runs the current single supervisor container in the agent pod. "sidecar" runs network enforcement in a dedicated sidecar and the process supervisor as a low-capability wrapper in the agent container. | | tolerations | list | `[]` | Tolerations for the gateway pod. | diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index e18e3b3586..b3e07ac644 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -50,13 +50,16 @@ supervisor: # supervisor as a low-capability wrapper in the agent container. topology: "combined" sidecar: - # -- UID for the long-running network sidecar in sidecar topology. The - # network init container installs nftables rules that exempt this UID. + # -- UID for relaxed long-running network sidecars in sidecar topology. + # Strict process/binary-aware sidecars run as UID 0 so Kubernetes grants + # the required /proc inspection capabilities into the effective set. The + # network init container installs nftables rules that exempt the effective + # sidecar UID. proxyUid: 1337 # -- Keep process/binary-aware network policy enabled in sidecar topology. - # When false, the network sidecar drops the SYS_PTRACE capability used for - # cross-UID /proc inspection and enforces endpoint/L7 policy without - # matching policy.binaries. + # When false, the network sidecar runs as proxyUid, drops the extra /proc + # inspection capabilities, and enforces endpoint/L7 policy without matching + # policy.binaries. processBinaryAwareNetworkPolicy: true # -- Image pull secrets attached to gateway and helper pods. diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 451a5ead5f..04b0856286 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -206,12 +206,15 @@ provider_spiffe_workload_api_socket_path = "/spiffe-workload-api/spire-agent.soc # sandbox_gid = 1500 [openshell.drivers.kubernetes.sidecar] -# UID used by the long-running network sidecar. In sidecar topology the -# network init container installs nftables rules that exempt this UID. +# UID used by relaxed long-running network sidecars. Strict process/binary-aware +# sidecars run as UID 0 so Kubernetes grants the required /proc inspection +# capabilities into the effective set. In sidecar topology the network init +# container installs nftables rules that exempt the effective sidecar UID. proxy_uid = 1337 # Keep process/binary-aware network policy enabled in sidecar topology. Set -# false to drop the sidecar's SYS_PTRACE capability used for cross-UID /proc -# inspection and enforce endpoint/L7 policy without matching policy.binaries. +# false to run the sidecar as proxy_uid, drop the sidecar's extra /proc +# inspection capabilities, and enforce endpoint/L7 policy without matching +# policy.binaries. process_binary_aware_network_policy = true ```