Spread egress-lane VMs over several CNI networks

docs/cluster.md

@@ -179,7 +179,7 @@ fully from it:
 
 | state | source of truth | survives restart |
 |---|---|---|
-| operator config (`tenants`, `secrets`, egress policies, `bridge`/`network`, `mesh`, `preview_secret`, `egress_ca`) | `config.json` (human/deploy-tool owned) | re-read at boot |
+| operator config (`tenants`, `secrets`, egress policies, `bridge`/`networks`, `mesh`, `preview_secret`, `egress_ca`) | `config.json` (human/deploy-tool owned) | re-read at boot |
 | API-applied pool targets (`PUT /v1/pools`) | `<data_dir>/pools.json` (machine owned) | yes |
 | claims | the claims journal + `Reconcile` | yes |
 | placement hints (warm counts, template sets) | gossip | rebuilt |

docs/deploy.md

@@ -68,7 +68,7 @@ sandboxd reads one JSON file (`-config`, default
 | `restore_mode` | unset | clone and wake-restore memory mode: `copy`, `ondemand`, or `mmap`; use `mmap` for dense pools |
 | `no_direct_io` | false | use buffered writable disks for Cloud Hypervisor cold boots and clones; recommended for dense ephemeral pools to avoid direct-I/O CoW journal contention |
 | `advertise_addr` | = `listen` | the host:port clients reach this node at; returned as a claim's owner address and gossiped to peers. Must be routable when `listen` is a wildcard |
-| `bridge` / `network` | unset | egress-lane attachment: a host bridge device, or a CNI conflist name. Mutually exclusive; with neither set the node serves only the no-network lane. [Guarded egress](egress.md) needs the bridge form and rejects a CNI network at load |
+| `bridge` / `networks` | unset | egress-lane attachment: a host bridge device, or a list of CNI conflist names. Mutually exclusive; with neither set the node serves only the no-network lane. Each conflist attaches its own Linux bridge and a bridge holds at most 1024 ports (kernel `BR_MAX_PORTS`), so N conflists raise the node's egress ceiling to N×1024 — VMs spread over them by a stable hash of the VM name, so size the list with headroom (the spread is statistical, not exact). [Guarded egress](egress.md) needs the bridge form and rejects a CNI network at load |
 | `egress_ca` | unset | [HTTPS-interception](egress.md#https-interception) PKI: `root_cert` (the cluster root baked into intercepted guests; may bundle old+new roots during rotation) plus this node's `intermediate_cert`/`intermediate_key` from `sandboxd ca issue-intermediate`. Required when any pool rule sets `intercept` |
 | `api_token` | unset | the operator (root) credential: when set, guards the node-level endpoints (Bearer) with full access, including release-by-id cleanup. Per-sandbox tokens guard ordinary sandbox-scoped calls |
 | `tenants` | unset | multi-tenant tokens next to `api_token`: `[{"name": "acme", "token": "…", "max_claims": 50}]`. A tenant token reaches the resource-creating verbs (claim, fork, promote, checkpoint, preview) and everything it creates is stamped with the tenant name; operator surfaces (`GET /v1/sandboxes` and the per-id reads under it, `GET /v1/info`, `PUT /v1/pools`, `POST/DELETE /v1/drain`, `/metrics`) answer it 403. `max_claims` (0 = unlimited) caps that tenant's live claims next to the node-wide cap. Requires `api_token` set (operator surfaces need it). Names and tokens must be unique, tokens distinct from `api_token`. On a cluster all nodes must carry the same tenants set (the SDK replays a tenant token across a redirect; a peer missing that tenant answers 401), and per-node caps mean a tenant's effective cluster limit is `max_claims` × nodes. Empty = exactly the single-token behavior |

docs/egress.md

@@ -84,7 +84,7 @@ domain policy first; the allow-list widens the IP gate only.
   bridge they do not share with an untrusted listener.
 - **Bridge lane only (egress lane).** A CNI network's tap lives in the VM netns,
   out of reach of the root-netns lock, so a guarded egress *lane* needs a bridge
-  and is rejected on a CNI `network`. None-lane policies ride the proxy and work
+  and is rejected on CNI `networks`. None-lane policies ride the proxy and work
   on either. A bridge egress lane locks every NIC default-deny, even with no
   policy configured.
 - **No custom NAT64/DNS64 prefix routed to the host.** The SSRF guard folds the

e2e/cmd/egresssmoke/main.go

@@ -23,6 +23,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/cocoonstack/sandbox/e2e/internal/harness"
 	sandbox "github.com/cocoonstack/sandbox/sdk/go"
 )
 
@@ -58,14 +59,10 @@ func run(addr, token, template, wantToken, netShape, reach, nicAddr, echo string
 	defer func() { _ = origin.Close() }()
 	target := fmt.Sprintf("http://%s:%d/", reach, port)
 
-	c, err := sandbox.Connect(addr, sandbox.WithAPIToken(token))
+	_, sb, err := harness.Claim(ctx, addr, token, template, sandbox.WithNetwork(sandbox.NetShape(netShape)), sandbox.WithSize(sandbox.Small))
 	if err != nil {
 		return err
 	}
-	sb, err := c.New(ctx, template, sandbox.WithNetwork(sandbox.NetShape(netShape)), sandbox.WithSize(sandbox.Small))
-	if err != nil {
-		return fmt.Errorf("claim: %w", err)
-	}
 	defer func() { _ = sb.Close() }()
 	fmt.Printf("  claimed %s-lane sandbox %s\n", netShape, sb.ID)
 

e2e/cmd/interceptsmoke/main.go

@@ -13,6 +13,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/cocoonstack/sandbox/e2e/internal/harness"
 	sandbox "github.com/cocoonstack/sandbox/sdk/go"
 )
 
@@ -36,14 +37,10 @@ func run(addr, token, template, echo, secret, issuer string) error {
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
 	defer cancel()
 
-	c, err := sandbox.Connect(addr, sandbox.WithAPIToken(token))
+	_, sb, err := harness.Claim(ctx, addr, token, template, sandbox.WithNetwork(sandbox.NetNone), sandbox.WithSize(sandbox.Small))
 	if err != nil {
 		return err
 	}
-	sb, err := c.New(ctx, template, sandbox.WithNetwork(sandbox.NetNone), sandbox.WithSize(sandbox.Small))
-	if err != nil {
-		return fmt.Errorf("claim: %w", err)
-	}
 	defer func() { _ = sb.Close() }()
 	fmt.Printf("  claimed none-lane sandbox %s\n", sb.ID)
 

e2e/cmd/lifecycle/main.go

@@ -12,6 +12,7 @@ import (
 	"os"
 	"time"
 
+	"github.com/cocoonstack/sandbox/e2e/internal/harness"
 	sandbox "github.com/cocoonstack/sandbox/sdk/go"
 )
 
@@ -34,15 +35,10 @@ func main() {
 
 func run(addr, token, template, netShape string, wait time.Duration) error {
 	ctx := context.Background()
-	client, err := sandbox.Connect(addr, sandbox.WithAPIToken(token))
+	client, sb, err := harness.Claim(ctx, addr, token, template, sandbox.WithNetwork(sandbox.NetShape(netShape)))
 	if err != nil {
 		return err
 	}
-
-	sb, err := client.New(ctx, template, sandbox.WithNetwork(sandbox.NetShape(netShape)))
-	if err != nil {
-		return fmt.Errorf("claim: %w", err)
-	}
 	id := sb.ID
 	marker := fmt.Appendf(nil, "archive-marker-%d", time.Now().UnixNano())
 	if err = sb.WriteFile(ctx, markerPath, marker, nil); err != nil {

e2e/cmd/pullbench/main.go

@@ -11,6 +11,7 @@ import (
 	"os"
 	"time"
 
+	"github.com/cocoonstack/sandbox/e2e/internal/harness"
 	sandbox "github.com/cocoonstack/sandbox/sdk/go"
 )
 
@@ -30,14 +31,10 @@ func main() {
 func run(addr, token, template string, size, n int) error {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
 	defer cancel()
-	client, err := sandbox.Connect(addr, sandbox.WithAPIToken(token))
+	_, sb, err := harness.Claim(ctx, addr, token, template, sandbox.WithNetwork(sandbox.NetNone))
 	if err != nil {
 		return err
 	}
-	sb, err := client.New(ctx, template, sandbox.WithNetwork(sandbox.NetNone))
-	if err != nil {
-		return fmt.Errorf("claim: %w", err)
-	}
 	defer func() { _ = sb.Close() }()
 
 	if _, err := sb.Exec(ctx, "sh", "-c",

e2e/cmd/pushbench/main.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"time"
 
+	"github.com/cocoonstack/sandbox/e2e/internal/harness"
 	sandbox "github.com/cocoonstack/sandbox/sdk/go"
 )
 
@@ -28,14 +29,10 @@ func main() {
 func run(addr, token, template string, size, n int) error {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
 	defer cancel()
-	client, err := sandbox.Connect(addr, sandbox.WithAPIToken(token))
+	_, sb, err := harness.Claim(ctx, addr, token, template, sandbox.WithNetwork(sandbox.NetNone))
 	if err != nil {
 		return err
 	}
-	sb, err := client.New(ctx, template, sandbox.WithNetwork(sandbox.NetNone))
-	if err != nil {
-		return fmt.Errorf("claim: %w", err)
-	}
 	defer func() { _ = sb.Close() }()
 
 	if _, err := sb.Exec(ctx, "mkdir", "-p", "/work/bench"); err != nil {

e2e/cmd/rpcbench/main.go

@@ -18,6 +18,7 @@ import (
 	"slices"
 	"time"
 
+	"github.com/cocoonstack/sandbox/e2e/internal/harness"
 	"github.com/cocoonstack/sandbox/protocol/wire"
 	sandbox "github.com/cocoonstack/sandbox/sdk/go"
 	"github.com/cocoonstack/sandbox/sdk/go/silkd"
@@ -38,14 +39,10 @@ func main() {
 func run(addr, token, template string, n int) error {
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
 	defer cancel()
-	client, err := sandbox.Connect(addr, sandbox.WithAPIToken(token))
+	_, sb, err := harness.Claim(ctx, addr, token, template, sandbox.WithNetwork(sandbox.NetNone))
 	if err != nil {
 		return err
 	}
-	sb, err := client.New(ctx, template, sandbox.WithNetwork(sandbox.NetNone))
-	if err != nil {
-		return fmt.Errorf("claim: %w", err)
-	}
 	defer func() { _ = sb.Close() }()
 
 	dial := func() (net.Conn, error) { return dialAgent(ctx, sb.Owner(), sb.ID, sb.Token()) }

e2e/cmd/smoke/main.go

@@ -21,6 +21,7 @@ import (
 	"testing/iotest"
 	"time"
 
+	"github.com/cocoonstack/sandbox/e2e/internal/harness"
 	"github.com/cocoonstack/sandbox/protocol/wire"
 	sandbox "github.com/cocoonstack/sandbox/sdk/go"
 )
@@ -48,14 +49,10 @@ func run(addr, token, template, lspTemplate string, egress bool) error {
 	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	defer cancel()
 
-	client, err := sandbox.Connect(addr, sandbox.WithAPIToken(token))
-	if err != nil {
-		return err
-	}
 	// Explicitly the no-network lane: the git step asserts its typed error.
-	sb, err := client.New(ctx, template, sandbox.WithNetwork(sandbox.NetNone))
+	client, sb, err := harness.Claim(ctx, addr, token, template, sandbox.WithNetwork(sandbox.NetNone))
 	if err != nil {
-		return fmt.Errorf("claim: %w", err)
+		return err
 	}
 	defer func() { _ = sb.Close() }()
 	// Info is node-local: on a cluster the claim may have been redirected, so

e2e/fakeengine_test.go

@@ -31,7 +31,7 @@ type fakeEngine struct {
 
 func newFakeEngine(dir string) *fakeEngine {
 	return &fakeEngine{
-		real:      engine.New("cocoon", "", "", false, ""),
+		real:      engine.New("cocoon", "", nil, false, ""),
 		dir:       dir,
 		listeners: map[string]io.Closer{},
 		socks:     map[string]string{},

e2e/internal/harness/harness.go

@@ -0,0 +1,22 @@
+// Package harness carries the claim prologue shared by the e2e tools.
+package harness
+
+import (
+	"context"
+	"fmt"
+
+	sandbox "github.com/cocoonstack/sandbox/sdk/go"
+)
+
+// Claim connects to a node and claims one sandbox; the caller owns Close.
+func Claim(ctx context.Context, addr, token, template string, opts ...sandbox.Option) (*sandbox.Client, *sandbox.Sandbox, error) {
+	client, err := sandbox.Connect(addr, sandbox.WithAPIToken(token))
+	if err != nil {
+		return nil, nil, err
+	}
+	sb, err := client.New(ctx, template, opts...)
+	if err != nil {
+		return nil, nil, fmt.Errorf("claim: %w", err)
+	}
+	return client, sb, nil
+}